Lines Matching full:zone
41 * The basic ideas stem from similar slab/zone based allocators whose algorithms
114 * This is the zone and keg from which all zones are spawned.
132 * One zone is for slab headers that can represent a larger number of items,
133 * making the slabs themselves more efficient, and the other zone is for
146 * The initial hash tables come out of this zone so they can be allocated
211 * This structure is passed as the zone ctor arg so that I don't have to create
230 uma_zone_t zone; member
293 static void bucket_cache_reclaim(uma_zone_t zone, bool, int);
300 static inline void item_dtor(uma_zone_t zone, void *item, int size,
303 static void zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata,
307 static void zone_timeout(uma_zone_t zone, void *);
315 static int zone_alloc_limit(uma_zone_t zone, int count, int flags);
316 static void zone_free_limit(uma_zone_t zone, int count);
319 static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
320 static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
324 static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item);
326 static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
341 static uint64_t uma_zone_get_allocs(uma_zone_t zone);
347 static uint64_t uma_keg_get_allocs(uma_keg_t zone);
351 static bool uma_dbg_zskip(uma_zone_t zone, void *mem);
352 static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
353 static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);
375 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats");
388 * Select the slab zone for an offpage slab with the given maximum item count.
411 * For each zone, calculate the memory required for each bucket, consisting
431 * Given a desired number of entries for a bucket, return the zone from which
463 bucket_alloc(uma_zone_t zone, void *udata, int flags) in bucket_alloc() argument
475 * To limit bucket recursion we store the original zone flags in bucket_alloc()
479 * a bucket for a bucket zone so we do not allow infinite bucket in bucket_alloc()
484 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) in bucket_alloc()
485 udata = (void *)(uintptr_t)zone->uz_flags; in bucket_alloc()
493 ubz = bucket_zone_lookup(atomic_load_16(&zone->uz_bucket_size)); in bucket_alloc()
494 if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0) in bucket_alloc()
503 zone->uz_bucket_size_max); in bucket_alloc()
505 CTR3(KTR_UMA, "bucket_alloc: zone %s(%p) allocated bucket %p", in bucket_alloc()
506 zone->uz_name, zone, bucket); in bucket_alloc()
513 bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata) in bucket_free() argument
518 bucket_drain(zone, bucket); in bucket_free()
524 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) in bucket_free()
525 udata = (void *)(uintptr_t)zone->uz_flags; in bucket_free()
545 kasan_mark_item_valid(uma_zone_t zone, void *item) in kasan_mark_item_valid() argument
551 if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0) in kasan_mark_item_valid()
554 sz = zone->uz_size; in kasan_mark_item_valid()
556 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { in kasan_mark_item_valid()
567 kasan_mark_item_invalid(uma_zone_t zone, void *item) in kasan_mark_item_invalid() argument
573 if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0) in kasan_mark_item_invalid()
576 sz = roundup2(zone->uz_size, KASAN_SHADOW_SCALE); in kasan_mark_item_invalid()
577 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { in kasan_mark_item_invalid()
613 kasan_mark_item_valid(uma_zone_t zone __unused, void *item __unused) in kasan_mark_item_valid()
618 kasan_mark_item_invalid(uma_zone_t zone __unused, void *item __unused) in kasan_mark_item_invalid()
635 kmsan_mark_item_uninitialized(uma_zone_t zone, void *item) in kmsan_mark_item_uninitialized() argument
641 if ((zone->uz_flags & in kmsan_mark_item_uninitialized()
650 * zone and thus cannot safely be marked by UMA. in kmsan_mark_item_uninitialized()
657 if (zone->uz_keg->uk_init != NULL) { in kmsan_mark_item_uninitialized()
666 sz = zone->uz_size; in kmsan_mark_item_uninitialized()
667 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { in kmsan_mark_item_uninitialized()
682 kmsan_mark_item_uninitialized(uma_zone_t zone __unused, void *item __unused) in kmsan_mark_item_uninitialized()
691 zone_domain_lock(uma_zone_t zone, int domain) in zone_domain_lock() argument
696 zdom = ZDOM_GET(zone, domain); in zone_domain_lock()
702 if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max) in zone_domain_lock()
703 zone->uz_bucket_size++; in zone_domain_lock()
712 zone_domain_lowest(uma_zone_t zone, int pref) in zone_domain_lowest() argument
721 nitems = ZDOM_GET(zone, i)->uzd_nitems; in zone_domain_lowest()
740 zone_domain_highest(uma_zone_t zone, int pref) in zone_domain_highest() argument
746 if (ZDOM_GET(zone, pref)->uzd_nitems > BUCKET_MAX) in zone_domain_highest()
752 nitems = ZDOM_GET(zone, i)->uzd_nitems; in zone_domain_highest()
787 * zone's caches. If a bucket is found the zone is not locked on return.
790 zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim) in zone_fetch_bucket() argument
803 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && in zone_fetch_bucket()
805 if (!smr_poll(zone->uz_smr, bucket->ub_seq, false)) in zone_fetch_bucket()
808 dtor = (zone->uz_dtor != NULL) || UMA_ALWAYS_CTORDTOR; in zone_fetch_bucket()
845 item_dtor(zone, bucket->ub_bucket[i], zone->uz_size, in zone_fetch_bucket()
853 * whether the bucket's contents should be counted as part of the zone's working
857 zone_put_bucket(uma_zone_t zone, int domain, uma_bucket_t bucket, void *udata, in zone_put_bucket() argument
865 zdom = zone_domain_lock(zone, domain); in zone_put_bucket()
871 if (__predict_true(zdom->uzd_nitems < zone->uz_bucket_max)) { in zone_put_bucket()
901 bucket_free(zone, bucket, udata); in zone_put_bucket()
1044 * Attempt to fetch a bucket from a zone on behalf of the current cpu cache.
1047 cache_fetch_bucket(uma_zone_t zone, uma_cache_t cache, int domain) in cache_fetch_bucket() argument
1056 zdom = ZDOM_GET(zone, domain); in cache_fetch_bucket()
1062 !smr_poll(zone->uz_smr, seq, false)) in cache_fetch_bucket()
1066 * Check the zone's cache of buckets. in cache_fetch_bucket()
1068 zdom = zone_domain_lock(zone, domain); in cache_fetch_bucket()
1069 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) in cache_fetch_bucket()
1077 zone_log_warning(uma_zone_t zone) in zone_log_warning() argument
1081 if (!zone_warnings || zone->uz_warning == NULL) in zone_log_warning()
1084 if (ratecheck(&zone->uz_ratecheck, &warninterval)) in zone_log_warning()
1085 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning); in zone_log_warning()
1089 zone_maxaction(uma_zone_t zone) in zone_maxaction() argument
1092 if (zone->uz_maxaction.ta_func != NULL) in zone_maxaction()
1093 taskqueue_enqueue(taskqueue_thread, &zone->uz_maxaction); in zone_maxaction()
1118 * Update the working set size estimates for the zone's bucket cache.
1170 zone_timeout(uma_zone_t zone, void *unused) in zone_timeout() argument
1175 if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) in zone_timeout()
1178 keg = zone->uz_keg; in zone_timeout()
1224 if ((zone->uz_flags & (UMA_ZONE_UNMANAGED | UMA_ZONE_NOTRIM)) == 0) { in zone_timeout()
1226 if (bucket_cache_reclaim_domain(zone, false, false, i) && in zone_timeout()
1227 (zone->uz_flags & UMA_ZFLAG_CACHE) == 0) in zone_timeout()
1228 keg_drain(zone->uz_keg, i); in zone_timeout()
1337 * zone The zone to free to, must be unlocked.
1344 bucket_drain(uma_zone_t zone, uma_bucket_t bucket) in bucket_drain() argument
1351 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && in bucket_drain()
1353 smr_wait(zone->uz_smr, bucket->ub_seq); in bucket_drain()
1356 item_dtor(zone, bucket->ub_bucket[i], in bucket_drain()
1357 zone->uz_size, NULL, SKIP_NONE); in bucket_drain()
1359 if (zone->uz_fini) in bucket_drain()
1361 kasan_mark_item_valid(zone, bucket->ub_bucket[i]); in bucket_drain()
1362 zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); in bucket_drain()
1363 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); in bucket_drain()
1365 zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); in bucket_drain()
1366 if (zone->uz_max_items > 0) in bucket_drain()
1367 zone_free_limit(zone, bucket->ub_cnt); in bucket_drain()
1375 * Drains the per cpu caches for a zone.
1377 * NOTE: This may only be called while the zone is being torn down, and not
1382 * zone The zone to drain, must be unlocked.
1388 cache_drain(uma_zone_t zone) in cache_drain() argument
1397 * tearing down the zone anyway. I.e., there will be no further use in cache_drain()
1400 * XXX: It would good to be able to assert that the zone is being in cache_drain()
1404 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in cache_drain()
1405 seq = smr_advance(zone->uz_smr); in cache_drain()
1407 cache = &zone->uz_cpu[cpu]; in cache_drain()
1410 bucket_free(zone, bucket, NULL); in cache_drain()
1414 bucket_free(zone, bucket, NULL); in cache_drain()
1419 bucket_free(zone, bucket, NULL); in cache_drain()
1422 bucket_cache_reclaim(zone, true, UMA_ANYDOMAIN); in cache_drain()
1426 cache_shrink(uma_zone_t zone, void *unused) in cache_shrink() argument
1429 if (zone->uz_flags & UMA_ZFLAG_INTERNAL) in cache_shrink()
1432 ZONE_LOCK(zone); in cache_shrink()
1433 zone->uz_bucket_size = in cache_shrink()
1434 (zone->uz_bucket_size_min + zone->uz_bucket_size) / 2; in cache_shrink()
1435 ZONE_UNLOCK(zone); in cache_shrink()
1439 cache_drain_safe_cpu(uma_zone_t zone, void *unused) in cache_drain_safe_cpu() argument
1445 if (zone->uz_flags & UMA_ZFLAG_INTERNAL) in cache_drain_safe_cpu()
1450 cache = &zone->uz_cpu[curcpu]; in cache_drain_safe_cpu()
1455 * Don't flush SMR zone buckets. This leaves the zone without a in cache_drain_safe_cpu()
1458 if ((zone->uz_flags & UMA_ZONE_SMR) == 0) { in cache_drain_safe_cpu()
1465 zone_free_bucket(zone, b1, NULL, domain, false); in cache_drain_safe_cpu()
1467 zone_free_bucket(zone, b2, NULL, domain, false); in cache_drain_safe_cpu()
1471 zone_free_bucket(zone, b3, NULL, domain, false); in cache_drain_safe_cpu()
1476 * Safely drain per-CPU caches of a zone(s) to alloc bucket.
1480 * Zone lock must not be held on call this function.
1483 pcpu_cache_drain_safe(uma_zone_t zone) in pcpu_cache_drain_safe() argument
1490 if (zone) in pcpu_cache_drain_safe()
1491 cache_shrink(zone, NULL); in pcpu_cache_drain_safe()
1500 if (zone) in pcpu_cache_drain_safe()
1501 cache_drain_safe_cpu(zone, NULL); in pcpu_cache_drain_safe()
1511 * Reclaim cached buckets from a zone. All buckets are reclaimed if the caller
1516 bucket_cache_reclaim_domain(uma_zone_t zone, bool drain, bool trim, int domain) in bucket_cache_reclaim_domain() argument
1527 zdom = ZDOM_GET(zone, domain); in bucket_cache_reclaim_domain()
1528 if ((zone->uz_flags & UMA_ZONE_SMR) == 0 || drain) { in bucket_cache_reclaim_domain()
1529 ZONE_CROSS_LOCK(zone); in bucket_cache_reclaim_domain()
1532 ZONE_CROSS_UNLOCK(zone); in bucket_cache_reclaim_domain()
1534 bucket_free(zone, bucket, NULL); in bucket_cache_reclaim_domain()
1538 * If we were asked to drain the zone, we are done only once in bucket_cache_reclaim_domain()
1540 * excess of the zone's estimated working set size. Multiple in bucket_cache_reclaim_domain()
1559 bucket = zone_fetch_bucket(zone, zdom, true); in bucket_cache_reclaim_domain()
1562 bucket_free(zone, bucket, NULL); in bucket_cache_reclaim_domain()
1571 bucket_cache_reclaim(uma_zone_t zone, bool drain, int domain) in bucket_cache_reclaim() argument
1576 * Shrink the zone bucket size to ensure that the per-CPU caches in bucket_cache_reclaim()
1579 if (zone->uz_bucket_size > zone->uz_bucket_size_min) in bucket_cache_reclaim()
1580 zone->uz_bucket_size--; in bucket_cache_reclaim()
1583 (zone->uz_flags & UMA_ZONE_ROUNDROBIN) == 0) { in bucket_cache_reclaim()
1584 bucket_cache_reclaim_domain(zone, drain, true, domain); in bucket_cache_reclaim()
1587 bucket_cache_reclaim_domain(zone, drain, true, i); in bucket_cache_reclaim()
1710 zone_reclaim(uma_zone_t zone, int domain, int waitok, bool drain) in zone_reclaim() argument
1714 * zone_dtor(), which removes the zone from global lists before in zone_reclaim()
1717 * The zone may be destroyed while sleeping, so only zone_dtor() should in zone_reclaim()
1720 ZONE_LOCK(zone); in zone_reclaim()
1722 while (zone->uz_reclaimers > 0) in zone_reclaim()
1723 msleep(zone, ZONE_LOCKPTR(zone), PVM, "zonedrain", 1); in zone_reclaim()
1725 zone->uz_reclaimers++; in zone_reclaim()
1726 ZONE_UNLOCK(zone); in zone_reclaim()
1727 bucket_cache_reclaim(zone, drain, domain); in zone_reclaim()
1729 if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) in zone_reclaim()
1730 keg_drain(zone->uz_keg, domain); in zone_reclaim()
1731 ZONE_LOCK(zone); in zone_reclaim()
1732 zone->uz_reclaimers--; in zone_reclaim()
1733 if (zone->uz_reclaimers == 0) in zone_reclaim()
1734 wakeup(zone); in zone_reclaim()
1735 ZONE_UNLOCK(zone); in zone_reclaim()
1752 keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, in keg_alloc_slab() argument
1780 * first time they are added to a zone. in keg_alloc_slab()
1796 /* zone is passed for legacy reasons. */ in keg_alloc_slab()
1798 mem = keg->uk_allocf(zone, size, domain, &sflags, aflags); in keg_alloc_slab()
1823 zone, slab); in keg_alloc_slab()
1876 startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in startup_alloc() argument
1942 page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in page_alloc() argument
1954 pcpu_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in pcpu_page_alloc() argument
2019 noobj_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags, in noobj_alloc() argument
2030 keg = zone->uz_keg; in noobj_alloc()
2069 contig_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in contig_alloc() argument
2080 uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags, in uma_small_alloc() argument
2127 * Frees pcpu zone allocations
2288 * keg The zone we should initialize
2469 uma_zone_t zone; in keg_ctor() local
2489 * The primary zone is passed to us at keg-creation time. in keg_ctor()
2491 zone = arg->zone; in keg_ctor()
2492 keg->uk_name = zone->uz_name; in keg_ctor()
2571 ("zone %s ipers %d rsize %d size %d slab won't fit", in keg_ctor()
2572 zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); in keg_ctor()
2578 CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)", keg, zone->uz_name, zone); in keg_ctor()
2580 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); in keg_ctor()
2589 zone_kva_available(uma_zone_t zone, void *unused) in zone_kva_available() argument
2593 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) in zone_kva_available()
2595 KEG_GET(zone, keg); in zone_kva_available()
2610 zone_alloc_counters(uma_zone_t zone, void *unused) in zone_alloc_counters() argument
2613 zone->uz_allocs = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2614 zone->uz_frees = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2615 zone->uz_fails = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2616 zone->uz_xdomain = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2620 zone_alloc_sysctl(uma_zone_t zone, void *unused) in zone_alloc_sysctl() argument
2627 static const char *nokeg = "cache zone"; in zone_alloc_sysctl()
2631 * Make a sysctl safe copy of the zone name by removing in zone_alloc_sysctl()
2635 if (zone->uz_namecnt != 0) { in zone_alloc_sysctl()
2637 for (i = 1, cnt = zone->uz_namecnt; cnt != 0; i++) in zone_alloc_sysctl()
2639 zone->uz_ctlname = malloc(strlen(zone->uz_name) + i + 1, in zone_alloc_sysctl()
2641 sprintf(zone->uz_ctlname, "%s_%d", zone->uz_name, in zone_alloc_sysctl()
2642 zone->uz_namecnt); in zone_alloc_sysctl()
2644 zone->uz_ctlname = strdup(zone->uz_name, M_UMA); in zone_alloc_sysctl()
2645 for (c = zone->uz_ctlname; *c != '\0'; c++) in zone_alloc_sysctl()
2652 zone->uz_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_vm_uma), in zone_alloc_sysctl()
2653 OID_AUTO, zone->uz_ctlname, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); in zone_alloc_sysctl()
2654 oid = zone->uz_oid; in zone_alloc_sysctl()
2656 "size", CTLFLAG_RD, &zone->uz_size, 0, "Allocation size"); in zone_alloc_sysctl()
2659 zone, 0, sysctl_handle_uma_zone_flags, "A", in zone_alloc_sysctl()
2662 "bucket_size", CTLFLAG_RD, &zone->uz_bucket_size, 0, in zone_alloc_sysctl()
2665 "bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0, in zone_alloc_sysctl()
2671 if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) in zone_alloc_sysctl()
2675 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, in zone_alloc_sysctl()
2677 keg = zone->uz_keg; in zone_alloc_sysctl()
2678 if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) { in zone_alloc_sysctl()
2722 * Information about zone limits. in zone_alloc_sysctl()
2724 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, in zone_alloc_sysctl()
2728 zone, 0, sysctl_handle_uma_zone_items, "QU", in zone_alloc_sysctl()
2731 "max_items", CTLFLAG_RD, &zone->uz_max_items, 0, in zone_alloc_sysctl()
2734 "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0, in zone_alloc_sysctl()
2737 "sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0, in zone_alloc_sysctl()
2738 "Total zone limit sleeps"); in zone_alloc_sysctl()
2740 "bucket_max", CTLFLAG_RD, &zone->uz_bucket_max, 0, in zone_alloc_sysctl()
2744 * Per-domain zone information. in zone_alloc_sysctl()
2746 domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), in zone_alloc_sysctl()
2749 zdom = ZDOM_GET(zone, i); in zone_alloc_sysctl()
2779 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, in zone_alloc_sysctl()
2783 zone, 1, sysctl_handle_uma_zone_cur, "I", in zone_alloc_sysctl()
2787 zone, 0, sysctl_handle_uma_zone_allocs, "QU", in zone_alloc_sysctl()
2791 zone, 0, sysctl_handle_uma_zone_frees, "QU", in zone_alloc_sysctl()
2794 "fails", CTLFLAG_RD, &zone->uz_fails, in zone_alloc_sysctl()
2797 "xdomain", CTLFLAG_RD, &zone->uz_xdomain, in zone_alloc_sysctl()
2807 zone_count(uma_zone_t zone, void *arg) in zone_count() argument
2817 if (strcmp(zone->uz_name, cnt->name) == 0) in zone_count()
2819 zone->uz_namecnt + 1); in zone_count()
2823 zone_update_caches(uma_zone_t zone) in zone_update_caches() argument
2828 cache_set_uz_size(&zone->uz_cpu[i], zone->uz_size); in zone_update_caches()
2829 cache_set_uz_flags(&zone->uz_cpu[i], zone->uz_flags); in zone_update_caches()
2834 * Zone header ctor. This initializes all fields, locks, etc.
2845 uma_zone_t zone = mem; in zone_ctor() local
2850 bzero(zone, size); in zone_ctor()
2851 zone->uz_name = arg->name; in zone_ctor()
2852 zone->uz_ctor = arg->ctor; in zone_ctor()
2853 zone->uz_dtor = arg->dtor; in zone_ctor()
2854 zone->uz_init = NULL; in zone_ctor()
2855 zone->uz_fini = NULL; in zone_ctor()
2856 zone->uz_sleeps = 0; in zone_ctor()
2857 zone->uz_bucket_size = 0; in zone_ctor()
2858 zone->uz_bucket_size_min = 0; in zone_ctor()
2859 zone->uz_bucket_size_max = BUCKET_MAX; in zone_ctor()
2860 zone->uz_flags = (arg->flags & UMA_ZONE_SMR); in zone_ctor()
2861 zone->uz_warning = NULL; in zone_ctor()
2863 zone->uz_bucket_max = ULONG_MAX; in zone_ctor()
2864 timevalclear(&zone->uz_ratecheck); in zone_ctor()
2870 zone->uz_namecnt = cnt.count; in zone_ctor()
2871 ZONE_CROSS_LOCK_INIT(zone); in zone_ctor()
2874 zdom = ZDOM_GET(zone, i); in zone_ctor()
2875 ZDOM_LOCK_INIT(zone, zdom, (arg->flags & UMA_ZONE_MTXCLASS)); in zone_ctor()
2881 zone->uz_flags |= UMA_ZFLAG_TRASH | UMA_ZFLAG_CTORDTOR; in zone_ctor()
2888 * This is a pure cache zone, no kegs. in zone_ctor()
2892 ("zone_ctor: Import specified for non-cache zone.")); in zone_ctor()
2893 zone->uz_flags = arg->flags; in zone_ctor()
2894 zone->uz_size = arg->size; in zone_ctor()
2895 zone->uz_import = arg->import; in zone_ctor()
2896 zone->uz_release = arg->release; in zone_ctor()
2897 zone->uz_arg = arg->arg; in zone_ctor()
2904 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) == 0) in zone_ctor()
2905 zone->uz_flags |= UMA_ZONE_ROUNDROBIN; in zone_ctor()
2908 LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link); in zone_ctor()
2914 * Use the regular zone/keg/slab allocator. in zone_ctor()
2916 zone->uz_import = zone_import; in zone_ctor()
2917 zone->uz_release = zone_release; in zone_ctor()
2918 zone->uz_arg = zone; in zone_ctor()
2922 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0, in zone_ctor()
2923 ("Secondary zone requested UMA_ZFLAG_INTERNAL")); in zone_ctor()
2924 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); in zone_ctor()
2925 zone->uz_init = arg->uminit; in zone_ctor()
2926 zone->uz_fini = arg->fini; in zone_ctor()
2927 zone->uz_flags |= UMA_ZONE_SECONDARY; in zone_ctor()
2929 ZONE_LOCK(zone); in zone_ctor()
2932 LIST_INSERT_AFTER(z, zone, uz_link); in zone_ctor()
2936 ZONE_UNLOCK(zone); in zone_ctor()
2939 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini, in zone_ctor()
2952 karg.zone = zone; in zone_ctor()
2960 zone->uz_keg = keg; in zone_ctor()
2961 zone->uz_size = keg->uk_size; in zone_ctor()
2962 zone->uz_flags |= (keg->uk_flags & in zone_ctor()
2967 zone_alloc_counters(zone, NULL); in zone_ctor()
2969 zone_alloc_sysctl(zone, NULL); in zone_ctor()
2971 zone->uz_allocs = EARLY_COUNTER; in zone_ctor()
2972 zone->uz_frees = EARLY_COUNTER; in zone_ctor()
2973 zone->uz_fails = EARLY_COUNTER; in zone_ctor()
2977 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in zone_ctor()
2978 zone->uz_smr = smr_create(zone->uz_name, 0, 0); in zone_ctor()
2982 ("Invalid zone flag combination")); in zone_ctor()
2984 zone->uz_bucket_size_max = zone->uz_bucket_size = 0; in zone_ctor()
2986 zone->uz_bucket_size = BUCKET_MAX; in zone_ctor()
2988 zone->uz_bucket_size = 0; in zone_ctor()
2990 zone->uz_bucket_size = bucket_select(zone->uz_size); in zone_ctor()
2991 zone->uz_bucket_size_min = zone->uz_bucket_size; in zone_ctor()
2992 if (zone->uz_dtor != NULL || zone->uz_ctor != NULL) in zone_ctor()
2993 zone->uz_flags |= UMA_ZFLAG_CTORDTOR; in zone_ctor()
2994 zone_update_caches(zone); in zone_ctor()
3030 * Zone header dtor.
3038 uma_zone_t zone; in zone_dtor() local
3042 zone = (uma_zone_t)arg; in zone_dtor()
3044 sysctl_remove_oid(zone->uz_oid, 1, 1); in zone_dtor()
3046 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) in zone_dtor()
3047 cache_drain(zone); in zone_dtor()
3050 LIST_REMOVE(zone, uz_link); in zone_dtor()
3052 if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) { in zone_dtor()
3053 keg = zone->uz_keg; in zone_dtor()
3056 zone_reclaim(zone, UMA_ANYDOMAIN, M_WAITOK, true); in zone_dtor()
3061 if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) { in zone_dtor()
3062 keg = zone->uz_keg; in zone_dtor()
3068 counter_u64_free(zone->uz_allocs); in zone_dtor()
3069 counter_u64_free(zone->uz_frees); in zone_dtor()
3070 counter_u64_free(zone->uz_fails); in zone_dtor()
3071 counter_u64_free(zone->uz_xdomain); in zone_dtor()
3072 free(zone->uz_ctlname, M_UMA); in zone_dtor()
3074 ZDOM_LOCK_FINI(ZDOM_GET(zone, i)); in zone_dtor()
3075 ZONE_CROSS_LOCK_FINI(zone); in zone_dtor()
3082 uma_zone_t zone; in zone_foreach_unlocked() local
3085 LIST_FOREACH(zone, &keg->uk_zones, uz_link) in zone_foreach_unlocked()
3086 zfunc(zone, arg); in zone_foreach_unlocked()
3088 LIST_FOREACH(zone, &uma_cachezones, uz_link) in zone_foreach_unlocked()
3089 zfunc(zone, arg); in zone_foreach_unlocked()
3093 * Traverses every zone in the system and calls a callback
3096 * zfunc A pointer to a function which accepts a zone
3138 /* Allocate the zone of zones, zone of kegs, and zone of zones keg. */ in uma_startup1()
3152 /* "manually" create the initial zone */ in uma_startup1()
3211 /* Set up radix zone to use noobj_alloc. */ in uma_startup2()
3271 uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, in uma_kcreate() argument
3281 args.zone = zone; in uma_kcreate()
3330 /* This stuff is essential for the zone ctor */ in uma_zcreate()
3418 uma_zdestroy(uma_zone_t zone) in uma_zdestroy() argument
3426 zone->uz_fini == NULL && zone->uz_release == zone_release) in uma_zdestroy()
3429 zone_free_item(zones, zone, NULL, SKIP_NONE); in uma_zdestroy()
3434 uma_zwait(uma_zone_t zone) in uma_zwait() argument
3437 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in uma_zwait()
3438 uma_zfree_smr(zone, uma_zalloc_smr(zone, M_WAITOK)); in uma_zwait()
3439 else if ((zone->uz_flags & UMA_ZONE_PCPU) != 0) in uma_zwait()
3440 uma_zfree_pcpu(zone, uma_zalloc_pcpu(zone, M_WAITOK)); in uma_zwait()
3442 uma_zfree(zone, uma_zalloc(zone, M_WAITOK)); in uma_zwait()
3446 uma_zalloc_pcpu_arg(uma_zone_t zone, void *udata, int flags) in uma_zalloc_pcpu_arg() argument
3452 MPASS(zone->uz_flags & UMA_ZONE_PCPU); in uma_zalloc_pcpu_arg()
3454 item = uma_zalloc_arg(zone, udata, flags & ~M_ZERO); in uma_zalloc_pcpu_arg()
3461 bzero(zpcpu_get_cpu(pcpu_item, i), zone->uz_size); in uma_zalloc_pcpu_arg()
3463 bzero(item, zone->uz_size); in uma_zalloc_pcpu_arg()
3473 uma_zfree_pcpu_arg(uma_zone_t zone, void *pcpu_item, void *udata) in uma_zfree_pcpu_arg() argument
3478 MPASS(zone->uz_flags & UMA_ZONE_PCPU); in uma_zfree_pcpu_arg()
3486 uma_zfree_arg(zone, item, udata); in uma_zfree_pcpu_arg()
3490 item_ctor(uma_zone_t zone, int uz_flags, int size, void *udata, int flags, in item_ctor() argument
3497 kasan_mark_item_valid(zone, item); in item_ctor()
3498 kmsan_mark_item_uninitialized(zone, item); in item_ctor()
3501 skipdbg = uma_dbg_zskip(zone, item); in item_ctor()
3503 zone->uz_ctor != trash_ctor) in item_ctor()
3504 trash_ctor(item, size, zone, flags); in item_ctor()
3509 __predict_false(zone->uz_ctor != NULL) && in item_ctor()
3510 zone->uz_ctor(item, size, udata, flags) != 0) { in item_ctor()
3511 counter_u64_add(zone->uz_fails, 1); in item_ctor()
3512 zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); in item_ctor()
3517 uma_dbg_alloc(zone, NULL, item); in item_ctor()
3526 item_dtor(uma_zone_t zone, void *item, int size, void *udata, in item_dtor() argument
3532 skipdbg = uma_dbg_zskip(zone, item); in item_dtor()
3534 if ((zone->uz_flags & UMA_ZONE_MALLOC) != 0) in item_dtor()
3535 uma_dbg_free(zone, udata, item); in item_dtor()
3537 uma_dbg_free(zone, NULL, item); in item_dtor()
3541 if (zone->uz_dtor != NULL) in item_dtor()
3542 zone->uz_dtor(item, size, udata); in item_dtor()
3544 if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && in item_dtor()
3545 zone->uz_dtor != trash_dtor) in item_dtor()
3546 trash_dtor(item, size, zone); in item_dtor()
3549 kasan_mark_item_invalid(zone, item); in item_dtor()
3571 uma_zalloc_debug(uma_zone_t zone, void **itemp, void *udata, int flags) in uma_zalloc_debug() argument
3579 "uma_zalloc_debug: zone \"%s\"", zone->uz_name); in uma_zalloc_debug()
3588 KASSERT((zone->uz_flags & UMA_ZONE_PCPU) == 0 || (flags & M_ZERO) == 0, in uma_zalloc_debug()
3589 ("uma_zalloc_debug: allocating from a pcpu zone with M_ZERO")); in uma_zalloc_debug()
3618 if ((zone->uz_flags & (UMA_ZONE_SMR | UMA_ZFLAG_CACHE)) == 0 && in uma_zalloc_debug()
3619 memguard_cmp_zone(zone)) { in uma_zalloc_debug()
3621 item = memguard_alloc(zone->uz_size, flags); in uma_zalloc_debug()
3624 if (zone->uz_init != NULL && in uma_zalloc_debug()
3625 zone->uz_init(item, zone->uz_size, flags) != 0) { in uma_zalloc_debug()
3629 if (zone->uz_ctor != NULL && in uma_zalloc_debug()
3630 zone->uz_ctor(item, zone->uz_size, udata, in uma_zalloc_debug()
3632 counter_u64_add(zone->uz_fails, 1); in uma_zalloc_debug()
3633 if (zone->uz_fini != NULL) in uma_zalloc_debug()
3634 zone->uz_fini(item, zone->uz_size); in uma_zalloc_debug()
3648 uma_zfree_debug(uma_zone_t zone, void *item, void *udata) in uma_zfree_debug() argument
3654 if ((zone->uz_flags & (UMA_ZONE_SMR | UMA_ZFLAG_CACHE)) == 0 && in uma_zfree_debug()
3656 if (zone->uz_dtor != NULL) in uma_zfree_debug()
3657 zone->uz_dtor(item, zone->uz_size, udata); in uma_zfree_debug()
3658 if (zone->uz_fini != NULL) in uma_zfree_debug()
3659 zone->uz_fini(item, zone->uz_size); in uma_zfree_debug()
3669 cache_alloc_item(uma_zone_t zone, uma_cache_t cache, uma_cache_bucket_t bucket, in cache_alloc_item() argument
3679 return (item_ctor(zone, uz_flags, size, udata, flags, item)); in cache_alloc_item()
3683 cache_alloc_retry(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) in cache_alloc_retry() argument
3688 while (cache_alloc(zone, cache, udata, flags)) { in cache_alloc_retry()
3689 cache = &zone->uz_cpu[curcpu]; in cache_alloc_retry()
3693 return (cache_alloc_item(zone, cache, bucket, udata, flags)); in cache_alloc_retry()
3700 if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH) in cache_alloc_retry()
3704 return (zone_alloc_item(zone, udata, domain, flags)); in cache_alloc_retry()
3709 uma_zalloc_smr(uma_zone_t zone, int flags) in uma_zalloc_smr() argument
3714 CTR3(KTR_UMA, "uma_zalloc_smr zone %s(%p) flags %d", zone->uz_name, in uma_zalloc_smr()
3715 zone, flags); in uma_zalloc_smr()
3720 KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, in uma_zalloc_smr()
3721 ("uma_zalloc_arg: called with non-SMR zone.")); in uma_zalloc_smr()
3722 if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) in uma_zalloc_smr()
3727 cache = &zone->uz_cpu[curcpu]; in uma_zalloc_smr()
3730 return (cache_alloc_retry(zone, cache, NULL, flags)); in uma_zalloc_smr()
3731 return (cache_alloc_item(zone, cache, bucket, NULL, flags)); in uma_zalloc_smr()
3736 uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) in uma_zalloc_arg() argument
3742 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); in uma_zalloc_arg()
3745 CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name, in uma_zalloc_arg()
3746 zone, flags); in uma_zalloc_arg()
3751 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zalloc_arg()
3752 ("uma_zalloc_arg: called with SMR zone.")); in uma_zalloc_arg()
3753 if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) in uma_zalloc_arg()
3764 * order to acquire the zone mutex if we are unable to allocate from in uma_zalloc_arg()
3769 cache = &zone->uz_cpu[curcpu]; in uma_zalloc_arg()
3772 return (cache_alloc_retry(zone, cache, udata, flags)); in uma_zalloc_arg()
3773 return (cache_alloc_item(zone, cache, bucket, udata, flags)); in uma_zalloc_arg()
3784 cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) in cache_alloc() argument
3815 bucket_free(zone, bucket, udata); in cache_alloc()
3820 * we must go back to the zone. This requires the zdom lock, so we in cache_alloc()
3830 domain = zone_domain_highest(zone, domain); in cache_alloc()
3831 bucket = cache_fetch_bucket(zone, cache, domain); in cache_alloc()
3832 if (bucket == NULL && zone->uz_bucket_size != 0 && !bucketdisable) { in cache_alloc()
3833 bucket = zone_alloc_bucket(zone, udata, domain, flags); in cache_alloc()
3839 CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p", in cache_alloc()
3840 zone->uz_name, zone, bucket); in cache_alloc()
3852 cache = &zone->uz_cpu[curcpu]; in cache_alloc()
3858 atomic_add_long(&ZDOM_GET(zone, domain)->uzd_imax, in cache_alloc()
3868 zone_put_bucket(zone, domain, bucket, udata, !new); in cache_alloc()
3875 uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) in uma_zalloc_domain() argument
3884 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); in uma_zalloc_domain()
3887 CTR4(KTR_UMA, "uma_zalloc_domain zone %s(%p) domain %d flags %d", in uma_zalloc_domain()
3888 zone->uz_name, zone, domain, flags); in uma_zalloc_domain()
3890 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zalloc_domain()
3891 ("uma_zalloc_domain: called with SMR zone.")); in uma_zalloc_domain()
3893 KASSERT((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0, in uma_zalloc_domain()
3894 ("uma_zalloc_domain: called with non-FIRSTTOUCH zone.")); in uma_zalloc_domain()
3897 return (uma_zalloc_arg(zone, udata, flags)); in uma_zalloc_domain()
3900 if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) in uma_zalloc_domain()
3911 zdom = zone_domain_lock(zone, domain); in uma_zalloc_domain()
3912 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) { in uma_zalloc_domain()
3918 zone_put_bucket(zone, domain, bucket, udata, true); in uma_zalloc_domain()
3919 item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, in uma_zalloc_domain()
3925 counter_u64_add(zone->uz_allocs, 1); in uma_zalloc_domain()
3930 return (zone_alloc_item(zone, udata, domain, flags)); in uma_zalloc_domain()
3932 return (uma_zalloc_arg(zone, udata, flags)); in uma_zalloc_domain()
3998 keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) in keg_fetch_slab() argument
4037 slab = keg_alloc_slab(keg, zone, domain, flags, aflags); in keg_fetch_slab()
4108 uma_zone_t zone; in zone_import() local
4116 zone = arg; in zone_import()
4118 keg = zone->uz_keg; in zone_import()
4121 if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL) in zone_import()
4141 * If the zone is striped we pick a new slab for every in zone_import()
4148 if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0 && in zone_import()
4164 zone_alloc_limit_hard(uma_zone_t zone, int count, int flags) in zone_alloc_limit_hard() argument
4176 zone_free_limit(zone, count); in zone_alloc_limit_hard()
4177 zone_log_warning(zone); in zone_alloc_limit_hard()
4178 zone_maxaction(zone); in zone_alloc_limit_hard()
4187 sleepq_lock(&zone->uz_max_items); in zone_alloc_limit_hard()
4188 old = zone->uz_items; in zone_alloc_limit_hard()
4192 max = zone->uz_max_items; in zone_alloc_limit_hard()
4198 } while (atomic_fcmpset_64(&zone->uz_items, &old, new) == 0); in zone_alloc_limit_hard()
4202 sleepq_release(&zone->uz_max_items); in zone_alloc_limit_hard()
4212 atomic_add_32(&zone->uz_sleepers, 1); in zone_alloc_limit_hard()
4213 atomic_add_64(&zone->uz_sleeps, 1); in zone_alloc_limit_hard()
4219 sleepq_add(&zone->uz_max_items, NULL, "zonelimit", 0, 0); in zone_alloc_limit_hard()
4220 sleepq_wait(&zone->uz_max_items, PVM); in zone_alloc_limit_hard()
4229 atomic_subtract_32(&zone->uz_sleepers, 1); in zone_alloc_limit_hard()
4230 old = atomic_fetchadd_64(&zone->uz_items, in zone_alloc_limit_hard()
4241 max = zone->uz_max_items; in zone_alloc_limit_hard()
4246 zone_free_limit(zone, total + count - max); in zone_alloc_limit_hard()
4249 wakeup_one(&zone->uz_max_items); in zone_alloc_limit_hard()
4261 zone_alloc_limit(uma_zone_t zone, int count, int flags) in zone_alloc_limit() argument
4266 max = zone->uz_max_items; in zone_alloc_limit()
4273 old = atomic_fetchadd_64(&zone->uz_items, count); in zone_alloc_limit()
4284 zone_free_limit(zone, (old + count) - max); in zone_alloc_limit()
4287 return (zone_alloc_limit_hard(zone, count, flags)); in zone_alloc_limit()
4294 zone_free_limit(uma_zone_t zone, int count) in zone_free_limit() argument
4304 old = atomic_fetchadd_64(&zone->uz_items, -count); in zone_free_limit()
4306 UZ_ITEMS_COUNT(old) - count >= zone->uz_max_items)) in zone_free_limit()
4313 wakeup_one(&zone->uz_max_items); in zone_free_limit()
4317 zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) in zone_alloc_bucket() argument
4322 CTR3(KTR_UMA, "zone_alloc_bucket zone %s(%p) domain %d", zone->uz_name, in zone_alloc_bucket()
4323 zone, domain); in zone_alloc_bucket()
4328 else if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0) in zone_alloc_bucket()
4331 if (zone->uz_max_items > 0) in zone_alloc_bucket()
4332 maxbucket = zone_alloc_limit(zone, zone->uz_bucket_size, in zone_alloc_bucket()
4335 maxbucket = zone->uz_bucket_size; in zone_alloc_bucket()
4340 bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); in zone_alloc_bucket()
4346 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket, in zone_alloc_bucket()
4352 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) { in zone_alloc_bucket()
4356 kasan_mark_item_valid(zone, bucket->ub_bucket[i]); in zone_alloc_bucket()
4357 error = zone->uz_init(bucket->ub_bucket[i], in zone_alloc_bucket()
4358 zone->uz_size, flags); in zone_alloc_bucket()
4359 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); in zone_alloc_bucket()
4369 zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i], in zone_alloc_bucket()
4381 bucket_free(zone, bucket, udata); in zone_alloc_bucket()
4382 counter_u64_add(zone->uz_fails, 1); in zone_alloc_bucket()
4386 if (zone->uz_max_items > 0 && cnt < maxbucket) in zone_alloc_bucket()
4387 zone_free_limit(zone, maxbucket - cnt); in zone_alloc_bucket()
4393 * Allocates a single item from a zone.
4396 * zone The zone to alloc for.
4407 zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags) in zone_alloc_item() argument
4411 if (zone->uz_max_items > 0 && zone_alloc_limit(zone, 1, flags) == 0) { in zone_alloc_item()
4412 counter_u64_add(zone->uz_fails, 1); in zone_alloc_item()
4420 if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1) in zone_alloc_item()
4424 * We have to call both the zone's init (not the keg's init) in zone_alloc_item()
4425 * and the zone's ctor. This is because the item is going from in zone_alloc_item()
4427 * to be both zone-init'd as well as zone-ctor'd. in zone_alloc_item()
4429 if (zone->uz_init != NULL) { in zone_alloc_item()
4432 kasan_mark_item_valid(zone, item); in zone_alloc_item()
4433 error = zone->uz_init(item, zone->uz_size, flags); in zone_alloc_item()
4434 kasan_mark_item_invalid(zone, item); in zone_alloc_item()
4436 zone_free_item(zone, item, udata, SKIP_FINI | SKIP_CNT); in zone_alloc_item()
4440 item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, flags, in zone_alloc_item()
4445 counter_u64_add(zone->uz_allocs, 1); in zone_alloc_item()
4447 zone->uz_name, zone); in zone_alloc_item()
4452 counter_u64_add(zone->uz_fails, 1); in zone_alloc_item()
4454 if (zone->uz_max_items > 0) in zone_alloc_item()
4455 zone_free_limit(zone, 1); in zone_alloc_item()
4457 zone->uz_name, zone); in zone_alloc_item()
4464 uma_zfree_smr(uma_zone_t zone, void *item) in uma_zfree_smr() argument
4473 CTR3(KTR_UMA, "uma_zfree_smr zone %s(%p) item %p", in uma_zfree_smr()
4474 zone->uz_name, zone, item); in uma_zfree_smr()
4477 KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, in uma_zfree_smr()
4478 ("uma_zfree_smr: called with non-SMR zone.")); in uma_zfree_smr()
4480 SMR_ASSERT_NOT_ENTERED(zone->uz_smr); in uma_zfree_smr()
4481 if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN) in uma_zfree_smr()
4484 cache = &zone->uz_cpu[curcpu]; in uma_zfree_smr()
4493 cache = &zone->uz_cpu[curcpu]; in uma_zfree_smr()
4507 } while (cache_free(zone, cache, NULL, itemdomain)); in uma_zfree_smr()
4513 zone_free_item(zone, item, NULL, SKIP_NONE); in uma_zfree_smr()
4518 uma_zfree_arg(uma_zone_t zone, void *item, void *udata) in uma_zfree_arg() argument
4525 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); in uma_zfree_arg()
4527 CTR3(KTR_UMA, "uma_zfree_arg zone %s(%p) item %p", in uma_zfree_arg()
4528 zone->uz_name, zone, item); in uma_zfree_arg()
4531 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zfree_arg()
4532 ("uma_zfree_arg: called with SMR zone.")); in uma_zfree_arg()
4533 if (uma_zfree_debug(zone, item, udata) == EJUSTRETURN) in uma_zfree_arg()
4545 cache = &zone->uz_cpu[curcpu]; in uma_zfree_arg()
4549 item_dtor(zone, item, cache_uz_size(cache), udata, SKIP_NONE); in uma_zfree_arg()
4556 if (atomic_load_32(&zone->uz_sleepers) > 0) in uma_zfree_arg()
4567 * order to acquire the zone mutex if we are unable to free to the in uma_zfree_arg()
4578 cache = &zone->uz_cpu[curcpu]; in uma_zfree_arg()
4602 } while (cache_free(zone, cache, udata, itemdomain)); in uma_zfree_arg()
4609 zone_free_item(zone, item, udata, SKIP_DTOR); in uma_zfree_arg()
4618 zone_free_cross(uma_zone_t zone, uma_bucket_t bucket, void *udata) in zone_free_cross() argument
4628 "uma_zfree: zone %s(%p) draining cross bucket %p", in zone_free_cross()
4629 zone->uz_name, zone, bucket); in zone_free_cross()
4636 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in zone_free_cross()
4637 seq = smr_advance(zone->uz_smr); in zone_free_cross()
4646 ZONE_CROSS_LOCK(zone); in zone_free_cross()
4650 zdom = ZDOM_GET(zone, domain); in zone_free_cross()
4662 ZONE_CROSS_UNLOCK(zone); in zone_free_cross()
4663 b = bucket_alloc(zone, udata, M_NOWAIT); in zone_free_cross()
4666 ZONE_CROSS_LOCK(zone); in zone_free_cross()
4685 ZONE_CROSS_UNLOCK(zone); in zone_free_cross()
4689 bucket_free(zone, bucket, udata); in zone_free_cross()
4693 bucket_free(zone, b, udata); in zone_free_cross()
4698 zone_put_bucket(zone, domain, b, udata, true); in zone_free_cross()
4704 zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata, in zone_free_bucket() argument
4715 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && in zone_free_bucket()
4717 zone_free_cross(zone, bucket, udata); in zone_free_bucket()
4723 * Attempt to save the bucket in the zone's domain bucket cache. in zone_free_bucket()
4726 "uma_zfree: zone %s(%p) putting bucket %p on free list", in zone_free_bucket()
4727 zone->uz_name, zone, bucket); in zone_free_bucket()
4729 if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0) in zone_free_bucket()
4730 itemdomain = zone_domain_lowest(zone, itemdomain); in zone_free_bucket()
4731 zone_put_bucket(zone, itemdomain, bucket, udata, ws); in zone_free_bucket()
4736 * existing full bucket either to the zone cache or back to the slab layer.
4743 cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, int itemdomain) in cache_free() argument
4750 if (zone->uz_bucket_size == 0) in cache_free()
4753 cache = &zone->uz_cpu[curcpu]; in cache_free()
4767 counter_u64_add(zone->uz_xdomain, in cache_free()
4785 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) { in cache_free()
4787 bucket->ub_seq = smr_advance(zone->uz_smr); in cache_free()
4788 newbucket = bucket_alloc(zone, udata, M_NOWAIT); in cache_free()
4790 bucket_drain(zone, bucket); in cache_free()
4795 newbucket = bucket_alloc(zone, udata, M_NOWAIT); in cache_free()
4798 zone_free_bucket(zone, bucket, udata, itemdomain, true); in cache_free()
4803 cache = &zone->uz_cpu[curcpu]; in cache_free()
4823 bucket_free(zone, bucket, udata); in cache_free()
4832 slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item) in slab_free_item() argument
4838 keg = zone->uz_keg; in slab_free_item()
4865 uma_zone_t zone; in zone_release() local
4872 zone = arg; in zone_release()
4873 keg = zone->uz_keg; in zone_release()
4875 if (__predict_false((zone->uz_flags & UMA_ZFLAG_HASH) != 0)) in zone_release()
4879 if (__predict_true((zone->uz_flags & UMA_ZFLAG_VTOSLAB) != 0)) { in zone_release()
4883 if ((zone->uz_flags & UMA_ZFLAG_HASH) != 0) in zone_release()
4893 slab_free_item(zone, slab, item); in zone_release()
4900 * Frees a single item to any zone.
4903 * zone The zone to free to
4909 zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) in zone_free_item() argument
4913 * If a free is sent directly to an SMR zone we have to in zone_free_item()
4918 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && skip == SKIP_NONE) in zone_free_item()
4919 smr_synchronize(zone->uz_smr); in zone_free_item()
4921 item_dtor(zone, item, zone->uz_size, udata, skip); in zone_free_item()
4923 if (skip < SKIP_FINI && zone->uz_fini) { in zone_free_item()
4924 kasan_mark_item_valid(zone, item); in zone_free_item()
4925 zone->uz_fini(item, zone->uz_size); in zone_free_item()
4926 kasan_mark_item_invalid(zone, item); in zone_free_item()
4929 zone->uz_release(zone->uz_arg, &item, 1); in zone_free_item()
4934 counter_u64_add(zone->uz_frees, 1); in zone_free_item()
4936 if (zone->uz_max_items > 0) in zone_free_item()
4937 zone_free_limit(zone, 1); in zone_free_item()
4942 uma_zone_set_max(uma_zone_t zone, int nitems) in uma_zone_set_max() argument
4949 uma_zone_set_maxcache(zone, nitems); in uma_zone_set_max()
4952 * XXX This can misbehave if the zone has any allocations with in uma_zone_set_max()
4956 ZONE_LOCK(zone); in uma_zone_set_max()
4957 if (zone->uz_max_items == 0) in uma_zone_set_max()
4958 ZONE_ASSERT_COLD(zone); in uma_zone_set_max()
4959 zone->uz_max_items = nitems; in uma_zone_set_max()
4960 zone->uz_flags |= UMA_ZFLAG_LIMIT; in uma_zone_set_max()
4961 zone_update_caches(zone); in uma_zone_set_max()
4963 wakeup(&zone->uz_max_items); in uma_zone_set_max()
4964 ZONE_UNLOCK(zone); in uma_zone_set_max()
4971 uma_zone_set_maxcache(uma_zone_t zone, int nitems) in uma_zone_set_maxcache() argument
4975 ZONE_LOCK(zone); in uma_zone_set_maxcache()
4979 * the zone. Each CPU gets at least two buckets, and for cross-domain in uma_zone_set_maxcache()
4987 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && vm_ndomains > 1) { in uma_zone_set_maxcache()
4998 zone->uz_bucket_size_max = zone->uz_bucket_size = bsize; in uma_zone_set_maxcache()
4999 if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) in uma_zone_set_maxcache()
5000 zone->uz_bucket_size_min = zone->uz_bucket_size_max; in uma_zone_set_maxcache()
5001 zone->uz_bucket_max = nitems - nb * bsize; in uma_zone_set_maxcache()
5002 ZONE_UNLOCK(zone); in uma_zone_set_maxcache()
5007 uma_zone_get_max(uma_zone_t zone) in uma_zone_get_max() argument
5011 nitems = atomic_load_64(&zone->uz_max_items); in uma_zone_get_max()
5018 uma_zone_set_warning(uma_zone_t zone, const char *warning) in uma_zone_set_warning() argument
5021 ZONE_ASSERT_COLD(zone); in uma_zone_set_warning()
5022 zone->uz_warning = warning; in uma_zone_set_warning()
5027 uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction) in uma_zone_set_maxaction() argument
5030 ZONE_ASSERT_COLD(zone); in uma_zone_set_maxaction()
5031 TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone); in uma_zone_set_maxaction()
5036 uma_zone_get_cur(uma_zone_t zone) in uma_zone_get_cur() argument
5042 if (zone->uz_allocs != EARLY_COUNTER && zone->uz_frees != EARLY_COUNTER) in uma_zone_get_cur()
5043 nitems = counter_u64_fetch(zone->uz_allocs) - in uma_zone_get_cur()
5044 counter_u64_fetch(zone->uz_frees); in uma_zone_get_cur()
5046 nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs) - in uma_zone_get_cur()
5047 atomic_load_64(&zone->uz_cpu[i].uc_frees); in uma_zone_get_cur()
5053 uma_zone_get_allocs(uma_zone_t zone) in uma_zone_get_allocs() argument
5059 if (zone->uz_allocs != EARLY_COUNTER) in uma_zone_get_allocs()
5060 nitems = counter_u64_fetch(zone->uz_allocs); in uma_zone_get_allocs()
5062 nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs); in uma_zone_get_allocs()
5068 uma_zone_get_frees(uma_zone_t zone) in uma_zone_get_frees() argument
5074 if (zone->uz_frees != EARLY_COUNTER) in uma_zone_get_frees()
5075 nitems = counter_u64_fetch(zone->uz_frees); in uma_zone_get_frees()
5077 nitems += atomic_load_64(&zone->uz_cpu[i].uc_frees); in uma_zone_get_frees()
5100 uma_zone_set_init(uma_zone_t zone, uma_init uminit) in uma_zone_set_init() argument
5104 KEG_GET(zone, keg); in uma_zone_set_init()
5111 uma_zone_set_fini(uma_zone_t zone, uma_fini fini) in uma_zone_set_fini() argument
5115 KEG_GET(zone, keg); in uma_zone_set_fini()
5122 uma_zone_set_zinit(uma_zone_t zone, uma_init zinit) in uma_zone_set_zinit() argument
5125 ZONE_ASSERT_COLD(zone); in uma_zone_set_zinit()
5126 zone->uz_init = zinit; in uma_zone_set_zinit()
5131 uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini) in uma_zone_set_zfini() argument
5134 ZONE_ASSERT_COLD(zone); in uma_zone_set_zfini()
5135 zone->uz_fini = zfini; in uma_zone_set_zfini()
5140 uma_zone_set_freef(uma_zone_t zone, uma_free freef) in uma_zone_set_freef() argument
5144 KEG_GET(zone, keg); in uma_zone_set_freef()
5151 uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) in uma_zone_set_allocf() argument
5155 KEG_GET(zone, keg); in uma_zone_set_allocf()
5162 uma_zone_set_smr(uma_zone_t zone, smr_t smr) in uma_zone_set_smr() argument
5165 ZONE_ASSERT_COLD(zone); in uma_zone_set_smr()
5168 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zone_set_smr()
5169 ("zone %p (%s) already uses SMR", zone, zone->uz_name)); in uma_zone_set_smr()
5170 zone->uz_flags |= UMA_ZONE_SMR; in uma_zone_set_smr()
5171 zone->uz_smr = smr; in uma_zone_set_smr()
5172 zone_update_caches(zone); in uma_zone_set_smr()
5176 uma_zone_get_smr(uma_zone_t zone) in uma_zone_get_smr() argument
5179 return (zone->uz_smr); in uma_zone_get_smr()
5184 uma_zone_reserve(uma_zone_t zone, int items) in uma_zone_reserve() argument
5188 KEG_GET(zone, keg); in uma_zone_reserve()
5195 uma_zone_reserve_kva(uma_zone_t zone, int count) argument
5201 KEG_GET(zone, keg);
5203 ZONE_ASSERT_COLD(zone);
5221 zone->uz_max_items = pages * keg->uk_ipers;
5228 zone->uz_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE;
5229 zone_update_caches(zone);
5236 uma_prealloc(uma_zone_t zone, int items) argument
5244 KEG_GET(zone, keg);
5251 slab = keg_alloc_slab(keg, zone, domain, M_WAITOK,
5276 uma_zone_memory(uma_zone_t zone) argument
5282 if (zone->uz_flags & UMA_ZFLAG_CACHE) {
5284 sz += ZDOM_GET(zone, i)->uzd_nitems;
5285 return (sz * zone->uz_size);
5288 sz += zone->uz_keg->uk_domain[i].ud_pages;
5299 uma_reclaim_domain_cb(uma_zone_t zone, void *arg) argument
5304 if ((zone->uz_flags & UMA_ZONE_UNMANAGED) != 0)
5307 (zone->uz_flags & UMA_ZONE_NOTRIM) !=0)
5310 uma_zone_reclaim_domain(zone, args->req, args->domain);
5341 * drain each zone's per-CPU buckets.
5353 * Some slabs may have been freed but this zone will be visited early
5393 uma_zone_reclaim(uma_zone_t zone, int req) argument
5395 uma_zone_reclaim_domain(zone, req, UMA_ANYDOMAIN);
5399 uma_zone_reclaim_domain(uma_zone_t zone, int req, int domain) argument
5403 zone_reclaim(zone, domain, M_NOWAIT, false);
5406 zone_reclaim(zone, domain, M_NOWAIT, true);
5409 pcpu_cache_drain_safe(zone);
5410 zone_reclaim(zone, domain, M_NOWAIT, true);
5419 uma_zone_exhausted(uma_zone_t zone) argument
5422 return (atomic_load_32(&zone->uz_sleepers) > 0);
5455 * Generate statistics across both the zone and its per-cpu cache's. Return
5458 * Note: does not update the zone statistics, as it can't safely clear the
5612 * A zone is secondary is it is not the first entry
5613 * on the keg's zone list.
5645 uma_zone_t zone = *(uma_zone_t *)arg1; local
5648 max = uma_zone_get_max(zone);
5653 uma_zone_set_max(zone, max);
5661 uma_zone_t zone; local
5669 zone = *(uma_zone_t *)arg1;
5671 zone = arg1;
5672 cur = uma_zone_get_cur(zone);
5679 uma_zone_t zone = arg1; local
5682 cur = uma_zone_get_allocs(zone);
5689 uma_zone_t zone = arg1; local
5692 cur = uma_zone_get_frees(zone);
5700 uma_zone_t zone = arg1; local
5704 if (zone->uz_flags != 0)
5705 sbuf_printf(&sbuf, "0x%b", zone->uz_flags, PRINT_UMA_ZFLAGS);
5738 uma_zone_t zone = arg1; local
5741 cur = UZ_ITEMS_COUNT(atomic_load_64(&zone->uz_items));
5747 uma_dbg_getslab(uma_zone_t zone, void *item) argument
5755 * zone is unlocked because the item's allocation state
5759 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0)
5761 if (zone->uz_flags & UMA_ZFLAG_VTOSLAB)
5763 keg = zone->uz_keg;
5774 uma_dbg_zskip(uma_zone_t zone, void *mem) argument
5777 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0)
5780 return (uma_dbg_kskip(zone->uz_keg, mem));
5814 uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item) argument
5820 slab = uma_dbg_getslab(zone, item);
5822 panic("uma: item %p did not belong to zone %s",
5823 item, zone->uz_name);
5825 keg = zone->uz_keg;
5830 panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)",
5831 item, zone, zone->uz_name, slab, freei);
5840 uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item) argument
5846 slab = uma_dbg_getslab(zone, item);
5848 panic("uma: Freed item %p did not belong to zone %s",
5849 item, zone->uz_name);
5851 keg = zone->uz_keg;
5855 panic("Invalid free of %p from zone %p(%s) slab %p(%d)",
5856 item, zone, zone->uz_name, slab, freei);
5859 panic("Unaligned free of %p from zone %p(%s) slab %p(%d)",
5860 item, zone, zone->uz_name, slab, freei);
5864 panic("Duplicate free of %p from zone %p(%s) slab %p(%d)",
5865 item, zone, zone->uz_name, slab, freei);
5918 db_printf(fmt_hdr, "Zone", "Size", "Used", "Free", "Requests",
5934 * zone, we have already printed all preceding
5976 db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free",