Lines Matching full:zone
41 * The basic ideas stem from similar slab/zone based allocators whose algorithms
114 * This is the zone and keg from which all zones are spawned.
132 * One zone is for slab headers that can represent a larger number of items,
133 * making the slabs themselves more efficient, and the other zone is for
146 * The initial hash tables come out of this zone so they can be allocated
211 * This structure is passed as the zone ctor arg so that I don't have to create
230 uma_zone_t zone; member
293 static void bucket_cache_reclaim(uma_zone_t zone, bool, int);
300 static inline void item_dtor(uma_zone_t zone, void *item, int size,
303 static void zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata,
307 static void zone_timeout(uma_zone_t zone, void *);
315 static int zone_alloc_limit(uma_zone_t zone, int count, int flags);
316 static void zone_free_limit(uma_zone_t zone, int count);
319 static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
320 static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
324 static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item);
326 static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
341 static uint64_t uma_zone_get_allocs(uma_zone_t zone);
347 static uint64_t uma_keg_get_allocs(uma_keg_t zone);
351 static bool uma_dbg_zskip(uma_zone_t zone, void *mem);
352 static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
353 static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);
375 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats");
388 * Select the slab zone for an offpage slab with the given maximum item count.
411 * For each zone, calculate the memory required for each bucket, consisting
431 * Given a desired number of entries for a bucket, return the zone from which
463 bucket_alloc(uma_zone_t zone, void *udata, int flags) in bucket_alloc() argument
475 * To limit bucket recursion we store the original zone flags in bucket_alloc()
479 * a bucket for a bucket zone so we do not allow infinite bucket in bucket_alloc()
484 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) in bucket_alloc()
485 udata = (void *)(uintptr_t)zone->uz_flags; in bucket_alloc()
493 ubz = bucket_zone_lookup(atomic_load_16(&zone->uz_bucket_size)); in bucket_alloc()
494 if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0) in bucket_alloc()
503 zone->uz_bucket_size_max); in bucket_alloc()
505 CTR3(KTR_UMA, "bucket_alloc: zone %s(%p) allocated bucket %p", in bucket_alloc()
506 zone->uz_name, zone, bucket); in bucket_alloc()
513 bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata) in bucket_free() argument
518 bucket_drain(zone, bucket); in bucket_free()
524 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) in bucket_free()
525 udata = (void *)(uintptr_t)zone->uz_flags; in bucket_free()
545 kasan_mark_item_valid(uma_zone_t zone, void *item) in kasan_mark_item_valid() argument
551 if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0) in kasan_mark_item_valid()
554 sz = zone->uz_size; in kasan_mark_item_valid()
556 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { in kasan_mark_item_valid()
567 kasan_mark_item_invalid(uma_zone_t zone, void *item) in kasan_mark_item_invalid() argument
573 if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0) in kasan_mark_item_invalid()
576 sz = roundup2(zone->uz_size, KASAN_SHADOW_SCALE); in kasan_mark_item_invalid()
577 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { in kasan_mark_item_invalid()
613 kasan_mark_item_valid(uma_zone_t zone __unused, void *item __unused) in kasan_mark_item_valid()
618 kasan_mark_item_invalid(uma_zone_t zone __unused, void *item __unused) in kasan_mark_item_invalid()
635 kmsan_mark_item_uninitialized(uma_zone_t zone, void *item) in kmsan_mark_item_uninitialized() argument
641 if ((zone->uz_flags & in kmsan_mark_item_uninitialized()
650 * zone and thus cannot safely be marked by UMA. in kmsan_mark_item_uninitialized()
657 if (zone->uz_keg->uk_init != NULL) { in kmsan_mark_item_uninitialized()
666 sz = zone->uz_size; in kmsan_mark_item_uninitialized()
667 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { in kmsan_mark_item_uninitialized()
682 kmsan_mark_item_uninitialized(uma_zone_t zone __unused, void *item __unused) in kmsan_mark_item_uninitialized()
691 zone_domain_lock(uma_zone_t zone, int domain) in zone_domain_lock() argument
696 zdom = ZDOM_GET(zone, domain); in zone_domain_lock()
702 if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max) in zone_domain_lock()
703 zone->uz_bucket_size++; in zone_domain_lock()
712 zone_domain_lowest(uma_zone_t zone, int pref) in zone_domain_lowest() argument
721 nitems = ZDOM_GET(zone, i)->uzd_nitems; in zone_domain_lowest()
740 zone_domain_highest(uma_zone_t zone, int pref) in zone_domain_highest() argument
746 if (ZDOM_GET(zone, pref)->uzd_nitems > BUCKET_MAX) in zone_domain_highest()
752 nitems = ZDOM_GET(zone, i)->uzd_nitems; in zone_domain_highest()
787 * zone's caches. If a bucket is found the zone is not locked on return.
790 zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim) in zone_fetch_bucket() argument
803 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && in zone_fetch_bucket()
805 if (!smr_poll(zone->uz_smr, bucket->ub_seq, false)) in zone_fetch_bucket()
808 dtor = (zone->uz_dtor != NULL) || UMA_ALWAYS_CTORDTOR; in zone_fetch_bucket()
845 item_dtor(zone, bucket->ub_bucket[i], zone->uz_size, in zone_fetch_bucket()
853 * whether the bucket's contents should be counted as part of the zone's working
857 zone_put_bucket(uma_zone_t zone, int domain, uma_bucket_t bucket, void *udata, in zone_put_bucket() argument
865 zdom = zone_domain_lock(zone, domain); in zone_put_bucket()
871 if (__predict_true(zdom->uzd_nitems < zone->uz_bucket_max)) { in zone_put_bucket()
901 bucket_free(zone, bucket, udata); in zone_put_bucket()
1044 * Attempt to fetch a bucket from a zone on behalf of the current cpu cache.
1047 cache_fetch_bucket(uma_zone_t zone, uma_cache_t cache, int domain) in cache_fetch_bucket() argument
1056 zdom = ZDOM_GET(zone, domain); in cache_fetch_bucket()
1062 !smr_poll(zone->uz_smr, seq, false)) in cache_fetch_bucket()
1066 * Check the zone's cache of buckets. in cache_fetch_bucket()
1068 zdom = zone_domain_lock(zone, domain); in cache_fetch_bucket()
1069 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) in cache_fetch_bucket()
1077 zone_log_warning(uma_zone_t zone) in zone_log_warning() argument
1081 if (!zone_warnings || zone->uz_warning == NULL) in zone_log_warning()
1084 if (ratecheck(&zone->uz_ratecheck, &warninterval)) in zone_log_warning()
1085 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning); in zone_log_warning()
1089 zone_maxaction(uma_zone_t zone) in zone_maxaction() argument
1092 if (zone->uz_maxaction.ta_func != NULL) in zone_maxaction()
1093 taskqueue_enqueue(taskqueue_thread, &zone->uz_maxaction); in zone_maxaction()
1118 * Update the working set size estimates for the zone's bucket cache.
1170 zone_timeout(uma_zone_t zone, void *unused) in zone_timeout() argument
1175 if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) in zone_timeout()
1178 keg = zone->uz_keg; in zone_timeout()
1224 if ((zone->uz_flags & (UMA_ZONE_UNMANAGED | UMA_ZONE_NOTRIM)) == 0) { in zone_timeout()
1226 if (bucket_cache_reclaim_domain(zone, false, false, i) && in zone_timeout()
1227 (zone->uz_flags & UMA_ZFLAG_CACHE) == 0) in zone_timeout()
1228 keg_drain(zone->uz_keg, i); in zone_timeout()
1337 * zone The zone to free to, must be unlocked.
1344 bucket_drain(uma_zone_t zone, uma_bucket_t bucket) in bucket_drain() argument
1351 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && in bucket_drain()
1353 smr_wait(zone->uz_smr, bucket->ub_seq); in bucket_drain()
1356 item_dtor(zone, bucket->ub_bucket[i], in bucket_drain()
1357 zone->uz_size, NULL, SKIP_NONE); in bucket_drain()
1359 if (zone->uz_fini) in bucket_drain()
1361 kasan_mark_item_valid(zone, bucket->ub_bucket[i]); in bucket_drain()
1362 zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); in bucket_drain()
1363 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); in bucket_drain()
1365 zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); in bucket_drain()
1366 if (zone->uz_max_items > 0) in bucket_drain()
1367 zone_free_limit(zone, bucket->ub_cnt); in bucket_drain()
1375 * Drains the per cpu caches for a zone.
1377 * NOTE: This may only be called while the zone is being torn down, and not
1382 * zone The zone to drain, must be unlocked.
1388 cache_drain(uma_zone_t zone) in cache_drain() argument
1397 * tearing down the zone anyway. I.e., there will be no further use in cache_drain()
1400 * XXX: It would good to be able to assert that the zone is being in cache_drain()
1404 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in cache_drain()
1405 seq = smr_advance(zone->uz_smr); in cache_drain()
1407 cache = &zone->uz_cpu[cpu]; in cache_drain()
1410 bucket_free(zone, bucket, NULL); in cache_drain()
1414 bucket_free(zone, bucket, NULL); in cache_drain()
1419 bucket_free(zone, bucket, NULL); in cache_drain()
1422 bucket_cache_reclaim(zone, true, UMA_ANYDOMAIN); in cache_drain()
1426 cache_shrink(uma_zone_t zone, void *unused) in cache_shrink() argument
1429 if (zone->uz_flags & UMA_ZFLAG_INTERNAL) in cache_shrink()
1432 ZONE_LOCK(zone); in cache_shrink()
1433 zone->uz_bucket_size = in cache_shrink()
1434 (zone->uz_bucket_size_min + zone->uz_bucket_size) / 2; in cache_shrink()
1435 ZONE_UNLOCK(zone); in cache_shrink()
1439 cache_drain_safe_cpu(uma_zone_t zone, void *unused) in cache_drain_safe_cpu() argument
1445 if (zone->uz_flags & UMA_ZFLAG_INTERNAL) in cache_drain_safe_cpu()
1450 cache = &zone->uz_cpu[curcpu]; in cache_drain_safe_cpu()
1455 * Don't flush SMR zone buckets. This leaves the zone without a in cache_drain_safe_cpu()
1458 if ((zone->uz_flags & UMA_ZONE_SMR) == 0) { in cache_drain_safe_cpu()
1465 zone_free_bucket(zone, b1, NULL, domain, false); in cache_drain_safe_cpu()
1467 zone_free_bucket(zone, b2, NULL, domain, false); in cache_drain_safe_cpu()
1471 zone_free_bucket(zone, b3, NULL, domain, false); in cache_drain_safe_cpu()
1476 * Safely drain per-CPU caches of a zone(s) to alloc bucket.
1480 * Zone lock must not be held on call this function.
1483 pcpu_cache_drain_safe(uma_zone_t zone) in pcpu_cache_drain_safe() argument
1490 if (zone) in pcpu_cache_drain_safe()
1491 cache_shrink(zone, NULL); in pcpu_cache_drain_safe()
1500 if (zone) in pcpu_cache_drain_safe()
1501 cache_drain_safe_cpu(zone, NULL); in pcpu_cache_drain_safe()
1511 * Reclaim cached buckets from a zone. All buckets are reclaimed if the caller
1516 bucket_cache_reclaim_domain(uma_zone_t zone, bool drain, bool trim, int domain) in bucket_cache_reclaim_domain() argument
1527 zdom = ZDOM_GET(zone, domain); in bucket_cache_reclaim_domain()
1528 if ((zone->uz_flags & UMA_ZONE_SMR) == 0 || drain) { in bucket_cache_reclaim_domain()
1529 ZONE_CROSS_LOCK(zone); in bucket_cache_reclaim_domain()
1532 ZONE_CROSS_UNLOCK(zone); in bucket_cache_reclaim_domain()
1534 bucket_free(zone, bucket, NULL); in bucket_cache_reclaim_domain()
1538 * If we were asked to drain the zone, we are done only once in bucket_cache_reclaim_domain()
1540 * excess of the zone's estimated working set size. Multiple in bucket_cache_reclaim_domain()
1559 bucket = zone_fetch_bucket(zone, zdom, true); in bucket_cache_reclaim_domain()
1562 bucket_free(zone, bucket, NULL); in bucket_cache_reclaim_domain()
1571 bucket_cache_reclaim(uma_zone_t zone, bool drain, int domain) in bucket_cache_reclaim() argument
1576 * Shrink the zone bucket size to ensure that the per-CPU caches in bucket_cache_reclaim()
1579 if (zone->uz_bucket_size > zone->uz_bucket_size_min) in bucket_cache_reclaim()
1580 zone->uz_bucket_size--; in bucket_cache_reclaim()
1583 (zone->uz_flags & UMA_ZONE_ROUNDROBIN) == 0) { in bucket_cache_reclaim()
1584 bucket_cache_reclaim_domain(zone, drain, true, domain); in bucket_cache_reclaim()
1587 bucket_cache_reclaim_domain(zone, drain, true, i); in bucket_cache_reclaim()
1710 zone_reclaim(uma_zone_t zone, int domain, int waitok, bool drain) in zone_reclaim() argument
1714 * zone_dtor(), which removes the zone from global lists before in zone_reclaim()
1717 * The zone may be destroyed while sleeping, so only zone_dtor() should in zone_reclaim()
1720 ZONE_LOCK(zone); in zone_reclaim()
1722 while (zone->uz_reclaimers > 0) in zone_reclaim()
1723 msleep(zone, ZONE_LOCKPTR(zone), PVM, "zonedrain", 1); in zone_reclaim()
1725 zone->uz_reclaimers++; in zone_reclaim()
1726 ZONE_UNLOCK(zone); in zone_reclaim()
1727 bucket_cache_reclaim(zone, drain, domain); in zone_reclaim()
1729 if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) in zone_reclaim()
1730 keg_drain(zone->uz_keg, domain); in zone_reclaim()
1731 ZONE_LOCK(zone); in zone_reclaim()
1732 zone->uz_reclaimers--; in zone_reclaim()
1733 if (zone->uz_reclaimers == 0) in zone_reclaim()
1734 wakeup(zone); in zone_reclaim()
1735 ZONE_UNLOCK(zone); in zone_reclaim()
1752 keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, in keg_alloc_slab() argument
1780 * first time they are added to a zone. in keg_alloc_slab()
1796 /* zone is passed for legacy reasons. */ in keg_alloc_slab()
1798 mem = keg->uk_allocf(zone, size, domain, &sflags, aflags); in keg_alloc_slab()
1823 zone, slab); in keg_alloc_slab()
1876 startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in startup_alloc() argument
1942 page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in page_alloc() argument
1954 pcpu_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in pcpu_page_alloc() argument
2019 noobj_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags, in noobj_alloc() argument
2030 keg = zone->uz_keg; in noobj_alloc()
2069 contig_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in contig_alloc() argument
2080 uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags, in uma_small_alloc() argument
2127 * Frees pcpu zone allocations
2288 * keg The zone we should initialize
2469 uma_zone_t zone; in keg_ctor() local
2489 * The primary zone is passed to us at keg-creation time. in keg_ctor()
2491 zone = arg->zone; in keg_ctor()
2492 keg->uk_name = zone->uz_name; in keg_ctor()
2571 ("zone %s ipers %d rsize %d size %d slab won't fit", in keg_ctor()
2572 zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); in keg_ctor()
2578 CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)", keg, zone->uz_name, zone); in keg_ctor()
2580 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); in keg_ctor()
2589 zone_kva_available(uma_zone_t zone, void *unused) in zone_kva_available() argument
2593 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) in zone_kva_available()
2595 KEG_GET(zone, keg); in zone_kva_available()
2610 zone_alloc_counters(uma_zone_t zone, void *unused) in zone_alloc_counters() argument
2613 zone->uz_allocs = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2614 zone->uz_frees = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2615 zone->uz_fails = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2616 zone->uz_xdomain = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2620 zone_alloc_sysctl(uma_zone_t zone, void *unused) in zone_alloc_sysctl() argument
2627 static const char *nokeg = "cache zone"; in zone_alloc_sysctl()
2631 * Make a sysctl safe copy of the zone name by removing in zone_alloc_sysctl()
2635 if (zone->uz_namecnt != 0) { in zone_alloc_sysctl()
2637 for (i = 1, cnt = zone->uz_namecnt; cnt != 0; i++) in zone_alloc_sysctl()
2639 zone->uz_ctlname = malloc(strlen(zone->uz_name) + i + 1, in zone_alloc_sysctl()
2641 sprintf(zone->uz_ctlname, "%s_%d", zone->uz_name, in zone_alloc_sysctl()
2642 zone->uz_namecnt); in zone_alloc_sysctl()
2644 zone->uz_ctlname = strdup(zone->uz_name, M_UMA); in zone_alloc_sysctl()
2645 for (c = zone->uz_ctlname; *c != '\0'; c++) in zone_alloc_sysctl()
2652 zone->uz_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_vm_uma), in zone_alloc_sysctl()
2653 OID_AUTO, zone->uz_ctlname, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); in zone_alloc_sysctl()
2654 oid = zone->uz_oid; in zone_alloc_sysctl()
2656 "size", CTLFLAG_RD, &zone->uz_size, 0, "Allocation size"); in zone_alloc_sysctl()
2659 zone, 0, sysctl_handle_uma_zone_flags, "A", in zone_alloc_sysctl()
2662 "bucket_size", CTLFLAG_RD, &zone->uz_bucket_size, 0, in zone_alloc_sysctl()
2665 "bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0, in zone_alloc_sysctl()
2671 if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) in zone_alloc_sysctl()
2675 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, in zone_alloc_sysctl()
2677 keg = zone->uz_keg; in zone_alloc_sysctl()
2678 if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) { in zone_alloc_sysctl()
2722 * Information about zone limits. in zone_alloc_sysctl()
2724 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, in zone_alloc_sysctl()
2728 zone, 0, sysctl_handle_uma_zone_items, "QU", in zone_alloc_sysctl()
2731 "max_items", CTLFLAG_RD, &zone->uz_max_items, 0, in zone_alloc_sysctl()
2734 "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0, in zone_alloc_sysctl()
2737 "sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0, in zone_alloc_sysctl()
2738 "Total zone limit sleeps"); in zone_alloc_sysctl()
2740 "bucket_max", CTLFLAG_RD, &zone->uz_bucket_max, 0, in zone_alloc_sysctl()
2744 * Per-domain zone information. in zone_alloc_sysctl()
2746 domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), in zone_alloc_sysctl()
2749 zdom = ZDOM_GET(zone, i); in zone_alloc_sysctl()
2779 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, in zone_alloc_sysctl()
2783 zone, 1, sysctl_handle_uma_zone_cur, "I", in zone_alloc_sysctl()
2787 zone, 0, sysctl_handle_uma_zone_allocs, "QU", in zone_alloc_sysctl()
2791 zone, 0, sysctl_handle_uma_zone_frees, "QU", in zone_alloc_sysctl()
2794 "fails", CTLFLAG_RD, &zone->uz_fails, in zone_alloc_sysctl()
2797 "xdomain", CTLFLAG_RD, &zone->uz_xdomain, in zone_alloc_sysctl()
2807 zone_count(uma_zone_t zone, void *arg) in zone_count() argument
2817 if (strcmp(zone->uz_name, cnt->name) == 0) in zone_count()
2819 zone->uz_namecnt + 1); in zone_count()
2823 zone_update_caches(uma_zone_t zone) in zone_update_caches() argument
2828 cache_set_uz_size(&zone->uz_cpu[i], zone->uz_size); in zone_update_caches()
2829 cache_set_uz_flags(&zone->uz_cpu[i], zone->uz_flags); in zone_update_caches()
2834 * Zone header ctor. This initializes all fields, locks, etc.
2845 uma_zone_t zone = mem; in zone_ctor() local
2850 bzero(zone, size); in zone_ctor()
2851 zone->uz_name = arg->name; in zone_ctor()
2852 zone->uz_ctor = arg->ctor; in zone_ctor()
2853 zone->uz_dtor = arg->dtor; in zone_ctor()
2854 zone->uz_init = NULL; in zone_ctor()
2855 zone->uz_fini = NULL; in zone_ctor()
2856 zone->uz_sleeps = 0; in zone_ctor()
2857 zone->uz_bucket_size = 0; in zone_ctor()
2858 zone->uz_bucket_size_min = 0; in zone_ctor()
2859 zone->uz_bucket_size_max = BUCKET_MAX; in zone_ctor()
2860 zone->uz_flags = (arg->flags & UMA_ZONE_SMR); in zone_ctor()
2861 zone->uz_warning = NULL; in zone_ctor()
2863 zone->uz_bucket_max = ULONG_MAX; in zone_ctor()
2864 timevalclear(&zone->uz_ratecheck); in zone_ctor()
2870 zone->uz_namecnt = cnt.count; in zone_ctor()
2871 ZONE_CROSS_LOCK_INIT(zone); in zone_ctor()
2874 zdom = ZDOM_GET(zone, i); in zone_ctor()
2875 ZDOM_LOCK_INIT(zone, zdom, (arg->flags & UMA_ZONE_MTXCLASS)); in zone_ctor()
2881 zone->uz_flags |= UMA_ZFLAG_TRASH | UMA_ZFLAG_CTORDTOR; in zone_ctor()
2888 * This is a pure cache zone, no kegs. in zone_ctor()
2892 ("zone_ctor: Import specified for non-cache zone.")); in zone_ctor()
2893 zone->uz_flags = arg->flags; in zone_ctor()
2894 zone->uz_size = arg->size; in zone_ctor()
2895 zone->uz_import = arg->import; in zone_ctor()
2896 zone->uz_release = arg->release; in zone_ctor()
2897 zone->uz_arg = arg->arg; in zone_ctor()
2904 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) == 0) in zone_ctor()
2905 zone->uz_flags |= UMA_ZONE_ROUNDROBIN; in zone_ctor()
2908 LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link); in zone_ctor()
2914 * Use the regular zone/keg/slab allocator. in zone_ctor()
2916 zone->uz_import = zone_import; in zone_ctor()
2917 zone->uz_release = zone_release; in zone_ctor()
2918 zone->uz_arg = zone; in zone_ctor()
2922 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0, in zone_ctor()
2923 ("Secondary zone requested UMA_ZFLAG_INTERNAL")); in zone_ctor()
2924 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); in zone_ctor()
2925 zone->uz_init = arg->uminit; in zone_ctor()
2926 zone->uz_fini = arg->fini; in zone_ctor()
2927 zone->uz_flags |= UMA_ZONE_SECONDARY; in zone_ctor()
2929 ZONE_LOCK(zone); in zone_ctor()
2932 LIST_INSERT_AFTER(z, zone, uz_link); in zone_ctor()
2936 ZONE_UNLOCK(zone); in zone_ctor()
2939 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini, in zone_ctor()
2952 karg.zone = zone; in zone_ctor()
2960 zone->uz_keg = keg; in zone_ctor()
2961 zone->uz_size = keg->uk_size; in zone_ctor()
2962 zone->uz_flags |= (keg->uk_flags & in zone_ctor()
2967 zone_alloc_counters(zone, NULL); in zone_ctor()
2969 zone_alloc_sysctl(zone, NULL); in zone_ctor()
2971 zone->uz_allocs = EARLY_COUNTER; in zone_ctor()
2972 zone->uz_frees = EARLY_COUNTER; in zone_ctor()
2973 zone->uz_fails = EARLY_COUNTER; in zone_ctor()
2977 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in zone_ctor()
2978 zone->uz_smr = smr_create(zone->uz_name, 0, 0); in zone_ctor()
2982 ("Invalid zone flag combination")); in zone_ctor()
2984 zone->uz_bucket_size_max = zone->uz_bucket_size = 0; in zone_ctor()
2986 zone->uz_bucket_size = BUCKET_MAX; in zone_ctor()
2988 zone->uz_bucket_size = 0; in zone_ctor()
2990 zone->uz_bucket_size = bucket_select(zone->uz_size); in zone_ctor()
2991 zone->uz_bucket_size_min = zone->uz_bucket_size; in zone_ctor()
2992 if (zone->uz_dtor != NULL || zone->uz_ctor != NULL) in zone_ctor()
2993 zone->uz_flags |= UMA_ZFLAG_CTORDTOR; in zone_ctor()
2994 zone_update_caches(zone); in zone_ctor()
3030 * Zone header dtor.
3038 uma_zone_t zone; in zone_dtor() local
3042 zone = (uma_zone_t)arg; in zone_dtor()
3044 sysctl_remove_oid(zone->uz_oid, 1, 1); in zone_dtor()
3046 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) in zone_dtor()
3047 cache_drain(zone); in zone_dtor()
3050 LIST_REMOVE(zone, uz_link); in zone_dtor()
3052 if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) { in zone_dtor()
3053 keg = zone->uz_keg; in zone_dtor()
3056 zone_reclaim(zone, UMA_ANYDOMAIN, M_WAITOK, true); in zone_dtor()
3061 if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) { in zone_dtor()
3062 keg = zone->uz_keg; in zone_dtor()
3068 counter_u64_free(zone->uz_allocs); in zone_dtor()
3069 counter_u64_free(zone->uz_frees); in zone_dtor()
3070 counter_u64_free(zone->uz_fails); in zone_dtor()
3071 counter_u64_free(zone->uz_xdomain); in zone_dtor()
3072 free(zone->uz_ctlname, M_UMA); in zone_dtor()
3074 ZDOM_LOCK_FINI(ZDOM_GET(zone, i)); in zone_dtor()
3075 ZONE_CROSS_LOCK_FINI(zone); in zone_dtor()
3082 uma_zone_t zone; in zone_foreach_unlocked() local
3085 LIST_FOREACH(zone, &keg->uk_zones, uz_link) in zone_foreach_unlocked()
3086 zfunc(zone, arg); in zone_foreach_unlocked()
3088 LIST_FOREACH(zone, &uma_cachezones, uz_link) in zone_foreach_unlocked()
3089 zfunc(zone, arg); in zone_foreach_unlocked()
3093 * Traverses every zone in the system and calls a callback
3096 * zfunc A pointer to a function which accepts a zone
3138 /* Allocate the zone of zones, zone of kegs, and zone of zones keg. */ in uma_startup1()
3152 /* "manually" create the initial zone */ in uma_startup1()
3211 /* Set up radix zone to use noobj_alloc. */ in uma_startup2()
3271 uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, in uma_kcreate() argument
3281 args.zone = zone; in uma_kcreate()
3330 /* This stuff is essential for the zone ctor */ in uma_zcreate()
3418 uma_zdestroy(uma_zone_t zone) in uma_zdestroy() argument
3426 zone->uz_fini == NULL && zone->uz_release == zone_release) in uma_zdestroy()
3429 zone_free_item(zones, zone, NULL, SKIP_NONE); in uma_zdestroy()
3434 uma_zwait(uma_zone_t zone) in uma_zwait() argument
3437 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in uma_zwait()
3438 uma_zfree_smr(zone, uma_zalloc_smr(zone, M_WAITOK)); in uma_zwait()
3439 else if ((zone->uz_flags & UMA_ZONE_PCPU) != 0) in uma_zwait()
3440 uma_zfree_pcpu(zone, uma_zalloc_pcpu(zone, M_WAITOK)); in uma_zwait()
3442 uma_zfree(zone, uma_zalloc(zone, M_WAITOK)); in uma_zwait()
3446 uma_zalloc_pcpu_arg(uma_zone_t zone, void *udata, int flags) in uma_zalloc_pcpu_arg() argument
3452 MPASS(zone->uz_flags & UMA_ZONE_PCPU); in uma_zalloc_pcpu_arg()
3454 item = uma_zalloc_arg(zone, udata, flags & ~M_ZERO); in uma_zalloc_pcpu_arg()
3461 bzero(zpcpu_get_cpu(pcpu_item, i), zone->uz_size); in uma_zalloc_pcpu_arg()
3463 bzero(item, zone->uz_size); in uma_zalloc_pcpu_arg()
3473 uma_zfree_pcpu_arg(uma_zone_t zone, void *pcpu_item, void *udata) in uma_zfree_pcpu_arg() argument
3478 MPASS(zone->uz_flags & UMA_ZONE_PCPU); in uma_zfree_pcpu_arg()
3486 uma_zfree_arg(zone, item, udata); in uma_zfree_pcpu_arg()
3490 item_ctor(uma_zone_t zone, int uz_flags, int size, void *udata, int flags, in item_ctor() argument
3497 kasan_mark_item_valid(zone, item); in item_ctor()
3498 kmsan_mark_item_uninitialized(zone, item); in item_ctor()
3501 skipdbg = uma_dbg_zskip(zone, item); in item_ctor()
3503 zone->uz_ctor != trash_ctor) in item_ctor()
3504 trash_ctor(item, size, zone, flags); in item_ctor()
3509 __predict_false(zone->uz_ctor != NULL) && in item_ctor()
3510 zone->uz_ctor(item, size, udata, flags) != 0) { in item_ctor()
3511 counter_u64_add(zone->uz_fails, 1); in item_ctor()
3512 zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); in item_ctor()
3517 uma_dbg_alloc(zone, NULL, item); in item_ctor()
3526 item_dtor(uma_zone_t zone, void *item, int size, void *udata, in item_dtor() argument
3532 skipdbg = uma_dbg_zskip(zone, item); in item_dtor()
3534 if ((zone->uz_flags & UMA_ZONE_MALLOC) != 0) in item_dtor()
3535 uma_dbg_free(zone, udata, item); in item_dtor()
3537 uma_dbg_free(zone, NULL, item); in item_dtor()
3541 if (zone->uz_dtor != NULL) in item_dtor()
3542 zone->uz_dtor(item, size, udata); in item_dtor()
3544 if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && in item_dtor()
3545 zone->uz_dtor != trash_dtor) in item_dtor()
3546 trash_dtor(item, size, zone); in item_dtor()
3549 kasan_mark_item_invalid(zone, item); in item_dtor()
3571 uma_zalloc_debug(uma_zone_t zone, void **itemp, void *udata, int flags) in uma_zalloc_debug() argument
3579 "uma_zalloc_debug: zone \"%s\"", zone->uz_name); in uma_zalloc_debug()
3588 KASSERT((zone->uz_flags & UMA_ZONE_PCPU) == 0 || (flags & M_ZERO) == 0, in uma_zalloc_debug()
3589 ("uma_zalloc_debug: allocating from a pcpu zone with M_ZERO")); in uma_zalloc_debug()
3618 if ((zone->uz_flags & (UMA_ZONE_SMR | UMA_ZFLAG_CACHE)) == 0 && in uma_zalloc_debug()
3619 memguard_cmp_zone(zone)) { in uma_zalloc_debug()
3621 item = memguard_alloc(zone->uz_size, flags); in uma_zalloc_debug()
3624 if (zone->uz_init != NULL && in uma_zalloc_debug()
3625 zone->uz_init(item, zone->uz_size, flags) != 0) { in uma_zalloc_debug()
3629 if (zone->uz_ctor != NULL && in uma_zalloc_debug()
3630 zone->uz_ctor(item, zone->uz_size, udata, in uma_zalloc_debug()
3632 counter_u64_add(zone->uz_fails, 1); in uma_zalloc_debug()
3633 if (zone->uz_fini != NULL) in uma_zalloc_debug()
3634 zone->uz_fini(item, zone->uz_size); in uma_zalloc_debug()
3648 uma_zfree_debug(uma_zone_t zone, void *item, void *udata) in uma_zfree_debug() argument
3654 if ((zone->uz_flags & (UMA_ZONE_SMR | UMA_ZFLAG_CACHE)) == 0 && in uma_zfree_debug()
3656 if (zone->uz_dtor != NULL) in uma_zfree_debug()
3657 zone->uz_dtor(item, zone->uz_size, udata); in uma_zfree_debug()
3658 if (zone->uz_fini != NULL) in uma_zfree_debug()
3659 zone->uz_fini(item, zone->uz_size); in uma_zfree_debug()
3669 cache_alloc_item(uma_zone_t zone, uma_cache_t cache, uma_cache_bucket_t bucket, in cache_alloc_item() argument
3679 return (item_ctor(zone, uz_flags, size, udata, flags, item)); in cache_alloc_item()
3683 cache_alloc_retry(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) in cache_alloc_retry() argument
3688 while (cache_alloc(zone, cache, udata, flags)) { in cache_alloc_retry()
3689 cache = &zone->uz_cpu[curcpu]; in cache_alloc_retry()
3693 return (cache_alloc_item(zone, cache, bucket, udata, flags)); in cache_alloc_retry()
3700 if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH) in cache_alloc_retry()
3704 return (zone_alloc_item(zone, udata, domain, flags)); in cache_alloc_retry()
3709 uma_zalloc_smr(uma_zone_t zone, int flags) in uma_zalloc_smr() argument
3714 CTR3(KTR_UMA, "uma_zalloc_smr zone %s(%p) flags %d", zone->uz_name, in uma_zalloc_smr()
3715 zone, flags); in uma_zalloc_smr()
3720 KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, in uma_zalloc_smr()
3721 ("uma_zalloc_arg: called with non-SMR zone.")); in uma_zalloc_smr()
3722 if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) in uma_zalloc_smr()
3727 cache = &zone->uz_cpu[curcpu]; in uma_zalloc_smr()
3730 return (cache_alloc_retry(zone, cache, NULL, flags)); in uma_zalloc_smr()
3731 return (cache_alloc_item(zone, cache, bucket, NULL, flags)); in uma_zalloc_smr()
3736 uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) in uma_zalloc_arg() argument
3742 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); in uma_zalloc_arg()
3745 CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name, in uma_zalloc_arg()
3746 zone, flags); in uma_zalloc_arg()
3751 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zalloc_arg()
3752 ("uma_zalloc_arg: called with SMR zone.")); in uma_zalloc_arg()
3753 if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) in uma_zalloc_arg()
3764 * order to acquire the zone mutex if we are unable to allocate from in uma_zalloc_arg()
3769 cache = &zone->uz_cpu[curcpu]; in uma_zalloc_arg()
3772 return (cache_alloc_retry(zone, cache, udata, flags)); in uma_zalloc_arg()
3773 return (cache_alloc_item(zone, cache, bucket, udata, flags)); in uma_zalloc_arg()
3784 cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) in cache_alloc() argument
3815 bucket_free(zone, bucket, udata); in cache_alloc()
3820 * we must go back to the zone. This requires the zdom lock, so we in cache_alloc()
3830 domain = zone_domain_highest(zone, domain); in cache_alloc()
3831 bucket = cache_fetch_bucket(zone, cache, domain); in cache_alloc()
3832 if (bucket == NULL && zone->uz_bucket_size != 0 && !bucketdisable) { in cache_alloc()
3833 bucket = zone_alloc_bucket(zone, udata, domain, flags); in cache_alloc()
3839 CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p", in cache_alloc()
3840 zone->uz_name, zone, bucket); in cache_alloc()
3852 cache = &zone->uz_cpu[curcpu]; in cache_alloc()
3858 atomic_add_long(&ZDOM_GET(zone, domain)->uzd_imax, in cache_alloc()
3868 zone_put_bucket(zone, domain, bucket, udata, !new); in cache_alloc()
3875 uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) in uma_zalloc_domain() argument
3884 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); in uma_zalloc_domain()
3887 CTR4(KTR_UMA, "uma_zalloc_domain zone %s(%p) domain %d flags %d", in uma_zalloc_domain()
3888 zone->uz_name, zone, domain, flags); in uma_zalloc_domain()
3890 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zalloc_domain()
3891 ("uma_zalloc_domain: called with SMR zone.")); in uma_zalloc_domain()
3893 KASSERT((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0, in uma_zalloc_domain()
3894 ("uma_zalloc_domain: called with non-FIRSTTOUCH zone.")); in uma_zalloc_domain()
3897 return (uma_zalloc_arg(zone, udata, flags)); in uma_zalloc_domain()
3900 if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) in uma_zalloc_domain()
3911 zdom = zone_domain_lock(zone, domain); in uma_zalloc_domain()
3912 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) { in uma_zalloc_domain()
3918 zone_put_bucket(zone, domain, bucket, udata, true); in uma_zalloc_domain()
3919 item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, in uma_zalloc_domain()
3925 counter_u64_add(zone->uz_allocs, 1); in uma_zalloc_domain()
3930 return (zone_alloc_item(zone, udata, domain, flags)); in uma_zalloc_domain()
3932 return (uma_zalloc_arg(zone, udata, flags)); in uma_zalloc_domain()
3998 keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) in keg_fetch_slab() argument
4032 slab = keg_alloc_slab(keg, zone, domain, flags, aflags); in keg_fetch_slab()
4098 uma_zone_t zone; in zone_import() local
4106 zone = arg; in zone_import()
4108 keg = zone->uz_keg; in zone_import()
4111 if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL) in zone_import()
4131 * If the zone is striped we pick a new slab for every in zone_import()
4138 if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0 && in zone_import()
4154 zone_alloc_limit_hard(uma_zone_t zone, int count, int flags) in zone_alloc_limit_hard() argument
4166 zone_free_limit(zone, count); in zone_alloc_limit_hard()
4167 zone_log_warning(zone); in zone_alloc_limit_hard()
4168 zone_maxaction(zone); in zone_alloc_limit_hard()
4177 sleepq_lock(&zone->uz_max_items); in zone_alloc_limit_hard()
4178 old = zone->uz_items; in zone_alloc_limit_hard()
4182 max = zone->uz_max_items; in zone_alloc_limit_hard()
4188 } while (atomic_fcmpset_64(&zone->uz_items, &old, new) == 0); in zone_alloc_limit_hard()
4192 sleepq_release(&zone->uz_max_items); in zone_alloc_limit_hard()
4202 atomic_add_32(&zone->uz_sleepers, 1); in zone_alloc_limit_hard()
4203 atomic_add_64(&zone->uz_sleeps, 1); in zone_alloc_limit_hard()
4209 sleepq_add(&zone->uz_max_items, NULL, "zonelimit", 0, 0); in zone_alloc_limit_hard()
4210 sleepq_wait(&zone->uz_max_items, PVM); in zone_alloc_limit_hard()
4219 atomic_subtract_32(&zone->uz_sleepers, 1); in zone_alloc_limit_hard()
4220 old = atomic_fetchadd_64(&zone->uz_items, in zone_alloc_limit_hard()
4231 max = zone->uz_max_items; in zone_alloc_limit_hard()
4236 zone_free_limit(zone, total + count - max); in zone_alloc_limit_hard()
4239 wakeup_one(&zone->uz_max_items); in zone_alloc_limit_hard()
4251 zone_alloc_limit(uma_zone_t zone, int count, int flags) in zone_alloc_limit() argument
4256 max = zone->uz_max_items; in zone_alloc_limit()
4263 old = atomic_fetchadd_64(&zone->uz_items, count); in zone_alloc_limit()
4274 zone_free_limit(zone, (old + count) - max); in zone_alloc_limit()
4277 return (zone_alloc_limit_hard(zone, count, flags)); in zone_alloc_limit()
4284 zone_free_limit(uma_zone_t zone, int count) in zone_free_limit() argument
4294 old = atomic_fetchadd_64(&zone->uz_items, -count); in zone_free_limit()
4296 UZ_ITEMS_COUNT(old) - count >= zone->uz_max_items)) in zone_free_limit()
4303 wakeup_one(&zone->uz_max_items); in zone_free_limit()
4307 zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) in zone_alloc_bucket() argument
4312 CTR3(KTR_UMA, "zone_alloc_bucket zone %s(%p) domain %d", zone->uz_name, in zone_alloc_bucket()
4313 zone, domain); in zone_alloc_bucket()
4318 else if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0) in zone_alloc_bucket()
4321 if (zone->uz_max_items > 0) in zone_alloc_bucket()
4322 maxbucket = zone_alloc_limit(zone, zone->uz_bucket_size, in zone_alloc_bucket()
4325 maxbucket = zone->uz_bucket_size; in zone_alloc_bucket()
4330 bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); in zone_alloc_bucket()
4336 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket, in zone_alloc_bucket()
4342 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) { in zone_alloc_bucket()
4346 kasan_mark_item_valid(zone, bucket->ub_bucket[i]); in zone_alloc_bucket()
4347 error = zone->uz_init(bucket->ub_bucket[i], in zone_alloc_bucket()
4348 zone->uz_size, flags); in zone_alloc_bucket()
4349 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); in zone_alloc_bucket()
4359 zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i], in zone_alloc_bucket()
4371 bucket_free(zone, bucket, udata); in zone_alloc_bucket()
4372 counter_u64_add(zone->uz_fails, 1); in zone_alloc_bucket()
4376 if (zone->uz_max_items > 0 && cnt < maxbucket) in zone_alloc_bucket()
4377 zone_free_limit(zone, maxbucket - cnt); in zone_alloc_bucket()
4383 * Allocates a single item from a zone.
4386 * zone The zone to alloc for.
4397 zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags) in zone_alloc_item() argument
4401 if (zone->uz_max_items > 0 && zone_alloc_limit(zone, 1, flags) == 0) { in zone_alloc_item()
4402 counter_u64_add(zone->uz_fails, 1); in zone_alloc_item()
4410 if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1) in zone_alloc_item()
4414 * We have to call both the zone's init (not the keg's init) in zone_alloc_item()
4415 * and the zone's ctor. This is because the item is going from in zone_alloc_item()
4417 * to be both zone-init'd as well as zone-ctor'd. in zone_alloc_item()
4419 if (zone->uz_init != NULL) { in zone_alloc_item()
4422 kasan_mark_item_valid(zone, item); in zone_alloc_item()
4423 error = zone->uz_init(item, zone->uz_size, flags); in zone_alloc_item()
4424 kasan_mark_item_invalid(zone, item); in zone_alloc_item()
4426 zone_free_item(zone, item, udata, SKIP_FINI | SKIP_CNT); in zone_alloc_item()
4430 item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, flags, in zone_alloc_item()
4435 counter_u64_add(zone->uz_allocs, 1); in zone_alloc_item()
4437 zone->uz_name, zone); in zone_alloc_item()
4442 counter_u64_add(zone->uz_fails, 1); in zone_alloc_item()
4444 if (zone->uz_max_items > 0) in zone_alloc_item()
4445 zone_free_limit(zone, 1); in zone_alloc_item()
4447 zone->uz_name, zone); in zone_alloc_item()
4454 uma_zfree_smr(uma_zone_t zone, void *item) in uma_zfree_smr() argument
4463 CTR3(KTR_UMA, "uma_zfree_smr zone %s(%p) item %p", in uma_zfree_smr()
4464 zone->uz_name, zone, item); in uma_zfree_smr()
4467 KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, in uma_zfree_smr()
4468 ("uma_zfree_smr: called with non-SMR zone.")); in uma_zfree_smr()
4470 SMR_ASSERT_NOT_ENTERED(zone->uz_smr); in uma_zfree_smr()
4471 if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN) in uma_zfree_smr()
4474 cache = &zone->uz_cpu[curcpu]; in uma_zfree_smr()
4483 cache = &zone->uz_cpu[curcpu]; in uma_zfree_smr()
4497 } while (cache_free(zone, cache, NULL, itemdomain)); in uma_zfree_smr()
4503 zone_free_item(zone, item, NULL, SKIP_NONE); in uma_zfree_smr()
4508 uma_zfree_arg(uma_zone_t zone, void *item, void *udata) in uma_zfree_arg() argument
4515 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); in uma_zfree_arg()
4517 CTR3(KTR_UMA, "uma_zfree_arg zone %s(%p) item %p", in uma_zfree_arg()
4518 zone->uz_name, zone, item); in uma_zfree_arg()
4521 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zfree_arg()
4522 ("uma_zfree_arg: called with SMR zone.")); in uma_zfree_arg()
4523 if (uma_zfree_debug(zone, item, udata) == EJUSTRETURN) in uma_zfree_arg()
4535 cache = &zone->uz_cpu[curcpu]; in uma_zfree_arg()
4539 item_dtor(zone, item, cache_uz_size(cache), udata, SKIP_NONE); in uma_zfree_arg()
4546 if (atomic_load_32(&zone->uz_sleepers) > 0) in uma_zfree_arg()
4557 * order to acquire the zone mutex if we are unable to free to the in uma_zfree_arg()
4568 cache = &zone->uz_cpu[curcpu]; in uma_zfree_arg()
4592 } while (cache_free(zone, cache, udata, itemdomain)); in uma_zfree_arg()
4599 zone_free_item(zone, item, udata, SKIP_DTOR); in uma_zfree_arg()
4608 zone_free_cross(uma_zone_t zone, uma_bucket_t bucket, void *udata) in zone_free_cross() argument
4618 "uma_zfree: zone %s(%p) draining cross bucket %p", in zone_free_cross()
4619 zone->uz_name, zone, bucket); in zone_free_cross()
4626 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in zone_free_cross()
4627 seq = smr_advance(zone->uz_smr); in zone_free_cross()
4636 ZONE_CROSS_LOCK(zone); in zone_free_cross()
4640 zdom = ZDOM_GET(zone, domain); in zone_free_cross()
4652 ZONE_CROSS_UNLOCK(zone); in zone_free_cross()
4653 b = bucket_alloc(zone, udata, M_NOWAIT); in zone_free_cross()
4656 ZONE_CROSS_LOCK(zone); in zone_free_cross()
4675 ZONE_CROSS_UNLOCK(zone); in zone_free_cross()
4679 bucket_free(zone, bucket, udata); in zone_free_cross()
4683 bucket_free(zone, b, udata); in zone_free_cross()
4688 zone_put_bucket(zone, domain, b, udata, true); in zone_free_cross()
4694 zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata, in zone_free_bucket() argument
4705 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && in zone_free_bucket()
4707 zone_free_cross(zone, bucket, udata); in zone_free_bucket()
4713 * Attempt to save the bucket in the zone's domain bucket cache. in zone_free_bucket()
4716 "uma_zfree: zone %s(%p) putting bucket %p on free list", in zone_free_bucket()
4717 zone->uz_name, zone, bucket); in zone_free_bucket()
4719 if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0) in zone_free_bucket()
4720 itemdomain = zone_domain_lowest(zone, itemdomain); in zone_free_bucket()
4721 zone_put_bucket(zone, itemdomain, bucket, udata, ws); in zone_free_bucket()
4726 * existing full bucket either to the zone cache or back to the slab layer.
4733 cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, int itemdomain) in cache_free() argument
4740 if (zone->uz_bucket_size == 0) in cache_free()
4743 cache = &zone->uz_cpu[curcpu]; in cache_free()
4757 counter_u64_add(zone->uz_xdomain, in cache_free()
4775 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) { in cache_free()
4777 bucket->ub_seq = smr_advance(zone->uz_smr); in cache_free()
4778 newbucket = bucket_alloc(zone, udata, M_NOWAIT); in cache_free()
4780 bucket_drain(zone, bucket); in cache_free()
4785 newbucket = bucket_alloc(zone, udata, M_NOWAIT); in cache_free()
4788 zone_free_bucket(zone, bucket, udata, itemdomain, true); in cache_free()
4793 cache = &zone->uz_cpu[curcpu]; in cache_free()
4813 bucket_free(zone, bucket, udata); in cache_free()
4822 slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item) in slab_free_item() argument
4828 keg = zone->uz_keg; in slab_free_item()
4855 uma_zone_t zone; in zone_release() local
4862 zone = arg; in zone_release()
4863 keg = zone->uz_keg; in zone_release()
4865 if (__predict_false((zone->uz_flags & UMA_ZFLAG_HASH) != 0)) in zone_release()
4869 if (__predict_true((zone->uz_flags & UMA_ZFLAG_VTOSLAB) != 0)) { in zone_release()
4873 if ((zone->uz_flags & UMA_ZFLAG_HASH) != 0) in zone_release()
4883 slab_free_item(zone, slab, item); in zone_release()
4890 * Frees a single item to any zone.
4893 * zone The zone to free to
4899 zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) in zone_free_item() argument
4903 * If a free is sent directly to an SMR zone we have to in zone_free_item()
4908 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && skip == SKIP_NONE) in zone_free_item()
4909 smr_synchronize(zone->uz_smr); in zone_free_item()
4911 item_dtor(zone, item, zone->uz_size, udata, skip); in zone_free_item()
4913 if (skip < SKIP_FINI && zone->uz_fini) { in zone_free_item()
4914 kasan_mark_item_valid(zone, item); in zone_free_item()
4915 zone->uz_fini(item, zone->uz_size); in zone_free_item()
4916 kasan_mark_item_invalid(zone, item); in zone_free_item()
4919 zone->uz_release(zone->uz_arg, &item, 1); in zone_free_item()
4924 counter_u64_add(zone->uz_frees, 1); in zone_free_item()
4926 if (zone->uz_max_items > 0) in zone_free_item()
4927 zone_free_limit(zone, 1); in zone_free_item()
4932 uma_zone_set_max(uma_zone_t zone, int nitems) in uma_zone_set_max() argument
4939 uma_zone_set_maxcache(zone, nitems); in uma_zone_set_max()
4942 * XXX This can misbehave if the zone has any allocations with in uma_zone_set_max()
4946 ZONE_LOCK(zone); in uma_zone_set_max()
4947 if (zone->uz_max_items == 0) in uma_zone_set_max()
4948 ZONE_ASSERT_COLD(zone); in uma_zone_set_max()
4949 zone->uz_max_items = nitems; in uma_zone_set_max()
4950 zone->uz_flags |= UMA_ZFLAG_LIMIT; in uma_zone_set_max()
4951 zone_update_caches(zone); in uma_zone_set_max()
4953 wakeup(&zone->uz_max_items); in uma_zone_set_max()
4954 ZONE_UNLOCK(zone); in uma_zone_set_max()
4961 uma_zone_set_maxcache(uma_zone_t zone, int nitems) in uma_zone_set_maxcache() argument
4965 ZONE_LOCK(zone); in uma_zone_set_maxcache()
4969 * the zone. Each CPU gets at least two buckets, and for cross-domain in uma_zone_set_maxcache()
4977 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && vm_ndomains > 1) { in uma_zone_set_maxcache()
4988 zone->uz_bucket_size_max = zone->uz_bucket_size = bsize; in uma_zone_set_maxcache()
4989 if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) in uma_zone_set_maxcache()
4990 zone->uz_bucket_size_min = zone->uz_bucket_size_max; in uma_zone_set_maxcache()
4991 zone->uz_bucket_max = nitems - nb * bsize; in uma_zone_set_maxcache()
4992 ZONE_UNLOCK(zone); in uma_zone_set_maxcache()
4997 uma_zone_get_max(uma_zone_t zone) in uma_zone_get_max() argument
5001 nitems = atomic_load_64(&zone->uz_max_items); in uma_zone_get_max()
5008 uma_zone_set_warning(uma_zone_t zone, const char *warning) in uma_zone_set_warning() argument
5011 ZONE_ASSERT_COLD(zone); in uma_zone_set_warning()
5012 zone->uz_warning = warning; in uma_zone_set_warning()
5017 uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction) in uma_zone_set_maxaction() argument
5020 ZONE_ASSERT_COLD(zone); in uma_zone_set_maxaction()
5021 TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone); in uma_zone_set_maxaction()
5026 uma_zone_get_cur(uma_zone_t zone) in uma_zone_get_cur() argument
5032 if (zone->uz_allocs != EARLY_COUNTER && zone->uz_frees != EARLY_COUNTER) in uma_zone_get_cur()
5033 nitems = counter_u64_fetch(zone->uz_allocs) - in uma_zone_get_cur()
5034 counter_u64_fetch(zone->uz_frees); in uma_zone_get_cur()
5036 nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs) - in uma_zone_get_cur()
5037 atomic_load_64(&zone->uz_cpu[i].uc_frees); in uma_zone_get_cur()
5043 uma_zone_get_allocs(uma_zone_t zone) in uma_zone_get_allocs() argument
5049 if (zone->uz_allocs != EARLY_COUNTER) in uma_zone_get_allocs()
5050 nitems = counter_u64_fetch(zone->uz_allocs); in uma_zone_get_allocs()
5052 nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs); in uma_zone_get_allocs()
5058 uma_zone_get_frees(uma_zone_t zone) in uma_zone_get_frees() argument
5064 if (zone->uz_frees != EARLY_COUNTER) in uma_zone_get_frees()
5065 nitems = counter_u64_fetch(zone->uz_frees); in uma_zone_get_frees()
5067 nitems += atomic_load_64(&zone->uz_cpu[i].uc_frees); in uma_zone_get_frees()
5090 uma_zone_set_init(uma_zone_t zone, uma_init uminit) in uma_zone_set_init() argument
5094 KEG_GET(zone, keg); in uma_zone_set_init()
5101 uma_zone_set_fini(uma_zone_t zone, uma_fini fini) in uma_zone_set_fini() argument
5105 KEG_GET(zone, keg); in uma_zone_set_fini()
5112 uma_zone_set_zinit(uma_zone_t zone, uma_init zinit) in uma_zone_set_zinit() argument
5115 ZONE_ASSERT_COLD(zone); in uma_zone_set_zinit()
5116 zone->uz_init = zinit; in uma_zone_set_zinit()
5121 uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini) in uma_zone_set_zfini() argument
5124 ZONE_ASSERT_COLD(zone); in uma_zone_set_zfini()
5125 zone->uz_fini = zfini; in uma_zone_set_zfini()
5130 uma_zone_set_freef(uma_zone_t zone, uma_free freef) in uma_zone_set_freef() argument
5134 KEG_GET(zone, keg); in uma_zone_set_freef()
5141 uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) in uma_zone_set_allocf() argument
5145 KEG_GET(zone, keg); in uma_zone_set_allocf()
5152 uma_zone_set_smr(uma_zone_t zone, smr_t smr) in uma_zone_set_smr() argument
5155 ZONE_ASSERT_COLD(zone); in uma_zone_set_smr()
5158 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zone_set_smr()
5159 ("zone %p (%s) already uses SMR", zone, zone->uz_name)); in uma_zone_set_smr()
5160 zone->uz_flags |= UMA_ZONE_SMR; in uma_zone_set_smr()
5161 zone->uz_smr = smr; in uma_zone_set_smr()
5162 zone_update_caches(zone); in uma_zone_set_smr()
5166 uma_zone_get_smr(uma_zone_t zone) in uma_zone_get_smr() argument
5169 return (zone->uz_smr); in uma_zone_get_smr()
5174 uma_zone_reserve(uma_zone_t zone, int items) in uma_zone_reserve() argument
5178 KEG_GET(zone, keg); in uma_zone_reserve()
5185 uma_zone_reserve_kva(uma_zone_t zone, int count) argument
5191 KEG_GET(zone, keg);
5193 ZONE_ASSERT_COLD(zone);
5211 zone->uz_max_items = pages * keg->uk_ipers;
5218 zone->uz_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE;
5219 zone_update_caches(zone);
5226 uma_prealloc(uma_zone_t zone, int items) argument
5234 KEG_GET(zone, keg);
5242 slab = keg_alloc_slab(keg, zone, domain, M_WAITOK,
5268 uma_zone_memory(uma_zone_t zone) argument
5274 if (zone->uz_flags & UMA_ZFLAG_CACHE) {
5276 sz += ZDOM_GET(zone, i)->uzd_nitems;
5277 return (sz * zone->uz_size);
5280 sz += zone->uz_keg->uk_domain[i].ud_pages;
5291 uma_reclaim_domain_cb(uma_zone_t zone, void *arg) argument
5296 if ((zone->uz_flags & UMA_ZONE_UNMANAGED) != 0)
5299 (zone->uz_flags & UMA_ZONE_NOTRIM) !=0)
5302 uma_zone_reclaim_domain(zone, args->req, args->domain);
5333 * drain each zone's per-CPU buckets.
5345 * Some slabs may have been freed but this zone will be visited early
5385 uma_zone_reclaim(uma_zone_t zone, int req) argument
5387 uma_zone_reclaim_domain(zone, req, UMA_ANYDOMAIN);
5391 uma_zone_reclaim_domain(uma_zone_t zone, int req, int domain) argument
5395 zone_reclaim(zone, domain, M_NOWAIT, false);
5398 zone_reclaim(zone, domain, M_NOWAIT, true);
5401 pcpu_cache_drain_safe(zone);
5402 zone_reclaim(zone, domain, M_NOWAIT, true);
5411 uma_zone_exhausted(uma_zone_t zone) argument
5414 return (atomic_load_32(&zone->uz_sleepers) > 0);
5447 * Generate statistics across both the zone and its per-cpu cache's. Return
5450 * Note: does not update the zone statistics, as it can't safely clear the
5604 * A zone is secondary is it is not the first entry
5605 * on the keg's zone list.
5637 uma_zone_t zone = *(uma_zone_t *)arg1; local
5640 max = uma_zone_get_max(zone);
5645 uma_zone_set_max(zone, max);
5653 uma_zone_t zone; local
5661 zone = *(uma_zone_t *)arg1;
5663 zone = arg1;
5664 cur = uma_zone_get_cur(zone);
5671 uma_zone_t zone = arg1; local
5674 cur = uma_zone_get_allocs(zone);
5681 uma_zone_t zone = arg1; local
5684 cur = uma_zone_get_frees(zone);
5692 uma_zone_t zone = arg1; local
5696 if (zone->uz_flags != 0)
5697 sbuf_printf(&sbuf, "0x%b", zone->uz_flags, PRINT_UMA_ZFLAGS);
5730 uma_zone_t zone = arg1; local
5733 cur = UZ_ITEMS_COUNT(atomic_load_64(&zone->uz_items));
5739 uma_dbg_getslab(uma_zone_t zone, void *item) argument
5747 * zone is unlocked because the item's allocation state
5751 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0)
5753 if (zone->uz_flags & UMA_ZFLAG_VTOSLAB)
5755 keg = zone->uz_keg;
5766 uma_dbg_zskip(uma_zone_t zone, void *mem) argument
5769 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0)
5772 return (uma_dbg_kskip(zone->uz_keg, mem));
5806 uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item) argument
5812 slab = uma_dbg_getslab(zone, item);
5814 panic("uma: item %p did not belong to zone %s",
5815 item, zone->uz_name);
5817 keg = zone->uz_keg;
5822 panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)",
5823 item, zone, zone->uz_name, slab, freei);
5832 uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item) argument
5838 slab = uma_dbg_getslab(zone, item);
5840 panic("uma: Freed item %p did not belong to zone %s",
5841 item, zone->uz_name);
5843 keg = zone->uz_keg;
5847 panic("Invalid free of %p from zone %p(%s) slab %p(%d)",
5848 item, zone, zone->uz_name, slab, freei);
5851 panic("Unaligned free of %p from zone %p(%s) slab %p(%d)",
5852 item, zone, zone->uz_name, slab, freei);
5856 panic("Duplicate free of %p from zone %p(%s) slab %p(%d)",
5857 item, zone, zone->uz_name, slab, freei);
5910 db_printf(fmt_hdr, "Zone", "Size", "Used", "Free", "Requests",
5926 * zone, we have already printed all preceding
5968 db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free",