Lines Matching refs:bucket
303 static void zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata,
466 uma_bucket_t bucket; in bucket_alloc() local
496 bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags); in bucket_alloc()
497 if (bucket) { in bucket_alloc()
499 bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); in bucket_alloc()
501 bucket->ub_cnt = 0; in bucket_alloc()
502 bucket->ub_entries = min(ubz->ubz_entries, in bucket_alloc()
504 bucket->ub_seq = SMR_SEQ_INVALID; in bucket_alloc()
506 zone->uz_name, zone, bucket); in bucket_alloc()
509 return (bucket); in bucket_alloc()
513 bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata) in bucket_free() argument
517 if (bucket->ub_cnt != 0) in bucket_free()
518 bucket_drain(zone, bucket); in bucket_free()
520 KASSERT(bucket->ub_cnt == 0, in bucket_free()
522 KASSERT(bucket->ub_seq == SMR_SEQ_INVALID, in bucket_free()
526 ubz = bucket_zone_lookup(bucket->ub_entries); in bucket_free()
527 uma_zfree_arg(ubz->ubz_zone, bucket, udata); in bucket_free()
792 uma_bucket_t bucket; in zone_fetch_bucket() local
799 if ((bucket = STAILQ_FIRST(&zdom->uzd_buckets)) == NULL) in zone_fetch_bucket()
804 bucket->ub_seq != SMR_SEQ_INVALID) { in zone_fetch_bucket()
805 if (!smr_poll(zone->uz_smr, bucket->ub_seq, false)) in zone_fetch_bucket()
807 bucket->ub_seq = SMR_SEQ_INVALID; in zone_fetch_bucket()
809 if (STAILQ_NEXT(bucket, ub_link) != NULL) in zone_fetch_bucket()
810 zdom->uzd_seq = STAILQ_NEXT(bucket, ub_link)->ub_seq; in zone_fetch_bucket()
814 KASSERT(zdom->uzd_nitems >= bucket->ub_cnt, in zone_fetch_bucket()
816 __func__, zdom->uzd_nitems, bucket->ub_cnt)); in zone_fetch_bucket()
817 KASSERT(bucket->ub_cnt > 0, in zone_fetch_bucket()
819 zdom->uzd_nitems -= bucket->ub_cnt; in zone_fetch_bucket()
826 cnt = lmin(zdom->uzd_bimin, bucket->ub_cnt); in zone_fetch_bucket()
829 zdom->uzd_imin -= lmin(zdom->uzd_imin, bucket->ub_cnt); in zone_fetch_bucket()
830 if (zdom->uzd_limin >= bucket->ub_cnt) { in zone_fetch_bucket()
831 zdom->uzd_limin -= bucket->ub_cnt; in zone_fetch_bucket()
844 for (i = 0; i < bucket->ub_cnt; i++) in zone_fetch_bucket()
845 item_dtor(zone, bucket->ub_bucket[i], zone->uz_size, in zone_fetch_bucket()
848 return (bucket); in zone_fetch_bucket()
857 zone_put_bucket(uma_zone_t zone, int domain, uma_bucket_t bucket, void *udata, in zone_put_bucket() argument
863 if (bucket->ub_cnt == 0) in zone_put_bucket()
870 zdom->uzd_nitems += bucket->ub_cnt; in zone_put_bucket()
879 atomic_add_long(&zdom->uzd_imax, bucket->ub_cnt); in zone_put_bucket()
880 zdom->uzd_imin += bucket->ub_cnt; in zone_put_bucket()
881 zdom->uzd_bimin += bucket->ub_cnt; in zone_put_bucket()
882 zdom->uzd_limin += bucket->ub_cnt; in zone_put_bucket()
885 zdom->uzd_seq = bucket->ub_seq; in zone_put_bucket()
891 if (bucket->ub_seq == SMR_SEQ_INVALID) in zone_put_bucket()
892 STAILQ_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link); in zone_put_bucket()
894 STAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link); in zone_put_bucket()
898 zdom->uzd_nitems -= bucket->ub_cnt; in zone_put_bucket()
901 bucket_free(zone, bucket, udata); in zone_put_bucket()
906 cache_bucket_pop(uma_cache_t cache, uma_cache_bucket_t bucket) in cache_bucket_pop() argument
912 bucket->ucb_cnt--; in cache_bucket_pop()
913 item = bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt]; in cache_bucket_pop()
915 bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt] = NULL; in cache_bucket_pop()
925 cache_bucket_push(uma_cache_t cache, uma_cache_bucket_t bucket, void *item) in cache_bucket_push() argument
929 KASSERT(bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt] == NULL, in cache_bucket_push()
932 bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt] = item; in cache_bucket_push()
933 bucket->ucb_cnt++; in cache_bucket_push()
941 cache_bucket_unload(uma_cache_bucket_t bucket) in cache_bucket_unload() argument
945 b = bucket->ucb_bucket; in cache_bucket_unload()
947 MPASS(b->ub_entries == bucket->ucb_entries); in cache_bucket_unload()
948 b->ub_cnt = bucket->ucb_cnt; in cache_bucket_unload()
949 bucket->ucb_bucket = NULL; in cache_bucket_unload()
950 bucket->ucb_entries = bucket->ucb_cnt = 0; in cache_bucket_unload()
981 cache_bucket_load(uma_cache_bucket_t bucket, uma_bucket_t b) in cache_bucket_load() argument
985 MPASS(bucket->ucb_bucket == NULL); in cache_bucket_load()
988 bucket->ucb_bucket = b; in cache_bucket_load()
989 bucket->ucb_cnt = b->ub_cnt; in cache_bucket_load()
990 bucket->ucb_entries = b->ub_entries; in cache_bucket_load()
1050 uma_bucket_t bucket; in cache_fetch_bucket() local
1069 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) in cache_fetch_bucket()
1070 return (bucket); in cache_fetch_bucket()
1344 bucket_drain(uma_zone_t zone, uma_bucket_t bucket) in bucket_drain() argument
1348 if (bucket->ub_cnt == 0) in bucket_drain()
1352 bucket->ub_seq != SMR_SEQ_INVALID) { in bucket_drain()
1353 smr_wait(zone->uz_smr, bucket->ub_seq); in bucket_drain()
1354 bucket->ub_seq = SMR_SEQ_INVALID; in bucket_drain()
1355 for (i = 0; i < bucket->ub_cnt; i++) in bucket_drain()
1356 item_dtor(zone, bucket->ub_bucket[i], in bucket_drain()
1360 for (i = 0; i < bucket->ub_cnt; i++) { in bucket_drain()
1361 kasan_mark_item_valid(zone, bucket->ub_bucket[i]); in bucket_drain()
1362 zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); in bucket_drain()
1363 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); in bucket_drain()
1365 zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); in bucket_drain()
1367 zone_free_limit(zone, bucket->ub_cnt); in bucket_drain()
1369 bzero(bucket->ub_bucket, sizeof(void *) * bucket->ub_cnt); in bucket_drain()
1371 bucket->ub_cnt = 0; in bucket_drain()
1391 uma_bucket_t bucket; in cache_drain() local
1408 bucket = cache_bucket_unload_alloc(cache); in cache_drain()
1409 if (bucket != NULL) in cache_drain()
1410 bucket_free(zone, bucket, NULL); in cache_drain()
1411 bucket = cache_bucket_unload_free(cache); in cache_drain()
1412 if (bucket != NULL) { in cache_drain()
1413 bucket->ub_seq = seq; in cache_drain()
1414 bucket_free(zone, bucket, NULL); in cache_drain()
1416 bucket = cache_bucket_unload_cross(cache); in cache_drain()
1417 if (bucket != NULL) { in cache_drain()
1418 bucket->ub_seq = seq; in cache_drain()
1419 bucket_free(zone, bucket, NULL); in cache_drain()
1519 uma_bucket_t bucket; in bucket_cache_reclaim_domain() local
1530 bucket = zdom->uzd_cross; in bucket_cache_reclaim_domain()
1533 if (bucket != NULL) in bucket_cache_reclaim_domain()
1534 bucket_free(zone, bucket, NULL); in bucket_cache_reclaim_domain()
1557 while ((bucket = STAILQ_FIRST(&zdom->uzd_buckets)) != NULL && in bucket_cache_reclaim_domain()
1558 zdom->uzd_nitems >= target + bucket->ub_cnt) { in bucket_cache_reclaim_domain()
1559 bucket = zone_fetch_bucket(zone, zdom, true); in bucket_cache_reclaim_domain()
1560 if (bucket == NULL) in bucket_cache_reclaim_domain()
1562 bucket_free(zone, bucket, NULL); in bucket_cache_reclaim_domain()
3669 cache_alloc_item(uma_zone_t zone, uma_cache_t cache, uma_cache_bucket_t bucket, in cache_alloc_item() argument
3675 item = cache_bucket_pop(cache, bucket); in cache_alloc_item()
3685 uma_cache_bucket_t bucket; in cache_alloc_retry() local
3690 bucket = &cache->uc_allocbucket; in cache_alloc_retry()
3691 if (__predict_false(bucket->ucb_cnt == 0)) in cache_alloc_retry()
3693 return (cache_alloc_item(zone, cache, bucket, udata, flags)); in cache_alloc_retry()
3711 uma_cache_bucket_t bucket; in uma_zalloc_smr() local
3728 bucket = &cache->uc_allocbucket; in uma_zalloc_smr()
3729 if (__predict_false(bucket->ucb_cnt == 0)) in uma_zalloc_smr()
3731 return (cache_alloc_item(zone, cache, bucket, NULL, flags)); in uma_zalloc_smr()
3738 uma_cache_bucket_t bucket; in uma_zalloc_arg() local
3770 bucket = &cache->uc_allocbucket; in uma_zalloc_arg()
3771 if (__predict_false(bucket->ucb_cnt == 0)) in uma_zalloc_arg()
3773 return (cache_alloc_item(zone, cache, bucket, udata, flags)); in uma_zalloc_arg()
3786 uma_bucket_t bucket; in cache_alloc() local
3809 bucket = cache_bucket_unload_alloc(cache); in cache_alloc()
3812 if (bucket != NULL) { in cache_alloc()
3813 KASSERT(bucket->ub_cnt == 0, in cache_alloc()
3815 bucket_free(zone, bucket, udata); in cache_alloc()
3831 bucket = cache_fetch_bucket(zone, cache, domain); in cache_alloc()
3832 if (bucket == NULL && zone->uz_bucket_size != 0 && !bucketdisable) { in cache_alloc()
3833 bucket = zone_alloc_bucket(zone, udata, domain, flags); in cache_alloc()
3840 zone->uz_name, zone, bucket); in cache_alloc()
3841 if (bucket == NULL) { in cache_alloc()
3859 bucket->ub_cnt); in cache_alloc()
3860 cache_bucket_load_alloc(cache, bucket); in cache_alloc()
3868 zone_put_bucket(zone, domain, bucket, udata, !new); in cache_alloc()
3878 uma_bucket_t bucket; in uma_zalloc_domain() local
3912 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) { in uma_zalloc_domain()
3913 item = bucket->ub_bucket[bucket->ub_cnt - 1]; in uma_zalloc_domain()
3915 bucket->ub_bucket[bucket->ub_cnt - 1] = NULL; in uma_zalloc_domain()
3917 bucket->ub_cnt--; in uma_zalloc_domain()
3918 zone_put_bucket(zone, domain, bucket, udata, true); in uma_zalloc_domain()
4105 zone_import(void *arg, void **bucket, int max, int domain, int flags) in zone_import() argument
4128 bucket[i++] = slab_alloc_item(keg, slab); in zone_import()
4319 uma_bucket_t bucket; in zone_alloc_bucket() local
4340 bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); in zone_alloc_bucket()
4341 if (bucket == NULL) { in zone_alloc_bucket()
4346 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket, in zone_alloc_bucket()
4347 MIN(maxbucket, bucket->ub_entries), domain, flags); in zone_alloc_bucket()
4352 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) { in zone_alloc_bucket()
4355 for (i = 0; i < bucket->ub_cnt; i++) { in zone_alloc_bucket()
4356 kasan_mark_item_valid(zone, bucket->ub_bucket[i]); in zone_alloc_bucket()
4357 error = zone->uz_init(bucket->ub_bucket[i], in zone_alloc_bucket()
4359 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); in zone_alloc_bucket()
4368 if (i != bucket->ub_cnt) { in zone_alloc_bucket()
4369 zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i], in zone_alloc_bucket()
4370 bucket->ub_cnt - i); in zone_alloc_bucket()
4372 bzero(&bucket->ub_bucket[i], in zone_alloc_bucket()
4373 sizeof(void *) * (bucket->ub_cnt - i)); in zone_alloc_bucket()
4375 bucket->ub_cnt = i; in zone_alloc_bucket()
4379 cnt = bucket->ub_cnt; in zone_alloc_bucket()
4380 if (bucket->ub_cnt == 0) { in zone_alloc_bucket()
4381 bucket_free(zone, bucket, udata); in zone_alloc_bucket()
4383 bucket = NULL; in zone_alloc_bucket()
4389 return (bucket); in zone_alloc_bucket()
4467 uma_cache_bucket_t bucket; in uma_zfree_smr() local
4495 bucket = &cache->uc_freebucket; in uma_zfree_smr()
4499 bucket = &cache->uc_crossbucket; in uma_zfree_smr()
4502 if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { in uma_zfree_smr()
4503 cache_bucket_push(cache, bucket, item); in uma_zfree_smr()
4521 uma_cache_bucket_t bucket; in uma_zfree_arg() local
4585 bucket = &cache->uc_allocbucket; in uma_zfree_arg()
4589 bucket = &cache->uc_crossbucket; in uma_zfree_arg()
4592 if (bucket->ucb_cnt == bucket->ucb_entries && in uma_zfree_arg()
4597 if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { in uma_zfree_arg()
4598 cache_bucket_push(cache, bucket, item); in uma_zfree_arg()
4618 zone_free_cross(uma_zone_t zone, uma_bucket_t bucket, void *udata) in zone_free_cross() argument
4629 zone->uz_name, zone, bucket); in zone_free_cross()
4647 for (; bucket->ub_cnt > 0; bucket->ub_cnt--) { in zone_free_cross()
4648 item = bucket->ub_bucket[bucket->ub_cnt - 1]; in zone_free_cross()
4687 if (bucket->ub_cnt == 0) in zone_free_cross()
4688 bucket->ub_seq = SMR_SEQ_INVALID; in zone_free_cross()
4689 bucket_free(zone, bucket, udata); in zone_free_cross()
4704 zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata, in zone_free_bucket() argument
4717 zone_free_cross(zone, bucket, udata); in zone_free_bucket()
4727 zone->uz_name, zone, bucket); in zone_free_bucket()
4731 zone_put_bucket(zone, itemdomain, bucket, udata, ws); in zone_free_bucket()
4746 uma_bucket_t newbucket, bucket; in cache_free() local
4772 bucket = cache_bucket_unload(cbucket); in cache_free()
4773 KASSERT(bucket == NULL || bucket->ub_cnt == bucket->ub_entries, in cache_free()
4786 if (bucket != NULL) in cache_free()
4787 bucket->ub_seq = smr_advance(zone->uz_smr); in cache_free()
4789 if (newbucket == NULL && bucket != NULL) { in cache_free()
4790 bucket_drain(zone, bucket); in cache_free()
4791 newbucket = bucket; in cache_free()
4792 bucket = NULL; in cache_free()
4797 if (bucket != NULL) in cache_free()
4798 zone_free_bucket(zone, bucket, udata, itemdomain, true); in cache_free()
4801 if ((bucket = newbucket) == NULL) in cache_free()
4813 cache_bucket_load_cross(cache, bucket); in cache_free()
4823 bucket_free(zone, bucket, udata); in cache_free()
4826 cache_bucket_load_free(cache, bucket); in cache_free()
4862 zone_release(void *arg, void **bucket, int cnt) in zone_release() argument
4878 item = bucket[i]; in zone_release()