Lines Matching +full:- +full:- +full:enable +full:- +full:debug +full:- +full:kmem
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
38 * - Magazines and Vmem: Extending the Slab Allocator
81 (VMEM_OPTVALUE - 1 + sizeof(vmem_size_t) * NBBY - VMEM_OPTORDER)
115 #define QC_POOL_TO_QCACHE(pool) ((qcache_t *)(pool->pr_qcache))
177 #define BT_ISSPAN_P(bt) ((bt)->bt_type <= BT_TYPE_SPAN_STATIC)
179 #define BT_END(bt) ((bt)->bt_start + (bt)->bt_size - 1)
184 &enable_vmem_check, 0, "Enable vmem check");
196 /* ---- misc */
197 #define VMEM_CONDVAR_INIT(vm, wchan) cv_init(&vm->vm_cv, wchan)
198 #define VMEM_CONDVAR_DESTROY(vm) cv_destroy(&vm->vm_cv)
199 #define VMEM_CONDVAR_WAIT(vm) cv_wait(&vm->vm_cv, &vm->vm_lock)
200 #define VMEM_CONDVAR_BROADCAST(vm) cv_broadcast(&vm->vm_cv)
202 #define VMEM_LOCK(vm) mtx_lock(&vm->vm_lock)
203 #define VMEM_TRYLOCK(vm) mtx_trylock(&vm->vm_lock)
204 #define VMEM_UNLOCK(vm) mtx_unlock(&vm->vm_lock)
205 #define VMEM_LOCK_INIT(vm, name) mtx_init(&vm->vm_lock, (name), NULL, MTX_DEF)
206 #define VMEM_LOCK_DESTROY(vm) mtx_destroy(&vm->vm_lock)
207 #define VMEM_ASSERT_LOCKED(vm) mtx_assert(&vm->vm_lock, MA_OWNED);
209 #define VMEM_ALIGNUP(addr, align) (-(-(addr) & -(align)))
212 ((((addr1) ^ (addr2)) & -(boundary)) != 0)
215 (vmem_size_t)1 << ((order) - (VMEM_OPTVALUE - VMEM_OPTORDER - 1)))
216 #define SIZE2ORDER(size) ((size) <= VMEM_OPTVALUE ? ((size) - 1) : \
217 (flsl(size) + (VMEM_OPTVALUE - VMEM_OPTORDER - 2)))
251 return (bt->bt_type == BT_TYPE_BUSY);
257 return (bt->bt_type == BT_TYPE_FREE);
277 if (vm != kernel_arena && vm->vm_arg != kernel_arena)
286 while (vm->vm_nfreetags < BT_MAXALLOC) {
296 LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
297 vm->vm_nfreetags++;
300 if (vm->vm_nfreetags < BT_MAXALLOC)
309 if (vm->vm_nfreetags >= BT_MAXALLOC)
323 bt = LIST_FIRST(&vm->vm_freetags);
326 vm->vm_nfreetags--;
332 * Trim the per-vmem free list. Returns with the lock released to
343 while (vm->vm_nfreetags > freelimit) {
344 bt = LIST_FIRST(&vm->vm_freetags);
346 vm->vm_nfreetags--;
361 MPASS(LIST_FIRST(&vm->vm_freetags) != bt);
362 LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
363 vm->vm_nfreetags++;
373 KASSERT(vm->vm_nfreetags >= BT_MAXALLOC,
374 ("%s: insufficient free tags %d", __func__, vm->vm_nfreetags));
375 vm->vm_nfreetags -= BT_MAXALLOC;
381 vm->vm_nfreetags += BT_MAXALLOC;
393 * freelist[n] ... [(1 << (n - 26)), (1 << (n - 25)) - 1]
400 const vmem_size_t qsize = size >> vm->vm_quantum_shift;
404 MPASS((size & vm->vm_quantum_mask) == 0);
408 return &vm->vm_freelist[idx];
422 const vmem_size_t qsize = size >> vm->vm_quantum_shift;
426 MPASS((size & vm->vm_quantum_mask) == 0);
435 return &vm->vm_freelist[idx];
438 /* ---- boundary tag hash */
447 list = &vm->vm_hashlist[hash % vm->vm_hashsize];
461 if (bt->bt_start == addr) {
474 MPASS(vm->vm_nbusytag > 0);
475 vm->vm_inuse -= bt->bt_size;
476 vm->vm_nbusytag--;
486 MPASS(bt->bt_type == BT_TYPE_BUSY);
488 list = bt_hashhead(vm, bt->bt_start);
490 vm->vm_nbusytag++;
491 vm->vm_inuse += bt->bt_size;
494 /* ---- boundary tag list */
500 MPASS(bt->bt_type != BT_TYPE_CURSOR);
501 TAILQ_REMOVE(&vm->vm_seglist, bt, bt_seglist);
509 TAILQ_INSERT_AFTER(&vm->vm_seglist, prev, bt, bt_seglist);
516 TAILQ_INSERT_TAIL(&vm->vm_seglist, bt, bt_seglist);
523 MPASS(bt->bt_type == BT_TYPE_FREE);
533 list = bt_freehead_tofree(vm, bt->bt_size);
537 /* ---- vmem internal functions */
556 if (vmem_xalloc(qc->qc_vmem, qc->qc_size, 0, 0, 0,
575 vmem_xfree(qc->qc_vmem, (vmem_addr_t)store[i], qc->qc_size);
586 MPASS((qcache_max & vm->vm_quantum_mask) == 0);
587 qcache_idx_max = MIN(qcache_max >> vm->vm_quantum_shift,
589 vm->vm_qcache_max = qcache_idx_max << vm->vm_quantum_shift;
591 qc = &vm->vm_qcache[i];
592 size = (i + 1) << vm->vm_quantum_shift;
593 snprintf(qc->qc_name, sizeof(qc->qc_name), "%s-%zu",
594 vm->vm_name, size);
595 qc->qc_vmem = vm;
596 qc->qc_size = size;
597 qc->qc_cache = uma_zcache_create(qc->qc_name, size,
599 MPASS(qc->qc_cache);
609 qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift;
611 uma_zdestroy(vm->vm_qcache[i].qc_cache);
620 qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift;
622 uma_zone_reclaim(vm->vm_qcache[i].qc_cache, UMA_RECLAIM_DRAIN);
714 * false restarts in UMA. vmem_bt_alloc() allocates from a per-domain
723 /* ---- rehash */
744 oldhashlist = vm->vm_hashlist;
745 oldhashsize = vm->vm_hashsize;
746 vm->vm_hashlist = newhashlist;
747 vm->vm_hashsize = newhashsize;
760 if (oldhashlist != vm->vm_hash0)
790 desired = 1 << flsl(vm->vm_nbusytag);
793 current = vm->vm_hashsize;
830 MPASS((size & vm->vm_quantum_mask) == 0);
832 if (vm->vm_releasefn == NULL) {
840 btprev = TAILQ_LAST(&vm->vm_seglist, vmem_seglist);
842 btprev->bt_start + btprev->bt_size != addr)
851 btspan->bt_type = type;
852 btspan->bt_start = addr;
853 btspan->bt_size = size;
858 btfree->bt_type = BT_TYPE_FREE;
859 btfree->bt_start = addr;
860 btfree->bt_size = size;
865 btprev->bt_size += size;
869 vm->vm_size += size;
878 * Drain per-cpu quantum caches.
886 MPASS(vm->vm_nbusytag == 0);
888 TAILQ_REMOVE(&vm->vm_seglist, &vm->vm_cursor, bt_seglist);
889 while ((bt = TAILQ_FIRST(&vm->vm_seglist)) != NULL)
892 if (vm->vm_hashlist != NULL && vm->vm_hashlist != vm->vm_hash0)
893 free(vm->vm_hashlist, M_VMEM);
908 if (vm->vm_importfn == NULL)
915 if (align != vm->vm_quantum_mask + 1)
917 size = roundup(size, vm->vm_import_quantum);
919 if (vm->vm_limit != 0 && vm->vm_limit < vm->vm_size + size)
924 error = (vm->vm_importfn)(vm->vm_arg, size, flags, &addr);
950 MPASS(bt->bt_size >= size); /* caller's responsibility */
957 start = bt->bt_start;
967 start = VMEM_ALIGNUP(start - phase, align) + phase;
968 if (start < bt->bt_start)
970 if (VMEM_CROSS_P(start, start + size - 1, nocross)) {
972 start = VMEM_ALIGNUP(start - phase, nocross) + phase;
974 if (start <= end && end - start >= size - 1) {
975 MPASS((start & (align - 1)) == phase);
976 MPASS(!VMEM_CROSS_P(start, start + size - 1, nocross));
978 MPASS(maxaddr == 0 || start + size - 1 <= maxaddr);
979 MPASS(bt->bt_start <= start);
980 MPASS(BT_END(bt) - start >= size - 1);
998 MPASS(bt->bt_type == BT_TYPE_FREE);
999 MPASS(bt->bt_size >= size);
1001 if (bt->bt_start != start) {
1003 btprev->bt_type = BT_TYPE_FREE;
1004 btprev->bt_start = bt->bt_start;
1005 btprev->bt_size = start - bt->bt_start;
1006 bt->bt_start = start;
1007 bt->bt_size -= btprev->bt_size;
1012 MPASS(bt->bt_start == start);
1013 if (bt->bt_size != size && bt->bt_size - size > vm->vm_quantum_mask) {
1016 btnew->bt_type = BT_TYPE_BUSY;
1017 btnew->bt_start = bt->bt_start;
1018 btnew->bt_size = size;
1019 bt->bt_start = bt->bt_start + size;
1020 bt->bt_size -= size;
1027 bt->bt_type = BT_TYPE_BUSY;
1030 MPASS(bt->bt_size >= size);
1052 if (vm->vm_qcache_max != 0 || vm->vm_reclaimfn != NULL) {
1053 avail = vm->vm_size - vm->vm_inuse;
1056 if (vm->vm_qcache_max != 0)
1058 if (vm->vm_reclaimfn != NULL)
1059 vm->vm_reclaimfn(vm, flags);
1063 if (vm->vm_size - vm->vm_inuse > avail)
1079 MPASS(bt->bt_type == BT_TYPE_FREE);
1081 if (vm->vm_releasefn == NULL)
1086 MPASS(prev->bt_type != BT_TYPE_FREE);
1088 if (prev->bt_type == BT_TYPE_SPAN && prev->bt_size == bt->bt_size) {
1092 MPASS(prev->bt_start == bt->bt_start);
1093 spanaddr = prev->bt_start;
1094 spansize = prev->bt_size;
1099 vm->vm_size -= spansize;
1102 vm->vm_releasefn(vm->vm_arg, spanaddr, spansize);
1130 for (cursor = &vm->vm_cursor, bt = TAILQ_NEXT(cursor, bt_seglist);
1133 bt = TAILQ_FIRST(&vm->vm_seglist);
1134 if (bt->bt_type == BT_TYPE_FREE && bt->bt_size >= size &&
1149 next->bt_type == BT_TYPE_FREE && prev->bt_type == BT_TYPE_FREE &&
1150 prev->bt_start + prev->bt_size == next->bt_start) {
1151 prev->bt_size += next->bt_size;
1159 if (error == ENOMEM && prev->bt_size >= size &&
1172 TAILQ_REMOVE(&vm->vm_seglist, cursor, bt_seglist);
1173 for (; bt != NULL && bt->bt_start < *addrp + size;
1179 TAILQ_INSERT_HEAD(&vm->vm_seglist, cursor, bt_seglist);
1194 /* ---- vmem API */
1202 KASSERT(vm->vm_size == 0, ("%s: arena is non-empty", __func__));
1203 vm->vm_importfn = importfn;
1204 vm->vm_releasefn = releasefn;
1205 vm->vm_arg = arg;
1206 vm->vm_import_quantum = import_quantum;
1215 vm->vm_limit = limit;
1224 vm->vm_reclaimfn = reclaimfn;
1238 MPASS((quantum & (quantum - 1)) == 0);
1244 vm->vm_nfreetags = 0;
1245 LIST_INIT(&vm->vm_freetags);
1246 strlcpy(vm->vm_name, name, sizeof(vm->vm_name));
1247 vm->vm_quantum_mask = quantum - 1;
1248 vm->vm_quantum_shift = flsl(quantum) - 1;
1249 vm->vm_nbusytag = 0;
1250 vm->vm_size = 0;
1251 vm->vm_limit = 0;
1252 vm->vm_inuse = 0;
1255 TAILQ_INIT(&vm->vm_seglist);
1256 vm->vm_cursor.bt_start = vm->vm_cursor.bt_size = 0;
1257 vm->vm_cursor.bt_type = BT_TYPE_CURSOR;
1258 TAILQ_INSERT_TAIL(&vm->vm_seglist, &vm->vm_cursor, bt_seglist);
1261 LIST_INIT(&vm->vm_freelist[i]);
1263 memset(&vm->vm_hash0, 0, sizeof(vm->vm_hash0));
1264 vm->vm_hashsize = VMEM_HASHSIZE_MIN;
1265 vm->vm_hashlist = vm->vm_hash0;
1315 return (size + vm->vm_quantum_mask) & ~vm->vm_quantum_mask;
1333 if (size <= vm->vm_qcache_max) {
1339 qc = &vm->vm_qcache[(size - 1) >> vm->vm_quantum_shift];
1340 *addrp = (vmem_addr_t)uma_zalloc(qc->qc_cache,
1372 MPASS((align & vm->vm_quantum_mask) == 0);
1373 MPASS((align & (align - 1)) == 0);
1374 MPASS((phase & vm->vm_quantum_mask) == 0);
1375 MPASS((nocross & vm->vm_quantum_mask) == 0);
1376 MPASS((nocross & (nocross - 1)) == 0);
1380 MPASS(!VMEM_CROSS_P(phase, phase + size - 1, nocross));
1385 align = vm->vm_quantum_mask + 1;
1389 * Next-fit allocations don't use the freelists.
1395 end = &vm->vm_freelist[VMEM_MAXORDER];
1417 if (bt->bt_size >= size) {
1467 if (size <= vm->vm_qcache_max &&
1469 qc = &vm->vm_qcache[(size - 1) >> vm->vm_quantum_shift];
1470 uma_zfree(qc->qc_cache, (void *)addr);
1486 MPASS(bt->bt_start == addr);
1487 MPASS(bt->bt_size == vmem_roundup_size(vm, size) ||
1488 bt->bt_size - vmem_roundup_size(vm, size) <= vm->vm_quantum_mask);
1489 MPASS(bt->bt_type == BT_TYPE_BUSY);
1491 bt->bt_type = BT_TYPE_FREE;
1495 if (t != NULL && t->bt_type == BT_TYPE_FREE) {
1496 MPASS(BT_END(bt) < t->bt_start); /* YYY */
1497 bt->bt_size += t->bt_size;
1502 if (t != NULL && t->bt_type == BT_TYPE_FREE) {
1503 MPASS(BT_END(t) < bt->bt_start); /* YYY */
1504 bt->bt_size += t->bt_size;
1505 bt->bt_start = t->bt_start;
1547 return vm->vm_inuse;
1549 return vm->vm_size - vm->vm_inuse;
1551 return vm->vm_size;
1554 for (i = VMEM_MAXORDER - 1; i >= 0; i--) {
1555 if (LIST_EMPTY(&vm->vm_freelist[i]))
1559 vm->vm_quantum_shift);
1568 /* ---- debug */
1601 bt, (intmax_t)bt->bt_start, (intmax_t)bt->bt_size,
1602 bt->bt_type, bt_type_string(bt->bt_type));
1611 (*pr)("vmem %p '%s'\n", vm, vm->vm_name);
1612 TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
1617 const struct vmem_freelist *fl = &vm->vm_freelist[i];
1640 TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
1644 if (bt->bt_start <= addr && addr <= BT_END(bt)) {
1665 (void *)addr, (void *)bt->bt_start,
1666 (vmem_size_t)(addr - bt->bt_start), vm->vm_name,
1667 (bt->bt_type == BT_TYPE_BUSY) ? "allocated" : "free");
1721 db_printf("vmem %p '%s'\n", vm, vm->vm_name);
1722 db_printf("\tquantum:\t%zu\n", vm->vm_quantum_mask + 1);
1723 db_printf("\tsize:\t%zu\n", vm->vm_size);
1724 db_printf("\tinuse:\t%zu\n", vm->vm_inuse);
1725 db_printf("\tfree:\t%zu\n", vm->vm_size - vm->vm_inuse);
1726 db_printf("\tbusy tags:\t%d\n", vm->vm_nbusytag);
1727 db_printf("\tfree tags:\t%d\n", vm->vm_nfreetags);
1733 TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
1734 ord = SIZE2ORDER(bt->bt_size >> vm->vm_quantum_shift);
1735 if (bt->bt_type == BT_TYPE_BUSY) {
1737 us[ord] += bt->bt_size;
1738 } else if (bt->bt_type == BT_TYPE_FREE) {
1740 fs[ord] += bt->bt_size;
1747 db_printf("\t%-15zu %zu\t%-15zu %zu\t%-16zu\n",
1748 ORDER2SIZE(ord) << vm->vm_quantum_shift,
1773 TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
1774 if (bt->bt_start > BT_END(bt)) {
1780 TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
1781 if (bt->bt_type == BT_TYPE_CURSOR) {
1782 if (bt->bt_start != 0 || bt->bt_size != 0) {
1788 TAILQ_FOREACH(bt2, &vm->vm_seglist, bt_seglist) {
1792 if (bt2->bt_type == BT_TYPE_CURSOR) {
1798 if (bt->bt_start <= BT_END(bt2) &&
1799 bt2->bt_start <= BT_END(bt)) {