Lines Matching refs:vmp
272 vmem_getseg(vmem_t *vmp) in vmem_getseg() argument
276 ASSERT(vmp->vm_nsegfree > 0); in vmem_getseg()
278 vsp = vmp->vm_segfree; in vmem_getseg()
279 vmp->vm_segfree = vsp->vs_knext; in vmem_getseg()
280 vmp->vm_nsegfree--; in vmem_getseg()
289 vmem_putseg(vmem_t *vmp, vmem_seg_t *vsp) in vmem_putseg() argument
291 vsp->vs_knext = vmp->vm_segfree; in vmem_putseg()
292 vmp->vm_segfree = vsp; in vmem_putseg()
293 vmp->vm_nsegfree++; in vmem_putseg()
300 vmem_freelist_insert(vmem_t *vmp, vmem_seg_t *vsp) in vmem_freelist_insert() argument
304 ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp); in vmem_freelist_insert()
306 vprev = (vmem_seg_t *)&vmp->vm_freelist[highbit(VS_SIZE(vsp)) - 1]; in vmem_freelist_insert()
308 vmp->vm_freemap |= VS_SIZE(vprev); in vmem_freelist_insert()
311 (void) cond_broadcast(&vmp->vm_cv); in vmem_freelist_insert()
318 vmem_freelist_delete(vmem_t *vmp, vmem_seg_t *vsp) in vmem_freelist_delete() argument
320 ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp); in vmem_freelist_delete()
328 ASSERT(vmp->vm_freemap & VS_SIZE(vsp->vs_kprev)); in vmem_freelist_delete()
329 vmp->vm_freemap ^= VS_SIZE(vsp->vs_kprev); in vmem_freelist_delete()
338 vmem_hash_insert(vmem_t *vmp, vmem_seg_t *vsp) in vmem_hash_insert() argument
343 bucket = VMEM_HASH(vmp, vsp->vs_start); in vmem_hash_insert()
356 vmp->vm_kstat.vk_alloc++; in vmem_hash_insert()
357 vmp->vm_kstat.vk_mem_inuse += VS_SIZE(vsp); in vmem_hash_insert()
364 vmem_hash_delete(vmem_t *vmp, uintptr_t addr, size_t size) in vmem_hash_delete() argument
368 prev_vspp = VMEM_HASH(vmp, addr); in vmem_hash_delete()
374 vmp->vm_kstat.vk_lookup++; in vmem_hash_delete()
380 vmp, addr, size); in vmem_hash_delete()
384 "(expect %lu)", vmp, addr, size, VS_SIZE(vsp)); in vmem_hash_delete()
387 vmp->vm_kstat.vk_free++; in vmem_hash_delete()
388 vmp->vm_kstat.vk_mem_inuse -= size; in vmem_hash_delete()
397 vmem_seg_create(vmem_t *vmp, vmem_seg_t *vprev, uintptr_t start, uintptr_t end) in vmem_seg_create() argument
399 vmem_seg_t *newseg = vmem_getseg(vmp); in vmem_seg_create()
415 vmem_seg_destroy(vmem_t *vmp, vmem_seg_t *vsp) in vmem_seg_destroy() argument
420 vmem_putseg(vmp, vsp); in vmem_seg_destroy()
427 vmem_span_create(vmem_t *vmp, void *vaddr, size_t size, uint8_t import) in vmem_span_create() argument
434 knext = &vmp->vm_seg0; in vmem_span_create()
435 if (!import && vmp->vm_source_alloc == NULL) { in vmem_span_create()
444 kend = &vmp->vm_seg0; in vmem_span_create()
453 ASSERT(MUTEX_HELD(&vmp->vm_lock)); in vmem_span_create()
455 if ((start | end) & (vmp->vm_quantum - 1)) { in vmem_span_create()
457 vmp, vaddr, size); in vmem_span_create()
460 span = vmem_seg_create(vmp, knext->vs_aprev, start, end); in vmem_span_create()
465 newseg = vmem_seg_create(vmp, span, start, end); in vmem_span_create()
466 vmem_freelist_insert(vmp, newseg); in vmem_span_create()
469 vmp->vm_kstat.vk_mem_import += size; in vmem_span_create()
470 vmp->vm_kstat.vk_mem_total += size; in vmem_span_create()
479 vmem_span_destroy(vmem_t *vmp, vmem_seg_t *vsp) in vmem_span_destroy() argument
484 ASSERT(MUTEX_HELD(&vmp->vm_lock)); in vmem_span_destroy()
488 vmp->vm_kstat.vk_mem_import -= size; in vmem_span_destroy()
489 vmp->vm_kstat.vk_mem_total -= size; in vmem_span_destroy()
493 vmem_seg_destroy(vmp, vsp); in vmem_span_destroy()
494 vmem_seg_destroy(vmp, span); in vmem_span_destroy()
503 vmem_seg_alloc(vmem_t *vmp, vmem_seg_t *vsp, uintptr_t addr, size_t size) in vmem_seg_alloc() argument
508 size_t realsize = P2ROUNDUP(size, vmp->vm_quantum); in vmem_seg_alloc()
511 ASSERT(P2PHASE(vs_start, vmp->vm_quantum) == 0); in vmem_seg_alloc()
512 ASSERT(P2PHASE(addr, vmp->vm_quantum) == 0); in vmem_seg_alloc()
525 vsp = vmem_seg_create(vmp, vsp->vs_aprev, addr, addr + size); in vmem_seg_alloc()
526 vmem_hash_insert(vmp, vsp); in vmem_seg_alloc()
530 vmem_freelist_delete(vmp, vsp); in vmem_seg_alloc()
533 vmem_freelist_insert(vmp, in vmem_seg_alloc()
534 vmem_seg_create(vmp, vsp, addr_end, vs_end)); in vmem_seg_alloc()
537 vmem_freelist_insert(vmp, in vmem_seg_alloc()
538 vmem_seg_create(vmp, vsp->vs_aprev, vs_start, addr)); in vmem_seg_alloc()
543 vmem_hash_insert(vmp, vsp); in vmem_seg_alloc()
561 vmem_populate(vmem_t *vmp, int vmflag) in vmem_populate() argument
570 while (vmp->vm_nsegfree < VMEM_MINFREE && in vmem_populate()
572 vmem_putseg(vmp, vsp); in vmem_populate()
574 if (vmp->vm_nsegfree >= VMEM_MINFREE) in vmem_populate()
581 ASSERT(vmp->vm_cflags & VMC_POPULATOR); in vmem_populate()
585 (void) mutex_unlock(&vmp->vm_lock); in vmem_populate()
617 (void) mutex_lock(&vmp->vm_lock); in vmem_populate()
618 vmp->vm_kstat.vk_populate_fail++; in vmem_populate()
634 (void) mutex_lock(&vmp->vm_lock); in vmem_populate()
640 while (vmp->vm_nsegfree < VMEM_MINFREE) in vmem_populate()
641 vmem_putseg(vmp, (vmem_seg_t *)(p + --nseg * vmem_seg_size)); in vmem_populate()
657 vmem_advance(vmem_t *vmp, vmem_seg_t *walker, vmem_seg_t *afterme) in vmem_advance() argument
675 vmem_freelist_delete(vmp, vnext); in vmem_advance()
676 vmem_freelist_delete(vmp, vprev); in vmem_advance()
678 vmem_freelist_insert(vmp, vprev); in vmem_advance()
679 vmem_seg_destroy(vmp, vnext); in vmem_advance()
691 vmp->vm_source_free != NULL && in vmem_advance()
697 vmem_freelist_delete(vmp, vsp); in vmem_advance()
698 vmem_span_destroy(vmp, vsp); in vmem_advance()
699 (void) mutex_unlock(&vmp->vm_lock); in vmem_advance()
700 vmp->vm_source_free(vmp->vm_source, vaddr, size); in vmem_advance()
701 (void) mutex_lock(&vmp->vm_lock); in vmem_advance()
713 vmem_nextfit_alloc(vmem_t *vmp, size_t size, int vmflag) in vmem_nextfit_alloc() argument
717 size_t realsize = P2ROUNDUP(size, vmp->vm_quantum); in vmem_nextfit_alloc()
720 (void) mutex_lock(&vmp->vm_lock); in vmem_nextfit_alloc()
722 if (vmp->vm_nsegfree < VMEM_MINFREE && !vmem_populate(vmp, vmflag)) { in vmem_nextfit_alloc()
723 (void) mutex_unlock(&vmp->vm_lock); in vmem_nextfit_alloc()
737 rotor = &vmp->vm_rotor; in vmem_nextfit_alloc()
744 vmem_hash_insert(vmp, in vmem_nextfit_alloc()
745 vmem_seg_create(vmp, rotor->vs_aprev, addr, addr + size)); in vmem_nextfit_alloc()
746 (void) mutex_unlock(&vmp->vm_lock); in vmem_nextfit_alloc()
755 vmp->vm_kstat.vk_search++; in vmem_nextfit_alloc()
770 vmem_advance(vmp, rotor, rotor->vs_anext); in vmem_nextfit_alloc()
779 if (vmp->vm_source_alloc != NULL || in vmem_nextfit_alloc()
781 (void) mutex_unlock(&vmp->vm_lock); in vmem_nextfit_alloc()
782 return (vmem_xalloc(vmp, size, vmp->vm_quantum, in vmem_nextfit_alloc()
785 vmp->vm_kstat.vk_wait++; in vmem_nextfit_alloc()
788 (void) cond_wait(&vmp->vm_cv, &vmp->vm_lock); in vmem_nextfit_alloc()
798 vsp = vmem_seg_alloc(vmp, vsp, addr, size); in vmem_nextfit_alloc()
806 vmem_advance(vmp, rotor, vsp); in vmem_nextfit_alloc()
807 (void) mutex_unlock(&vmp->vm_lock); in vmem_nextfit_alloc()
817 vmem_xalloc(vmem_t *vmp, size_t size, size_t align, size_t phase, in vmem_xalloc() argument
830 (void *)vmp, size, align, phase, nocross, in vmem_xalloc()
834 align = vmp->vm_quantum; in vmem_xalloc()
836 if ((align | phase | nocross) & (vmp->vm_quantum - 1)) { in vmem_xalloc()
839 (void *)vmp, size, align, phase, nocross, in vmem_xalloc()
847 (void *)vmp, size, align, phase, nocross, in vmem_xalloc()
851 if ((mtbf = vmem_mtbf | vmp->vm_mtbf) != 0 && gethrtime() % mtbf == 0 && in vmem_xalloc()
855 (void) mutex_lock(&vmp->vm_lock); in vmem_xalloc()
859 if (vmp->vm_nsegfree < VMEM_MINFREE && in vmem_xalloc()
860 !vmem_populate(vmp, vmflag)) in vmem_xalloc()
882 flist = lowbit(P2ALIGN(vmp->vm_freemap, size)); in vmem_xalloc()
885 if ((vmp->vm_freemap >> hb) == 0 || in vmem_xalloc()
889 flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb)); in vmem_xalloc()
893 vmp->vm_freelist[flist - 1].vs_knext; in vmem_xalloc()
895 vmp->vm_kstat.vk_search++; in vmem_xalloc()
907 flist = lowbit(P2ALIGN(vmp->vm_freemap, in vmem_xalloc()
911 vsp = (vmem_seg_t *)&vmp->vm_freelist[flist]; in vmem_xalloc()
937 if (vmp->vm_source_alloc != NULL && nocross == 0 && in vmem_xalloc()
940 MAX(align, vmp->vm_source->vm_quantum)); in vmem_xalloc()
942 (void) mutex_unlock(&vmp->vm_lock); in vmem_xalloc()
959 ASSERT(vmp->vm_nsegfree >= resv); in vmem_xalloc()
960 vmp->vm_nsegfree -= resv; /* reserve our segs */ in vmem_xalloc()
961 (void) mutex_unlock(&vmp->vm_lock); in vmem_xalloc()
962 vaddr = vmp->vm_source_alloc(vmp->vm_source, asize, in vmem_xalloc()
964 (void) mutex_lock(&vmp->vm_lock); in vmem_xalloc()
965 vmp->vm_nsegfree += resv; /* claim reservation */ in vmem_xalloc()
967 vbest = vmem_span_create(vmp, vaddr, asize, 1); in vmem_xalloc()
972 (void) mutex_unlock(&vmp->vm_lock); in vmem_xalloc()
974 (void) mutex_lock(&vmp->vm_lock); in vmem_xalloc()
977 vmp->vm_kstat.vk_wait++; in vmem_xalloc()
980 (void) cond_wait(&vmp->vm_cv, &vmp->vm_lock); in vmem_xalloc()
989 (void) vmem_seg_alloc(vmp, vbest, addr, size); in vmem_xalloc()
990 (void) mutex_unlock(&vmp->vm_lock); in vmem_xalloc()
997 vmp->vm_kstat.vk_fail++; in vmem_xalloc()
998 (void) mutex_unlock(&vmp->vm_lock); in vmem_xalloc()
1002 (void *)vmp, size, align, phase, nocross, in vmem_xalloc()
1013 vmem_xfree(vmem_t *vmp, void *vaddr, size_t size) in vmem_xfree() argument
1017 (void) mutex_lock(&vmp->vm_lock); in vmem_xfree()
1019 vsp = vmem_hash_delete(vmp, (uintptr_t)vaddr, size); in vmem_xfree()
1020 vsp->vs_end = P2ROUNDUP(vsp->vs_end, vmp->vm_quantum); in vmem_xfree()
1028 vmem_freelist_delete(vmp, vnext); in vmem_xfree()
1030 vmem_seg_destroy(vmp, vnext); in vmem_xfree()
1039 vmem_freelist_delete(vmp, vprev); in vmem_xfree()
1041 vmem_seg_destroy(vmp, vsp); in vmem_xfree()
1048 if (vsp->vs_aprev->vs_import && vmp->vm_source_free != NULL && in vmem_xfree()
1054 vmem_span_destroy(vmp, vsp); in vmem_xfree()
1055 (void) mutex_unlock(&vmp->vm_lock); in vmem_xfree()
1056 vmp->vm_source_free(vmp->vm_source, vaddr, size); in vmem_xfree()
1058 vmem_freelist_insert(vmp, vsp); in vmem_xfree()
1059 (void) mutex_unlock(&vmp->vm_lock); in vmem_xfree()
1071 vmem_alloc(vmem_t *vmp, size_t size, int vmflag) in vmem_alloc() argument
1080 if (size - 1 < vmp->vm_qcache_max) { in vmem_alloc()
1082 return (_umem_cache_alloc(vmp->vm_qcache[(size - 1) >> in vmem_alloc()
1083 vmp->vm_qshift], UMEM_DEFAULT)); in vmem_alloc()
1086 if ((mtbf = vmem_mtbf | vmp->vm_mtbf) != 0 && gethrtime() % mtbf == 0 && in vmem_alloc()
1091 return (vmem_nextfit_alloc(vmp, size, vmflag)); in vmem_alloc()
1094 return (vmem_xalloc(vmp, size, vmp->vm_quantum, 0, 0, in vmem_alloc()
1100 (void) mutex_lock(&vmp->vm_lock); in vmem_alloc()
1102 if (vmp->vm_nsegfree >= VMEM_MINFREE || vmem_populate(vmp, vmflag)) { in vmem_alloc()
1104 flist = lowbit(P2ALIGN(vmp->vm_freemap, size)); in vmem_alloc()
1106 flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb)); in vmem_alloc()
1110 (void) mutex_unlock(&vmp->vm_lock); in vmem_alloc()
1111 return (vmem_xalloc(vmp, size, vmp->vm_quantum, in vmem_alloc()
1116 vsp = vmp->vm_freelist[flist].vs_knext; in vmem_alloc()
1118 (void) vmem_seg_alloc(vmp, vsp, addr, size); in vmem_alloc()
1119 (void) mutex_unlock(&vmp->vm_lock); in vmem_alloc()
1127 vmem_free(vmem_t *vmp, void *vaddr, size_t size) in vmem_free() argument
1129 if (size - 1 < vmp->vm_qcache_max) in vmem_free()
1130 _umem_cache_free(vmp->vm_qcache[(size - 1) >> vmp->vm_qshift], in vmem_free()
1133 vmem_xfree(vmp, vaddr, size); in vmem_free()
1140 vmem_contains(vmem_t *vmp, void *vaddr, size_t size) in vmem_contains() argument
1145 vmem_seg_t *seg0 = &vmp->vm_seg0; in vmem_contains()
1147 (void) mutex_lock(&vmp->vm_lock); in vmem_contains()
1148 vmp->vm_kstat.vk_contains++; in vmem_contains()
1150 vmp->vm_kstat.vk_contains_search++; in vmem_contains()
1155 (void) mutex_unlock(&vmp->vm_lock); in vmem_contains()
1163 vmem_add(vmem_t *vmp, void *vaddr, size_t size, int vmflag) in vmem_add() argument
1167 vmp, vaddr, size); in vmem_add()
1170 ASSERT(!vmem_contains(vmp, vaddr, size)); in vmem_add()
1172 (void) mutex_lock(&vmp->vm_lock); in vmem_add()
1173 if (vmem_populate(vmp, vmflag)) in vmem_add()
1174 (void) vmem_span_create(vmp, vaddr, size, 0); in vmem_add()
1177 (void) cond_broadcast(&vmp->vm_cv); in vmem_add()
1178 (void) mutex_unlock(&vmp->vm_lock); in vmem_add()
1198 vmem_extend_unlocked(vmem_t *vmp, uintptr_t addr, uintptr_t endaddr) in vmem_extend_unlocked() argument
1203 vmem_seg_t *end = &vmp->vm_seg0; in vmem_extend_unlocked()
1205 ASSERT(MUTEX_HELD(&vmp->vm_lock)); in vmem_extend_unlocked()
1216 return (vmem_span_create(vmp, (void *)addr, endaddr - addr, 0)); in vmem_extend_unlocked()
1242 vmem_freelist_delete(vmp, prevseg); in vmem_extend_unlocked()
1245 vmem_freelist_delete(vmp, nextseg); in vmem_extend_unlocked()
1247 vmem_seg_destroy(vmp, nextseg); in vmem_extend_unlocked()
1248 vmem_seg_destroy(vmp, span); in vmem_extend_unlocked()
1258 vmem_seg_destroy(vmp, span); in vmem_extend_unlocked()
1260 vmem_freelist_delete(vmp, prevseg); in vmem_extend_unlocked()
1271 vmem_seg_destroy(vmp, span); in vmem_extend_unlocked()
1273 vmem_freelist_delete(vmp, nextseg); in vmem_extend_unlocked()
1294 vmem_freelist_delete(vmp, oldseg); in vmem_extend_unlocked()
1298 vsp = vmem_seg_create(vmp, oldseg, addr, endaddr); in vmem_extend_unlocked()
1307 vmem_freelist_delete(vmp, oldseg); in vmem_extend_unlocked()
1311 vsp = vmem_seg_create(vmp, span, addr, endaddr); in vmem_extend_unlocked()
1313 vmem_freelist_insert(vmp, vsp); in vmem_extend_unlocked()
1314 vmp->vm_kstat.vk_mem_total += (endaddr - addr); in vmem_extend_unlocked()
1324 _vmem_extend_alloc(vmem_t *vmp, void *vaddr, size_t size, size_t alloc, in _vmem_extend_alloc() argument
1333 ASSERT(((addr | size | alloc) & (vmp->vm_quantum - 1)) == 0); in _vmem_extend_alloc()
1335 ASSERT(!vmem_contains(vmp, vaddr, size)); in _vmem_extend_alloc()
1337 (void) mutex_lock(&vmp->vm_lock); in _vmem_extend_alloc()
1338 if (!vmem_populate(vmp, vmflag)) { in _vmem_extend_alloc()
1339 (void) mutex_unlock(&vmp->vm_lock); in _vmem_extend_alloc()
1345 if (vmp->vm_source_alloc != NULL) in _vmem_extend_alloc()
1346 vsp = vmem_span_create(vmp, vaddr, size, 0); in _vmem_extend_alloc()
1348 vsp = vmem_extend_unlocked(vmp, addr, endaddr); in _vmem_extend_alloc()
1353 (void) vmem_seg_alloc(vmp, vsp, addr, alloc); in _vmem_extend_alloc()
1356 (void) cond_broadcast(&vmp->vm_cv); in _vmem_extend_alloc()
1357 (void) mutex_unlock(&vmp->vm_lock); in _vmem_extend_alloc()
1371 vmem_walk(vmem_t *vmp, int typemask, in vmem_walk() argument
1375 vmem_seg_t *seg0 = &vmp->vm_seg0; in vmem_walk()
1384 (void) mutex_lock(&vmp->vm_lock); in vmem_walk()
1391 vmem_advance(vmp, &walker, vsp); in vmem_walk()
1392 (void) mutex_unlock(&vmp->vm_lock); in vmem_walk()
1394 (void) mutex_lock(&vmp->vm_lock); in vmem_walk()
1401 vmem_advance(vmp, &walker, NULL); in vmem_walk()
1402 (void) mutex_unlock(&vmp->vm_lock); in vmem_walk()
1413 vmem_size(vmem_t *vmp, int typemask) in vmem_size() argument
1418 size += vmp->vm_kstat.vk_mem_inuse; in vmem_size()
1420 size += vmp->vm_kstat.vk_mem_total - in vmem_size()
1421 vmp->vm_kstat.vk_mem_inuse; in vmem_size()
1441 vmem_t *vmp, *cur, **vmpp; in vmem_create() local
1447 vmp = vmem_alloc(vmem_vmem_arena, sizeof (vmem_t), in vmem_create()
1451 vmp = &vmem0[id - 1]; in vmem_create()
1454 if (vmp == NULL) in vmem_create()
1456 bzero(vmp, sizeof (vmem_t)); in vmem_create()
1458 (void) snprintf(vmp->vm_name, VMEM_NAMELEN, "%s", name); in vmem_create()
1459 (void) mutex_init(&vmp->vm_lock, USYNC_THREAD, NULL); in vmem_create()
1460 (void) cond_init(&vmp->vm_cv, USYNC_THREAD, NULL); in vmem_create()
1461 vmp->vm_cflags = vmflag; in vmem_create()
1464 vmp->vm_quantum = quantum; in vmem_create()
1465 vmp->vm_qshift = highbit(quantum) - 1; in vmem_create()
1466 nqcache = MIN(qcache_max >> vmp->vm_qshift, VMEM_NQCACHE_MAX); in vmem_create()
1469 vfp = &vmp->vm_freelist[i]; in vmem_create()
1475 vmp->vm_freelist[0].vs_kprev = NULL; in vmem_create()
1476 vmp->vm_freelist[VMEM_FREELISTS].vs_knext = NULL; in vmem_create()
1477 vmp->vm_freelist[VMEM_FREELISTS].vs_end = 0; in vmem_create()
1478 vmp->vm_hash_table = vmp->vm_hash0; in vmem_create()
1479 vmp->vm_hash_mask = VMEM_HASH_INITIAL - 1; in vmem_create()
1480 vmp->vm_hash_shift = highbit(vmp->vm_hash_mask); in vmem_create()
1482 vsp = &vmp->vm_seg0; in vmem_create()
1489 vsp = &vmp->vm_rotor; in vmem_create()
1491 VMEM_INSERT(&vmp->vm_seg0, vsp, a); in vmem_create()
1493 vmp->vm_id = id; in vmem_create()
1495 vmp->vm_kstat.vk_source_id = source->vm_id; in vmem_create()
1496 vmp->vm_source = source; in vmem_create()
1497 vmp->vm_source_alloc = afunc; in vmem_create()
1498 vmp->vm_source_free = ffunc; in vmem_create()
1501 vmp->vm_qcache_max = nqcache << vmp->vm_qshift; in vmem_create()
1505 vmp->vm_name, (long)((i + 1) * quantum)); in vmem_create()
1506 vmp->vm_qcache[i] = umem_cache_create(buf, in vmem_create()
1508 NULL, vmp, UMC_QCACHE | UMC_NOTOUCH); in vmem_create()
1509 if (vmp->vm_qcache[i] == NULL) { in vmem_create()
1510 vmp->vm_qcache_max = i * quantum; in vmem_create()
1520 *vmpp = vmp; in vmem_create()
1523 if (vmp->vm_cflags & VMC_POPULATOR) { in vmem_create()
1526 vmem_populator[pop_id - 1] = vmp; in vmem_create()
1527 (void) mutex_lock(&vmp->vm_lock); in vmem_create()
1528 (void) vmem_populate(vmp, vmflag | VM_PANIC); in vmem_create()
1529 (void) mutex_unlock(&vmp->vm_lock); in vmem_create()
1532 if ((base || size) && vmem_add(vmp, base, size, vmflag) == NULL) { in vmem_create()
1533 vmem_destroy(vmp); in vmem_create()
1537 return (vmp); in vmem_create()
1544 vmem_destroy(vmem_t *vmp) in vmem_destroy() argument
1547 vmem_seg_t *seg0 = &vmp->vm_seg0; in vmem_destroy()
1554 while ((cur = *vmpp) != vmp) in vmem_destroy()
1556 *vmpp = vmp->vm_next; in vmem_destroy()
1560 if (vmp->vm_qcache[i]) in vmem_destroy()
1561 umem_cache_destroy(vmp->vm_qcache[i]); in vmem_destroy()
1563 leaked = vmem_size(vmp, VMEM_ALLOC); in vmem_destroy()
1566 vmp->vm_name, leaked); in vmem_destroy()
1568 if (vmp->vm_hash_table != vmp->vm_hash0) in vmem_destroy()
1569 vmem_free(vmem_hash_arena, vmp->vm_hash_table, in vmem_destroy()
1570 (vmp->vm_hash_mask + 1) * sizeof (void *)); in vmem_destroy()
1576 VMEM_DELETE(&vmp->vm_rotor, a); in vmem_destroy()
1580 while (vmp->vm_nsegfree > 0) in vmem_destroy()
1581 vmem_putseg_global(vmem_getseg(vmp)); in vmem_destroy()
1583 (void) mutex_destroy(&vmp->vm_lock); in vmem_destroy()
1584 (void) cond_destroy(&vmp->vm_cv); in vmem_destroy()
1585 vmem_free(vmem_vmem_arena, vmp, sizeof (vmem_t)); in vmem_destroy()
1592 vmem_hash_rescale(vmem_t *vmp) in vmem_hash_rescale() argument
1597 nseg = (size_t)(vmp->vm_kstat.vk_alloc - vmp->vm_kstat.vk_free); in vmem_hash_rescale()
1600 old_size = vmp->vm_hash_mask + 1; in vmem_hash_rescale()
1611 (void) mutex_lock(&vmp->vm_lock); in vmem_hash_rescale()
1613 old_size = vmp->vm_hash_mask + 1; in vmem_hash_rescale()
1614 old_table = vmp->vm_hash_table; in vmem_hash_rescale()
1616 vmp->vm_hash_mask = new_size - 1; in vmem_hash_rescale()
1617 vmp->vm_hash_table = new_table; in vmem_hash_rescale()
1618 vmp->vm_hash_shift = highbit(vmp->vm_hash_mask); in vmem_hash_rescale()
1625 vmem_seg_t **hash_bucket = VMEM_HASH(vmp, addr); in vmem_hash_rescale()
1632 (void) mutex_unlock(&vmp->vm_lock); in vmem_hash_rescale()
1634 if (old_table != vmp->vm_hash0) in vmem_hash_rescale()
1646 vmem_t *vmp; in vmem_update() local
1649 for (vmp = vmem_list; vmp != NULL; vmp = vmp->vm_next) { in vmem_update()
1655 (void) cond_broadcast(&vmp->vm_cv); in vmem_update()
1660 vmem_hash_rescale(vmp); in vmem_update()