Lines Matching refs:vmp
271 vmem_getseg(vmem_t *vmp) in vmem_getseg() argument
275 ASSERT(vmp->vm_nsegfree > 0); in vmem_getseg()
277 vsp = vmp->vm_segfree; in vmem_getseg()
278 vmp->vm_segfree = vsp->vs_knext; in vmem_getseg()
279 vmp->vm_nsegfree--; in vmem_getseg()
288 vmem_putseg(vmem_t *vmp, vmem_seg_t *vsp) in vmem_putseg() argument
290 vsp->vs_knext = vmp->vm_segfree; in vmem_putseg()
291 vmp->vm_segfree = vsp; in vmem_putseg()
292 vmp->vm_nsegfree++; in vmem_putseg()
299 vmem_freelist_insert(vmem_t *vmp, vmem_seg_t *vsp) in vmem_freelist_insert() argument
303 ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp); in vmem_freelist_insert()
305 vprev = (vmem_seg_t *)&vmp->vm_freelist[highbit(VS_SIZE(vsp)) - 1]; in vmem_freelist_insert()
307 vmp->vm_freemap |= VS_SIZE(vprev); in vmem_freelist_insert()
310 (void) cond_broadcast(&vmp->vm_cv); in vmem_freelist_insert()
317 vmem_freelist_delete(vmem_t *vmp, vmem_seg_t *vsp) in vmem_freelist_delete() argument
319 ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp); in vmem_freelist_delete()
327 ASSERT(vmp->vm_freemap & VS_SIZE(vsp->vs_kprev)); in vmem_freelist_delete()
328 vmp->vm_freemap ^= VS_SIZE(vsp->vs_kprev); in vmem_freelist_delete()
337 vmem_hash_insert(vmem_t *vmp, vmem_seg_t *vsp) in vmem_hash_insert() argument
342 bucket = VMEM_HASH(vmp, vsp->vs_start); in vmem_hash_insert()
355 vmp->vm_kstat.vk_alloc++; in vmem_hash_insert()
356 vmp->vm_kstat.vk_mem_inuse += VS_SIZE(vsp); in vmem_hash_insert()
363 vmem_hash_delete(vmem_t *vmp, uintptr_t addr, size_t size) in vmem_hash_delete() argument
367 prev_vspp = VMEM_HASH(vmp, addr); in vmem_hash_delete()
373 vmp->vm_kstat.vk_lookup++; in vmem_hash_delete()
379 vmp, addr, size); in vmem_hash_delete()
383 "(expect %lu)", vmp, addr, size, VS_SIZE(vsp)); in vmem_hash_delete()
386 vmp->vm_kstat.vk_free++; in vmem_hash_delete()
387 vmp->vm_kstat.vk_mem_inuse -= size; in vmem_hash_delete()
396 vmem_seg_create(vmem_t *vmp, vmem_seg_t *vprev, uintptr_t start, uintptr_t end) in vmem_seg_create() argument
398 vmem_seg_t *newseg = vmem_getseg(vmp); in vmem_seg_create()
414 vmem_seg_destroy(vmem_t *vmp, vmem_seg_t *vsp) in vmem_seg_destroy() argument
419 vmem_putseg(vmp, vsp); in vmem_seg_destroy()
426 vmem_span_create(vmem_t *vmp, void *vaddr, size_t size, uint8_t import) in vmem_span_create() argument
433 knext = &vmp->vm_seg0; in vmem_span_create()
434 if (!import && vmp->vm_source_alloc == NULL) { in vmem_span_create()
443 kend = &vmp->vm_seg0; in vmem_span_create()
452 ASSERT(MUTEX_HELD(&vmp->vm_lock)); in vmem_span_create()
454 if ((start | end) & (vmp->vm_quantum - 1)) { in vmem_span_create()
456 vmp, vaddr, size); in vmem_span_create()
459 span = vmem_seg_create(vmp, knext->vs_aprev, start, end); in vmem_span_create()
463 newseg = vmem_seg_create(vmp, span, start, end); in vmem_span_create()
464 vmem_freelist_insert(vmp, newseg); in vmem_span_create()
468 vmp->vm_kstat.vk_mem_import += size; in vmem_span_create()
469 vmp->vm_kstat.vk_mem_total += size; in vmem_span_create()
478 vmem_span_destroy(vmem_t *vmp, vmem_seg_t *vsp) in vmem_span_destroy() argument
483 ASSERT(MUTEX_HELD(&vmp->vm_lock)); in vmem_span_destroy()
487 vmp->vm_kstat.vk_mem_import -= size; in vmem_span_destroy()
488 vmp->vm_kstat.vk_mem_total -= size; in vmem_span_destroy()
492 vmem_seg_destroy(vmp, vsp); in vmem_span_destroy()
493 vmem_seg_destroy(vmp, span); in vmem_span_destroy()
502 vmem_seg_alloc(vmem_t *vmp, vmem_seg_t *vsp, uintptr_t addr, size_t size) in vmem_seg_alloc() argument
507 size_t realsize = P2ROUNDUP(size, vmp->vm_quantum); in vmem_seg_alloc()
510 ASSERT(P2PHASE(vs_start, vmp->vm_quantum) == 0); in vmem_seg_alloc()
511 ASSERT(P2PHASE(addr, vmp->vm_quantum) == 0); in vmem_seg_alloc()
524 vsp = vmem_seg_create(vmp, vsp->vs_aprev, addr, addr + size); in vmem_seg_alloc()
525 vmem_hash_insert(vmp, vsp); in vmem_seg_alloc()
529 vmem_freelist_delete(vmp, vsp); in vmem_seg_alloc()
532 vmem_freelist_insert(vmp, in vmem_seg_alloc()
533 vmem_seg_create(vmp, vsp, addr_end, vs_end)); in vmem_seg_alloc()
536 vmem_freelist_insert(vmp, in vmem_seg_alloc()
537 vmem_seg_create(vmp, vsp->vs_aprev, vs_start, addr)); in vmem_seg_alloc()
542 vmem_hash_insert(vmp, vsp); in vmem_seg_alloc()
560 vmem_populate(vmem_t *vmp, int vmflag) in vmem_populate() argument
569 while (vmp->vm_nsegfree < VMEM_MINFREE && in vmem_populate()
571 vmem_putseg(vmp, vsp); in vmem_populate()
573 if (vmp->vm_nsegfree >= VMEM_MINFREE) in vmem_populate()
580 ASSERT(vmp->vm_cflags & VMC_POPULATOR); in vmem_populate()
584 (void) mutex_unlock(&vmp->vm_lock); in vmem_populate()
616 (void) mutex_lock(&vmp->vm_lock); in vmem_populate()
617 vmp->vm_kstat.vk_populate_fail++; in vmem_populate()
633 (void) mutex_lock(&vmp->vm_lock); in vmem_populate()
639 while (vmp->vm_nsegfree < VMEM_MINFREE) in vmem_populate()
640 vmem_putseg(vmp, (vmem_seg_t *)(p + --nseg * vmem_seg_size)); in vmem_populate()
656 vmem_advance(vmem_t *vmp, vmem_seg_t *walker, vmem_seg_t *afterme) in vmem_advance() argument
674 vmem_freelist_delete(vmp, vnext); in vmem_advance()
675 vmem_freelist_delete(vmp, vprev); in vmem_advance()
677 vmem_freelist_insert(vmp, vprev); in vmem_advance()
678 vmem_seg_destroy(vmp, vnext); in vmem_advance()
689 if (vsp != NULL && vsp->vs_import && vmp->vm_source_free != NULL && in vmem_advance()
695 vmem_freelist_delete(vmp, vsp); in vmem_advance()
696 vmem_span_destroy(vmp, vsp); in vmem_advance()
697 (void) mutex_unlock(&vmp->vm_lock); in vmem_advance()
698 vmp->vm_source_free(vmp->vm_source, vaddr, size); in vmem_advance()
699 (void) mutex_lock(&vmp->vm_lock); in vmem_advance()
711 vmem_nextfit_alloc(vmem_t *vmp, size_t size, int vmflag) in vmem_nextfit_alloc() argument
715 size_t realsize = P2ROUNDUP(size, vmp->vm_quantum); in vmem_nextfit_alloc()
718 (void) mutex_lock(&vmp->vm_lock); in vmem_nextfit_alloc()
720 if (vmp->vm_nsegfree < VMEM_MINFREE && !vmem_populate(vmp, vmflag)) { in vmem_nextfit_alloc()
721 (void) mutex_unlock(&vmp->vm_lock); in vmem_nextfit_alloc()
735 rotor = &vmp->vm_rotor; in vmem_nextfit_alloc()
742 vmem_hash_insert(vmp, in vmem_nextfit_alloc()
743 vmem_seg_create(vmp, rotor->vs_aprev, addr, addr + size)); in vmem_nextfit_alloc()
744 (void) mutex_unlock(&vmp->vm_lock); in vmem_nextfit_alloc()
753 vmp->vm_kstat.vk_search++; in vmem_nextfit_alloc()
768 vmem_advance(vmp, rotor, rotor->vs_anext); in vmem_nextfit_alloc()
777 if (vmp->vm_source_alloc != NULL || in vmem_nextfit_alloc()
779 (void) mutex_unlock(&vmp->vm_lock); in vmem_nextfit_alloc()
780 return (vmem_xalloc(vmp, size, vmp->vm_quantum, in vmem_nextfit_alloc()
783 vmp->vm_kstat.vk_wait++; in vmem_nextfit_alloc()
786 (void) cond_wait(&vmp->vm_cv, &vmp->vm_lock); in vmem_nextfit_alloc()
796 vsp = vmem_seg_alloc(vmp, vsp, addr, size); in vmem_nextfit_alloc()
804 vmem_advance(vmp, rotor, vsp); in vmem_nextfit_alloc()
805 (void) mutex_unlock(&vmp->vm_lock); in vmem_nextfit_alloc()
815 vmem_xalloc(vmem_t *vmp, size_t size, size_t align, size_t phase, in vmem_xalloc() argument
828 (void *)vmp, size, align, phase, nocross, in vmem_xalloc()
832 align = vmp->vm_quantum; in vmem_xalloc()
834 if ((align | phase | nocross) & (vmp->vm_quantum - 1)) { in vmem_xalloc()
837 (void *)vmp, size, align, phase, nocross, in vmem_xalloc()
845 (void *)vmp, size, align, phase, nocross, in vmem_xalloc()
849 if ((mtbf = vmem_mtbf | vmp->vm_mtbf) != 0 && gethrtime() % mtbf == 0 && in vmem_xalloc()
853 (void) mutex_lock(&vmp->vm_lock); in vmem_xalloc()
857 if (vmp->vm_nsegfree < VMEM_MINFREE && in vmem_xalloc()
858 !vmem_populate(vmp, vmflag)) in vmem_xalloc()
880 flist = lowbit(P2ALIGN(vmp->vm_freemap, size)); in vmem_xalloc()
883 if ((vmp->vm_freemap >> hb) == 0 || in vmem_xalloc()
887 flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb)); in vmem_xalloc()
891 vmp->vm_freelist[flist - 1].vs_knext; in vmem_xalloc()
893 vmp->vm_kstat.vk_search++; in vmem_xalloc()
905 flist = lowbit(P2ALIGN(vmp->vm_freemap, in vmem_xalloc()
909 vsp = (vmem_seg_t *)&vmp->vm_freelist[flist]; in vmem_xalloc()
935 if (vmp->vm_source_alloc != NULL && nocross == 0 && in vmem_xalloc()
938 MAX(align, vmp->vm_source->vm_quantum)); in vmem_xalloc()
940 (void) mutex_unlock(&vmp->vm_lock); in vmem_xalloc()
957 ASSERT(vmp->vm_nsegfree >= resv); in vmem_xalloc()
958 vmp->vm_nsegfree -= resv; /* reserve our segs */ in vmem_xalloc()
959 (void) mutex_unlock(&vmp->vm_lock); in vmem_xalloc()
960 vaddr = vmp->vm_source_alloc(vmp->vm_source, asize, in vmem_xalloc()
962 (void) mutex_lock(&vmp->vm_lock); in vmem_xalloc()
963 vmp->vm_nsegfree += resv; /* claim reservation */ in vmem_xalloc()
965 vbest = vmem_span_create(vmp, vaddr, asize, 1); in vmem_xalloc()
970 (void) mutex_unlock(&vmp->vm_lock); in vmem_xalloc()
972 (void) mutex_lock(&vmp->vm_lock); in vmem_xalloc()
975 vmp->vm_kstat.vk_wait++; in vmem_xalloc()
978 (void) cond_wait(&vmp->vm_cv, &vmp->vm_lock); in vmem_xalloc()
984 (void) vmem_seg_alloc(vmp, vbest, addr, size); in vmem_xalloc()
985 (void) mutex_unlock(&vmp->vm_lock); in vmem_xalloc()
992 vmp->vm_kstat.vk_fail++; in vmem_xalloc()
993 (void) mutex_unlock(&vmp->vm_lock); in vmem_xalloc()
997 (void *)vmp, size, align, phase, nocross, in vmem_xalloc()
1008 vmem_xfree(vmem_t *vmp, void *vaddr, size_t size) in vmem_xfree() argument
1012 (void) mutex_lock(&vmp->vm_lock); in vmem_xfree()
1014 vsp = vmem_hash_delete(vmp, (uintptr_t)vaddr, size); in vmem_xfree()
1015 vsp->vs_end = P2ROUNDUP(vsp->vs_end, vmp->vm_quantum); in vmem_xfree()
1023 vmem_freelist_delete(vmp, vnext); in vmem_xfree()
1025 vmem_seg_destroy(vmp, vnext); in vmem_xfree()
1034 vmem_freelist_delete(vmp, vprev); in vmem_xfree()
1036 vmem_seg_destroy(vmp, vsp); in vmem_xfree()
1043 if (vsp->vs_import && vmp->vm_source_free != NULL && in vmem_xfree()
1049 vmem_span_destroy(vmp, vsp); in vmem_xfree()
1050 (void) mutex_unlock(&vmp->vm_lock); in vmem_xfree()
1051 vmp->vm_source_free(vmp->vm_source, vaddr, size); in vmem_xfree()
1053 vmem_freelist_insert(vmp, vsp); in vmem_xfree()
1054 (void) mutex_unlock(&vmp->vm_lock); in vmem_xfree()
1066 vmem_alloc(vmem_t *vmp, size_t size, int vmflag) in vmem_alloc() argument
1075 if (size - 1 < vmp->vm_qcache_max) { in vmem_alloc()
1077 return (_umem_cache_alloc(vmp->vm_qcache[(size - 1) >> in vmem_alloc()
1078 vmp->vm_qshift], UMEM_DEFAULT)); in vmem_alloc()
1081 if ((mtbf = vmem_mtbf | vmp->vm_mtbf) != 0 && gethrtime() % mtbf == 0 && in vmem_alloc()
1086 return (vmem_nextfit_alloc(vmp, size, vmflag)); in vmem_alloc()
1089 return (vmem_xalloc(vmp, size, vmp->vm_quantum, 0, 0, in vmem_alloc()
1095 (void) mutex_lock(&vmp->vm_lock); in vmem_alloc()
1097 if (vmp->vm_nsegfree >= VMEM_MINFREE || vmem_populate(vmp, vmflag)) { in vmem_alloc()
1099 flist = lowbit(P2ALIGN(vmp->vm_freemap, size)); in vmem_alloc()
1101 flist = lowbit(P2ALIGN(vmp->vm_freemap, 1UL << hb)); in vmem_alloc()
1105 (void) mutex_unlock(&vmp->vm_lock); in vmem_alloc()
1106 return (vmem_xalloc(vmp, size, vmp->vm_quantum, in vmem_alloc()
1111 vsp = vmp->vm_freelist[flist].vs_knext; in vmem_alloc()
1113 (void) vmem_seg_alloc(vmp, vsp, addr, size); in vmem_alloc()
1114 (void) mutex_unlock(&vmp->vm_lock); in vmem_alloc()
1122 vmem_free(vmem_t *vmp, void *vaddr, size_t size) in vmem_free() argument
1124 if (size - 1 < vmp->vm_qcache_max) in vmem_free()
1125 _umem_cache_free(vmp->vm_qcache[(size - 1) >> vmp->vm_qshift], in vmem_free()
1128 vmem_xfree(vmp, vaddr, size); in vmem_free()
1135 vmem_contains(vmem_t *vmp, void *vaddr, size_t size) in vmem_contains() argument
1140 vmem_seg_t *seg0 = &vmp->vm_seg0; in vmem_contains()
1142 (void) mutex_lock(&vmp->vm_lock); in vmem_contains()
1143 vmp->vm_kstat.vk_contains++; in vmem_contains()
1145 vmp->vm_kstat.vk_contains_search++; in vmem_contains()
1150 (void) mutex_unlock(&vmp->vm_lock); in vmem_contains()
1158 vmem_add(vmem_t *vmp, void *vaddr, size_t size, int vmflag) in vmem_add() argument
1162 vmp, vaddr, size); in vmem_add()
1165 ASSERT(!vmem_contains(vmp, vaddr, size)); in vmem_add()
1167 (void) mutex_lock(&vmp->vm_lock); in vmem_add()
1168 if (vmem_populate(vmp, vmflag)) in vmem_add()
1169 (void) vmem_span_create(vmp, vaddr, size, 0); in vmem_add()
1172 (void) cond_broadcast(&vmp->vm_cv); in vmem_add()
1173 (void) mutex_unlock(&vmp->vm_lock); in vmem_add()
1193 vmem_extend_unlocked(vmem_t *vmp, uintptr_t addr, uintptr_t endaddr) in vmem_extend_unlocked() argument
1198 vmem_seg_t *end = &vmp->vm_seg0; in vmem_extend_unlocked()
1200 ASSERT(MUTEX_HELD(&vmp->vm_lock)); in vmem_extend_unlocked()
1211 return (vmem_span_create(vmp, (void *)addr, endaddr - addr, 0)); in vmem_extend_unlocked()
1237 vmem_freelist_delete(vmp, prevseg); in vmem_extend_unlocked()
1240 vmem_freelist_delete(vmp, nextseg); in vmem_extend_unlocked()
1242 vmem_seg_destroy(vmp, nextseg); in vmem_extend_unlocked()
1243 vmem_seg_destroy(vmp, span); in vmem_extend_unlocked()
1253 vmem_seg_destroy(vmp, span); in vmem_extend_unlocked()
1255 vmem_freelist_delete(vmp, prevseg); in vmem_extend_unlocked()
1266 vmem_seg_destroy(vmp, span); in vmem_extend_unlocked()
1268 vmem_freelist_delete(vmp, nextseg); in vmem_extend_unlocked()
1289 vmem_freelist_delete(vmp, oldseg); in vmem_extend_unlocked()
1293 vsp = vmem_seg_create(vmp, oldseg, addr, endaddr); in vmem_extend_unlocked()
1302 vmem_freelist_delete(vmp, oldseg); in vmem_extend_unlocked()
1306 vsp = vmem_seg_create(vmp, span, addr, endaddr); in vmem_extend_unlocked()
1308 vmem_freelist_insert(vmp, vsp); in vmem_extend_unlocked()
1309 vmp->vm_kstat.vk_mem_total += (endaddr - addr); in vmem_extend_unlocked()
1319 _vmem_extend_alloc(vmem_t *vmp, void *vaddr, size_t size, size_t alloc, in _vmem_extend_alloc() argument
1328 ASSERT(((addr | size | alloc) & (vmp->vm_quantum - 1)) == 0); in _vmem_extend_alloc()
1330 ASSERT(!vmem_contains(vmp, vaddr, size)); in _vmem_extend_alloc()
1332 (void) mutex_lock(&vmp->vm_lock); in _vmem_extend_alloc()
1333 if (!vmem_populate(vmp, vmflag)) { in _vmem_extend_alloc()
1334 (void) mutex_unlock(&vmp->vm_lock); in _vmem_extend_alloc()
1340 if (vmp->vm_source_alloc != NULL) in _vmem_extend_alloc()
1341 vsp = vmem_span_create(vmp, vaddr, size, 0); in _vmem_extend_alloc()
1343 vsp = vmem_extend_unlocked(vmp, addr, endaddr); in _vmem_extend_alloc()
1348 (void) vmem_seg_alloc(vmp, vsp, addr, alloc); in _vmem_extend_alloc()
1351 (void) cond_broadcast(&vmp->vm_cv); in _vmem_extend_alloc()
1352 (void) mutex_unlock(&vmp->vm_lock); in _vmem_extend_alloc()
1366 vmem_walk(vmem_t *vmp, int typemask, in vmem_walk() argument
1370 vmem_seg_t *seg0 = &vmp->vm_seg0; in vmem_walk()
1379 (void) mutex_lock(&vmp->vm_lock); in vmem_walk()
1386 vmem_advance(vmp, &walker, vsp); in vmem_walk()
1387 (void) mutex_unlock(&vmp->vm_lock); in vmem_walk()
1389 (void) mutex_lock(&vmp->vm_lock); in vmem_walk()
1396 vmem_advance(vmp, &walker, NULL); in vmem_walk()
1397 (void) mutex_unlock(&vmp->vm_lock); in vmem_walk()
1408 vmem_size(vmem_t *vmp, int typemask) in vmem_size() argument
1413 size += vmp->vm_kstat.vk_mem_inuse; in vmem_size()
1415 size += vmp->vm_kstat.vk_mem_total - in vmem_size()
1416 vmp->vm_kstat.vk_mem_inuse; in vmem_size()
1436 vmem_t *vmp, *cur, **vmpp; in vmem_create() local
1442 vmp = vmem_alloc(vmem_vmem_arena, sizeof (vmem_t), in vmem_create()
1446 vmp = &vmem0[id - 1]; in vmem_create()
1449 if (vmp == NULL) in vmem_create()
1451 bzero(vmp, sizeof (vmem_t)); in vmem_create()
1453 (void) snprintf(vmp->vm_name, VMEM_NAMELEN, "%s", name); in vmem_create()
1454 (void) mutex_init(&vmp->vm_lock, USYNC_THREAD, NULL); in vmem_create()
1455 (void) cond_init(&vmp->vm_cv, USYNC_THREAD, NULL); in vmem_create()
1456 vmp->vm_cflags = vmflag; in vmem_create()
1459 vmp->vm_quantum = quantum; in vmem_create()
1460 vmp->vm_qshift = highbit(quantum) - 1; in vmem_create()
1461 nqcache = MIN(qcache_max >> vmp->vm_qshift, VMEM_NQCACHE_MAX); in vmem_create()
1464 vfp = &vmp->vm_freelist[i]; in vmem_create()
1470 vmp->vm_freelist[0].vs_kprev = NULL; in vmem_create()
1471 vmp->vm_freelist[VMEM_FREELISTS].vs_knext = NULL; in vmem_create()
1472 vmp->vm_freelist[VMEM_FREELISTS].vs_end = 0; in vmem_create()
1473 vmp->vm_hash_table = vmp->vm_hash0; in vmem_create()
1474 vmp->vm_hash_mask = VMEM_HASH_INITIAL - 1; in vmem_create()
1475 vmp->vm_hash_shift = highbit(vmp->vm_hash_mask); in vmem_create()
1477 vsp = &vmp->vm_seg0; in vmem_create()
1484 vsp = &vmp->vm_rotor; in vmem_create()
1486 VMEM_INSERT(&vmp->vm_seg0, vsp, a); in vmem_create()
1488 vmp->vm_id = id; in vmem_create()
1490 vmp->vm_kstat.vk_source_id = source->vm_id; in vmem_create()
1491 vmp->vm_source = source; in vmem_create()
1492 vmp->vm_source_alloc = afunc; in vmem_create()
1493 vmp->vm_source_free = ffunc; in vmem_create()
1496 vmp->vm_qcache_max = nqcache << vmp->vm_qshift; in vmem_create()
1500 vmp->vm_name, (long)((i + 1) * quantum)); in vmem_create()
1501 vmp->vm_qcache[i] = umem_cache_create(buf, in vmem_create()
1503 NULL, vmp, UMC_QCACHE | UMC_NOTOUCH); in vmem_create()
1504 if (vmp->vm_qcache[i] == NULL) { in vmem_create()
1505 vmp->vm_qcache_max = i * quantum; in vmem_create()
1515 *vmpp = vmp; in vmem_create()
1518 if (vmp->vm_cflags & VMC_POPULATOR) { in vmem_create()
1521 vmem_populator[pop_id - 1] = vmp; in vmem_create()
1522 (void) mutex_lock(&vmp->vm_lock); in vmem_create()
1523 (void) vmem_populate(vmp, vmflag | VM_PANIC); in vmem_create()
1524 (void) mutex_unlock(&vmp->vm_lock); in vmem_create()
1527 if ((base || size) && vmem_add(vmp, base, size, vmflag) == NULL) { in vmem_create()
1528 vmem_destroy(vmp); in vmem_create()
1532 return (vmp); in vmem_create()
1539 vmem_destroy(vmem_t *vmp) in vmem_destroy() argument
1542 vmem_seg_t *seg0 = &vmp->vm_seg0; in vmem_destroy()
1549 while ((cur = *vmpp) != vmp) in vmem_destroy()
1551 *vmpp = vmp->vm_next; in vmem_destroy()
1555 if (vmp->vm_qcache[i]) in vmem_destroy()
1556 umem_cache_destroy(vmp->vm_qcache[i]); in vmem_destroy()
1558 leaked = vmem_size(vmp, VMEM_ALLOC); in vmem_destroy()
1561 vmp->vm_name, leaked); in vmem_destroy()
1563 if (vmp->vm_hash_table != vmp->vm_hash0) in vmem_destroy()
1564 vmem_free(vmem_hash_arena, vmp->vm_hash_table, in vmem_destroy()
1565 (vmp->vm_hash_mask + 1) * sizeof (void *)); in vmem_destroy()
1571 VMEM_DELETE(&vmp->vm_rotor, a); in vmem_destroy()
1575 while (vmp->vm_nsegfree > 0) in vmem_destroy()
1576 vmem_putseg_global(vmem_getseg(vmp)); in vmem_destroy()
1578 (void) mutex_destroy(&vmp->vm_lock); in vmem_destroy()
1579 (void) cond_destroy(&vmp->vm_cv); in vmem_destroy()
1580 vmem_free(vmem_vmem_arena, vmp, sizeof (vmem_t)); in vmem_destroy()
1587 vmem_hash_rescale(vmem_t *vmp) in vmem_hash_rescale() argument
1592 nseg = (size_t)(vmp->vm_kstat.vk_alloc - vmp->vm_kstat.vk_free); in vmem_hash_rescale()
1595 old_size = vmp->vm_hash_mask + 1; in vmem_hash_rescale()
1606 (void) mutex_lock(&vmp->vm_lock); in vmem_hash_rescale()
1608 old_size = vmp->vm_hash_mask + 1; in vmem_hash_rescale()
1609 old_table = vmp->vm_hash_table; in vmem_hash_rescale()
1611 vmp->vm_hash_mask = new_size - 1; in vmem_hash_rescale()
1612 vmp->vm_hash_table = new_table; in vmem_hash_rescale()
1613 vmp->vm_hash_shift = highbit(vmp->vm_hash_mask); in vmem_hash_rescale()
1620 vmem_seg_t **hash_bucket = VMEM_HASH(vmp, addr); in vmem_hash_rescale()
1627 (void) mutex_unlock(&vmp->vm_lock); in vmem_hash_rescale()
1629 if (old_table != vmp->vm_hash0) in vmem_hash_rescale()
1641 vmem_t *vmp; in vmem_update() local
1644 for (vmp = vmem_list; vmp != NULL; vmp = vmp->vm_next) { in vmem_update()
1650 (void) cond_broadcast(&vmp->vm_cv); in vmem_update()
1655 vmem_hash_rescale(vmp); in vmem_update()