Lines Matching refs:vsp
222 #define VMEM_INSERT(vprev, vsp, type) \ argument
225 (vsp)->vs_##type##next = (vnext); \
226 (vsp)->vs_##type##prev = (vprev); \
227 (vprev)->vs_##type##next = (vsp); \
228 (vnext)->vs_##type##prev = (vsp); \
231 #define VMEM_DELETE(vsp, type) \ argument
233 vmem_seg_t *vprev = (vsp)->vs_##type##prev; \
234 vmem_seg_t *vnext = (vsp)->vs_##type##next; \
245 vmem_seg_t *vsp; in vmem_getseg_global() local
248 if ((vsp = vmem_segfree) != NULL) in vmem_getseg_global()
249 vmem_segfree = vsp->vs_knext; in vmem_getseg_global()
252 return (vsp); in vmem_getseg_global()
259 vmem_putseg_global(vmem_seg_t *vsp) in vmem_putseg_global() argument
262 vsp->vs_knext = vmem_segfree; in vmem_putseg_global()
263 vmem_segfree = vsp; in vmem_putseg_global()
273 vmem_seg_t *vsp; in vmem_getseg() local
277 vsp = vmp->vm_segfree; in vmem_getseg()
278 vmp->vm_segfree = vsp->vs_knext; in vmem_getseg()
281 return (vsp); in vmem_getseg()
288 vmem_putseg(vmem_t *vmp, vmem_seg_t *vsp) in vmem_putseg() argument
290 vsp->vs_knext = vmp->vm_segfree; in vmem_putseg()
291 vmp->vm_segfree = vsp; in vmem_putseg()
299 vmem_freelist_insert(vmem_t *vmp, vmem_seg_t *vsp) in vmem_freelist_insert() argument
303 ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp); in vmem_freelist_insert()
305 vprev = (vmem_seg_t *)&vmp->vm_freelist[highbit(VS_SIZE(vsp)) - 1]; in vmem_freelist_insert()
306 vsp->vs_type = VMEM_FREE; in vmem_freelist_insert()
308 VMEM_INSERT(vprev, vsp, k); in vmem_freelist_insert()
317 vmem_freelist_delete(vmem_t *vmp, vmem_seg_t *vsp) in vmem_freelist_delete() argument
319 ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp); in vmem_freelist_delete()
320 ASSERT(vsp->vs_type == VMEM_FREE); in vmem_freelist_delete()
322 if (vsp->vs_knext->vs_start == 0 && vsp->vs_kprev->vs_start == 0) { in vmem_freelist_delete()
327 ASSERT(vmp->vm_freemap & VS_SIZE(vsp->vs_kprev)); in vmem_freelist_delete()
328 vmp->vm_freemap ^= VS_SIZE(vsp->vs_kprev); in vmem_freelist_delete()
330 VMEM_DELETE(vsp, k); in vmem_freelist_delete()
337 vmem_hash_insert(vmem_t *vmp, vmem_seg_t *vsp) in vmem_hash_insert() argument
341 vsp->vs_type = VMEM_ALLOC; in vmem_hash_insert()
342 bucket = VMEM_HASH(vmp, vsp->vs_start); in vmem_hash_insert()
343 vsp->vs_knext = *bucket; in vmem_hash_insert()
344 *bucket = vsp; in vmem_hash_insert()
347 vsp->vs_depth = (uint8_t)getpcstack(vsp->vs_stack, in vmem_hash_insert()
349 vsp->vs_thread = thr_self(); in vmem_hash_insert()
350 vsp->vs_timestamp = gethrtime(); in vmem_hash_insert()
352 vsp->vs_depth = 0; in vmem_hash_insert()
356 vmp->vm_kstat.vk_mem_inuse += VS_SIZE(vsp); in vmem_hash_insert()
365 vmem_seg_t *vsp, **prev_vspp; in vmem_hash_delete() local
368 while ((vsp = *prev_vspp) != NULL) { in vmem_hash_delete()
369 if (vsp->vs_start == addr) { in vmem_hash_delete()
370 *prev_vspp = vsp->vs_knext; in vmem_hash_delete()
374 prev_vspp = &vsp->vs_knext; in vmem_hash_delete()
377 if (vsp == NULL) { in vmem_hash_delete()
381 if (VS_SIZE(vsp) != size) { in vmem_hash_delete()
383 "(expect %lu)", vmp, addr, size, VS_SIZE(vsp)); in vmem_hash_delete()
389 return (vsp); in vmem_hash_delete()
414 vmem_seg_destroy(vmem_t *vmp, vmem_seg_t *vsp) in vmem_seg_destroy() argument
416 ASSERT(vsp->vs_type != VMEM_ROTOR); in vmem_seg_destroy()
417 VMEM_DELETE(vsp, a); in vmem_seg_destroy()
419 vmem_putseg(vmp, vsp); in vmem_seg_destroy()
478 vmem_span_destroy(vmem_t *vmp, vmem_seg_t *vsp) in vmem_span_destroy() argument
480 vmem_seg_t *span = vsp->vs_aprev; in vmem_span_destroy()
481 size_t size = VS_SIZE(vsp); in vmem_span_destroy()
486 if (vsp->vs_import) in vmem_span_destroy()
492 vmem_seg_destroy(vmp, vsp); in vmem_span_destroy()
502 vmem_seg_alloc(vmem_t *vmp, vmem_seg_t *vsp, uintptr_t addr, size_t size) in vmem_seg_alloc() argument
504 uintptr_t vs_start = vsp->vs_start; in vmem_seg_alloc()
505 uintptr_t vs_end = vsp->vs_end; in vmem_seg_alloc()
512 ASSERT(vsp->vs_type == VMEM_FREE); in vmem_seg_alloc()
523 vsp->vs_start = addr_end; in vmem_seg_alloc()
524 vsp = vmem_seg_create(vmp, vsp->vs_aprev, addr, addr + size); in vmem_seg_alloc()
525 vmem_hash_insert(vmp, vsp); in vmem_seg_alloc()
526 return (vsp); in vmem_seg_alloc()
529 vmem_freelist_delete(vmp, vsp); in vmem_seg_alloc()
533 vmem_seg_create(vmp, vsp, addr_end, vs_end)); in vmem_seg_alloc()
537 vmem_seg_create(vmp, vsp->vs_aprev, vs_start, addr)); in vmem_seg_alloc()
539 vsp->vs_start = addr; in vmem_seg_alloc()
540 vsp->vs_end = addr + size; in vmem_seg_alloc()
542 vmem_hash_insert(vmp, vsp); in vmem_seg_alloc()
543 return (vsp); in vmem_seg_alloc()
563 vmem_seg_t *vsp; in vmem_populate() local
570 (vsp = vmem_getseg_global()) != NULL) in vmem_populate()
571 vmem_putseg(vmp, vsp); in vmem_populate()
660 vmem_seg_t *vsp = NULL; in vmem_advance() local
680 vsp = vprev; in vmem_advance()
682 vsp = vnext; in vmem_advance()
689 if (vsp != NULL && vsp->vs_import && vmp->vm_source_free != NULL && in vmem_advance()
690 vsp->vs_aprev->vs_type == VMEM_SPAN && in vmem_advance()
691 vsp->vs_anext->vs_type == VMEM_SPAN) { in vmem_advance()
692 void *vaddr = (void *)vsp->vs_start; in vmem_advance()
693 size_t size = VS_SIZE(vsp); in vmem_advance()
694 ASSERT(size == VS_SIZE(vsp->vs_aprev)); in vmem_advance()
695 vmem_freelist_delete(vmp, vsp); in vmem_advance()
696 vmem_span_destroy(vmp, vsp); in vmem_advance()
713 vmem_seg_t *vsp, *rotor; in vmem_nextfit_alloc() local
736 vsp = rotor->vs_anext; in vmem_nextfit_alloc()
737 if (vsp->vs_type == VMEM_FREE && (vs_size = VS_SIZE(vsp)) > realsize && in vmem_nextfit_alloc()
740 addr = vsp->vs_start; in vmem_nextfit_alloc()
741 vsp->vs_start = addr + realsize; in vmem_nextfit_alloc()
754 if (vsp->vs_type == VMEM_FREE && VS_SIZE(vsp) >= size) in vmem_nextfit_alloc()
756 vsp = vsp->vs_anext; in vmem_nextfit_alloc()
757 if (vsp == rotor) { in vmem_nextfit_alloc()
769 vsp = rotor->vs_aprev; in vmem_nextfit_alloc()
770 if (vsp->vs_type == VMEM_FREE && VS_SIZE(vsp) >= size) in vmem_nextfit_alloc()
788 vsp = rotor->vs_anext; in vmem_nextfit_alloc()
795 addr = vsp->vs_start; in vmem_nextfit_alloc()
796 vsp = vmem_seg_alloc(vmp, vsp, addr, size); in vmem_nextfit_alloc()
797 ASSERT(vsp->vs_type == VMEM_ALLOC && in vmem_nextfit_alloc()
798 vsp->vs_start == addr && vsp->vs_end == addr + size); in vmem_nextfit_alloc()
804 vmem_advance(vmp, rotor, vsp); in vmem_nextfit_alloc()
818 vmem_seg_t *vsp; in vmem_xalloc() local
890 for (vbest = NULL, vsp = (flist == 0) ? NULL : in vmem_xalloc()
892 vsp != NULL; vsp = vsp->vs_knext) { in vmem_xalloc()
894 if (vsp->vs_start == 0) { in vmem_xalloc()
906 VS_SIZE(vsp))); in vmem_xalloc()
909 vsp = (vmem_seg_t *)&vmp->vm_freelist[flist]; in vmem_xalloc()
910 ASSERT(vsp->vs_knext->vs_type == VMEM_FREE); in vmem_xalloc()
913 if (vsp->vs_end - 1 < (uintptr_t)minaddr) in vmem_xalloc()
915 if (vsp->vs_start > (uintptr_t)maxaddr - 1) in vmem_xalloc()
917 start = MAX(vsp->vs_start, (uintptr_t)minaddr); in vmem_xalloc()
918 end = MIN(vsp->vs_end - 1, (uintptr_t)maxaddr - 1) + 1; in vmem_xalloc()
924 (vbest != NULL && VS_SIZE(vsp) >= VS_SIZE(vbest))) in vmem_xalloc()
926 vbest = vsp; in vmem_xalloc()
1010 vmem_seg_t *vsp, *vnext, *vprev; in vmem_xfree() local
1014 vsp = vmem_hash_delete(vmp, (uintptr_t)vaddr, size); in vmem_xfree()
1015 vsp->vs_end = P2ROUNDUP(vsp->vs_end, vmp->vm_quantum); in vmem_xfree()
1020 vnext = vsp->vs_anext; in vmem_xfree()
1022 ASSERT(vsp->vs_end == vnext->vs_start); in vmem_xfree()
1024 vsp->vs_end = vnext->vs_end; in vmem_xfree()
1031 vprev = vsp->vs_aprev; in vmem_xfree()
1033 ASSERT(vprev->vs_end == vsp->vs_start); in vmem_xfree()
1035 vprev->vs_end = vsp->vs_end; in vmem_xfree()
1036 vmem_seg_destroy(vmp, vsp); in vmem_xfree()
1037 vsp = vprev; in vmem_xfree()
1043 if (vsp->vs_import && vmp->vm_source_free != NULL && in vmem_xfree()
1044 vsp->vs_aprev->vs_type == VMEM_SPAN && in vmem_xfree()
1045 vsp->vs_anext->vs_type == VMEM_SPAN) { in vmem_xfree()
1046 vaddr = (void *)vsp->vs_start; in vmem_xfree()
1047 size = VS_SIZE(vsp); in vmem_xfree()
1048 ASSERT(size == VS_SIZE(vsp->vs_aprev)); in vmem_xfree()
1049 vmem_span_destroy(vmp, vsp); in vmem_xfree()
1053 vmem_freelist_insert(vmp, vsp); in vmem_xfree()
1068 vmem_seg_t *vsp; in vmem_alloc() local
1111 vsp = vmp->vm_freelist[flist].vs_knext; in vmem_alloc()
1112 addr = vsp->vs_start; in vmem_alloc()
1113 (void) vmem_seg_alloc(vmp, vsp, addr, size); in vmem_alloc()
1139 vmem_seg_t *vsp; in vmem_contains() local
1144 for (vsp = seg0->vs_knext; vsp != seg0; vsp = vsp->vs_knext) { in vmem_contains()
1146 ASSERT(vsp->vs_type == VMEM_SPAN); in vmem_contains()
1147 if (start >= vsp->vs_start && end - 1 <= vsp->vs_end - 1) in vmem_contains()
1151 return (vsp != seg0); in vmem_contains()
1196 vmem_seg_t *vsp; in vmem_extend_unlocked() local
1245 vsp = prevseg; in vmem_extend_unlocked()
1258 vsp = prevseg; in vmem_extend_unlocked()
1271 vsp = nextseg; in vmem_extend_unlocked()
1280 vsp = span; in vmem_extend_unlocked()
1291 vsp = oldseg; in vmem_extend_unlocked()
1293 vsp = vmem_seg_create(vmp, oldseg, addr, endaddr); in vmem_extend_unlocked()
1304 vsp = oldseg; in vmem_extend_unlocked()
1306 vsp = vmem_seg_create(vmp, span, addr, endaddr); in vmem_extend_unlocked()
1308 vmem_freelist_insert(vmp, vsp); in vmem_extend_unlocked()
1310 return (vsp); in vmem_extend_unlocked()
1324 vmem_seg_t *vsp; in _vmem_extend_alloc() local
1341 vsp = vmem_span_create(vmp, vaddr, size, 0); in _vmem_extend_alloc()
1343 vsp = vmem_extend_unlocked(vmp, addr, endaddr); in _vmem_extend_alloc()
1345 ASSERT(VS_SIZE(vsp) >= alloc); in _vmem_extend_alloc()
1347 addr = vsp->vs_start; in _vmem_extend_alloc()
1348 (void) vmem_seg_alloc(vmp, vsp, addr, alloc); in _vmem_extend_alloc()
1369 vmem_seg_t *vsp; in vmem_walk() local
1381 for (vsp = seg0->vs_anext; vsp != seg0; vsp = vsp->vs_anext) { in vmem_walk()
1382 if (vsp->vs_type & typemask) { in vmem_walk()
1383 void *start = (void *)vsp->vs_start; in vmem_walk()
1384 size_t size = VS_SIZE(vsp); in vmem_walk()
1386 vmem_advance(vmp, &walker, vsp); in vmem_walk()
1390 vsp = &walker; in vmem_walk()
1437 vmem_seg_t *vsp; in vmem_create() local
1477 vsp = &vmp->vm_seg0; in vmem_create()
1478 vsp->vs_anext = vsp; in vmem_create()
1479 vsp->vs_aprev = vsp; in vmem_create()
1480 vsp->vs_knext = vsp; in vmem_create()
1481 vsp->vs_kprev = vsp; in vmem_create()
1482 vsp->vs_type = VMEM_SPAN; in vmem_create()
1484 vsp = &vmp->vm_rotor; in vmem_create()
1485 vsp->vs_type = VMEM_ROTOR; in vmem_create()
1486 VMEM_INSERT(&vmp->vm_seg0, vsp, a); in vmem_create()
1543 vmem_seg_t *vsp; in vmem_destroy() local
1572 for (vsp = seg0->vs_anext; vsp != seg0; vsp = vsp->vs_anext) in vmem_destroy()
1573 vmem_putseg_global(vsp); in vmem_destroy()
1589 vmem_seg_t **old_table, **new_table, *vsp; in vmem_hash_rescale() local
1616 vsp = old_table[h]; in vmem_hash_rescale()
1617 while (vsp != NULL) { in vmem_hash_rescale()
1618 uintptr_t addr = vsp->vs_start; in vmem_hash_rescale()
1619 vmem_seg_t *next_vsp = vsp->vs_knext; in vmem_hash_rescale()
1621 vsp->vs_knext = *hash_bucket; in vmem_hash_rescale()
1622 *hash_bucket = vsp; in vmem_hash_rescale()
1623 vsp = next_vsp; in vmem_hash_rescale()