Lines Matching defs:vsp
352 #define VMEM_INSERT(vprev, vsp, type) \
355 (vsp)->vs_##type##next = (vnext); \
356 (vsp)->vs_##type##prev = (vprev); \
357 (vprev)->vs_##type##next = (vsp); \
358 (vnext)->vs_##type##prev = (vsp); \
361 #define VMEM_DELETE(vsp, type) \
363 vmem_seg_t *vprev = (vsp)->vs_##type##prev; \
364 vmem_seg_t *vnext = (vsp)->vs_##type##next; \
375 vmem_seg_t *vsp;
378 if ((vsp = vmem_segfree) != NULL)
379 vmem_segfree = vsp->vs_knext;
382 return (vsp);
389 vmem_putseg_global(vmem_seg_t *vsp)
392 vsp->vs_knext = vmem_segfree;
393 vmem_segfree = vsp;
403 vmem_seg_t *vsp;
407 vsp = vmp->vm_segfree;
408 vmp->vm_segfree = vsp->vs_knext;
411 return (vsp);
418 vmem_putseg(vmem_t *vmp, vmem_seg_t *vsp)
420 vsp->vs_knext = vmp->vm_segfree;
421 vmp->vm_segfree = vsp;
426 * Add vsp to the appropriate freelist.
429 vmem_freelist_insert(vmem_t *vmp, vmem_seg_t *vsp)
433 ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp);
435 vprev = (vmem_seg_t *)&vmp->vm_freelist[highbit(VS_SIZE(vsp)) - 1];
436 vsp->vs_type = VMEM_FREE;
438 VMEM_INSERT(vprev, vsp, k);
444 * Take vsp from the freelist.
447 vmem_freelist_delete(vmem_t *vmp, vmem_seg_t *vsp)
449 ASSERT(*VMEM_HASH(vmp, vsp->vs_start) != vsp);
450 ASSERT(vsp->vs_type == VMEM_FREE);
452 if (vsp->vs_knext->vs_start == 0 && vsp->vs_kprev->vs_start == 0) {
454 * The segments on both sides of 'vsp' are freelist heads,
455 * so taking vsp leaves the freelist at vsp->vs_kprev empty.
457 ASSERT(vmp->vm_freemap & VS_SIZE(vsp->vs_kprev));
458 vmp->vm_freemap ^= VS_SIZE(vsp->vs_kprev);
460 VMEM_DELETE(vsp, k);
464 * Add vsp to the allocated-segment hash table and update kstats.
467 vmem_hash_insert(vmem_t *vmp, vmem_seg_t *vsp)
471 vsp->vs_type = VMEM_ALLOC;
472 bucket = VMEM_HASH(vmp, vsp->vs_start);
473 vsp->vs_knext = *bucket;
474 *bucket = vsp;
477 vsp->vs_depth = (uint8_t)getpcstack(vsp->vs_stack,
479 vsp->vs_thread = curthread;
480 vsp->vs_timestamp = gethrtime();
482 vsp->vs_depth = 0;
486 vmp->vm_kstat.vk_mem_inuse.value.ui64 += VS_SIZE(vsp);
490 * Remove vsp from the allocated-segment hash table and update kstats.
495 vmem_seg_t *vsp, **prev_vspp;
498 while ((vsp = *prev_vspp) != NULL) {
499 if (vsp->vs_start == addr) {
500 *prev_vspp = vsp->vs_knext;
504 prev_vspp = &vsp->vs_knext;
507 if (vsp == NULL)
510 if (VS_SIZE(vsp) != size)
512 (void *)vmp, addr, size, VS_SIZE(vsp));
517 return (vsp);
539 * Remove segment vsp from the arena.
542 vmem_seg_destroy(vmem_t *vmp, vmem_seg_t *vsp)
544 ASSERT(vsp->vs_type != VMEM_ROTOR);
545 VMEM_DELETE(vsp, a);
547 vmem_putseg(vmp, vsp);
582 * Remove span vsp from vmp and update kstats.
585 vmem_span_destroy(vmem_t *vmp, vmem_seg_t *vsp)
587 vmem_seg_t *span = vsp->vs_aprev;
588 size_t size = VS_SIZE(vsp);
599 vmem_seg_destroy(vmp, vsp);
604 * Allocate the subrange [addr, addr + size) from segment vsp.
609 vmem_seg_alloc(vmem_t *vmp, vmem_seg_t *vsp, uintptr_t addr, size_t size)
611 uintptr_t vs_start = vsp->vs_start;
612 uintptr_t vs_end = vsp->vs_end;
619 ASSERT(vsp->vs_type == VMEM_FREE);
630 vsp->vs_start = addr_end;
631 vsp = vmem_seg_create(vmp, vsp->vs_aprev, addr, addr + size);
632 vmem_hash_insert(vmp, vsp);
633 return (vsp);
636 vmem_freelist_delete(vmp, vsp);
640 vmem_seg_create(vmp, vsp, addr_end, vs_end));
644 vmem_seg_create(vmp, vsp->vs_aprev, vs_start, addr));
646 vsp->vs_start = addr;
647 vsp->vs_end = addr + size;
649 vmem_hash_insert(vmp, vsp);
650 return (vsp);
673 vmem_seg_t *vsp;
680 (vsp = vmem_getseg_global()) != NULL)
681 vmem_putseg(vmp, vsp);
764 vmem_seg_t *vsp = NULL;
784 vsp = vprev;
786 vsp = vnext;
790 * vsp could represent a complete imported span,
793 if (vsp != NULL && vsp->vs_aprev->vs_import &&
795 vsp->vs_aprev->vs_type == VMEM_SPAN &&
796 vsp->vs_anext->vs_type == VMEM_SPAN) {
797 void *vaddr = (void *)vsp->vs_start;
798 size_t size = VS_SIZE(vsp);
799 ASSERT(size == VS_SIZE(vsp->vs_aprev));
800 vmem_freelist_delete(vmp, vsp);
801 vmem_span_destroy(vmp, vsp);
818 vmem_seg_t *vsp, *rotor;
841 vsp = rotor->vs_anext;
842 if (vsp->vs_type == VMEM_FREE && (vs_size = VS_SIZE(vsp)) > realsize &&
845 addr = vsp->vs_start;
846 vsp->vs_start = addr + realsize;
859 if (vsp->vs_type == VMEM_FREE && VS_SIZE(vsp) >= size)
861 vsp = vsp->vs_anext;
862 if (vsp == rotor) {
872 vsp = rotor->vs_aprev;
873 if (vsp->vs_type == VMEM_FREE && VS_SIZE(vsp) >= size)
888 vsp = rotor->vs_anext;
895 addr = vsp->vs_start;
896 vsp = vmem_seg_alloc(vmp, vsp, addr, size);
897 ASSERT(vsp->vs_type == VMEM_ALLOC &&
898 vsp->vs_start == addr && vsp->vs_end == addr + size);
904 vmem_advance(vmp, rotor, vsp);
940 vmem_seg_t *vsp;
1007 for (vbest = NULL, vsp = (flist == 0) ? NULL :
1009 vsp != NULL; vsp = vsp->vs_knext) {
1011 if (vsp->vs_start == 0) {
1023 VS_SIZE(vsp)));
1026 vsp = (vmem_seg_t *)&vmp->vm_freelist[flist];
1027 ASSERT(vsp->vs_knext->vs_type == VMEM_FREE);
1030 if (vsp->vs_end - 1 < (uintptr_t)minaddr)
1032 if (vsp->vs_start > (uintptr_t)maxaddr - 1)
1034 start = MAX(vsp->vs_start, (uintptr_t)minaddr);
1035 end = MIN(vsp->vs_end - 1, (uintptr_t)maxaddr - 1) + 1;
1041 (vbest != NULL && VS_SIZE(vsp) >= VS_SIZE(vbest)))
1043 vbest = vsp;
1207 vmem_seg_t *vsp, *vnext, *vprev;
1211 vsp = vmem_hash_delete(vmp, (uintptr_t)vaddr, size);
1212 vsp->vs_end = P2ROUNDUP(vsp->vs_end, vmp->vm_quantum);
1217 vnext = vsp->vs_anext;
1219 ASSERT(vsp->vs_end == vnext->vs_start);
1221 vsp->vs_end = vnext->vs_end;
1228 vprev = vsp->vs_aprev;
1230 ASSERT(vprev->vs_end == vsp->vs_start);
1232 vprev->vs_end = vsp->vs_end;
1233 vmem_seg_destroy(vmp, vsp);
1234 vsp = vprev;
1240 if (vsp->vs_aprev->vs_import && vmp->vm_source_free != NULL &&
1241 vsp->vs_aprev->vs_type == VMEM_SPAN &&
1242 vsp->vs_anext->vs_type == VMEM_SPAN) {
1243 vaddr = (void *)vsp->vs_start;
1244 size = VS_SIZE(vsp);
1245 ASSERT(size == VS_SIZE(vsp->vs_aprev));
1246 vmem_span_destroy(vmp, vsp);
1250 vmem_freelist_insert(vmp, vsp);
1265 vmem_seg_t *vsp;
1305 vsp = vmp->vm_freelist[flist].vs_knext;
1306 addr = vsp->vs_start;
1308 addr += vsp->vs_end - (addr + size);
1310 (void) vmem_seg_alloc(vmp, vsp, addr, size);
1336 vmem_seg_t *vsp;
1341 for (vsp = seg0->vs_knext; vsp != seg0; vsp = vsp->vs_knext) {
1343 ASSERT(vsp->vs_type == VMEM_SPAN);
1344 if (start >= vsp->vs_start && end - 1 <= vsp->vs_end - 1)
1348 return (vsp != seg0);
1384 vmem_seg_t *vsp;
1396 for (vsp = seg0->vs_anext; vsp != seg0; vsp = vsp->vs_anext) {
1397 if (vsp->vs_type & typemask) {
1398 void *start = (void *)vsp->vs_start;
1399 size_t size = VS_SIZE(vsp);
1401 vmem_advance(vmp, &walker, vsp);
1405 vsp = &walker;
1453 vmem_seg_t *vsp;
1497 vsp = &vmp->vm_seg0;
1498 vsp->vs_anext = vsp;
1499 vsp->vs_aprev = vsp;
1500 vsp->vs_knext = vsp;
1501 vsp->vs_kprev = vsp;
1502 vsp->vs_type = VMEM_SPAN;
1504 vsp = &vmp->vm_rotor;
1505 vsp->vs_type = VMEM_ROTOR;
1506 VMEM_INSERT(&vmp->vm_seg0, vsp, a);
1604 vmem_seg_t *vsp, *anext;
1634 for (vsp = seg0->vs_anext; vsp != seg0; vsp = anext) {
1635 anext = vsp->vs_anext;
1636 vmem_putseg_global(vsp);
1661 vmem_seg_t **old_table, **new_table, *vsp;
1690 vsp = old_table[h];
1691 while (vsp != NULL) {
1692 uintptr_t addr = vsp->vs_start;
1693 vmem_seg_t *next_vsp = vsp->vs_knext;
1695 vsp->vs_knext = *hash_bucket;
1696 *hash_bucket = vsp;
1697 vsp = next_vsp;