Lines Matching full:vb
2579 * regular operations: Purge if vb->free is less than 1/4 of the capacity.
2700 struct vmap_block *vb; in new_vmap_block() local
2709 vb = kmalloc_node(sizeof(struct vmap_block), gfp_mask, node); in new_vmap_block()
2710 if (unlikely(!vb)) in new_vmap_block()
2718 kfree(vb); in new_vmap_block()
2723 spin_lock_init(&vb->lock); in new_vmap_block()
2724 vb->va = va; in new_vmap_block()
2727 bitmap_zero(vb->used_map, VMAP_BBMAP_BITS); in new_vmap_block()
2728 vb->free = VMAP_BBMAP_BITS - (1UL << order); in new_vmap_block()
2729 vb->dirty = 0; in new_vmap_block()
2730 vb->dirty_min = VMAP_BBMAP_BITS; in new_vmap_block()
2731 vb->dirty_max = 0; in new_vmap_block()
2732 bitmap_set(vb->used_map, 0, (1UL << order)); in new_vmap_block()
2733 INIT_LIST_HEAD(&vb->free_list); in new_vmap_block()
2734 vb->cpu = raw_smp_processor_id(); in new_vmap_block()
2738 err = xa_insert(xa, vb_idx, vb, gfp_mask); in new_vmap_block()
2740 kfree(vb); in new_vmap_block()
2746 * rather than vb->cpu due to task migration, which in new_vmap_block()
2751 vbq = per_cpu_ptr(&vmap_block_queue, vb->cpu); in new_vmap_block()
2753 list_add_tail_rcu(&vb->free_list, &vbq->free); in new_vmap_block()
2759 static void free_vmap_block(struct vmap_block *vb) in free_vmap_block() argument
2765 xa = addr_to_vb_xa(vb->va->va_start); in free_vmap_block()
2766 tmp = xa_erase(xa, addr_to_vb_idx(vb->va->va_start)); in free_vmap_block()
2767 BUG_ON(tmp != vb); in free_vmap_block()
2769 vn = addr_to_node(vb->va->va_start); in free_vmap_block()
2771 unlink_va(vb->va, &vn->busy.root); in free_vmap_block()
2774 free_vmap_area_noflush(vb->va); in free_vmap_block()
2775 kfree_rcu(vb, rcu_head); in free_vmap_block()
2778 static bool purge_fragmented_block(struct vmap_block *vb, in purge_fragmented_block() argument
2781 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, vb->cpu); in purge_fragmented_block()
2783 if (vb->free + vb->dirty != VMAP_BBMAP_BITS || in purge_fragmented_block()
2784 vb->dirty == VMAP_BBMAP_BITS) in purge_fragmented_block()
2788 if (!(force_purge || vb->free < VMAP_PURGE_THRESHOLD)) in purge_fragmented_block()
2792 WRITE_ONCE(vb->free, 0); in purge_fragmented_block()
2794 WRITE_ONCE(vb->dirty, VMAP_BBMAP_BITS); in purge_fragmented_block()
2795 vb->dirty_min = 0; in purge_fragmented_block()
2796 vb->dirty_max = VMAP_BBMAP_BITS; in purge_fragmented_block()
2798 list_del_rcu(&vb->free_list); in purge_fragmented_block()
2800 list_add_tail(&vb->purge, purge_list); in purge_fragmented_block()
2806 struct vmap_block *vb, *n_vb; in free_purged_blocks() local
2808 list_for_each_entry_safe(vb, n_vb, purge_list, purge) { in free_purged_blocks()
2809 list_del(&vb->purge); in free_purged_blocks()
2810 free_vmap_block(vb); in free_purged_blocks()
2817 struct vmap_block *vb; in purge_fragmented_blocks() local
2821 list_for_each_entry_rcu(vb, &vbq->free, free_list) { in purge_fragmented_blocks()
2822 unsigned long free = READ_ONCE(vb->free); in purge_fragmented_blocks()
2823 unsigned long dirty = READ_ONCE(vb->dirty); in purge_fragmented_blocks()
2829 spin_lock(&vb->lock); in purge_fragmented_blocks()
2830 purge_fragmented_block(vb, &purge, true); in purge_fragmented_blocks()
2831 spin_unlock(&vb->lock); in purge_fragmented_blocks()
2848 struct vmap_block *vb; in vb_alloc() local
2866 list_for_each_entry_rcu(vb, &vbq->free, free_list) { in vb_alloc()
2869 if (READ_ONCE(vb->free) < (1UL << order)) in vb_alloc()
2872 spin_lock(&vb->lock); in vb_alloc()
2873 if (vb->free < (1UL << order)) { in vb_alloc()
2874 spin_unlock(&vb->lock); in vb_alloc()
2878 pages_off = VMAP_BBMAP_BITS - vb->free; in vb_alloc()
2879 vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); in vb_alloc()
2880 WRITE_ONCE(vb->free, vb->free - (1UL << order)); in vb_alloc()
2881 bitmap_set(vb->used_map, pages_off, (1UL << order)); in vb_alloc()
2882 if (vb->free == 0) { in vb_alloc()
2884 list_del_rcu(&vb->free_list); in vb_alloc()
2888 spin_unlock(&vb->lock); in vb_alloc()
2905 struct vmap_block *vb; in vb_free() local
2917 vb = xa_load(xa, addr_to_vb_idx(addr)); in vb_free()
2919 spin_lock(&vb->lock); in vb_free()
2920 bitmap_clear(vb->used_map, offset, (1UL << order)); in vb_free()
2921 spin_unlock(&vb->lock); in vb_free()
2928 spin_lock(&vb->lock); in vb_free()
2931 vb->dirty_min = min(vb->dirty_min, offset); in vb_free()
2932 vb->dirty_max = max(vb->dirty_max, offset + (1UL << order)); in vb_free()
2934 WRITE_ONCE(vb->dirty, vb->dirty + (1UL << order)); in vb_free()
2935 if (vb->dirty == VMAP_BBMAP_BITS) { in vb_free()
2936 BUG_ON(vb->free); in vb_free()
2937 spin_unlock(&vb->lock); in vb_free()
2938 free_vmap_block(vb); in vb_free()
2940 spin_unlock(&vb->lock); in vb_free()
2955 struct vmap_block *vb; in _vm_unmap_aliases() local
2959 xa_for_each(&vbq->vmap_blocks, idx, vb) { in _vm_unmap_aliases()
2960 spin_lock(&vb->lock); in _vm_unmap_aliases()
2967 if (!purge_fragmented_block(vb, &purge_list, false) && in _vm_unmap_aliases()
2968 vb->dirty_max && vb->dirty != VMAP_BBMAP_BITS) { in _vm_unmap_aliases()
2969 unsigned long va_start = vb->va->va_start; in _vm_unmap_aliases()
2972 s = va_start + (vb->dirty_min << PAGE_SHIFT); in _vm_unmap_aliases()
2973 e = va_start + (vb->dirty_max << PAGE_SHIFT); in _vm_unmap_aliases()
2979 vb->dirty_min = VMAP_BBMAP_BITS; in _vm_unmap_aliases()
2980 vb->dirty_max = 0; in _vm_unmap_aliases()
2984 spin_unlock(&vb->lock); in _vm_unmap_aliases()
4495 struct vmap_block *vb; in vmap_ram_vread_iter() local
4516 vb = xa_load(xa, addr_to_vb_idx((unsigned long)addr)); in vmap_ram_vread_iter()
4517 if (!vb) in vmap_ram_vread_iter()
4520 spin_lock(&vb->lock); in vmap_ram_vread_iter()
4521 if (bitmap_empty(vb->used_map, VMAP_BBMAP_BITS)) { in vmap_ram_vread_iter()
4522 spin_unlock(&vb->lock); in vmap_ram_vread_iter()
4526 for_each_set_bitrange(rs, re, vb->used_map, VMAP_BBMAP_BITS) { in vmap_ram_vread_iter()
4532 start = vmap_block_vaddr(vb->va->va_start, rs); in vmap_ram_vread_iter()
4560 spin_unlock(&vb->lock); in vmap_ram_vread_iter()
4567 spin_unlock(&vb->lock); in vmap_ram_vread_iter()