Lines Matching refs:vma
29 struct vm_area_struct *vma; member
35 static bool anon_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags) in anon_can_userfault() argument
43 static struct folio *anon_alloc_folio(struct vm_area_struct *vma, in anon_alloc_folio() argument
46 struct folio *folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, in anon_alloc_folio()
52 if (mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL)) { in anon_alloc_folio()
65 static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) in vma_uffd_ops() argument
67 if (vma_is_anonymous(vma)) in vma_uffd_ops()
69 return vma->vm_ops ? vma->vm_ops->uffd_ops : NULL; in vma_uffd_ops()
94 struct vm_area_struct *vma; in find_vma_and_prepare_anon() local
97 vma = vma_lookup(mm, addr); in find_vma_and_prepare_anon()
98 if (!vma) in find_vma_and_prepare_anon()
99 vma = ERR_PTR(-ENOENT); in find_vma_and_prepare_anon()
100 else if (!(vma->vm_flags & VM_SHARED) && in find_vma_and_prepare_anon()
101 unlikely(anon_vma_prepare(vma))) in find_vma_and_prepare_anon()
102 vma = ERR_PTR(-ENOMEM); in find_vma_and_prepare_anon()
104 return vma; in find_vma_and_prepare_anon()
121 struct vm_area_struct *vma; in uffd_lock_vma() local
123 vma = lock_vma_under_rcu(mm, address); in uffd_lock_vma()
124 if (vma) { in uffd_lock_vma()
129 if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma->anon_vma)) in uffd_lock_vma()
130 vma_end_read(vma); in uffd_lock_vma()
132 return vma; in uffd_lock_vma()
136 vma = find_vma_and_prepare_anon(mm, address); in uffd_lock_vma()
137 if (!IS_ERR(vma)) { in uffd_lock_vma()
138 bool locked = vma_start_read_locked(vma); in uffd_lock_vma()
141 vma = ERR_PTR(-EAGAIN); in uffd_lock_vma()
145 return vma; in uffd_lock_vma()
162 static void uffd_mfill_unlock(struct vm_area_struct *vma) in uffd_mfill_unlock() argument
164 vma_end_read(vma); in uffd_mfill_unlock()
189 static void uffd_mfill_unlock(struct vm_area_struct *vma) in uffd_mfill_unlock() argument
191 mmap_read_unlock(vma->vm_mm); in uffd_mfill_unlock()
197 if (!state->vma) in mfill_put_vma()
201 uffd_mfill_unlock(state->vma); in mfill_put_vma()
202 state->vma = NULL; in mfill_put_vma()
227 state->vma = dst_vma; in mfill_get_vma()
449 const struct vm_uffd_ops *orig_ops = vma_uffd_ops(state->vma); in mfill_copy_folio_retry()
475 if (vma_uffd_ops(state->vma) != orig_ops) in mfill_copy_folio_retry()
494 folio = ops->alloc_folio(state->vma, state->dst_addr); in __mfill_atomic_pte()
526 ret = ops->filemap_add(folio, state->vma, state->dst_addr); in __mfill_atomic_pte()
531 ret = mfill_atomic_install_pte(state->pmd, state->vma, dst_addr, in __mfill_atomic_pte()
540 ops->filemap_remove(folio, state->vma); in __mfill_atomic_pte()
548 const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); in mfill_atomic_pte_copy()
559 if (!(state->vma->vm_flags & VM_SHARED)) in mfill_atomic_pte_copy()
567 const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); in mfill_atomic_pte_zeroed_folio()
574 struct vm_area_struct *dst_vma = state->vma; in mfill_atomic_pte_zeropage()
611 struct vm_area_struct *dst_vma = state->vma; in mfill_atomic_pte_continue()
654 struct vm_area_struct *dst_vma = state->vma; in mfill_atomic_pte_poison()
915 if (is_vm_hugetlb_page(state.vma)) in mfill_atomic()
916 return mfill_atomic_hugetlb(ctx, state.vma, dst_start, in mfill_atomic()
1616 static inline bool vma_move_compatible(struct vm_area_struct *vma) in vma_move_compatible() argument
1618 return !(vma->vm_flags & (VM_PFNMAP | VM_IO | VM_HUGETLB | in vma_move_compatible()
1665 struct vm_area_struct *vma; in find_vmas_mm_locked() local
1668 vma = find_vma_and_prepare_anon(mm, dst_start); in find_vmas_mm_locked()
1669 if (IS_ERR(vma)) in find_vmas_mm_locked()
1670 return PTR_ERR(vma); in find_vmas_mm_locked()
1672 *dst_vmap = vma; in find_vmas_mm_locked()
1674 if (src_start >= vma->vm_start && src_start < vma->vm_end) in find_vmas_mm_locked()
1677 vma = vma_lookup(mm, src_start); in find_vmas_mm_locked()
1678 if (!vma) in find_vmas_mm_locked()
1681 *src_vmap = vma; in find_vmas_mm_locked()
1692 struct vm_area_struct *vma; in uffd_move_lock() local
1695 vma = uffd_lock_vma(mm, dst_start); in uffd_move_lock()
1696 if (IS_ERR(vma)) in uffd_move_lock()
1697 return PTR_ERR(vma); in uffd_move_lock()
1699 *dst_vmap = vma; in uffd_move_lock()
1704 if (src_start >= vma->vm_start && src_start < vma->vm_end) { in uffd_move_lock()
1705 *src_vmap = vma; in uffd_move_lock()
2027 bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags, in vma_can_userfault() argument
2030 const struct vm_uffd_ops *ops = vma_uffd_ops(vma); in vma_can_userfault()
2032 if (vma->vm_flags & VM_DROPPABLE) in vma_can_userfault()
2053 !vma_is_anonymous(vma)) in vma_can_userfault()
2056 return ops->can_userfault(vma, vm_flags); in vma_can_userfault()
2059 static void userfaultfd_set_vm_flags(struct vm_area_struct *vma, in userfaultfd_set_vm_flags() argument
2062 const bool uffd_wp_changed = (vma->vm_flags ^ vm_flags) & VM_UFFD_WP; in userfaultfd_set_vm_flags()
2064 vm_flags_reset(vma, vm_flags); in userfaultfd_set_vm_flags()
2070 if ((vma->vm_flags & VM_SHARED) && uffd_wp_changed) in userfaultfd_set_vm_flags()
2071 vma_set_page_prot(vma); in userfaultfd_set_vm_flags()
2074 static void userfaultfd_set_ctx(struct vm_area_struct *vma, in userfaultfd_set_ctx() argument
2078 vma_start_write(vma); in userfaultfd_set_ctx()
2079 vma->vm_userfaultfd_ctx = (struct vm_userfaultfd_ctx){ctx}; in userfaultfd_set_ctx()
2080 userfaultfd_set_vm_flags(vma, in userfaultfd_set_ctx()
2081 (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags); in userfaultfd_set_ctx()
2084 void userfaultfd_reset_ctx(struct vm_area_struct *vma) in userfaultfd_reset_ctx() argument
2086 userfaultfd_set_ctx(vma, NULL, 0); in userfaultfd_reset_ctx()
2091 struct vm_area_struct *vma, in userfaultfd_clear_vma() argument
2097 vma_flags_t new_vma_flags = vma->flags; in userfaultfd_clear_vma()
2105 if (start == vma->vm_start && end == vma->vm_end) in userfaultfd_clear_vma()
2109 if (userfaultfd_wp(vma)) in userfaultfd_clear_vma()
2110 uffd_wp_range(vma, start, end - start, false); in userfaultfd_clear_vma()
2112 ret = vma_modify_flags_uffd(vmi, prev, vma, start, end, in userfaultfd_clear_vma()
2129 struct vm_area_struct *vma, in userfaultfd_register_range() argument
2140 if (vma->vm_start < start) in userfaultfd_register_range()
2141 prev = vma; in userfaultfd_register_range()
2143 for_each_vma_range(vmi, vma, end) { in userfaultfd_register_range()
2146 VM_WARN_ON_ONCE(!vma_can_userfault(vma, vm_flags, wp_async)); in userfaultfd_register_range()
2147 VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx && in userfaultfd_register_range()
2148 vma->vm_userfaultfd_ctx.ctx != ctx); in userfaultfd_register_range()
2149 VM_WARN_ON_ONCE(!vma_test(vma, VMA_MAYWRITE_BIT)); in userfaultfd_register_range()
2155 if (vma->vm_userfaultfd_ctx.ctx == ctx && in userfaultfd_register_range()
2156 vma_test_all_mask(vma, vma_flags)) in userfaultfd_register_range()
2159 if (vma->vm_start > start) in userfaultfd_register_range()
2160 start = vma->vm_start; in userfaultfd_register_range()
2161 vma_end = min(end, vma->vm_end); in userfaultfd_register_range()
2163 new_vma_flags = vma->flags; in userfaultfd_register_range()
2167 vma = vma_modify_flags_uffd(&vmi, prev, vma, start, vma_end, in userfaultfd_register_range()
2171 if (IS_ERR(vma)) in userfaultfd_register_range()
2172 return PTR_ERR(vma); in userfaultfd_register_range()
2179 userfaultfd_set_ctx(vma, ctx, vm_flags); in userfaultfd_register_range()
2181 if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma)) in userfaultfd_register_range()
2182 hugetlb_unshare_all_pmds(vma); in userfaultfd_register_range()
2185 prev = vma; in userfaultfd_register_range()
2186 start = vma->vm_end; in userfaultfd_register_range()
2195 struct vm_area_struct *vma; in userfaultfd_release_new() local
2200 for_each_vma(vmi, vma) { in userfaultfd_release_new()
2201 if (vma->vm_userfaultfd_ctx.ctx == ctx) in userfaultfd_release_new()
2202 userfaultfd_reset_ctx(vma); in userfaultfd_release_new()
2210 struct vm_area_struct *vma, *prev; in userfaultfd_release_all() local
2226 for_each_vma(vmi, vma) { in userfaultfd_release_all()
2228 VM_WARN_ON_ONCE(!!vma->vm_userfaultfd_ctx.ctx ^ in userfaultfd_release_all()
2229 !!(vma->vm_flags & __VM_UFFD_FLAGS)); in userfaultfd_release_all()
2230 if (vma->vm_userfaultfd_ctx.ctx != ctx) { in userfaultfd_release_all()
2231 prev = vma; in userfaultfd_release_all()
2235 vma = userfaultfd_clear_vma(&vmi, prev, vma, in userfaultfd_release_all()
2236 vma->vm_start, vma->vm_end); in userfaultfd_release_all()
2237 prev = vma; in userfaultfd_release_all()