/linux/mm/ |
H A D | vma.c | 12 struct vma_iterator *vmi; member 36 .vmi = vmi_, \ 48 .vmi = (map_)->vmi, \ 288 static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi, in vma_complete() argument 309 vma_iter_store(vmi, vp->insert); in vma_complete() 450 __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, in __split_vma() argument 478 vma_iter_config(vmi, new->vm_start, new->vm_end); in __split_vma() 479 if (vma_iter_prealloc(vmi, new)) in __split_vma() 512 vma_complete(&vp, vmi, vma->vm_mm); in __split_vma() 517 vma_next(vmi); in __split_vma() [all …]
|
H A D | mmap.c | 123 struct vma_iterator vmi; in SYSCALL_DEFINE1() local 166 vma_iter_init(&vmi, mm, newbrk); in SYSCALL_DEFINE1() 167 brkvma = vma_find(&vmi, oldbrk); in SYSCALL_DEFINE1() 176 if (do_vmi_align_munmap(&vmi, brkvma, mm, newbrk, oldbrk, &uf, in SYSCALL_DEFINE1() 190 vma_iter_init(&vmi, mm, oldbrk); in SYSCALL_DEFINE1() 191 next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap); in SYSCALL_DEFINE1() 195 brkvma = vma_prev_limit(&vmi, mm->start_brk); in SYSCALL_DEFINE1() 197 if (do_brk_flags(&vmi, brkvma, oldbrk, newbrk - oldbrk, 0) < 0) in SYSCALL_DEFINE1() 935 VMA_ITERATOR(vmi, mm, addr); in find_vma_prev() 937 vma = vma_iter_load(&vmi); in find_vma_prev() [all …]
|
H A D | debug.c | 271 vmg->vmi, vmg->vmi ? vma_iter_addr(vmg->vmi) : 0, in dump_vmg() 272 vmg->vmi ? vma_iter_end(vmg->vmi) : 0, in dump_vmg() 313 if (vmg->vmi) { in dump_vmg() 315 vma_iter_dump_tree(vmg->vmi); in dump_vmg() 368 void vma_iter_dump_tree(const struct vma_iterator *vmi) in vma_iter_dump_tree() argument 371 mas_dump(&vmi->mas); in vma_iter_dump_tree() 372 mt_dump(vmi->mas.tree, mt_dump_hex); in vma_iter_dump_tree()
|
H A D | nommu.c | 573 VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_start); in delete_vma_from_mm() 575 vma_iter_config(&vmi, vma->vm_start, vma->vm_end); in delete_vma_from_mm() 576 if (vma_iter_prealloc(&vmi, NULL)) { in delete_vma_from_mm() 584 vma_iter_clear(&vmi); in delete_vma_from_mm() 616 VMA_ITERATOR(vmi, mm, addr); in find_vma() 618 return vma_iter_load(&vmi); in find_vma() 663 VMA_ITERATOR(vmi, mm, addr); in find_vma_exact() 665 vma = vma_iter_load(&vmi); in find_vma_exact() 1017 VMA_ITERATOR(vmi, current->mm, 0); in do_mmap() 1187 vma_iter_config(&vmi, vma->vm_start, vma->vm_end); in do_mmap() [all …]
|
H A D | mlock.c | 466 static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, in mlock_fixup() argument 481 vma = vma_modify_flags(vmi, *prev, vma, start, end, newflags); in mlock_fixup() 519 VMA_ITERATOR(vmi, current->mm, start); in apply_vma_lock_flags() 528 vma = vma_iter_load(&vmi); in apply_vma_lock_flags() 532 prev = vma_prev(&vmi); in apply_vma_lock_flags() 538 for_each_vma_range(vmi, vma, end) { in apply_vma_lock_flags() 551 error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, newflags); in apply_vma_lock_flags() 554 tmp = vma_iter_end(&vmi); in apply_vma_lock_flags() 577 VMA_ITERATOR(vmi, mm, start); in count_mm_mlocked_page_nr() 585 for_each_vma_range(vmi, vma, end) { in count_mm_mlocked_page_nr() [all …]
|
H A D | mremap.c | 713 struct vma_iterator vmi; in move_vma() local 830 vma_iter_init(&vmi, mm, old_addr); in move_vma() 831 if (do_vmi_munmap(&vmi, mm, old_addr, old_len, uf_unmap, false) < 0) { in move_vma() 847 vma = vma_prev(&vmi); in move_vma() 852 vma = vma_next(&vmi); in move_vma() 1147 VMA_ITERATOR(vmi, mm, addr + new_len); in SYSCALL_DEFINE5() 1154 ret = do_vmi_munmap(&vmi, mm, addr + new_len, old_len - new_len, in SYSCALL_DEFINE5() 1178 VMA_ITERATOR(vmi, mm, vma->vm_end); in SYSCALL_DEFINE5() 1198 vma = vma_merge_extend(&vmi, vma, delta); in SYSCALL_DEFINE5()
|
H A D | userfaultfd.c | 942 VMA_ITERATOR(vmi, dst_mm, start); in mwriteprotect_range() 966 for_each_vma_range(vmi, dst_vma, end) { in mwriteprotect_range() 1894 struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi, in userfaultfd_clear_vma() argument 1906 ret = vma_modify_flags_uffd(vmi, prev, vma, start, end, in userfaultfd_clear_vma() 1928 VMA_ITERATOR(vmi, ctx->mm, start); in userfaultfd_register_range() 1929 struct vm_area_struct *prev = vma_prev(&vmi); in userfaultfd_register_range() 1936 for_each_vma_range(vmi, vma, end) { in userfaultfd_register_range() 1957 vma = vma_modify_flags_uffd(&vmi, prev, vma, start, vma_end, in userfaultfd_register_range() 1985 VMA_ITERATOR(vmi, mm, 0); in userfaultfd_release_new() 1989 for_each_vma(vmi, vma) { in userfaultfd_release_new() [all …]
|
H A D | mempolicy.c | 418 VMA_ITERATOR(vmi, mm, 0); in mpol_rebind_mm() 421 for_each_vma(vmi, vma) { in mpol_rebind_mm() 819 static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma, in mbind_range() argument 838 vma = vma_modify_policy(vmi, *prev, vma, vmstart, vmend, new_pol); in mbind_range() 1272 struct vma_iterator vmi; in do_mbind() local 1337 vma_iter_init(&vmi, mm, start); in do_mbind() 1338 prev = vma_prev(&vmi); in do_mbind() 1339 for_each_vma_range(vmi, vma, end) { in do_mbind() 1340 err = mbind_range(&vmi, vma, &prev, start, end, new); in do_mbind() 1372 vma_iter_init(&vmi, mm, start); in do_mbind() [all …]
|
/linux/fs/proc/ |
H A D | task_nommu.c | 23 VMA_ITERATOR(vmi, mm, 0); in task_mem() 29 for_each_vma(vmi, vma) { in task_mem() 83 VMA_ITERATOR(vmi, mm, 0); in task_vsize() 88 for_each_vma(vmi, vma) in task_vsize() 98 VMA_ITERATOR(vmi, mm, 0); in task_statm() 104 for_each_vma(vmi, vma) { in task_statm()
|
/linux/arch/xtensa/kernel/ |
H A D | syscall.c | 62 struct vma_iterator vmi; in arch_get_unmapped_area() local 84 vma_iter_init(&vmi, current->mm, addr); in arch_get_unmapped_area() 85 for_each_vma(vmi, vmm) { in arch_get_unmapped_area()
|
/linux/arch/powerpc/mm/book3s32/ |
H A D | tlb.c | 84 VMA_ITERATOR(vmi, mm, 0); in hash__flush_tlb_mm() 92 for_each_vma(vmi, mp) in hash__flush_tlb_mm()
|
/linux/include/linux/ |
H A D | mm.h | 1009 struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) in vma_find() argument 1011 return mas_find(&vmi->mas, max - 1); in vma_find() 1014 static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) in vma_next() argument 1020 return mas_find(&vmi->mas, ULONG_MAX); in vma_next() 1024 struct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi) in vma_iter_next_range() argument 1026 return mas_next_range(&vmi->mas, ULONG_MAX); in vma_iter_next_range() 1030 static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) in vma_prev() argument 1032 return mas_prev(&vmi->mas, 0); in vma_prev() 1035 static inline int vma_iter_clear_gfp(struct vma_iterator *vmi, in vma_iter_clear_gfp() argument 1038 __mas_set_range(&vmi->mas, start, end - 1); in vma_iter_clear_gfp() [all …]
|
/linux/arch/loongarch/kernel/ |
H A D | vdso.c | 136 VMA_ITERATOR(vmi, mm, 0); in vdso_join_timens() 139 for_each_vma(vmi, vma) { in vdso_join_timens()
|
/linux/fs/ |
H A D | userfaultfd.c | 1251 struct vma_iterator vmi; in userfaultfd_register() local 1296 vma_iter_init(&vmi, mm, start); in userfaultfd_register() 1297 vma = vma_find(&vmi, end); in userfaultfd_register() 1375 } for_each_vma_range(vmi, cur, end); in userfaultfd_register() 1423 struct vma_iterator vmi; in userfaultfd_unregister() local 1444 vma_iter_init(&vmi, mm, start); in userfaultfd_unregister() 1445 vma = vma_find(&vmi, end); in userfaultfd_unregister() 1482 } for_each_vma_range(vmi, cur, end); in userfaultfd_unregister() 1485 vma_iter_set(&vmi, start); in userfaultfd_unregister() 1486 prev = vma_prev(&vmi); in userfaultfd_unregister() [all...] |
H A D | coredump.c | 1168 static struct vm_area_struct *coredump_next_vma(struct vma_iterator *vmi, in coredump_next_vma() argument 1175 vma = vma_next(vmi); in coredump_next_vma() 1215 VMA_ITERATOR(vmi, mm, 0); in dump_vma_snapshot() 1236 while ((vma = coredump_next_vma(&vmi, vma, gate_vma)) != NULL) { in dump_vma_snapshot()
|
/linux/arch/um/kernel/ |
H A D | tlb.c | 218 VMA_ITERATOR(vmi, mm, 0); in flush_tlb_mm() 220 for_each_vma(vmi, vma) in flush_tlb_mm()
|
/linux/arch/s390/kernel/ |
H A D | vdso.c | 51 VMA_ITERATOR(vmi, mm, 0); in vdso_join_timens() 55 for_each_vma(vmi, vma) { in vdso_join_timens()
|
/linux/arch/powerpc/mm/book3s64/ |
H A D | subpage_prot.c | 155 VMA_ITERATOR(vmi, mm, addr); in subpage_mark_vma_nohuge() 161 for_each_vma_range(vmi, vma, addr + len) { in subpage_mark_vma_nohuge()
|
/linux/arch/parisc/kernel/ |
H A D | cache.c | 717 VMA_ITERATOR(vmi, mm, 0); in mm_total_size() 719 for_each_vma(vmi, vma) { in mm_total_size() 730 VMA_ITERATOR(vmi, mm, 0); in flush_cache_mm() 750 for_each_vma(vmi, vma) in flush_cache_mm()
|
/linux/kernel/ |
H A D | fork.c | 640 VMA_ITERATOR(vmi, mm, 0); in dup_mmap() 664 mt_clear_in_rcu(vmi.mas.tree); in dup_mmap() 665 for_each_vma(vmi, mpnt) { in dup_mmap() 670 retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start, in dup_mmap() 724 vma_iter_bulk_store(&vmi, tmp); in dup_mmap() 751 mpnt = vma_next(&vmi); in dup_mmap() 758 vma_iter_free(&vmi); in dup_mmap() 760 mt_set_in_rcu(vmi.mas.tree); in dup_mmap() 773 mas_set_range(&vmi.mas, mpnt->vm_start, mpnt->vm_end - 1); in dup_mmap() 774 mas_store(&vmi.mas, XA_ZERO_ENTRY); in dup_mmap() [all …]
|
H A D | acct.c | 594 VMA_ITERATOR(vmi, mm, 0); in acct_collect() 598 for_each_vma(vmi, vma) in acct_collect()
|
/linux/drivers/misc/cxl/ |
H A D | fault.c | 297 VMA_ITERATOR(vmi, mm, 0); in cxl_prefault_vma() 302 for_each_vma(vmi, vma) { in cxl_prefault_vma()
|
/linux/arch/arm64/kernel/ |
H A D | vdso.c | 126 VMA_ITERATOR(vmi, mm, 0); in vdso_join_timens() 130 for_each_vma(vmi, vma) { in vdso_join_timens()
|
/linux/drivers/gpu/drm/i915/gem/ |
H A D | i915_gem_userptr.c | 403 VMA_ITERATOR(vmi, mm, addr); in probe_range() 408 for_each_vma_range(vmi, vma, end) { in probe_range()
|
/linux/ipc/ |
H A D | shm.c | 1732 VMA_ITERATOR(vmi, mm, addr); in ksys_shmdt() 1764 for_each_vma(vmi, vma) { in ksys_shmdt() 1781 do_vmi_align_munmap(&vmi, vma, mm, vma->vm_start, in ksys_shmdt() 1790 vma = vma_next(&vmi); in ksys_shmdt() 1806 do_vmi_align_munmap(&vmi, vma, mm, vma->vm_start, in ksys_shmdt() 1810 vma = vma_next(&vmi); in ksys_shmdt()
|