| /linux/mm/ |
| H A D | userfaultfd.c | 91 bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) in validate_dst_vma() argument
94 if (dst_end > dst_vma->vm_end) in validate_dst_vma()
102 if (!dst_vma->vm_userfaultfd_ctx.ctx) in validate_dst_vma()
170 struct vm_area_struct *dst_vma; in uffd_mfill_lock() local
172 dst_vma = uffd_lock_vma(dst_mm, dst_start); in uffd_mfill_lock()
173 if (IS_ERR(dst_vma) || validate_dst_vma(dst_vma, dst_start + len)) in uffd_mfill_lock()
174 return dst_vma; in uffd_mfill_lock()
176 vma_end_read(dst_vma); in uffd_mfill_lock()
191 struct vm_area_struct *dst_vma; in uffd_mfill_lock() local
194 dst_vma = find_vma_and_prepare_anon(dst_mm, dst_start); in uffd_mfill_lock()
[all …]
|
| H A D | memory.c | 1009 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma, in copy_nonpresent_pte() argument
1035 vma_is_cow_mapping(dst_vma)) { in copy_nonpresent_pte()
1066 folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma); in copy_nonpresent_pte()
1076 vma_is_cow_mapping(dst_vma)) { in copy_nonpresent_pte()
1096 pte_marker marker = copy_pte_marker(entry, dst_vma); in copy_nonpresent_pte()
1103 if (!userfaultfd_protected(dst_vma)) in copy_nonpresent_pte()
1122 copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, in copy_present_page() argument
1143 folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE); in copy_present_page()
1144 folio_add_lru_vma(new_folio, dst_vma); in copy_present_page()
1148 pte = folio_mk_pte(new_folio, dst_vma->vm_page_prot); in copy_present_page()
[all …]
|
| H A D | hugetlb.c | 4887 struct vm_area_struct *dst_vma, in copy_hugetlb_page_range() argument
4927 dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz); in copy_hugetlb_page_range()
4975 if (!userfaultfd_protected(dst_vma)) in copy_hugetlb_page_range()
4979 const pte_marker marker = copy_pte_marker(softleaf, dst_vma); in copy_hugetlb_page_range()
5008 new_folio = alloc_hugetlb_folio(dst_vma, addr, false); in copy_hugetlb_page_range()
5015 addr, dst_vma); in copy_hugetlb_page_range()
5018 restore_reserve_on_error(h, dst_vma, addr, new_folio); in copy_hugetlb_page_range()
5029 restore_reserve_on_error(h, dst_vma, addr, in copy_hugetlb_page_range()
5035 hugetlb_install_folio(dst_vma, dst_pte, addr, in copy_hugetlb_page_range()
5041 if (!userfaultfd_protected(dst_vma)) { in copy_hugetlb_page_range()
[all …]
|
| H A D | huge_memory.c | 1897 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, in copy_huge_non_present_pmd() argument
1939 dst_vma, src_vma); in copy_huge_non_present_pmd()
1945 if (!userfaultfd_protected(dst_vma)) in copy_huge_non_present_pmd()
1952 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) in copy_huge_pmd() argument
1981 if (!vma_is_anonymous(dst_vma)) in copy_huge_pmd()
1998 dst_vma, src_vma, pmd, pgtable); in copy_huge_pmd()
2027 if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, dst_vma, src_vma))) { in copy_huge_pmd()
2042 if (!userfaultfd_protected(dst_vma)) { in copy_huge_pmd()
2044 pmd = pmd_modify(pmd, dst_vma->vm_page_prot); in copy_huge_pmd()
2891 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, in move_pages_huge_pmd() argument
[all …]
|
| /linux/include/linux/ |
| H A D | rmap.h | 493 struct page *page, int nr_pages, struct vm_area_struct *dst_vma, in __folio_dup_file_rmap() argument
512 folio_add_large_mapcount(folio, orig_nr_pages, dst_vma); in __folio_dup_file_rmap()
517 folio_inc_large_mapcount(folio, dst_vma); in __folio_dup_file_rmap()
536 struct page *page, int nr_pages, struct vm_area_struct *dst_vma) in folio_dup_file_rmap_ptes() argument
538 __folio_dup_file_rmap(folio, page, nr_pages, dst_vma, PGTABLE_LEVEL_PTE); in folio_dup_file_rmap_ptes()
542 struct page *page, struct vm_area_struct *dst_vma) in folio_dup_file_rmap_pte() argument
544 __folio_dup_file_rmap(folio, page, 1, dst_vma, PGTABLE_LEVEL_PTE); in folio_dup_file_rmap_pte()
558 struct page *page, struct vm_area_struct *dst_vma) in folio_dup_file_rmap_pmd() argument
561 __folio_dup_file_rmap(folio, page, HPAGE_PMD_NR, dst_vma, PGTABLE_LEVEL_PTE); in folio_dup_file_rmap_pmd()
568 struct page *page, int nr_pages, struct vm_area_struct *dst_vma, in __folio_try_dup_anon_rmap() argument
[all …]
|
| H A D | mm.h | 3234 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);
|