Lines Matching full:pte
102 * Return true if the original pte was a uffd-wp pte marker (so the pte was
150 * Transitioning a PTE from 'old' to 'young' can be expensive on in arch_wants_old_prefaulted_pte()
425 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte) in pmd_install() argument
432 * Ensure all pte setup (eg. pte page lock and page clearing) are in pmd_install()
433 * visible before the pte is made visible to other CPUs by being in pmd_install()
445 pmd_populate(mm, pmd, *pte); in pmd_install()
446 *pte = NULL; in pmd_install()
608 * Dumping the PTE would be nice, but it's tricky with CONFIG_HIGHPTE, in __print_bad_page_map_pgtable()
611 * already take care of printing the PTE. in __print_bad_page_map_pgtable()
619 * PFN-mapped pte in a region that doesn't allow it.
682 pte_t pte, struct page *page) in print_bad_pte() argument
684 auto entry = pte_val(pte); in print_bad_pte()
813 * vm_normal_page() - Get the "struct page" associated with a PTE
814 * @vma: The VMA mapping the @pte.
815 * @addr: The address where the @pte is mapped.
816 * @pte: The PTE.
818 * Get the "struct page" associated with a PTE. See __vm_normal_page()
825 pte_t pte) in vm_normal_page() argument
827 auto entry = pte_val(pte); in vm_normal_page()
829 return __vm_normal_page(vma, addr, pte_pfn(pte), pte_special(pte), in vm_normal_page()
834 * vm_normal_folio() - Get the "struct folio" associated with a PTE
835 * @vma: The VMA mapping the @pte.
836 * @addr: The address where the @pte is mapped.
837 * @pte: The PTE.
839 * Get the "struct folio" associated with a PTE. See __vm_normal_page()
846 pte_t pte) in vm_normal_folio() argument
848 struct page *page = vm_normal_page(vma, addr, pte); in vm_normal_folio()
862 * Get the "struct page" associated with a PTE. See __vm_normal_page()
883 * Get the "struct folio" associated with a PTE. See __vm_normal_page()
927 * @ptep: pte pointer into the locked page table mapping the folio page
928 * @orig_pte: pte value at @ptep
930 * Restore a device-exclusive non-swap entry to an ordinary present pte.
935 * Locking the folio makes sure that anybody who just converted the pte to
951 pte_t pte; in restore_exclusive_pte() local
955 pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot))); in restore_exclusive_pte()
957 pte = pte_mksoft_dirty(pte); in restore_exclusive_pte()
960 pte = pte_mkuffd(pte); in restore_exclusive_pte()
964 pte = pte_modify(pte, PAGE_NONE); in restore_exclusive_pte()
967 can_change_pte_writable(vma, address, pte)) { in restore_exclusive_pte()
969 pte = pte_mkdirty(pte); in restore_exclusive_pte()
970 pte = pte_mkwrite(pte, vma); in restore_exclusive_pte()
972 set_pte_at(vma->vm_mm, address, ptep, pte); in restore_exclusive_pte()
982 * Tries to restore an exclusive pte if the page lock can be acquired without
1014 pte_t pte = orig_pte; in copy_nonpresent_pte() local
1025 pte = pte_swp_clear_exclusive(orig_pte); in copy_nonpresent_pte()
1026 set_pte_at(src_mm, addr, src_pte, pte); in copy_nonpresent_pte()
1043 pte = softleaf_to_pte(entry); in copy_nonpresent_pte()
1045 pte = pte_swp_mksoft_dirty(pte); in copy_nonpresent_pte()
1047 pte = pte_swp_mkuffd(pte); in copy_nonpresent_pte()
1048 set_pte_at(src_mm, addr, src_pte, pte); in copy_nonpresent_pte()
1079 pte = swp_entry_to_pte(entry); in copy_nonpresent_pte()
1081 pte = pte_swp_mkuffd(pte); in copy_nonpresent_pte()
1082 set_pte_at(src_mm, addr, src_pte, pte); in copy_nonpresent_pte()
1087 * original entry then copying as for a present pte. Device in copy_nonpresent_pte()
1104 pte = pte_swp_clear_uffd(pte); in copy_nonpresent_pte()
1105 set_pte_at(dst_mm, addr, dst_pte, pte); in copy_nonpresent_pte()
1114 * and re-use the pte the traditional way.
1127 pte_t pte; in copy_present_page() local
1148 pte = folio_mk_pte(new_folio, dst_vma->vm_page_prot); in copy_present_page()
1149 pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); in copy_present_page()
1151 /* The uffd bit needs to be delivered to the dest pte as well */ in copy_present_page()
1152 pte = pte_mkuffd(pte); in copy_present_page()
1155 pte = pte_modify(pte, PAGE_NONE); in copy_present_page()
1157 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); in copy_present_page()
1163 pte_t pte, unsigned long addr, int nr) in __copy_present_ptes() argument
1170 * child is not RWP-armed, pte_modify(pte, dst_vma->vm_page_prot) can in __copy_present_ptes()
1171 * silently drop _PAGE_RW from a resolved (no-marker) writable PTE, in __copy_present_ptes()
1172 * so a later pte_write(pte) check would skip the COW wrprotect and in __copy_present_ptes()
1175 writable = pte_write(pte); in __copy_present_ptes()
1180 * pte_uffd(pte) so unrelated PAGE_NONE markers (e.g. NUMA balancing) in __copy_present_ptes()
1184 if (userfaultfd_rwp(src_vma) && pte_uffd(pte)) in __copy_present_ptes()
1185 pte = pte_modify(pte, dst_vma->vm_page_prot); in __copy_present_ptes()
1186 pte = pte_clear_uffd(pte); in __copy_present_ptes()
1192 pte = pte_wrprotect(pte); in __copy_present_ptes()
1197 pte = pte_mkclean(pte); in __copy_present_ptes()
1198 pte = pte_mkold(pte); in __copy_present_ptes()
1200 set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr); in __copy_present_ptes()
1204 * Copy one present PTE, trying to batch-process subsequent PTEs that map
1207 * Returns -EAGAIN if one preallocated page is required to copy the next PTE.
1212 pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, in copy_present_ptes() argument
1220 page = vm_normal_page(src_vma, addr, pte); in copy_present_ptes()
1237 nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); in copy_present_ptes()
1251 __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, in copy_present_ptes()
1279 __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, addr, 1); in copy_present_ptes()
1344 * the PTE page is stable, and there is no need to get pmdval and do in copy_pte_range()
1396 * the now present pte. in copy_pte_range()
1405 * If we need a pre-allocated page for this pte, drop the in copy_pte_range()
1416 * could only happen if one pinned pte changed. in copy_pte_range()
1689 * @pte: Old value of the entry pointed to by @ptep.
1706 unsigned long addr, pte_t *ptep, pte_t pte, in cond_install_uffd_wp_ptes() argument
1714 /* The current status of the pte should be "cleared" before calling */ in cond_install_uffd_wp_ptes()
1720 * page, or in TTU where the present pte will be quickly replaced in cond_install_uffd_wp_ptes()
1721 * with a swap pte. There's no way of leaking the bit. in cond_install_uffd_wp_ptes()
1726 /* A uffd-wp wr-protected normal pte */ in cond_install_uffd_wp_ptes()
1727 if (unlikely(pte_present(pte) && pte_uffd(pte))) in cond_install_uffd_wp_ptes()
1731 * A uffd-wp wr-protected swap pte. Note: this should even cover an in cond_install_uffd_wp_ptes()
1732 * existing pte marker with uffd-wp bit set. in cond_install_uffd_wp_ptes()
1734 if (unlikely(pte_swp_uffd_any(pte))) in cond_install_uffd_wp_ptes()
1753 * This function makes sure that we'll replace the none pte with an uffd-wp
1754 * swap special pte marker when necessary. Must be with the pgtable lock held.
1760 unsigned long addr, pte_t *pte, int nr, in zap_install_uffd_wp_if_needed() argument
1766 return cond_install_uffd_wp_ptes(vma, addr, pte, pteval, nr); in zap_install_uffd_wp_if_needed()
1771 struct page *page, pte_t *pte, pte_t ptent, unsigned int nr, in zap_present_folio_ptes() argument
1779 ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm); in zap_present_folio_ptes()
1792 clear_full_ptes(mm, addr, pte, nr, tlb->fullmm); in zap_present_folio_ptes()
1795 /* Checking a single PTE in a batch is sufficient. */ in zap_present_folio_ptes()
1797 tlb_remove_tlb_entries(tlb, pte, nr, addr); in zap_present_folio_ptes()
1799 *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, in zap_present_folio_ptes()
1815 * Zap or skip at least one present PTE, trying to batch-process subsequent
1821 struct vm_area_struct *vma, pte_t *pte, pte_t ptent, in zap_present_ptes() argument
1834 ptep_get_and_clear_full(mm, addr, pte, tlb->fullmm); in zap_present_ptes()
1836 tlb_remove_tlb_entry(tlb, pte, addr); in zap_present_ptes()
1839 pte, 1, details, ptent); in zap_present_ptes()
1855 nr = folio_pte_batch(folio, pte, ptent, max_nr); in zap_present_ptes()
1856 zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, in zap_present_ptes()
1861 zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, 1, addr, in zap_present_ptes()
1867 struct vm_area_struct *vma, pte_t *pte, pte_t ptent, in zap_nonpresent_ptes() argument
1898 nr = swap_pte_batch(pte, max_nr, ptent); in zap_nonpresent_ptes()
1916 * Ordinary zapping should not remove guard PTE in zap_nonpresent_ptes()
1917 * markers. Only do so if we should remove PTE markers in zap_nonpresent_ptes()
1931 clear_nonpresent_ptes(vma->vm_mm, addr, pte, nr); in zap_nonpresent_ptes()
1932 *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent); in zap_nonpresent_ptes()
1938 struct vm_area_struct *vma, pte_t *pte, in do_zap_pte_range() argument
1944 pte_t ptent = ptep_get(pte); in do_zap_pte_range()
1951 ptent = ptep_get(pte + nr); in do_zap_pte_range()
1958 pte += nr; in do_zap_pte_range()
1963 nr += zap_present_ptes(tlb, vma, pte, ptent, max_nr, addr, in do_zap_pte_range()
1967 nr += zap_nonpresent_ptes(tlb, vma, pte, ptent, max_nr, addr, in do_zap_pte_range()
2001 pte_t *start_pte, *pte; in zap_pte_table_if_empty() local
2011 for (i = 0, pte = start_pte; i < PTRS_PER_PTE; i++, pte++) { in zap_pte_table_if_empty()
2012 if (!pte_none(ptep_get(pte))) in zap_pte_table_if_empty()
2042 pte_t *pte; in zap_pte_range() local
2051 start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); in zap_pte_range()
2052 if (!pte) in zap_pte_range()
2065 nr = do_zap_pte_range(tlb, vma, pte, addr, end, details, rss, in zap_pte_range()
2074 } while (pte += nr, addr += PAGE_SIZE * nr, addr != end); in zap_pte_range()
2077 * Fast path: try to hold the pmd lock and unmap the PTE page. in zap_pte_range()
2079 * If the pte lock was released midway (retry case), or if the attempt in zap_pte_range()
2080 * to hold the pmd lock failed, then we need to recheck all pte entries in zap_pte_range()
2081 * to ensure they are still none, thereby preventing the pte entries in zap_pte_range()
2482 static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, in insert_page_into_pte_locked() argument
2487 pte_t pteval = ptep_get(pte); in insert_page_into_pte_locked()
2500 if (ptep_set_access_flags(vma, addr, pte, pteval, 1)) in insert_page_into_pte_locked()
2501 update_mmu_cache(vma, addr, pte); in insert_page_into_pte_locked()
2519 set_pte_at(vma->vm_mm, addr, pte, pteval); in insert_page_into_pte_locked()
2527 pte_t *pte; in insert_page() local
2534 pte = get_locked_pte(vma->vm_mm, addr, &ptl); in insert_page()
2535 if (!pte) in insert_page()
2537 retval = insert_page_into_pte_locked(vma, pte, addr, page, prot, in insert_page()
2539 pte_unmap_unlock(pte, ptl); in insert_page()
2544 static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte, in insert_page_in_batch_locked() argument
2552 return insert_page_into_pte_locked(vma, pte, addr, page, prot, false); in insert_page_in_batch_locked()
2562 pte_t *start_pte, *pte; in insert_pages() local
2578 /* Allocate the PTE if necessary; takes PMD lock once only. */ in insert_pages()
2592 for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) { in insert_pages()
2593 int err = insert_page_in_batch_locked(vma, pte, in insert_pages()
2805 pte_t *pte, entry; in insert_pfn() local
2808 pte = get_locked_pte(mm, addr, &ptl); in insert_pfn()
2809 if (!pte) in insert_pfn()
2811 entry = ptep_get(pte); in insert_pfn()
2818 * case we are creating a writable PTE for a shared in insert_pfn()
2830 if (ptep_set_access_flags(vma, addr, pte, entry, 1)) in insert_pfn()
2831 update_mmu_cache(vma, addr, pte); in insert_pfn()
2844 set_pte_at(mm, addr, pte, entry); in insert_pfn()
2845 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */ in insert_pfn()
2848 pte_unmap_unlock(pte, ptl); in insert_pfn()
2971 * If we don't have pte special, then we have to use the pfn_valid() in __vm_insert_mixed()
2975 * without pte special, it would there be refcounted as a normal page. in __vm_insert_mixed()
3027 * If the insertion of PTE failed because someone else already added a
3046 pte_t *pte, *mapped_pte; in remap_pte_range() local
3050 mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); in remap_pte_range()
3051 if (!pte) in remap_pte_range()
3055 BUG_ON(!pte_none(ptep_get(pte))); in remap_pte_range()
3060 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); in remap_pte_range()
3062 } while (pte++, addr += PAGE_SIZE, addr != end); in remap_pte_range()
3447 pte_t *pte, *mapped_pte; in apply_to_pte_range() local
3452 mapped_pte = pte = (mm == &init_mm) ? in apply_to_pte_range()
3455 if (!pte) in apply_to_pte_range()
3458 mapped_pte = pte = (mm == &init_mm) ? in apply_to_pte_range()
3461 if (!pte) in apply_to_pte_range()
3469 if (create || !pte_none(ptep_get(pte))) { in apply_to_pte_range()
3470 err = fn(pte, addr, data); in apply_to_pte_range()
3474 } while (pte++, addr += PAGE_SIZE, addr != end); in apply_to_pte_range()
3662 * parts, do_swap_page must check under lock before unmapping the pte and
3672 same = pte_same(ptep_get(vmf->pte), vmf->orig_pte); in pte_unmap_same()
3676 pte_unmap(vmf->pte); in pte_unmap_same()
3677 vmf->pte = NULL; in pte_unmap_same()
3717 vmf->pte = NULL; in __wp_page_copy_user()
3721 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); in __wp_page_copy_user()
3722 if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { in __wp_page_copy_user()
3727 if (vmf->pte) in __wp_page_copy_user()
3728 update_mmu_tlb(vma, addr, vmf->pte); in __wp_page_copy_user()
3734 if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0)) in __wp_page_copy_user()
3735 update_mmu_cache_range(vmf, vma, addr, vmf->pte, 1); in __wp_page_copy_user()
3745 if (vmf->pte) in __wp_page_copy_user()
3749 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); in __wp_page_copy_user()
3750 if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { in __wp_page_copy_user()
3751 /* The PTE changed under us, update local tlb */ in __wp_page_copy_user()
3752 if (vmf->pte) in __wp_page_copy_user()
3753 update_mmu_tlb(vma, addr, vmf->pte); in __wp_page_copy_user()
3776 if (vmf->pte) in __wp_page_copy_user()
3777 pte_unmap_unlock(vmf->pte, vmf->ptl); in __wp_page_copy_user()
3914 if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1)) in wp_page_reuse()
3915 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); in wp_page_reuse()
3916 pte_unmap_unlock(vmf->pte, vmf->ptl); in wp_page_reuse()
3979 * - Take the PTL. If the pte changed, bail out and release the allocated page
3980 * - If the pte is still the way we remember it, update the page table and all
4041 * Re-check the pte - we dropped the lock in wp_page_copy()
4043 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); in wp_page_copy()
4044 if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { in wp_page_copy()
4067 * Clear the pte entry and flush it first, before updating the in wp_page_copy()
4068 * pte with the new entry, to keep TLBs on different CPUs in in wp_page_copy()
4069 * sync. This code used to set the new PTE then flush TLBs, but in wp_page_copy()
4070 * that left a window where the new PTE could be loaded into in wp_page_copy()
4071 * some TLBs while the old PTE remains in others. in wp_page_copy()
4073 ptep_clear_flush(vma, vmf->address, vmf->pte); in wp_page_copy()
4077 set_pte_at(mm, vmf->address, vmf->pte, entry); in wp_page_copy()
4078 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); in wp_page_copy()
4081 * Only after switching the pte to the new page may in wp_page_copy()
4084 * before the pte is switched to the new page, and in wp_page_copy()
4085 * "reuse" the old page writing into it while our pte in wp_page_copy()
4108 pte_unmap_unlock(vmf->pte, vmf->ptl); in wp_page_copy()
4109 } else if (vmf->pte) { in wp_page_copy()
4110 update_mmu_tlb(vma, vmf->address, vmf->pte); in wp_page_copy()
4111 pte_unmap_unlock(vmf->pte, vmf->ptl); in wp_page_copy()
4137 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
4144 * shared mapping due to PTE being read-only once the mapped page is prepared.
4145 * It handles locking of PTE and modifying it.
4150 * Return: %0 on success, %VM_FAULT_NOPAGE when PTE got changed before
4151 * we acquired PTE lock.
4156 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address, in finish_mkwrite_fault()
4158 if (!vmf->pte) in finish_mkwrite_fault()
4164 if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) { in finish_mkwrite_fault()
4165 update_mmu_tlb(vmf->vma, vmf->address, vmf->pte); in finish_mkwrite_fault()
4166 pte_unmap_unlock(vmf->pte, vmf->ptl); in finish_mkwrite_fault()
4184 pte_unmap_unlock(vmf->pte, vmf->ptl); in wp_pfn_shared()
4210 pte_unmap_unlock(vmf->pte, vmf->ptl); in wp_page_shared()
4377 * FAULT_FLAG_VMA_LOCK) and pte both mapped and locked. We return with
4378 * the same lock still held, but pte unmapped and unlocked.
4386 pte_t pte; in do_wp_page() local
4389 if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { in do_wp_page()
4391 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_wp_page()
4400 pte = pte_clear_uffd(ptep_get(vmf->pte)); in do_wp_page()
4402 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); in do_wp_page()
4407 vmf->orig_pte = pte; in do_wp_page()
4455 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_wp_page()
4467 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_wp_page()
4598 * Restore a potential device exclusive pte to a working pte entry
4628 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, in remove_device_exclusive_entry()
4630 if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) in remove_device_exclusive_entry()
4632 vmf->pte, vmf->orig_pte); in remove_device_exclusive_entry()
4634 if (vmf->pte) in remove_device_exclusive_entry()
4635 pte_unmap_unlock(vmf->pte, vmf->ptl); in remove_device_exclusive_entry()
4677 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in pte_marker_clear()
4679 if (!vmf->pte) in pte_marker_clear()
4682 * Be careful so that we will only recover a special uffd-wp pte into a in pte_marker_clear()
4683 * none pte. Otherwise it means the pte could have changed, so retry. in pte_marker_clear()
4685 * This should also cover the case where e.g. the pte changed in pte_marker_clear()
4687 * So pte_is_marker() check is not enough to safely drop the pte. in pte_marker_clear()
4689 if (pte_same(vmf->orig_pte, ptep_get(vmf->pte))) in pte_marker_clear()
4690 pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte); in pte_marker_clear()
4691 pte_unmap_unlock(vmf->pte, vmf->ptl); in pte_marker_clear()
4704 * This is actually a page-missing access, but with uffd-wp special pte
4705 * installed. It means this pte was wr-protected before being unmapped.
4725 * PTE markers should never be empty. If anything weird happened, in handle_pte_marker()
4742 /* This is an unknown pte marker */ in handle_pte_marker()
4754 pte_t pte; in can_swapin_thp() local
4758 pte = ptep_get(ptep); in can_swapin_thp()
4760 if (!pte_same(pte, pte_move_swp_offset(vmf->orig_pte, -idx))) in can_swapin_thp()
4767 if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages) in can_swapin_thp()
4802 pte_t *pte; in thp_swapin_suitable_orders() local
4834 pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in thp_swapin_suitable_orders()
4836 if (unlikely(!pte)) in thp_swapin_suitable_orders()
4846 if (can_swapin_thp(vmf, pte + pte_index(addr), 1 << order)) in thp_swapin_suitable_orders()
4851 pte_unmap_unlock(pte, ptl); in thp_swapin_suitable_orders()
4875 * FAULT_FLAG_VMA_LOCK), and pte mapped but not yet locked.
4876 * We return with pte unmapped and unlocked.
4891 pte_t pte; in do_swap_page() local
4921 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_swap_page()
4923 if (unlikely(!vmf->pte || in do_swap_page()
4924 !pte_same(ptep_get(vmf->pte), in do_swap_page()
4936 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_swap_page()
4942 pte_unmap(vmf->pte); in do_swap_page()
4975 * Back out if somebody else faulted in this pte in do_swap_page()
4976 * while we released the pte lock. in do_swap_page()
4978 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_swap_page()
4980 if (likely(vmf->pte && in do_swap_page()
4981 pte_same(ptep_get(vmf->pte), vmf->orig_pte))) in do_swap_page()
5038 * Back out if somebody else already faulted in this pte. in do_swap_page()
5040 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, in do_swap_page()
5042 if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) in do_swap_page()
5053 ptep = vmf->pte; in do_swap_page()
5067 folio_ptep = vmf->pte - idx; in do_swap_page()
5083 * PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte in do_swap_page()
5118 * The can_swapin_thp check above ensures all PTE have in do_swap_page()
5119 * same exclusiveness. Checking just one PTE is fine. in do_swap_page()
5163 pte = mk_pte(page, vma->vm_page_prot); in do_swap_page()
5165 pte = pte_mksoft_dirty(pte); in do_swap_page()
5167 pte = pte_mkuffd(pte); in do_swap_page()
5175 pte = pte_modify(pte, PAGE_NONE); in do_swap_page()
5183 * Skip the write upgrade for an RWP-restored pte: it must stay in do_swap_page()
5189 (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && in do_swap_page()
5190 !pte_needs_soft_dirty_wp(vma, pte)) { in do_swap_page()
5191 pte = pte_mkwrite(pte, vma); in do_swap_page()
5193 pte = pte_mkdirty(pte); in do_swap_page()
5199 vmf->orig_pte = pte_advance_pfn(pte, page_idx); in do_swap_page()
5223 (pte_write(pte) && !PageAnonExclusive(page))); in do_swap_page()
5224 set_ptes(vma->vm_mm, address, ptep, pte, nr_pages); in do_swap_page()
5226 pte, pte, nr_pages); in do_swap_page()
5251 * For an RWP-restored pte, leave it PROT_NONE and let the write in do_swap_page()
5255 if ((vmf->flags & FAULT_FLAG_WRITE) && !pte_write(pte) && !rwp_restore) { in do_swap_page()
5265 if (vmf->pte) in do_swap_page()
5266 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_swap_page()
5272 if (vmf->pte) in do_swap_page()
5273 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_swap_page()
5289 static bool pte_range_none(pte_t *pte, int nr_pages) in pte_range_none() argument
5294 if (!pte_none(ptep_get_lockless(pte + i))) in pte_range_none()
5308 pte_t *pte; in alloc_anon_folio() local
5331 pte = pte_offset_map(vmf->pmd, vmf->address & PMD_MASK); in alloc_anon_folio()
5332 if (!pte) in alloc_anon_folio()
5343 if (pte_range_none(pte + pte_index(addr), 1 << order)) in alloc_anon_folio()
5348 pte_unmap(pte); in alloc_anon_folio()
5390 void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, in map_anon_folio_pte_nopf() argument
5407 set_ptes(vma->vm_mm, addr, pte, entry, nr_pages); in map_anon_folio_pte_nopf()
5408 update_mmu_cache_range(NULL, vma, addr, pte, nr_pages); in map_anon_folio_pte_nopf()
5411 static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, in map_anon_folio_pte_pf() argument
5416 map_anon_folio_pte_nopf(folio, pte, vma, addr, uffd_wp); in map_anon_folio_pte_pf()
5423 * FAULT_FLAG_VMA_LOCK), and pte unmapped and unlocked.
5424 * We return with the lock still held, but pte unmapped and unlocked.
5452 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_anonymous_page()
5454 if (!vmf->pte) in do_anonymous_page()
5457 update_mmu_tlb(vma, vmf->address, vmf->pte); in do_anonymous_page()
5465 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_anonymous_page()
5470 set_pte_at(vma->vm_mm, addr, vmf->pte, entry); in do_anonymous_page()
5473 update_mmu_cache(vma, addr, vmf->pte); in do_anonymous_page()
5498 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); in do_anonymous_page()
5499 if (!vmf->pte) in do_anonymous_page()
5502 update_mmu_tlb(vma, addr, vmf->pte); in do_anonymous_page()
5504 } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) { in do_anonymous_page()
5505 update_mmu_tlb_range(vma, addr, vmf->pte, nr_pages); in do_anonymous_page()
5515 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_anonymous_page()
5519 map_anon_folio_pte_pf(folio, vmf->pte, vma, addr, in do_anonymous_page()
5522 if (vmf->pte) in do_anonymous_page()
5523 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_anonymous_page()
5545 * Preallocate pte before we take folio lock because this might lead to in __do_fault()
5619 * PMD mappings, but PTE-mapped THP are fine. So let's simply refuse any in do_set_pmd()
5637 * check. This kind of THP just can be PTE mapped. Access to in do_set_pmd()
5645 * related to pte entry. Use the preallocated table for that. in do_set_pmd()
5694 * @page: The first page to create a PTE for.
5696 * @addr: The first address to create a PTE for.
5728 set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr); in set_pte_range()
5731 update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr); in set_pte_range()
5737 return !pte_same(ptep_get(vmf->pte), vmf->orig_pte); in vmf_pte_changed()
5739 return !pte_none(ptep_get(vmf->pte)); in vmf_pte_changed()
5748 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
5753 * reference of a page being mapped (for the PTE which maps it).
5847 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in finish_fault()
5849 if (!vmf->pte) in finish_fault()
5854 update_mmu_tlb(vma, addr, vmf->pte); in finish_fault()
5857 } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) { in finish_fault()
5859 pte_unmap_unlock(vmf->pte, vmf->ptl); in finish_fault()
5870 pte_unmap_unlock(vmf->pte, vmf->ptl); in finish_fault()
5923 * map_pages() and acquire a PTE lock only once.
5943 /* The PTE offset of the start address, clamped to the VMA. */ in do_fault_around()
5947 /* The PTE offset of the end address, clamped to the VMA and PTE. */ in do_fault_around()
6114 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in do_fault()
6116 if (unlikely(!vmf->pte)) in do_fault()
6120 * Make sure this is not a temporary clearing of pte in do_fault()
6122 * of pte involves: take ptl, clearing the pte so that in do_fault()
6126 if (unlikely(pte_none(ptep_get(vmf->pte)))) in do_fault()
6131 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_fault()
6159 * pte_dirty has unpredictable behaviour between PTE scan updates, in numa_migrate_check()
6199 pte_t pte, old_pte; in numa_rebuild_single_mapping() local
6202 pte = pte_modify(old_pte, vma->vm_page_prot); in numa_rebuild_single_mapping()
6203 pte = pte_mkyoung(pte); in numa_rebuild_single_mapping()
6205 pte = pte_mkwrite(pte, vma); in numa_rebuild_single_mapping()
6206 ptep_modify_prot_commit(vma, fault_addr, fault_pte, old_pte, pte); in numa_rebuild_single_mapping()
6224 start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT); in numa_rebuild_large_mapping()
6261 pte_t pte; in do_uffd_rwp() local
6264 /* Sync mode: unmap PTE and deliver to userfaultfd handler */ in do_uffd_rwp()
6265 pte_unmap(vmf->pte); in do_uffd_rwp()
6270 if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { in do_uffd_rwp()
6271 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_uffd_rwp()
6274 pte = pte_modify(vmf->orig_pte, vmf->vma->vm_page_prot); in do_uffd_rwp()
6276 pte = pte_clear_uffd(pte); in do_uffd_rwp()
6277 pte = pte_mkyoung(pte); in do_uffd_rwp()
6278 if (!pte_write(pte) && in do_uffd_rwp()
6280 can_change_pte_writable(vmf->vma, vmf->address, pte)) in do_uffd_rwp()
6281 pte = pte_mkwrite(pte, vmf->vma); in do_uffd_rwp()
6282 set_pte_at(vmf->vma->vm_mm, vmf->address, vmf->pte, pte); in do_uffd_rwp()
6283 update_mmu_cache(vmf->vma, vmf->address, vmf->pte); in do_uffd_rwp()
6284 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_uffd_rwp()
6297 pte_t pte, old_pte; in do_numa_page() local
6301 * The pte cannot be used safely until we verify, while holding the page in do_numa_page()
6305 /* Read the live PTE from the page tables: */ in do_numa_page()
6306 old_pte = ptep_get(vmf->pte); in do_numa_page()
6309 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_numa_page()
6313 pte = pte_modify(old_pte, vma->vm_page_prot); in do_numa_page()
6316 * Detect now whether the PTE could be writable; this information in do_numa_page()
6319 writable = pte_write(pte); in do_numa_page()
6321 can_change_pte_writable(vma, vmf->address, pte)) in do_numa_page()
6324 folio = vm_normal_folio(vma, vmf->address, pte); in do_numa_page()
6340 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_numa_page()
6353 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_numa_page()
6355 if (unlikely(!vmf->pte)) in do_numa_page()
6357 if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { in do_numa_page()
6358 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_numa_page()
6367 numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable, in do_numa_page()
6370 numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte, in do_numa_page()
6372 pte_unmap_unlock(vmf->pte, vmf->ptl); in do_numa_page()
6415 /* COW or write-notify handled on pte level: split pmd. */ in wp_huge_pmd()
6465 * changed the page table before the PTE is checked on CPU A. Most of
6492 vmf->pte); in fix_spurious_fault()
6525 vmf->pte = NULL; in handle_pte_fault()
6536 * Use the maywrite version to indicate that vmf->pte may be in handle_pte_fault()
6543 vmf->pte = pte_offset_map_rw_nolock(vmf->vma->vm_mm, vmf->pmd, in handle_pte_fault()
6546 if (unlikely(!vmf->pte)) in handle_pte_fault()
6548 vmf->orig_pte = ptep_get_lockless(vmf->pte); in handle_pte_fault()
6552 pte_unmap(vmf->pte); in handle_pte_fault()
6553 vmf->pte = NULL; in handle_pte_fault()
6557 if (!vmf->pte) in handle_pte_fault()
6566 * VM_UFFD_RWP VMA, a protnone PTE without the uffd bit is in handle_pte_fault()
6576 if (unlikely(!pte_same(ptep_get(vmf->pte), entry))) { in handle_pte_fault()
6577 update_mmu_tlb(vmf->vma, vmf->address, vmf->pte); in handle_pte_fault()
6587 if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry, in handle_pte_fault()
6590 vmf->pte, 1); in handle_pte_fault()
6594 pte_unmap_unlock(vmf->pte, vmf->ptl); in handle_pte_fault()
7027 * If the PTE maps a refcounted page, callers are responsible to protect
7035 * This function must not be used to modify PTE content.
7049 pte_t *ptep, pte; in follow_pfnmap_start() local
7113 pte = ptep_get(ptep); in follow_pfnmap_start()
7114 if (!pte_present(pte)) in follow_pfnmap_start()
7116 pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte), in follow_pfnmap_start()
7117 pte_pfn(pte), PAGE_MASK, pte_write(pte), in follow_pfnmap_start()
7118 pte_special(pte)); in follow_pfnmap_start()