1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * mm/mprotect.c 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * (C) Copyright 1994 Linus Torvalds 61da177e4SLinus Torvalds * (C) Copyright 2002 Christoph Hellwig 71da177e4SLinus Torvalds * 8046c6884SAlan Cox * Address space accounting code <alan@lxorguk.ukuu.org.uk> 91da177e4SLinus Torvalds * (C) Copyright 2002 Red Hat Inc, All Rights Reserved 101da177e4SLinus Torvalds */ 111da177e4SLinus Torvalds 12a520110eSChristoph Hellwig #include <linux/pagewalk.h> 131da177e4SLinus Torvalds #include <linux/hugetlb.h> 141da177e4SLinus Torvalds #include <linux/shm.h> 151da177e4SLinus Torvalds #include <linux/mman.h> 161da177e4SLinus Torvalds #include <linux/fs.h> 171da177e4SLinus Torvalds #include <linux/highmem.h> 181da177e4SLinus Torvalds #include <linux/security.h> 191da177e4SLinus Torvalds #include <linux/mempolicy.h> 201da177e4SLinus Torvalds #include <linux/personality.h> 211da177e4SLinus Torvalds #include <linux/syscalls.h> 220697212aSChristoph Lameter #include <linux/swap.h> 230697212aSChristoph Lameter #include <linux/swapops.h> 24cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h> 2564cdd548SKOSAKI Motohiro #include <linux/migrate.h> 26cdd6c482SIngo Molnar #include <linux/perf_event.h> 27e8c24d3aSDave Hansen #include <linux/pkeys.h> 2864a9a34eSMel Gorman #include <linux/ksm.h> 297c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 3009a913a7SMel Gorman #include <linux/mm_inline.h> 31ca5999fdSMike Rapoport #include <linux/pgtable.h> 32a1a3a2fcSHuang Ying #include <linux/sched/sysctl.h> 33fe2567ebSPeter Xu #include <linux/userfaultfd_k.h> 34*467b171aSAneesh Kumar K.V #include <linux/memory-tiers.h> 351da177e4SLinus Torvalds #include <asm/cacheflush.h> 36e8c24d3aSDave Hansen #include <asm/mmu_context.h> 371da177e4SLinus Torvalds #include <asm/tlbflush.h> 384a18419fSNadav Amit #include <asm/tlb.h> 391da177e4SLinus Torvalds 4036f88188SKirill A. Shutemov #include "internal.h" 4136f88188SKirill A. Shutemov 4264fe24a3SDavid Hildenbrand static inline bool can_change_pte_writable(struct vm_area_struct *vma, 4364fe24a3SDavid Hildenbrand unsigned long addr, pte_t pte) 4464fe24a3SDavid Hildenbrand { 4564fe24a3SDavid Hildenbrand struct page *page; 4664fe24a3SDavid Hildenbrand 4764fe24a3SDavid Hildenbrand VM_BUG_ON(!(vma->vm_flags & VM_WRITE) || pte_write(pte)); 4864fe24a3SDavid Hildenbrand 4964fe24a3SDavid Hildenbrand if (pte_protnone(pte) || !pte_dirty(pte)) 5064fe24a3SDavid Hildenbrand return false; 5164fe24a3SDavid Hildenbrand 5264fe24a3SDavid Hildenbrand /* Do we need write faults for softdirty tracking? */ 5376aefad6SPeter Xu if (vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte)) 5464fe24a3SDavid Hildenbrand return false; 5564fe24a3SDavid Hildenbrand 5664fe24a3SDavid Hildenbrand /* Do we need write faults for uffd-wp tracking? */ 5764fe24a3SDavid Hildenbrand if (userfaultfd_pte_wp(vma, pte)) 5864fe24a3SDavid Hildenbrand return false; 5964fe24a3SDavid Hildenbrand 6064fe24a3SDavid Hildenbrand if (!(vma->vm_flags & VM_SHARED)) { 6164fe24a3SDavid Hildenbrand /* 6264fe24a3SDavid Hildenbrand * We can only special-case on exclusive anonymous pages, 6364fe24a3SDavid Hildenbrand * because we know that our write-fault handler similarly would 6464fe24a3SDavid Hildenbrand * map them writable without any additional checks while holding 6564fe24a3SDavid Hildenbrand * the PT lock. 6664fe24a3SDavid Hildenbrand */ 6764fe24a3SDavid Hildenbrand page = vm_normal_page(vma, addr, pte); 6864fe24a3SDavid Hildenbrand if (!page || !PageAnon(page) || !PageAnonExclusive(page)) 6964fe24a3SDavid Hildenbrand return false; 7064fe24a3SDavid Hildenbrand } 7164fe24a3SDavid Hildenbrand 7264fe24a3SDavid Hildenbrand return true; 7364fe24a3SDavid Hildenbrand } 7464fe24a3SDavid Hildenbrand 754a18419fSNadav Amit static unsigned long change_pte_range(struct mmu_gather *tlb, 764a18419fSNadav Amit struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, 774a18419fSNadav Amit unsigned long end, pgprot_t newprot, unsigned long cp_flags) 781da177e4SLinus Torvalds { 790697212aSChristoph Lameter pte_t *pte, oldpte; 80705e87c0SHugh Dickins spinlock_t *ptl; 817da4d641SPeter Zijlstra unsigned long pages = 0; 823e321587SAndi Kleen int target_node = NUMA_NO_NODE; 8358705444SPeter Xu bool prot_numa = cp_flags & MM_CP_PROT_NUMA; 84292924b2SPeter Xu bool uffd_wp = cp_flags & MM_CP_UFFD_WP; 85292924b2SPeter Xu bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; 861da177e4SLinus Torvalds 874a18419fSNadav Amit tlb_change_page_size(tlb, PAGE_SIZE); 884a18419fSNadav Amit 89175ad4f1SAndrea Arcangeli /* 90c1e8d7c6SMichel Lespinasse * Can be called with only the mmap_lock for reading by 91175ad4f1SAndrea Arcangeli * prot_numa so we must check the pmd isn't constantly 92175ad4f1SAndrea Arcangeli * changing from under us from pmd_none to pmd_trans_huge 93175ad4f1SAndrea Arcangeli * and/or the other way around. 94175ad4f1SAndrea Arcangeli */ 95175ad4f1SAndrea Arcangeli if (pmd_trans_unstable(pmd)) 96175ad4f1SAndrea Arcangeli return 0; 97175ad4f1SAndrea Arcangeli 98175ad4f1SAndrea Arcangeli /* 99175ad4f1SAndrea Arcangeli * The pmd points to a regular pte so the pmd can't change 100c1e8d7c6SMichel Lespinasse * from under us even if the mmap_lock is only hold for 101175ad4f1SAndrea Arcangeli * reading. 102175ad4f1SAndrea Arcangeli */ 103175ad4f1SAndrea Arcangeli pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1041ad9f620SMel Gorman 1053e321587SAndi Kleen /* Get target node for single threaded private VMAs */ 1063e321587SAndi Kleen if (prot_numa && !(vma->vm_flags & VM_SHARED) && 1073e321587SAndi Kleen atomic_read(&vma->vm_mm->mm_users) == 1) 1083e321587SAndi Kleen target_node = numa_node_id(); 1093e321587SAndi Kleen 1103ea27719SMel Gorman flush_tlb_batched_pending(vma->vm_mm); 1116606c3e0SZachary Amsden arch_enter_lazy_mmu_mode(); 1121da177e4SLinus Torvalds do { 1130697212aSChristoph Lameter oldpte = *pte; 1140697212aSChristoph Lameter if (pte_present(oldpte)) { 1151da177e4SLinus Torvalds pte_t ptent; 116b191f9b1SMel Gorman bool preserve_write = prot_numa && pte_write(oldpte); 1171da177e4SLinus Torvalds 118e944fd67SMel Gorman /* 119e944fd67SMel Gorman * Avoid trapping faults against the zero or KSM 120e944fd67SMel Gorman * pages. See similar comment in change_huge_pmd. 121e944fd67SMel Gorman */ 122e944fd67SMel Gorman if (prot_numa) { 123e944fd67SMel Gorman struct page *page; 124a1a3a2fcSHuang Ying int nid; 12533024536SHuang Ying bool toptier; 126e944fd67SMel Gorman 127a818f536SHuang Ying /* Avoid TLB flush if possible */ 128a818f536SHuang Ying if (pte_protnone(oldpte)) 129a818f536SHuang Ying continue; 130a818f536SHuang Ying 131e944fd67SMel Gorman page = vm_normal_page(vma, addr, oldpte); 1323218f871SAlex Sierra if (!page || is_zone_device_page(page) || PageKsm(page)) 133e944fd67SMel Gorman continue; 13410c1045fSMel Gorman 135859d4adcSHenry Willard /* Also skip shared copy-on-write pages */ 136859d4adcSHenry Willard if (is_cow_mapping(vma->vm_flags) && 13780d47f5dSLinus Torvalds page_count(page) != 1) 138859d4adcSHenry Willard continue; 139859d4adcSHenry Willard 14009a913a7SMel Gorman /* 14109a913a7SMel Gorman * While migration can move some dirty pages, 14209a913a7SMel Gorman * it cannot move them all from MIGRATE_ASYNC 14309a913a7SMel Gorman * context. 14409a913a7SMel Gorman */ 1459de4f22aSHuang Ying if (page_is_file_lru(page) && PageDirty(page)) 14609a913a7SMel Gorman continue; 14709a913a7SMel Gorman 1483e321587SAndi Kleen /* 1493e321587SAndi Kleen * Don't mess with PTEs if page is already on the node 1503e321587SAndi Kleen * a single-threaded process is running on. 1513e321587SAndi Kleen */ 152a1a3a2fcSHuang Ying nid = page_to_nid(page); 153a1a3a2fcSHuang Ying if (target_node == nid) 154a1a3a2fcSHuang Ying continue; 15533024536SHuang Ying toptier = node_is_toptier(nid); 156a1a3a2fcSHuang Ying 157a1a3a2fcSHuang Ying /* 158a1a3a2fcSHuang Ying * Skip scanning top tier node if normal numa 159a1a3a2fcSHuang Ying * balancing is disabled 160a1a3a2fcSHuang Ying */ 161a1a3a2fcSHuang Ying if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) && 16233024536SHuang Ying toptier) 1633e321587SAndi Kleen continue; 16433024536SHuang Ying if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && 16533024536SHuang Ying !toptier) 16633024536SHuang Ying xchg_page_access_time(page, 16733024536SHuang Ying jiffies_to_msecs(jiffies)); 168e944fd67SMel Gorman } 169e944fd67SMel Gorman 17004a86453SAneesh Kumar K.V oldpte = ptep_modify_prot_start(vma, addr, pte); 17104a86453SAneesh Kumar K.V ptent = pte_modify(oldpte, newprot); 172b191f9b1SMel Gorman if (preserve_write) 173288bc549SAneesh Kumar K.V ptent = pte_mk_savedwrite(ptent); 1748a0516edSMel Gorman 175292924b2SPeter Xu if (uffd_wp) { 176292924b2SPeter Xu ptent = pte_wrprotect(ptent); 177292924b2SPeter Xu ptent = pte_mkuffd_wp(ptent); 178292924b2SPeter Xu } else if (uffd_wp_resolve) { 179292924b2SPeter Xu ptent = pte_clear_uffd_wp(ptent); 180292924b2SPeter Xu } 181292924b2SPeter Xu 18264fe24a3SDavid Hildenbrand /* 18364fe24a3SDavid Hildenbrand * In some writable, shared mappings, we might want 18464fe24a3SDavid Hildenbrand * to catch actual write access -- see 18564fe24a3SDavid Hildenbrand * vma_wants_writenotify(). 18664fe24a3SDavid Hildenbrand * 18764fe24a3SDavid Hildenbrand * In all writable, private mappings, we have to 18864fe24a3SDavid Hildenbrand * properly handle COW. 18964fe24a3SDavid Hildenbrand * 19064fe24a3SDavid Hildenbrand * In both cases, we can sometimes still change PTEs 19164fe24a3SDavid Hildenbrand * writable and avoid the write-fault handler, for 19264fe24a3SDavid Hildenbrand * example, if a PTE is already dirty and no other 19364fe24a3SDavid Hildenbrand * COW or special handling is required. 19464fe24a3SDavid Hildenbrand */ 19564fe24a3SDavid Hildenbrand if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && 19664fe24a3SDavid Hildenbrand !pte_write(ptent) && 19764fe24a3SDavid Hildenbrand can_change_pte_writable(vma, addr, ptent)) 1989d85d586SAneesh Kumar K.V ptent = pte_mkwrite(ptent); 19964fe24a3SDavid Hildenbrand 20004a86453SAneesh Kumar K.V ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent); 201c9fe6656SNadav Amit if (pte_needs_flush(oldpte, ptent)) 2024a18419fSNadav Amit tlb_flush_pte_range(tlb, addr, PAGE_SIZE); 2034b10e7d5SMel Gorman pages++; 204f45ec5ffSPeter Xu } else if (is_swap_pte(oldpte)) { 2050697212aSChristoph Lameter swp_entry_t entry = pte_to_swp_entry(oldpte); 206f45ec5ffSPeter Xu pte_t newpte; 2070697212aSChristoph Lameter 2084dd845b5SAlistair Popple if (is_writable_migration_entry(entry)) { 2093d2f78f0SPeter Xu struct page *page = pfn_swap_entry_to_page(entry); 2103d2f78f0SPeter Xu 2110697212aSChristoph Lameter /* 2120697212aSChristoph Lameter * A protection check is difficult so 2130697212aSChristoph Lameter * just be safe and disable write 2140697212aSChristoph Lameter */ 2156c287605SDavid Hildenbrand if (PageAnon(page)) 2166c287605SDavid Hildenbrand entry = make_readable_exclusive_migration_entry( 2174dd845b5SAlistair Popple swp_offset(entry)); 2186c287605SDavid Hildenbrand else 2196c287605SDavid Hildenbrand entry = make_readable_migration_entry(swp_offset(entry)); 220c3d16e16SCyrill Gorcunov newpte = swp_entry_to_pte(entry); 221c3d16e16SCyrill Gorcunov if (pte_swp_soft_dirty(oldpte)) 222c3d16e16SCyrill Gorcunov newpte = pte_swp_mksoft_dirty(newpte); 223f45ec5ffSPeter Xu if (pte_swp_uffd_wp(oldpte)) 224f45ec5ffSPeter Xu newpte = pte_swp_mkuffd_wp(newpte); 2254dd845b5SAlistair Popple } else if (is_writable_device_private_entry(entry)) { 2265042db43SJérôme Glisse /* 2275042db43SJérôme Glisse * We do not preserve soft-dirtiness. See 2285042db43SJérôme Glisse * copy_one_pte() for explanation. 2295042db43SJérôme Glisse */ 2304dd845b5SAlistair Popple entry = make_readable_device_private_entry( 2314dd845b5SAlistair Popple swp_offset(entry)); 2325042db43SJérôme Glisse newpte = swp_entry_to_pte(entry); 233f45ec5ffSPeter Xu if (pte_swp_uffd_wp(oldpte)) 234f45ec5ffSPeter Xu newpte = pte_swp_mkuffd_wp(newpte); 235b756a3b5SAlistair Popple } else if (is_writable_device_exclusive_entry(entry)) { 236b756a3b5SAlistair Popple entry = make_readable_device_exclusive_entry( 237b756a3b5SAlistair Popple swp_offset(entry)); 238b756a3b5SAlistair Popple newpte = swp_entry_to_pte(entry); 239b756a3b5SAlistair Popple if (pte_swp_soft_dirty(oldpte)) 240b756a3b5SAlistair Popple newpte = pte_swp_mksoft_dirty(newpte); 241b756a3b5SAlistair Popple if (pte_swp_uffd_wp(oldpte)) 242b756a3b5SAlistair Popple newpte = pte_swp_mkuffd_wp(newpte); 243fe2567ebSPeter Xu } else if (pte_marker_entry_uffd_wp(entry)) { 244fe2567ebSPeter Xu /* 245fe2567ebSPeter Xu * If this is uffd-wp pte marker and we'd like 246fe2567ebSPeter Xu * to unprotect it, drop it; the next page 247fe2567ebSPeter Xu * fault will trigger without uffd trapping. 248fe2567ebSPeter Xu */ 249fe2567ebSPeter Xu if (uffd_wp_resolve) { 250fe2567ebSPeter Xu pte_clear(vma->vm_mm, addr, pte); 251fe2567ebSPeter Xu pages++; 252fe2567ebSPeter Xu } 2535c041f5dSPeter Xu continue; 254f45ec5ffSPeter Xu } else { 255f45ec5ffSPeter Xu newpte = oldpte; 256f45ec5ffSPeter Xu } 2575042db43SJérôme Glisse 258f45ec5ffSPeter Xu if (uffd_wp) 259f45ec5ffSPeter Xu newpte = pte_swp_mkuffd_wp(newpte); 260f45ec5ffSPeter Xu else if (uffd_wp_resolve) 261f45ec5ffSPeter Xu newpte = pte_swp_clear_uffd_wp(newpte); 262f45ec5ffSPeter Xu 263f45ec5ffSPeter Xu if (!pte_same(oldpte, newpte)) { 264f45ec5ffSPeter Xu set_pte_at(vma->vm_mm, addr, pte, newpte); 2655042db43SJérôme Glisse pages++; 2665042db43SJérôme Glisse } 267fe2567ebSPeter Xu } else { 268fe2567ebSPeter Xu /* It must be an none page, or what else?.. */ 269fe2567ebSPeter Xu WARN_ON_ONCE(!pte_none(oldpte)); 270fe2567ebSPeter Xu if (unlikely(uffd_wp && !vma_is_anonymous(vma))) { 271fe2567ebSPeter Xu /* 272fe2567ebSPeter Xu * For file-backed mem, we need to be able to 273fe2567ebSPeter Xu * wr-protect a none pte, because even if the 274fe2567ebSPeter Xu * pte is none, the page/swap cache could 275fe2567ebSPeter Xu * exist. Doing that by install a marker. 276fe2567ebSPeter Xu */ 277fe2567ebSPeter Xu set_pte_at(vma->vm_mm, addr, pte, 278fe2567ebSPeter Xu make_pte_marker(PTE_MARKER_UFFD_WP)); 279fe2567ebSPeter Xu pages++; 280fe2567ebSPeter Xu } 281e920e14cSMel Gorman } 2821da177e4SLinus Torvalds } while (pte++, addr += PAGE_SIZE, addr != end); 2836606c3e0SZachary Amsden arch_leave_lazy_mmu_mode(); 284705e87c0SHugh Dickins pte_unmap_unlock(pte - 1, ptl); 2857da4d641SPeter Zijlstra 2867da4d641SPeter Zijlstra return pages; 2871da177e4SLinus Torvalds } 2881da177e4SLinus Torvalds 2898b272b3cSMel Gorman /* 2908b272b3cSMel Gorman * Used when setting automatic NUMA hinting protection where it is 2918b272b3cSMel Gorman * critical that a numa hinting PMD is not confused with a bad PMD. 2928b272b3cSMel Gorman */ 2938b272b3cSMel Gorman static inline int pmd_none_or_clear_bad_unless_trans_huge(pmd_t *pmd) 2948b272b3cSMel Gorman { 2958b272b3cSMel Gorman pmd_t pmdval = pmd_read_atomic(pmd); 2968b272b3cSMel Gorman 2978b272b3cSMel Gorman /* See pmd_none_or_trans_huge_or_clear_bad for info on barrier */ 2988b272b3cSMel Gorman #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2998b272b3cSMel Gorman barrier(); 3008b272b3cSMel Gorman #endif 3018b272b3cSMel Gorman 3028b272b3cSMel Gorman if (pmd_none(pmdval)) 3038b272b3cSMel Gorman return 1; 3048b272b3cSMel Gorman if (pmd_trans_huge(pmdval)) 3058b272b3cSMel Gorman return 0; 3068b272b3cSMel Gorman if (unlikely(pmd_bad(pmdval))) { 3078b272b3cSMel Gorman pmd_clear_bad(pmd); 3088b272b3cSMel Gorman return 1; 3098b272b3cSMel Gorman } 3108b272b3cSMel Gorman 3118b272b3cSMel Gorman return 0; 3128b272b3cSMel Gorman } 3138b272b3cSMel Gorman 314fe2567ebSPeter Xu /* Return true if we're uffd wr-protecting file-backed memory, or false */ 315fe2567ebSPeter Xu static inline bool 316fe2567ebSPeter Xu uffd_wp_protect_file(struct vm_area_struct *vma, unsigned long cp_flags) 317fe2567ebSPeter Xu { 318fe2567ebSPeter Xu return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma); 319fe2567ebSPeter Xu } 320fe2567ebSPeter Xu 321fe2567ebSPeter Xu /* 322fe2567ebSPeter Xu * If wr-protecting the range for file-backed, populate pgtable for the case 323fe2567ebSPeter Xu * when pgtable is empty but page cache exists. When {pte|pmd|...}_alloc() 324fe2567ebSPeter Xu * failed it means no memory, we don't have a better option but stop. 325fe2567ebSPeter Xu */ 326fe2567ebSPeter Xu #define change_pmd_prepare(vma, pmd, cp_flags) \ 327fe2567ebSPeter Xu do { \ 328fe2567ebSPeter Xu if (unlikely(uffd_wp_protect_file(vma, cp_flags))) { \ 329fe2567ebSPeter Xu if (WARN_ON_ONCE(pte_alloc(vma->vm_mm, pmd))) \ 330fe2567ebSPeter Xu break; \ 331fe2567ebSPeter Xu } \ 332fe2567ebSPeter Xu } while (0) 333fe2567ebSPeter Xu /* 334fe2567ebSPeter Xu * This is the general pud/p4d/pgd version of change_pmd_prepare(). We need to 335fe2567ebSPeter Xu * have separate change_pmd_prepare() because pte_alloc() returns 0 on success, 336fe2567ebSPeter Xu * while {pmd|pud|p4d}_alloc() returns the valid pointer on success. 337fe2567ebSPeter Xu */ 338fe2567ebSPeter Xu #define change_prepare(vma, high, low, addr, cp_flags) \ 339fe2567ebSPeter Xu do { \ 340fe2567ebSPeter Xu if (unlikely(uffd_wp_protect_file(vma, cp_flags))) { \ 341fe2567ebSPeter Xu low##_t *p = low##_alloc(vma->vm_mm, high, addr); \ 342fe2567ebSPeter Xu if (WARN_ON_ONCE(p == NULL)) \ 343fe2567ebSPeter Xu break; \ 344fe2567ebSPeter Xu } \ 345fe2567ebSPeter Xu } while (0) 346fe2567ebSPeter Xu 3474a18419fSNadav Amit static inline unsigned long change_pmd_range(struct mmu_gather *tlb, 3484a18419fSNadav Amit struct vm_area_struct *vma, pud_t *pud, unsigned long addr, 3494a18419fSNadav Amit unsigned long end, pgprot_t newprot, unsigned long cp_flags) 3501da177e4SLinus Torvalds { 3511da177e4SLinus Torvalds pmd_t *pmd; 3521da177e4SLinus Torvalds unsigned long next; 3537da4d641SPeter Zijlstra unsigned long pages = 0; 35472403b4aSMel Gorman unsigned long nr_huge_updates = 0; 355ac46d4f3SJérôme Glisse struct mmu_notifier_range range; 356ac46d4f3SJérôme Glisse 357ac46d4f3SJérôme Glisse range.start = 0; 3581da177e4SLinus Torvalds 3591da177e4SLinus Torvalds pmd = pmd_offset(pud, addr); 3601da177e4SLinus Torvalds do { 36125cbbef1SMel Gorman unsigned long this_pages; 36225cbbef1SMel Gorman 3631da177e4SLinus Torvalds next = pmd_addr_end(addr, end); 3648b272b3cSMel Gorman 365fe2567ebSPeter Xu change_pmd_prepare(vma, pmd, cp_flags); 3668b272b3cSMel Gorman /* 367c1e8d7c6SMichel Lespinasse * Automatic NUMA balancing walks the tables with mmap_lock 3688b272b3cSMel Gorman * held for read. It's possible a parallel update to occur 3698b272b3cSMel Gorman * between pmd_trans_huge() and a pmd_none_or_clear_bad() 3708b272b3cSMel Gorman * check leading to a false positive and clearing. 3718b272b3cSMel Gorman * Hence, it's necessary to atomically read the PMD value 3728b272b3cSMel Gorman * for all the checks. 3738b272b3cSMel Gorman */ 3748b272b3cSMel Gorman if (!is_swap_pmd(*pmd) && !pmd_devmap(*pmd) && 3758b272b3cSMel Gorman pmd_none_or_clear_bad_unless_trans_huge(pmd)) 3764991c09cSAnshuman Khandual goto next; 377a5338093SRik van Riel 378a5338093SRik van Riel /* invoke the mmu notifier if the pmd is populated */ 379ac46d4f3SJérôme Glisse if (!range.start) { 3807269f999SJérôme Glisse mmu_notifier_range_init(&range, 3817269f999SJérôme Glisse MMU_NOTIFY_PROTECTION_VMA, 0, 3826f4f13e8SJérôme Glisse vma, vma->vm_mm, addr, end); 383ac46d4f3SJérôme Glisse mmu_notifier_invalidate_range_start(&range); 384a5338093SRik van Riel } 385a5338093SRik van Riel 38684c3fc4eSZi Yan if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { 387019c2d8bSPeter Xu if ((next - addr != HPAGE_PMD_SIZE) || 388019c2d8bSPeter Xu uffd_wp_protect_file(vma, cp_flags)) { 389fd60775aSDavid Rientjes __split_huge_pmd(vma, pmd, addr, false, NULL); 390019c2d8bSPeter Xu /* 391019c2d8bSPeter Xu * For file-backed, the pmd could have been 392019c2d8bSPeter Xu * cleared; make sure pmd populated if 393019c2d8bSPeter Xu * necessary, then fall-through to pte level. 394019c2d8bSPeter Xu */ 395019c2d8bSPeter Xu change_pmd_prepare(vma, pmd, cp_flags); 3966b9116a6SKirill A. Shutemov } else { 3974a18419fSNadav Amit /* 3984a18419fSNadav Amit * change_huge_pmd() does not defer TLB flushes, 3994a18419fSNadav Amit * so no need to propagate the tlb argument. 4004a18419fSNadav Amit */ 4014a18419fSNadav Amit int nr_ptes = change_huge_pmd(tlb, vma, pmd, 4024a18419fSNadav Amit addr, newprot, cp_flags); 403f123d74aSMel Gorman 404f123d74aSMel Gorman if (nr_ptes) { 40572403b4aSMel Gorman if (nr_ptes == HPAGE_PMD_NR) { 40672403b4aSMel Gorman pages += HPAGE_PMD_NR; 40772403b4aSMel Gorman nr_huge_updates++; 40872403b4aSMel Gorman } 4091ad9f620SMel Gorman 4101ad9f620SMel Gorman /* huge pmd was handled */ 4114991c09cSAnshuman Khandual goto next; 4127da4d641SPeter Zijlstra } 413f123d74aSMel Gorman } 41488a9ab6eSRik van Riel /* fall through, the trans huge pmd just split */ 415cd7548abSJohannes Weiner } 4164a18419fSNadav Amit this_pages = change_pte_range(tlb, vma, pmd, addr, next, 4174a18419fSNadav Amit newprot, cp_flags); 41825cbbef1SMel Gorman pages += this_pages; 4194991c09cSAnshuman Khandual next: 4204991c09cSAnshuman Khandual cond_resched(); 4211da177e4SLinus Torvalds } while (pmd++, addr = next, addr != end); 4227da4d641SPeter Zijlstra 423ac46d4f3SJérôme Glisse if (range.start) 424ac46d4f3SJérôme Glisse mmu_notifier_invalidate_range_end(&range); 425a5338093SRik van Riel 42672403b4aSMel Gorman if (nr_huge_updates) 42772403b4aSMel Gorman count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates); 4287da4d641SPeter Zijlstra return pages; 4291da177e4SLinus Torvalds } 4301da177e4SLinus Torvalds 4314a18419fSNadav Amit static inline unsigned long change_pud_range(struct mmu_gather *tlb, 4324a18419fSNadav Amit struct vm_area_struct *vma, p4d_t *p4d, unsigned long addr, 4334a18419fSNadav Amit unsigned long end, pgprot_t newprot, unsigned long cp_flags) 4341da177e4SLinus Torvalds { 4351da177e4SLinus Torvalds pud_t *pud; 4361da177e4SLinus Torvalds unsigned long next; 4377da4d641SPeter Zijlstra unsigned long pages = 0; 4381da177e4SLinus Torvalds 439c2febafcSKirill A. Shutemov pud = pud_offset(p4d, addr); 4401da177e4SLinus Torvalds do { 4411da177e4SLinus Torvalds next = pud_addr_end(addr, end); 442fe2567ebSPeter Xu change_prepare(vma, pud, pmd, addr, cp_flags); 4431da177e4SLinus Torvalds if (pud_none_or_clear_bad(pud)) 4441da177e4SLinus Torvalds continue; 4454a18419fSNadav Amit pages += change_pmd_range(tlb, vma, pud, addr, next, newprot, 44658705444SPeter Xu cp_flags); 4471da177e4SLinus Torvalds } while (pud++, addr = next, addr != end); 4487da4d641SPeter Zijlstra 4497da4d641SPeter Zijlstra return pages; 4501da177e4SLinus Torvalds } 4511da177e4SLinus Torvalds 4524a18419fSNadav Amit static inline unsigned long change_p4d_range(struct mmu_gather *tlb, 4534a18419fSNadav Amit struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr, 4544a18419fSNadav Amit unsigned long end, pgprot_t newprot, unsigned long cp_flags) 455c2febafcSKirill A. Shutemov { 456c2febafcSKirill A. Shutemov p4d_t *p4d; 457c2febafcSKirill A. Shutemov unsigned long next; 458c2febafcSKirill A. Shutemov unsigned long pages = 0; 459c2febafcSKirill A. Shutemov 460c2febafcSKirill A. Shutemov p4d = p4d_offset(pgd, addr); 461c2febafcSKirill A. Shutemov do { 462c2febafcSKirill A. Shutemov next = p4d_addr_end(addr, end); 463fe2567ebSPeter Xu change_prepare(vma, p4d, pud, addr, cp_flags); 464c2febafcSKirill A. Shutemov if (p4d_none_or_clear_bad(p4d)) 465c2febafcSKirill A. Shutemov continue; 4664a18419fSNadav Amit pages += change_pud_range(tlb, vma, p4d, addr, next, newprot, 46758705444SPeter Xu cp_flags); 468c2febafcSKirill A. Shutemov } while (p4d++, addr = next, addr != end); 469c2febafcSKirill A. Shutemov 470c2febafcSKirill A. Shutemov return pages; 471c2febafcSKirill A. Shutemov } 472c2febafcSKirill A. Shutemov 4734a18419fSNadav Amit static unsigned long change_protection_range(struct mmu_gather *tlb, 4744a18419fSNadav Amit struct vm_area_struct *vma, unsigned long addr, 4754a18419fSNadav Amit unsigned long end, pgprot_t newprot, unsigned long cp_flags) 4761da177e4SLinus Torvalds { 4771da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 4781da177e4SLinus Torvalds pgd_t *pgd; 4791da177e4SLinus Torvalds unsigned long next; 4807da4d641SPeter Zijlstra unsigned long pages = 0; 4811da177e4SLinus Torvalds 4821da177e4SLinus Torvalds BUG_ON(addr >= end); 4831da177e4SLinus Torvalds pgd = pgd_offset(mm, addr); 4844a18419fSNadav Amit tlb_start_vma(tlb, vma); 4851da177e4SLinus Torvalds do { 4861da177e4SLinus Torvalds next = pgd_addr_end(addr, end); 487fe2567ebSPeter Xu change_prepare(vma, pgd, p4d, addr, cp_flags); 4881da177e4SLinus Torvalds if (pgd_none_or_clear_bad(pgd)) 4891da177e4SLinus Torvalds continue; 4904a18419fSNadav Amit pages += change_p4d_range(tlb, vma, pgd, addr, next, newprot, 49158705444SPeter Xu cp_flags); 4921da177e4SLinus Torvalds } while (pgd++, addr = next, addr != end); 4937da4d641SPeter Zijlstra 4944a18419fSNadav Amit tlb_end_vma(tlb, vma); 4957da4d641SPeter Zijlstra 4967da4d641SPeter Zijlstra return pages; 4977da4d641SPeter Zijlstra } 4987da4d641SPeter Zijlstra 4994a18419fSNadav Amit unsigned long change_protection(struct mmu_gather *tlb, 5004a18419fSNadav Amit struct vm_area_struct *vma, unsigned long start, 5017da4d641SPeter Zijlstra unsigned long end, pgprot_t newprot, 50258705444SPeter Xu unsigned long cp_flags) 5037da4d641SPeter Zijlstra { 5047da4d641SPeter Zijlstra unsigned long pages; 5057da4d641SPeter Zijlstra 506292924b2SPeter Xu BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL); 507292924b2SPeter Xu 5087da4d641SPeter Zijlstra if (is_vm_hugetlb_page(vma)) 5095a90d5a1SPeter Xu pages = hugetlb_change_protection(vma, start, end, newprot, 5105a90d5a1SPeter Xu cp_flags); 5117da4d641SPeter Zijlstra else 5124a18419fSNadav Amit pages = change_protection_range(tlb, vma, start, end, newprot, 51358705444SPeter Xu cp_flags); 5147da4d641SPeter Zijlstra 5157da4d641SPeter Zijlstra return pages; 5161da177e4SLinus Torvalds } 5171da177e4SLinus Torvalds 51842e4089cSAndi Kleen static int prot_none_pte_entry(pte_t *pte, unsigned long addr, 51942e4089cSAndi Kleen unsigned long next, struct mm_walk *walk) 52042e4089cSAndi Kleen { 52142e4089cSAndi Kleen return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ? 52242e4089cSAndi Kleen 0 : -EACCES; 52342e4089cSAndi Kleen } 52442e4089cSAndi Kleen 52542e4089cSAndi Kleen static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask, 52642e4089cSAndi Kleen unsigned long addr, unsigned long next, 52742e4089cSAndi Kleen struct mm_walk *walk) 52842e4089cSAndi Kleen { 52942e4089cSAndi Kleen return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ? 53042e4089cSAndi Kleen 0 : -EACCES; 53142e4089cSAndi Kleen } 53242e4089cSAndi Kleen 53342e4089cSAndi Kleen static int prot_none_test(unsigned long addr, unsigned long next, 53442e4089cSAndi Kleen struct mm_walk *walk) 53542e4089cSAndi Kleen { 53642e4089cSAndi Kleen return 0; 53742e4089cSAndi Kleen } 53842e4089cSAndi Kleen 5397b86ac33SChristoph Hellwig static const struct mm_walk_ops prot_none_walk_ops = { 54042e4089cSAndi Kleen .pte_entry = prot_none_pte_entry, 54142e4089cSAndi Kleen .hugetlb_entry = prot_none_hugetlb_entry, 54242e4089cSAndi Kleen .test_walk = prot_none_test, 54342e4089cSAndi Kleen }; 54442e4089cSAndi Kleen 545b6a2fea3SOllie Wild int 5464a18419fSNadav Amit mprotect_fixup(struct mmu_gather *tlb, struct vm_area_struct *vma, 5474a18419fSNadav Amit struct vm_area_struct **pprev, unsigned long start, 5484a18419fSNadav Amit unsigned long end, unsigned long newflags) 5491da177e4SLinus Torvalds { 5501da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 5511da177e4SLinus Torvalds unsigned long oldflags = vma->vm_flags; 5521da177e4SLinus Torvalds long nrpages = (end - start) >> PAGE_SHIFT; 5531da177e4SLinus Torvalds unsigned long charged = 0; 55464fe24a3SDavid Hildenbrand bool try_change_writable; 5551da177e4SLinus Torvalds pgoff_t pgoff; 5561da177e4SLinus Torvalds int error; 5571da177e4SLinus Torvalds 5581da177e4SLinus Torvalds if (newflags == oldflags) { 5591da177e4SLinus Torvalds *pprev = vma; 5601da177e4SLinus Torvalds return 0; 5611da177e4SLinus Torvalds } 5621da177e4SLinus Torvalds 5631da177e4SLinus Torvalds /* 56442e4089cSAndi Kleen * Do PROT_NONE PFN permission checks here when we can still 56542e4089cSAndi Kleen * bail out without undoing a lot of state. This is a rather 56642e4089cSAndi Kleen * uncommon case, so doesn't need to be very optimized. 56742e4089cSAndi Kleen */ 56842e4089cSAndi Kleen if (arch_has_pfn_modify_check() && 56942e4089cSAndi Kleen (vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 5706cb4d9a2SAnshuman Khandual (newflags & VM_ACCESS_FLAGS) == 0) { 5717b86ac33SChristoph Hellwig pgprot_t new_pgprot = vm_get_page_prot(newflags); 5727b86ac33SChristoph Hellwig 5737b86ac33SChristoph Hellwig error = walk_page_range(current->mm, start, end, 5747b86ac33SChristoph Hellwig &prot_none_walk_ops, &new_pgprot); 57542e4089cSAndi Kleen if (error) 57642e4089cSAndi Kleen return error; 57742e4089cSAndi Kleen } 57842e4089cSAndi Kleen 57942e4089cSAndi Kleen /* 5801da177e4SLinus Torvalds * If we make a private mapping writable we increase our commit; 5811da177e4SLinus Torvalds * but (without finer accounting) cannot reduce our commit if we 5825a6fe125SMel Gorman * make it unwritable again. hugetlb mapping were accounted for 5835a6fe125SMel Gorman * even if read-only so there is no need to account for them here 5841da177e4SLinus Torvalds */ 5851da177e4SLinus Torvalds if (newflags & VM_WRITE) { 58684638335SKonstantin Khlebnikov /* Check space limits when area turns into data. */ 58784638335SKonstantin Khlebnikov if (!may_expand_vm(mm, newflags, nrpages) && 58884638335SKonstantin Khlebnikov may_expand_vm(mm, oldflags, nrpages)) 58984638335SKonstantin Khlebnikov return -ENOMEM; 5905a6fe125SMel Gorman if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| 591cdfd4325SAndy Whitcroft VM_SHARED|VM_NORESERVE))) { 5921da177e4SLinus Torvalds charged = nrpages; 593191c5424SAl Viro if (security_vm_enough_memory_mm(mm, charged)) 5941da177e4SLinus Torvalds return -ENOMEM; 5951da177e4SLinus Torvalds newflags |= VM_ACCOUNT; 5961da177e4SLinus Torvalds } 5971da177e4SLinus Torvalds } 5981da177e4SLinus Torvalds 5991da177e4SLinus Torvalds /* 6001da177e4SLinus Torvalds * First try to merge with previous and/or next vma. 6011da177e4SLinus Torvalds */ 6021da177e4SLinus Torvalds pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 6031da177e4SLinus Torvalds *pprev = vma_merge(mm, *pprev, start, end, newflags, 60419a809afSAndrea Arcangeli vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), 6055c26f6acSSuren Baghdasaryan vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 6061da177e4SLinus Torvalds if (*pprev) { 6071da177e4SLinus Torvalds vma = *pprev; 608e86f15eeSAndrea Arcangeli VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY); 6091da177e4SLinus Torvalds goto success; 6101da177e4SLinus Torvalds } 6111da177e4SLinus Torvalds 6121da177e4SLinus Torvalds *pprev = vma; 6131da177e4SLinus Torvalds 6141da177e4SLinus Torvalds if (start != vma->vm_start) { 6151da177e4SLinus Torvalds error = split_vma(mm, vma, start, 1); 6161da177e4SLinus Torvalds if (error) 6171da177e4SLinus Torvalds goto fail; 6181da177e4SLinus Torvalds } 6191da177e4SLinus Torvalds 6201da177e4SLinus Torvalds if (end != vma->vm_end) { 6211da177e4SLinus Torvalds error = split_vma(mm, vma, end, 0); 6221da177e4SLinus Torvalds if (error) 6231da177e4SLinus Torvalds goto fail; 6241da177e4SLinus Torvalds } 6251da177e4SLinus Torvalds 6261da177e4SLinus Torvalds success: 6271da177e4SLinus Torvalds /* 628c1e8d7c6SMichel Lespinasse * vm_flags and vm_page_prot are protected by the mmap_lock 6291da177e4SLinus Torvalds * held in write mode. 6301da177e4SLinus Torvalds */ 6311da177e4SLinus Torvalds vma->vm_flags = newflags; 63264fe24a3SDavid Hildenbrand /* 63364fe24a3SDavid Hildenbrand * We want to check manually if we can change individual PTEs writable 63464fe24a3SDavid Hildenbrand * if we can't do that automatically for all PTEs in a mapping. For 63564fe24a3SDavid Hildenbrand * private mappings, that's always the case when we have write 63664fe24a3SDavid Hildenbrand * permissions as we properly have to handle COW. 63764fe24a3SDavid Hildenbrand */ 63864fe24a3SDavid Hildenbrand if (vma->vm_flags & VM_SHARED) 63964fe24a3SDavid Hildenbrand try_change_writable = vma_wants_writenotify(vma, vma->vm_page_prot); 64064fe24a3SDavid Hildenbrand else 64164fe24a3SDavid Hildenbrand try_change_writable = !!(vma->vm_flags & VM_WRITE); 64264e45507SPeter Feiner vma_set_page_prot(vma); 643d08b3851SPeter Zijlstra 6444a18419fSNadav Amit change_protection(tlb, vma, start, end, vma->vm_page_prot, 64564fe24a3SDavid Hildenbrand try_change_writable ? MM_CP_TRY_CHANGE_WRITABLE : 0); 6467da4d641SPeter Zijlstra 64736f88188SKirill A. Shutemov /* 64836f88188SKirill A. Shutemov * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major 64936f88188SKirill A. Shutemov * fault on access. 65036f88188SKirill A. Shutemov */ 65136f88188SKirill A. Shutemov if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED && 65236f88188SKirill A. Shutemov (newflags & VM_WRITE)) { 65336f88188SKirill A. Shutemov populate_vma_page_range(vma, start, end, NULL); 65436f88188SKirill A. Shutemov } 65536f88188SKirill A. Shutemov 65684638335SKonstantin Khlebnikov vm_stat_account(mm, oldflags, -nrpages); 65784638335SKonstantin Khlebnikov vm_stat_account(mm, newflags, nrpages); 65863bfd738SPekka Enberg perf_event_mmap(vma); 6591da177e4SLinus Torvalds return 0; 6601da177e4SLinus Torvalds 6611da177e4SLinus Torvalds fail: 6621da177e4SLinus Torvalds vm_unacct_memory(charged); 6631da177e4SLinus Torvalds return error; 6641da177e4SLinus Torvalds } 6651da177e4SLinus Torvalds 6667d06d9c9SDave Hansen /* 6677d06d9c9SDave Hansen * pkey==-1 when doing a legacy mprotect() 6687d06d9c9SDave Hansen */ 6697d06d9c9SDave Hansen static int do_mprotect_pkey(unsigned long start, size_t len, 6707d06d9c9SDave Hansen unsigned long prot, int pkey) 6711da177e4SLinus Torvalds { 67262b5f7d0SDave Hansen unsigned long nstart, end, tmp, reqprot; 6731da177e4SLinus Torvalds struct vm_area_struct *vma, *prev; 67448725bbcSXiu Jianfeng int error; 6751da177e4SLinus Torvalds const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); 676f138556dSPiotr Kwapulinski const bool rier = (current->personality & READ_IMPLIES_EXEC) && 677f138556dSPiotr Kwapulinski (prot & PROT_READ); 6784a18419fSNadav Amit struct mmu_gather tlb; 679f138556dSPiotr Kwapulinski 680057d3389SAndrey Konovalov start = untagged_addr(start); 681057d3389SAndrey Konovalov 6821da177e4SLinus Torvalds prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); 6831da177e4SLinus Torvalds if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ 6841da177e4SLinus Torvalds return -EINVAL; 6851da177e4SLinus Torvalds 6861da177e4SLinus Torvalds if (start & ~PAGE_MASK) 6871da177e4SLinus Torvalds return -EINVAL; 6881da177e4SLinus Torvalds if (!len) 6891da177e4SLinus Torvalds return 0; 6901da177e4SLinus Torvalds len = PAGE_ALIGN(len); 6911da177e4SLinus Torvalds end = start + len; 6921da177e4SLinus Torvalds if (end <= start) 6931da177e4SLinus Torvalds return -ENOMEM; 6949035cf9aSKhalid Aziz if (!arch_validate_prot(prot, start)) 6951da177e4SLinus Torvalds return -EINVAL; 6961da177e4SLinus Torvalds 6971da177e4SLinus Torvalds reqprot = prot; 6981da177e4SLinus Torvalds 699d8ed45c5SMichel Lespinasse if (mmap_write_lock_killable(current->mm)) 700dc0ef0dfSMichal Hocko return -EINTR; 7011da177e4SLinus Torvalds 702e8c24d3aSDave Hansen /* 703e8c24d3aSDave Hansen * If userspace did not allocate the pkey, do not let 704e8c24d3aSDave Hansen * them use it here. 705e8c24d3aSDave Hansen */ 706e8c24d3aSDave Hansen error = -EINVAL; 707e8c24d3aSDave Hansen if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) 708e8c24d3aSDave Hansen goto out; 709e8c24d3aSDave Hansen 710097d5910SLinus Torvalds vma = find_vma(current->mm, start); 7111da177e4SLinus Torvalds error = -ENOMEM; 7121da177e4SLinus Torvalds if (!vma) 7131da177e4SLinus Torvalds goto out; 7146af5fa0dSLiu Song 7151da177e4SLinus Torvalds if (unlikely(grows & PROT_GROWSDOWN)) { 7161da177e4SLinus Torvalds if (vma->vm_start >= end) 7171da177e4SLinus Torvalds goto out; 7181da177e4SLinus Torvalds start = vma->vm_start; 7191da177e4SLinus Torvalds error = -EINVAL; 7201da177e4SLinus Torvalds if (!(vma->vm_flags & VM_GROWSDOWN)) 7211da177e4SLinus Torvalds goto out; 7227d12efaeSAndrew Morton } else { 7231da177e4SLinus Torvalds if (vma->vm_start > start) 7241da177e4SLinus Torvalds goto out; 7251da177e4SLinus Torvalds if (unlikely(grows & PROT_GROWSUP)) { 7261da177e4SLinus Torvalds end = vma->vm_end; 7271da177e4SLinus Torvalds error = -EINVAL; 7281da177e4SLinus Torvalds if (!(vma->vm_flags & VM_GROWSUP)) 7291da177e4SLinus Torvalds goto out; 7301da177e4SLinus Torvalds } 7311da177e4SLinus Torvalds } 7326af5fa0dSLiu Song 7331da177e4SLinus Torvalds if (start > vma->vm_start) 7341da177e4SLinus Torvalds prev = vma; 7356af5fa0dSLiu Song else 7366af5fa0dSLiu Song prev = vma->vm_prev; 7371da177e4SLinus Torvalds 7384a18419fSNadav Amit tlb_gather_mmu(&tlb, current->mm); 7391da177e4SLinus Torvalds for (nstart = start ; ; ) { 740a8502b67SDave Hansen unsigned long mask_off_old_flags; 7411da177e4SLinus Torvalds unsigned long newflags; 7427d06d9c9SDave Hansen int new_vma_pkey; 7431da177e4SLinus Torvalds 7441da177e4SLinus Torvalds /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ 7451da177e4SLinus Torvalds 746f138556dSPiotr Kwapulinski /* Does the application expect PROT_READ to imply PROT_EXEC */ 747f138556dSPiotr Kwapulinski if (rier && (vma->vm_flags & VM_MAYEXEC)) 748f138556dSPiotr Kwapulinski prot |= PROT_EXEC; 749f138556dSPiotr Kwapulinski 750a8502b67SDave Hansen /* 751a8502b67SDave Hansen * Each mprotect() call explicitly passes r/w/x permissions. 752a8502b67SDave Hansen * If a permission is not passed to mprotect(), it must be 753a8502b67SDave Hansen * cleared from the VMA. 754a8502b67SDave Hansen */ 755a8502b67SDave Hansen mask_off_old_flags = VM_READ | VM_WRITE | VM_EXEC | 7562c2d57b5SKhalid Aziz VM_FLAGS_CLEAR; 757a8502b67SDave Hansen 7587d06d9c9SDave Hansen new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey); 7597d06d9c9SDave Hansen newflags = calc_vm_prot_bits(prot, new_vma_pkey); 760a8502b67SDave Hansen newflags |= (vma->vm_flags & ~mask_off_old_flags); 7611da177e4SLinus Torvalds 7627e2cff42SPaolo 'Blaisorblade' Giarrusso /* newflags >> 4 shift VM_MAY% in place of VM_% */ 7636cb4d9a2SAnshuman Khandual if ((newflags & ~(newflags >> 4)) & VM_ACCESS_FLAGS) { 7641da177e4SLinus Torvalds error = -EACCES; 7654a18419fSNadav Amit break; 7661da177e4SLinus Torvalds } 7671da177e4SLinus Torvalds 768c462ac28SCatalin Marinas /* Allow architectures to sanity-check the new flags */ 769c462ac28SCatalin Marinas if (!arch_validate_flags(newflags)) { 770c462ac28SCatalin Marinas error = -EINVAL; 7714a18419fSNadav Amit break; 772c462ac28SCatalin Marinas } 773c462ac28SCatalin Marinas 7741da177e4SLinus Torvalds error = security_file_mprotect(vma, reqprot, prot); 7751da177e4SLinus Torvalds if (error) 7764a18419fSNadav Amit break; 7771da177e4SLinus Torvalds 7781da177e4SLinus Torvalds tmp = vma->vm_end; 7791da177e4SLinus Torvalds if (tmp > end) 7801da177e4SLinus Torvalds tmp = end; 78195bb7c42SSean Christopherson 782dbf53f75STianjia Zhang if (vma->vm_ops && vma->vm_ops->mprotect) { 78395bb7c42SSean Christopherson error = vma->vm_ops->mprotect(vma, nstart, tmp, newflags); 78495bb7c42SSean Christopherson if (error) 7854a18419fSNadav Amit break; 786dbf53f75STianjia Zhang } 78795bb7c42SSean Christopherson 7884a18419fSNadav Amit error = mprotect_fixup(&tlb, vma, &prev, nstart, tmp, newflags); 7891da177e4SLinus Torvalds if (error) 7904a18419fSNadav Amit break; 79195bb7c42SSean Christopherson 7921da177e4SLinus Torvalds nstart = tmp; 7931da177e4SLinus Torvalds 7941da177e4SLinus Torvalds if (nstart < prev->vm_end) 7951da177e4SLinus Torvalds nstart = prev->vm_end; 7961da177e4SLinus Torvalds if (nstart >= end) 7974a18419fSNadav Amit break; 7981da177e4SLinus Torvalds 7991da177e4SLinus Torvalds vma = prev->vm_next; 8001da177e4SLinus Torvalds if (!vma || vma->vm_start != nstart) { 8011da177e4SLinus Torvalds error = -ENOMEM; 8024a18419fSNadav Amit break; 8031da177e4SLinus Torvalds } 804f138556dSPiotr Kwapulinski prot = reqprot; 8051da177e4SLinus Torvalds } 8064a18419fSNadav Amit tlb_finish_mmu(&tlb); 8071da177e4SLinus Torvalds out: 808d8ed45c5SMichel Lespinasse mmap_write_unlock(current->mm); 8091da177e4SLinus Torvalds return error; 8101da177e4SLinus Torvalds } 8117d06d9c9SDave Hansen 8127d06d9c9SDave Hansen SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len, 8137d06d9c9SDave Hansen unsigned long, prot) 8147d06d9c9SDave Hansen { 8157d06d9c9SDave Hansen return do_mprotect_pkey(start, len, prot, -1); 8167d06d9c9SDave Hansen } 8177d06d9c9SDave Hansen 818c7142aeaSHeiko Carstens #ifdef CONFIG_ARCH_HAS_PKEYS 819c7142aeaSHeiko Carstens 8207d06d9c9SDave Hansen SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len, 8217d06d9c9SDave Hansen unsigned long, prot, int, pkey) 8227d06d9c9SDave Hansen { 8237d06d9c9SDave Hansen return do_mprotect_pkey(start, len, prot, pkey); 8247d06d9c9SDave Hansen } 825e8c24d3aSDave Hansen 826e8c24d3aSDave Hansen SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val) 827e8c24d3aSDave Hansen { 828e8c24d3aSDave Hansen int pkey; 829e8c24d3aSDave Hansen int ret; 830e8c24d3aSDave Hansen 831e8c24d3aSDave Hansen /* No flags supported yet. */ 832e8c24d3aSDave Hansen if (flags) 833e8c24d3aSDave Hansen return -EINVAL; 834e8c24d3aSDave Hansen /* check for unsupported init values */ 835e8c24d3aSDave Hansen if (init_val & ~PKEY_ACCESS_MASK) 836e8c24d3aSDave Hansen return -EINVAL; 837e8c24d3aSDave Hansen 838d8ed45c5SMichel Lespinasse mmap_write_lock(current->mm); 839e8c24d3aSDave Hansen pkey = mm_pkey_alloc(current->mm); 840e8c24d3aSDave Hansen 841e8c24d3aSDave Hansen ret = -ENOSPC; 842e8c24d3aSDave Hansen if (pkey == -1) 843e8c24d3aSDave Hansen goto out; 844e8c24d3aSDave Hansen 845e8c24d3aSDave Hansen ret = arch_set_user_pkey_access(current, pkey, init_val); 846e8c24d3aSDave Hansen if (ret) { 847e8c24d3aSDave Hansen mm_pkey_free(current->mm, pkey); 848e8c24d3aSDave Hansen goto out; 849e8c24d3aSDave Hansen } 850e8c24d3aSDave Hansen ret = pkey; 851e8c24d3aSDave Hansen out: 852d8ed45c5SMichel Lespinasse mmap_write_unlock(current->mm); 853e8c24d3aSDave Hansen return ret; 854e8c24d3aSDave Hansen } 855e8c24d3aSDave Hansen 856e8c24d3aSDave Hansen SYSCALL_DEFINE1(pkey_free, int, pkey) 857e8c24d3aSDave Hansen { 858e8c24d3aSDave Hansen int ret; 859e8c24d3aSDave Hansen 860d8ed45c5SMichel Lespinasse mmap_write_lock(current->mm); 861e8c24d3aSDave Hansen ret = mm_pkey_free(current->mm, pkey); 862d8ed45c5SMichel Lespinasse mmap_write_unlock(current->mm); 863e8c24d3aSDave Hansen 864e8c24d3aSDave Hansen /* 865f0953a1bSIngo Molnar * We could provide warnings or errors if any VMA still 866e8c24d3aSDave Hansen * has the pkey set here. 867e8c24d3aSDave Hansen */ 868e8c24d3aSDave Hansen return ret; 869e8c24d3aSDave Hansen } 870c7142aeaSHeiko Carstens 871c7142aeaSHeiko Carstens #endif /* CONFIG_ARCH_HAS_PKEYS */ 872