1c1a4de99SAndrea Arcangeli /* 2c1a4de99SAndrea Arcangeli * mm/userfaultfd.c 3c1a4de99SAndrea Arcangeli * 4c1a4de99SAndrea Arcangeli * Copyright (C) 2015 Red Hat, Inc. 5c1a4de99SAndrea Arcangeli * 6c1a4de99SAndrea Arcangeli * This work is licensed under the terms of the GNU GPL, version 2. See 7c1a4de99SAndrea Arcangeli * the COPYING file in the top-level directory. 8c1a4de99SAndrea Arcangeli */ 9c1a4de99SAndrea Arcangeli 10c1a4de99SAndrea Arcangeli #include <linux/mm.h> 11174cd4b1SIngo Molnar #include <linux/sched/signal.h> 12c1a4de99SAndrea Arcangeli #include <linux/pagemap.h> 13c1a4de99SAndrea Arcangeli #include <linux/rmap.h> 14c1a4de99SAndrea Arcangeli #include <linux/swap.h> 15c1a4de99SAndrea Arcangeli #include <linux/swapops.h> 16c1a4de99SAndrea Arcangeli #include <linux/userfaultfd_k.h> 17c1a4de99SAndrea Arcangeli #include <linux/mmu_notifier.h> 1860d4d2d2SMike Kravetz #include <linux/hugetlb.h> 1926071cedSMike Rapoport #include <linux/shmem_fs.h> 20c1a4de99SAndrea Arcangeli #include <asm/tlbflush.h> 21c1a4de99SAndrea Arcangeli #include "internal.h" 22c1a4de99SAndrea Arcangeli 23c1a4de99SAndrea Arcangeli static int mcopy_atomic_pte(struct mm_struct *dst_mm, 24c1a4de99SAndrea Arcangeli pmd_t *dst_pmd, 25c1a4de99SAndrea Arcangeli struct vm_area_struct *dst_vma, 26c1a4de99SAndrea Arcangeli unsigned long dst_addr, 27b6ebaedbSAndrea Arcangeli unsigned long src_addr, 28b6ebaedbSAndrea Arcangeli struct page **pagep) 29c1a4de99SAndrea Arcangeli { 30c1a4de99SAndrea Arcangeli struct mem_cgroup *memcg; 31c1a4de99SAndrea Arcangeli pte_t _dst_pte, *dst_pte; 32c1a4de99SAndrea Arcangeli spinlock_t *ptl; 33c1a4de99SAndrea Arcangeli void *page_kaddr; 34c1a4de99SAndrea Arcangeli int ret; 35b6ebaedbSAndrea Arcangeli struct page *page; 36e2a50c1fSAndrea Arcangeli pgoff_t offset, max_off; 37e2a50c1fSAndrea Arcangeli struct inode *inode; 38c1a4de99SAndrea Arcangeli 39b6ebaedbSAndrea Arcangeli if (!*pagep) { 40c1a4de99SAndrea Arcangeli ret = -ENOMEM; 41c1a4de99SAndrea Arcangeli page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, dst_vma, dst_addr); 42c1a4de99SAndrea Arcangeli if (!page) 43c1a4de99SAndrea Arcangeli goto out; 44c1a4de99SAndrea Arcangeli 45b6ebaedbSAndrea Arcangeli page_kaddr = kmap_atomic(page); 46b6ebaedbSAndrea Arcangeli ret = copy_from_user(page_kaddr, 47b6ebaedbSAndrea Arcangeli (const void __user *) src_addr, 48b6ebaedbSAndrea Arcangeli PAGE_SIZE); 49b6ebaedbSAndrea Arcangeli kunmap_atomic(page_kaddr); 50b6ebaedbSAndrea Arcangeli 51b6ebaedbSAndrea Arcangeli /* fallback to copy_from_user outside mmap_sem */ 52b6ebaedbSAndrea Arcangeli if (unlikely(ret)) { 539e368259SAndrea Arcangeli ret = -ENOENT; 54b6ebaedbSAndrea Arcangeli *pagep = page; 55b6ebaedbSAndrea Arcangeli /* don't free the page */ 56b6ebaedbSAndrea Arcangeli goto out; 57b6ebaedbSAndrea Arcangeli } 58b6ebaedbSAndrea Arcangeli } else { 59b6ebaedbSAndrea Arcangeli page = *pagep; 60b6ebaedbSAndrea Arcangeli *pagep = NULL; 61b6ebaedbSAndrea Arcangeli } 62c1a4de99SAndrea Arcangeli 63c1a4de99SAndrea Arcangeli /* 64c1a4de99SAndrea Arcangeli * The memory barrier inside __SetPageUptodate makes sure that 65c1a4de99SAndrea Arcangeli * preceeding stores to the page contents become visible before 66c1a4de99SAndrea Arcangeli * the set_pte_at() write. 67c1a4de99SAndrea Arcangeli */ 68c1a4de99SAndrea Arcangeli __SetPageUptodate(page); 69c1a4de99SAndrea Arcangeli 70c1a4de99SAndrea Arcangeli ret = -ENOMEM; 71f627c2f5SKirill A. Shutemov if (mem_cgroup_try_charge(page, dst_mm, GFP_KERNEL, &memcg, false)) 72c1a4de99SAndrea Arcangeli goto out_release; 73c1a4de99SAndrea Arcangeli 74c1a4de99SAndrea Arcangeli _dst_pte = mk_pte(page, dst_vma->vm_page_prot); 75c1a4de99SAndrea Arcangeli if (dst_vma->vm_flags & VM_WRITE) 76c1a4de99SAndrea Arcangeli _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte)); 77c1a4de99SAndrea Arcangeli 78c1a4de99SAndrea Arcangeli dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); 79e2a50c1fSAndrea Arcangeli if (dst_vma->vm_file) { 80e2a50c1fSAndrea Arcangeli /* the shmem MAP_PRIVATE case requires checking the i_size */ 81e2a50c1fSAndrea Arcangeli inode = dst_vma->vm_file->f_inode; 82e2a50c1fSAndrea Arcangeli offset = linear_page_index(dst_vma, dst_addr); 83e2a50c1fSAndrea Arcangeli max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 84e2a50c1fSAndrea Arcangeli ret = -EFAULT; 85e2a50c1fSAndrea Arcangeli if (unlikely(offset >= max_off)) 86e2a50c1fSAndrea Arcangeli goto out_release_uncharge_unlock; 87e2a50c1fSAndrea Arcangeli } 88e2a50c1fSAndrea Arcangeli ret = -EEXIST; 89c1a4de99SAndrea Arcangeli if (!pte_none(*dst_pte)) 90c1a4de99SAndrea Arcangeli goto out_release_uncharge_unlock; 91c1a4de99SAndrea Arcangeli 92c1a4de99SAndrea Arcangeli inc_mm_counter(dst_mm, MM_ANONPAGES); 93d281ee61SKirill A. Shutemov page_add_new_anon_rmap(page, dst_vma, dst_addr, false); 94f627c2f5SKirill A. Shutemov mem_cgroup_commit_charge(page, memcg, false, false); 95c1a4de99SAndrea Arcangeli lru_cache_add_active_or_unevictable(page, dst_vma); 96c1a4de99SAndrea Arcangeli 97c1a4de99SAndrea Arcangeli set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); 98c1a4de99SAndrea Arcangeli 99c1a4de99SAndrea Arcangeli /* No need to invalidate - it was non-present before */ 100c1a4de99SAndrea Arcangeli update_mmu_cache(dst_vma, dst_addr, dst_pte); 101c1a4de99SAndrea Arcangeli 102c1a4de99SAndrea Arcangeli pte_unmap_unlock(dst_pte, ptl); 103c1a4de99SAndrea Arcangeli ret = 0; 104c1a4de99SAndrea Arcangeli out: 105c1a4de99SAndrea Arcangeli return ret; 106c1a4de99SAndrea Arcangeli out_release_uncharge_unlock: 107c1a4de99SAndrea Arcangeli pte_unmap_unlock(dst_pte, ptl); 108f627c2f5SKirill A. Shutemov mem_cgroup_cancel_charge(page, memcg, false); 109c1a4de99SAndrea Arcangeli out_release: 11009cbfeafSKirill A. Shutemov put_page(page); 111c1a4de99SAndrea Arcangeli goto out; 112c1a4de99SAndrea Arcangeli } 113c1a4de99SAndrea Arcangeli 114c1a4de99SAndrea Arcangeli static int mfill_zeropage_pte(struct mm_struct *dst_mm, 115c1a4de99SAndrea Arcangeli pmd_t *dst_pmd, 116c1a4de99SAndrea Arcangeli struct vm_area_struct *dst_vma, 117c1a4de99SAndrea Arcangeli unsigned long dst_addr) 118c1a4de99SAndrea Arcangeli { 119c1a4de99SAndrea Arcangeli pte_t _dst_pte, *dst_pte; 120c1a4de99SAndrea Arcangeli spinlock_t *ptl; 121c1a4de99SAndrea Arcangeli int ret; 122e2a50c1fSAndrea Arcangeli pgoff_t offset, max_off; 123e2a50c1fSAndrea Arcangeli struct inode *inode; 124c1a4de99SAndrea Arcangeli 125c1a4de99SAndrea Arcangeli _dst_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr), 126c1a4de99SAndrea Arcangeli dst_vma->vm_page_prot)); 127c1a4de99SAndrea Arcangeli dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); 128e2a50c1fSAndrea Arcangeli if (dst_vma->vm_file) { 129e2a50c1fSAndrea Arcangeli /* the shmem MAP_PRIVATE case requires checking the i_size */ 130e2a50c1fSAndrea Arcangeli inode = dst_vma->vm_file->f_inode; 131e2a50c1fSAndrea Arcangeli offset = linear_page_index(dst_vma, dst_addr); 132e2a50c1fSAndrea Arcangeli max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 133e2a50c1fSAndrea Arcangeli ret = -EFAULT; 134e2a50c1fSAndrea Arcangeli if (unlikely(offset >= max_off)) 135e2a50c1fSAndrea Arcangeli goto out_unlock; 136e2a50c1fSAndrea Arcangeli } 137e2a50c1fSAndrea Arcangeli ret = -EEXIST; 138c1a4de99SAndrea Arcangeli if (!pte_none(*dst_pte)) 139c1a4de99SAndrea Arcangeli goto out_unlock; 140c1a4de99SAndrea Arcangeli set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); 141c1a4de99SAndrea Arcangeli /* No need to invalidate - it was non-present before */ 142c1a4de99SAndrea Arcangeli update_mmu_cache(dst_vma, dst_addr, dst_pte); 143c1a4de99SAndrea Arcangeli ret = 0; 144c1a4de99SAndrea Arcangeli out_unlock: 145c1a4de99SAndrea Arcangeli pte_unmap_unlock(dst_pte, ptl); 146c1a4de99SAndrea Arcangeli return ret; 147c1a4de99SAndrea Arcangeli } 148c1a4de99SAndrea Arcangeli 149c1a4de99SAndrea Arcangeli static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address) 150c1a4de99SAndrea Arcangeli { 151c1a4de99SAndrea Arcangeli pgd_t *pgd; 152c2febafcSKirill A. Shutemov p4d_t *p4d; 153c1a4de99SAndrea Arcangeli pud_t *pud; 154c1a4de99SAndrea Arcangeli 155c1a4de99SAndrea Arcangeli pgd = pgd_offset(mm, address); 156c2febafcSKirill A. Shutemov p4d = p4d_alloc(mm, pgd, address); 157c2febafcSKirill A. Shutemov if (!p4d) 158c2febafcSKirill A. Shutemov return NULL; 159c2febafcSKirill A. Shutemov pud = pud_alloc(mm, p4d, address); 160c2febafcSKirill A. Shutemov if (!pud) 161c2febafcSKirill A. Shutemov return NULL; 162c1a4de99SAndrea Arcangeli /* 163c1a4de99SAndrea Arcangeli * Note that we didn't run this because the pmd was 164c1a4de99SAndrea Arcangeli * missing, the *pmd may be already established and in 165c1a4de99SAndrea Arcangeli * turn it may also be a trans_huge_pmd. 166c1a4de99SAndrea Arcangeli */ 167c2febafcSKirill A. Shutemov return pmd_alloc(mm, pud, address); 168c1a4de99SAndrea Arcangeli } 169c1a4de99SAndrea Arcangeli 17060d4d2d2SMike Kravetz #ifdef CONFIG_HUGETLB_PAGE 17160d4d2d2SMike Kravetz /* 17260d4d2d2SMike Kravetz * __mcopy_atomic processing for HUGETLB vmas. Note that this routine is 17360d4d2d2SMike Kravetz * called with mmap_sem held, it will release mmap_sem before returning. 17460d4d2d2SMike Kravetz */ 17560d4d2d2SMike Kravetz static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, 17660d4d2d2SMike Kravetz struct vm_area_struct *dst_vma, 17760d4d2d2SMike Kravetz unsigned long dst_start, 17860d4d2d2SMike Kravetz unsigned long src_start, 17960d4d2d2SMike Kravetz unsigned long len, 18060d4d2d2SMike Kravetz bool zeropage) 18160d4d2d2SMike Kravetz { 1821c9e8defSMike Kravetz int vm_alloc_shared = dst_vma->vm_flags & VM_SHARED; 1831c9e8defSMike Kravetz int vm_shared = dst_vma->vm_flags & VM_SHARED; 18460d4d2d2SMike Kravetz ssize_t err; 18560d4d2d2SMike Kravetz pte_t *dst_pte; 18660d4d2d2SMike Kravetz unsigned long src_addr, dst_addr; 18760d4d2d2SMike Kravetz long copied; 18860d4d2d2SMike Kravetz struct page *page; 18960d4d2d2SMike Kravetz struct hstate *h; 19060d4d2d2SMike Kravetz unsigned long vma_hpagesize; 19160d4d2d2SMike Kravetz pgoff_t idx; 19260d4d2d2SMike Kravetz u32 hash; 19360d4d2d2SMike Kravetz struct address_space *mapping; 19460d4d2d2SMike Kravetz 19560d4d2d2SMike Kravetz /* 19660d4d2d2SMike Kravetz * There is no default zero huge page for all huge page sizes as 19760d4d2d2SMike Kravetz * supported by hugetlb. A PMD_SIZE huge pages may exist as used 19860d4d2d2SMike Kravetz * by THP. Since we can not reliably insert a zero page, this 19960d4d2d2SMike Kravetz * feature is not supported. 20060d4d2d2SMike Kravetz */ 20160d4d2d2SMike Kravetz if (zeropage) { 20260d4d2d2SMike Kravetz up_read(&dst_mm->mmap_sem); 20360d4d2d2SMike Kravetz return -EINVAL; 20460d4d2d2SMike Kravetz } 20560d4d2d2SMike Kravetz 20660d4d2d2SMike Kravetz src_addr = src_start; 20760d4d2d2SMike Kravetz dst_addr = dst_start; 20860d4d2d2SMike Kravetz copied = 0; 20960d4d2d2SMike Kravetz page = NULL; 21060d4d2d2SMike Kravetz vma_hpagesize = vma_kernel_pagesize(dst_vma); 21160d4d2d2SMike Kravetz 21260d4d2d2SMike Kravetz /* 21360d4d2d2SMike Kravetz * Validate alignment based on huge page size 21460d4d2d2SMike Kravetz */ 21560d4d2d2SMike Kravetz err = -EINVAL; 21660d4d2d2SMike Kravetz if (dst_start & (vma_hpagesize - 1) || len & (vma_hpagesize - 1)) 21760d4d2d2SMike Kravetz goto out_unlock; 21860d4d2d2SMike Kravetz 21960d4d2d2SMike Kravetz retry: 22060d4d2d2SMike Kravetz /* 22160d4d2d2SMike Kravetz * On routine entry dst_vma is set. If we had to drop mmap_sem and 22260d4d2d2SMike Kravetz * retry, dst_vma will be set to NULL and we must lookup again. 22360d4d2d2SMike Kravetz */ 22460d4d2d2SMike Kravetz if (!dst_vma) { 22527d02568SMike Rapoport err = -ENOENT; 22660d4d2d2SMike Kravetz dst_vma = find_vma(dst_mm, dst_start); 22760d4d2d2SMike Kravetz if (!dst_vma || !is_vm_hugetlb_page(dst_vma)) 22860d4d2d2SMike Kravetz goto out_unlock; 22927d02568SMike Rapoport /* 23029ec9066SAndrea Arcangeli * Check the vma is registered in uffd, this is 23129ec9066SAndrea Arcangeli * required to enforce the VM_MAYWRITE check done at 23229ec9066SAndrea Arcangeli * uffd registration time. 23327d02568SMike Rapoport */ 23427d02568SMike Rapoport if (!dst_vma->vm_userfaultfd_ctx.ctx) 23560d4d2d2SMike Kravetz goto out_unlock; 23660d4d2d2SMike Kravetz 23760d4d2d2SMike Kravetz if (dst_start < dst_vma->vm_start || 23860d4d2d2SMike Kravetz dst_start + len > dst_vma->vm_end) 23960d4d2d2SMike Kravetz goto out_unlock; 2401c9e8defSMike Kravetz 24127d02568SMike Rapoport err = -EINVAL; 24227d02568SMike Rapoport if (vma_hpagesize != vma_kernel_pagesize(dst_vma)) 24327d02568SMike Rapoport goto out_unlock; 24427d02568SMike Rapoport 2451c9e8defSMike Kravetz vm_shared = dst_vma->vm_flags & VM_SHARED; 24660d4d2d2SMike Kravetz } 24760d4d2d2SMike Kravetz 24860d4d2d2SMike Kravetz if (WARN_ON(dst_addr & (vma_hpagesize - 1) || 24960d4d2d2SMike Kravetz (len - copied) & (vma_hpagesize - 1))) 25060d4d2d2SMike Kravetz goto out_unlock; 25160d4d2d2SMike Kravetz 25260d4d2d2SMike Kravetz /* 2531c9e8defSMike Kravetz * If not shared, ensure the dst_vma has a anon_vma. 25460d4d2d2SMike Kravetz */ 25560d4d2d2SMike Kravetz err = -ENOMEM; 2561c9e8defSMike Kravetz if (!vm_shared) { 25760d4d2d2SMike Kravetz if (unlikely(anon_vma_prepare(dst_vma))) 25860d4d2d2SMike Kravetz goto out_unlock; 2591c9e8defSMike Kravetz } 26060d4d2d2SMike Kravetz 26160d4d2d2SMike Kravetz h = hstate_vma(dst_vma); 26260d4d2d2SMike Kravetz 26360d4d2d2SMike Kravetz while (src_addr < src_start + len) { 26460d4d2d2SMike Kravetz pte_t dst_pteval; 26560d4d2d2SMike Kravetz 26660d4d2d2SMike Kravetz BUG_ON(dst_addr >= dst_start + len); 26760d4d2d2SMike Kravetz VM_BUG_ON(dst_addr & ~huge_page_mask(h)); 26860d4d2d2SMike Kravetz 26960d4d2d2SMike Kravetz /* 270b43a9990SMike Kravetz * Serialize via i_mmap_rwsem and hugetlb_fault_mutex. 271b43a9990SMike Kravetz * i_mmap_rwsem ensures the dst_pte remains valid even 272b43a9990SMike Kravetz * in the case of shared pmds. fault mutex prevents 273b43a9990SMike Kravetz * races with other faulting threads. 27460d4d2d2SMike Kravetz */ 27560d4d2d2SMike Kravetz mapping = dst_vma->vm_file->f_mapping; 276b43a9990SMike Kravetz i_mmap_lock_read(mapping); 277b43a9990SMike Kravetz idx = linear_page_index(dst_vma, dst_addr); 27860d4d2d2SMike Kravetz hash = hugetlb_fault_mutex_hash(h, dst_mm, dst_vma, mapping, 27960d4d2d2SMike Kravetz idx, dst_addr); 28060d4d2d2SMike Kravetz mutex_lock(&hugetlb_fault_mutex_table[hash]); 28160d4d2d2SMike Kravetz 28260d4d2d2SMike Kravetz err = -ENOMEM; 28360d4d2d2SMike Kravetz dst_pte = huge_pte_alloc(dst_mm, dst_addr, huge_page_size(h)); 28460d4d2d2SMike Kravetz if (!dst_pte) { 28560d4d2d2SMike Kravetz mutex_unlock(&hugetlb_fault_mutex_table[hash]); 286b43a9990SMike Kravetz i_mmap_unlock_read(mapping); 28760d4d2d2SMike Kravetz goto out_unlock; 28860d4d2d2SMike Kravetz } 28960d4d2d2SMike Kravetz 29060d4d2d2SMike Kravetz err = -EEXIST; 29160d4d2d2SMike Kravetz dst_pteval = huge_ptep_get(dst_pte); 29260d4d2d2SMike Kravetz if (!huge_pte_none(dst_pteval)) { 29360d4d2d2SMike Kravetz mutex_unlock(&hugetlb_fault_mutex_table[hash]); 294b43a9990SMike Kravetz i_mmap_unlock_read(mapping); 29560d4d2d2SMike Kravetz goto out_unlock; 29660d4d2d2SMike Kravetz } 29760d4d2d2SMike Kravetz 29860d4d2d2SMike Kravetz err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma, 29960d4d2d2SMike Kravetz dst_addr, src_addr, &page); 30060d4d2d2SMike Kravetz 30160d4d2d2SMike Kravetz mutex_unlock(&hugetlb_fault_mutex_table[hash]); 302b43a9990SMike Kravetz i_mmap_unlock_read(mapping); 3031c9e8defSMike Kravetz vm_alloc_shared = vm_shared; 30460d4d2d2SMike Kravetz 30560d4d2d2SMike Kravetz cond_resched(); 30660d4d2d2SMike Kravetz 3079e368259SAndrea Arcangeli if (unlikely(err == -ENOENT)) { 30860d4d2d2SMike Kravetz up_read(&dst_mm->mmap_sem); 30960d4d2d2SMike Kravetz BUG_ON(!page); 31060d4d2d2SMike Kravetz 31160d4d2d2SMike Kravetz err = copy_huge_page_from_user(page, 31260d4d2d2SMike Kravetz (const void __user *)src_addr, 313810a56b9SMike Kravetz pages_per_huge_page(h), true); 31460d4d2d2SMike Kravetz if (unlikely(err)) { 31560d4d2d2SMike Kravetz err = -EFAULT; 31660d4d2d2SMike Kravetz goto out; 31760d4d2d2SMike Kravetz } 31860d4d2d2SMike Kravetz down_read(&dst_mm->mmap_sem); 31960d4d2d2SMike Kravetz 32060d4d2d2SMike Kravetz dst_vma = NULL; 32160d4d2d2SMike Kravetz goto retry; 32260d4d2d2SMike Kravetz } else 32360d4d2d2SMike Kravetz BUG_ON(page); 32460d4d2d2SMike Kravetz 32560d4d2d2SMike Kravetz if (!err) { 32660d4d2d2SMike Kravetz dst_addr += vma_hpagesize; 32760d4d2d2SMike Kravetz src_addr += vma_hpagesize; 32860d4d2d2SMike Kravetz copied += vma_hpagesize; 32960d4d2d2SMike Kravetz 33060d4d2d2SMike Kravetz if (fatal_signal_pending(current)) 33160d4d2d2SMike Kravetz err = -EINTR; 33260d4d2d2SMike Kravetz } 33360d4d2d2SMike Kravetz if (err) 33460d4d2d2SMike Kravetz break; 33560d4d2d2SMike Kravetz } 33660d4d2d2SMike Kravetz 33760d4d2d2SMike Kravetz out_unlock: 33860d4d2d2SMike Kravetz up_read(&dst_mm->mmap_sem); 33960d4d2d2SMike Kravetz out: 34021205bf8SMike Kravetz if (page) { 34121205bf8SMike Kravetz /* 34221205bf8SMike Kravetz * We encountered an error and are about to free a newly 3431c9e8defSMike Kravetz * allocated huge page. 3441c9e8defSMike Kravetz * 3451c9e8defSMike Kravetz * Reservation handling is very subtle, and is different for 3461c9e8defSMike Kravetz * private and shared mappings. See the routine 3471c9e8defSMike Kravetz * restore_reserve_on_error for details. Unfortunately, we 3481c9e8defSMike Kravetz * can not call restore_reserve_on_error now as it would 3491c9e8defSMike Kravetz * require holding mmap_sem. 3501c9e8defSMike Kravetz * 3511c9e8defSMike Kravetz * If a reservation for the page existed in the reservation 3521c9e8defSMike Kravetz * map of a private mapping, the map was modified to indicate 3531c9e8defSMike Kravetz * the reservation was consumed when the page was allocated. 3541c9e8defSMike Kravetz * We clear the PagePrivate flag now so that the global 35521205bf8SMike Kravetz * reserve count will not be incremented in free_huge_page. 35621205bf8SMike Kravetz * The reservation map will still indicate the reservation 35721205bf8SMike Kravetz * was consumed and possibly prevent later page allocation. 3581c9e8defSMike Kravetz * This is better than leaking a global reservation. If no 3591c9e8defSMike Kravetz * reservation existed, it is still safe to clear PagePrivate 3601c9e8defSMike Kravetz * as no adjustments to reservation counts were made during 3611c9e8defSMike Kravetz * allocation. 3621c9e8defSMike Kravetz * 3631c9e8defSMike Kravetz * The reservation map for shared mappings indicates which 3641c9e8defSMike Kravetz * pages have reservations. When a huge page is allocated 3651c9e8defSMike Kravetz * for an address with a reservation, no change is made to 3661c9e8defSMike Kravetz * the reserve map. In this case PagePrivate will be set 3671c9e8defSMike Kravetz * to indicate that the global reservation count should be 3681c9e8defSMike Kravetz * incremented when the page is freed. This is the desired 3691c9e8defSMike Kravetz * behavior. However, when a huge page is allocated for an 3701c9e8defSMike Kravetz * address without a reservation a reservation entry is added 3711c9e8defSMike Kravetz * to the reservation map, and PagePrivate will not be set. 3721c9e8defSMike Kravetz * When the page is freed, the global reserve count will NOT 3731c9e8defSMike Kravetz * be incremented and it will appear as though we have leaked 3741c9e8defSMike Kravetz * reserved page. In this case, set PagePrivate so that the 3751c9e8defSMike Kravetz * global reserve count will be incremented to match the 3761c9e8defSMike Kravetz * reservation map entry which was created. 3771c9e8defSMike Kravetz * 3781c9e8defSMike Kravetz * Note that vm_alloc_shared is based on the flags of the vma 3791c9e8defSMike Kravetz * for which the page was originally allocated. dst_vma could 3801c9e8defSMike Kravetz * be different or NULL on error. 38121205bf8SMike Kravetz */ 3821c9e8defSMike Kravetz if (vm_alloc_shared) 3831c9e8defSMike Kravetz SetPagePrivate(page); 3841c9e8defSMike Kravetz else 38521205bf8SMike Kravetz ClearPagePrivate(page); 38660d4d2d2SMike Kravetz put_page(page); 38721205bf8SMike Kravetz } 38860d4d2d2SMike Kravetz BUG_ON(copied < 0); 38960d4d2d2SMike Kravetz BUG_ON(err > 0); 39060d4d2d2SMike Kravetz BUG_ON(!copied && !err); 39160d4d2d2SMike Kravetz return copied ? copied : err; 39260d4d2d2SMike Kravetz } 39360d4d2d2SMike Kravetz #else /* !CONFIG_HUGETLB_PAGE */ 39460d4d2d2SMike Kravetz /* fail at build time if gcc attempts to use this */ 39560d4d2d2SMike Kravetz extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, 39660d4d2d2SMike Kravetz struct vm_area_struct *dst_vma, 39760d4d2d2SMike Kravetz unsigned long dst_start, 39860d4d2d2SMike Kravetz unsigned long src_start, 39960d4d2d2SMike Kravetz unsigned long len, 40060d4d2d2SMike Kravetz bool zeropage); 40160d4d2d2SMike Kravetz #endif /* CONFIG_HUGETLB_PAGE */ 40260d4d2d2SMike Kravetz 4033217d3c7SMike Rapoport static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, 4043217d3c7SMike Rapoport pmd_t *dst_pmd, 4053217d3c7SMike Rapoport struct vm_area_struct *dst_vma, 4063217d3c7SMike Rapoport unsigned long dst_addr, 4073217d3c7SMike Rapoport unsigned long src_addr, 4083217d3c7SMike Rapoport struct page **page, 4093217d3c7SMike Rapoport bool zeropage) 4103217d3c7SMike Rapoport { 4113217d3c7SMike Rapoport ssize_t err; 4123217d3c7SMike Rapoport 4135b51072eSAndrea Arcangeli /* 4145b51072eSAndrea Arcangeli * The normal page fault path for a shmem will invoke the 4155b51072eSAndrea Arcangeli * fault, fill the hole in the file and COW it right away. The 4165b51072eSAndrea Arcangeli * result generates plain anonymous memory. So when we are 4175b51072eSAndrea Arcangeli * asked to fill an hole in a MAP_PRIVATE shmem mapping, we'll 4185b51072eSAndrea Arcangeli * generate anonymous memory directly without actually filling 4195b51072eSAndrea Arcangeli * the hole. For the MAP_PRIVATE case the robustness check 4205b51072eSAndrea Arcangeli * only happens in the pagetable (to verify it's still none) 4215b51072eSAndrea Arcangeli * and not in the radix tree. 4225b51072eSAndrea Arcangeli */ 4235b51072eSAndrea Arcangeli if (!(dst_vma->vm_flags & VM_SHARED)) { 4243217d3c7SMike Rapoport if (!zeropage) 4253217d3c7SMike Rapoport err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma, 4263217d3c7SMike Rapoport dst_addr, src_addr, page); 4273217d3c7SMike Rapoport else 4283217d3c7SMike Rapoport err = mfill_zeropage_pte(dst_mm, dst_pmd, 4293217d3c7SMike Rapoport dst_vma, dst_addr); 4303217d3c7SMike Rapoport } else { 4318fb44e54SMike Rapoport if (!zeropage) 4323217d3c7SMike Rapoport err = shmem_mcopy_atomic_pte(dst_mm, dst_pmd, 4333217d3c7SMike Rapoport dst_vma, dst_addr, 4343217d3c7SMike Rapoport src_addr, page); 4358fb44e54SMike Rapoport else 4368fb44e54SMike Rapoport err = shmem_mfill_zeropage_pte(dst_mm, dst_pmd, 4378fb44e54SMike Rapoport dst_vma, dst_addr); 4383217d3c7SMike Rapoport } 4393217d3c7SMike Rapoport 4403217d3c7SMike Rapoport return err; 4413217d3c7SMike Rapoport } 4423217d3c7SMike Rapoport 443c1a4de99SAndrea Arcangeli static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm, 444c1a4de99SAndrea Arcangeli unsigned long dst_start, 445c1a4de99SAndrea Arcangeli unsigned long src_start, 446c1a4de99SAndrea Arcangeli unsigned long len, 447df2cc96eSMike Rapoport bool zeropage, 448df2cc96eSMike Rapoport bool *mmap_changing) 449c1a4de99SAndrea Arcangeli { 450c1a4de99SAndrea Arcangeli struct vm_area_struct *dst_vma; 451c1a4de99SAndrea Arcangeli ssize_t err; 452c1a4de99SAndrea Arcangeli pmd_t *dst_pmd; 453c1a4de99SAndrea Arcangeli unsigned long src_addr, dst_addr; 454b6ebaedbSAndrea Arcangeli long copied; 455b6ebaedbSAndrea Arcangeli struct page *page; 456c1a4de99SAndrea Arcangeli 457c1a4de99SAndrea Arcangeli /* 458c1a4de99SAndrea Arcangeli * Sanitize the command parameters: 459c1a4de99SAndrea Arcangeli */ 460c1a4de99SAndrea Arcangeli BUG_ON(dst_start & ~PAGE_MASK); 461c1a4de99SAndrea Arcangeli BUG_ON(len & ~PAGE_MASK); 462c1a4de99SAndrea Arcangeli 463c1a4de99SAndrea Arcangeli /* Does the address range wrap, or is the span zero-sized? */ 464c1a4de99SAndrea Arcangeli BUG_ON(src_start + len <= src_start); 465c1a4de99SAndrea Arcangeli BUG_ON(dst_start + len <= dst_start); 466c1a4de99SAndrea Arcangeli 467b6ebaedbSAndrea Arcangeli src_addr = src_start; 468b6ebaedbSAndrea Arcangeli dst_addr = dst_start; 469b6ebaedbSAndrea Arcangeli copied = 0; 470b6ebaedbSAndrea Arcangeli page = NULL; 471b6ebaedbSAndrea Arcangeli retry: 472c1a4de99SAndrea Arcangeli down_read(&dst_mm->mmap_sem); 473c1a4de99SAndrea Arcangeli 474c1a4de99SAndrea Arcangeli /* 475df2cc96eSMike Rapoport * If memory mappings are changing because of non-cooperative 476df2cc96eSMike Rapoport * operation (e.g. mremap) running in parallel, bail out and 477df2cc96eSMike Rapoport * request the user to retry later 478df2cc96eSMike Rapoport */ 479df2cc96eSMike Rapoport err = -EAGAIN; 480df2cc96eSMike Rapoport if (mmap_changing && READ_ONCE(*mmap_changing)) 481df2cc96eSMike Rapoport goto out_unlock; 482df2cc96eSMike Rapoport 483df2cc96eSMike Rapoport /* 484c1a4de99SAndrea Arcangeli * Make sure the vma is not shared, that the dst range is 485c1a4de99SAndrea Arcangeli * both valid and fully within a single existing vma. 486c1a4de99SAndrea Arcangeli */ 48727d02568SMike Rapoport err = -ENOENT; 488c1a4de99SAndrea Arcangeli dst_vma = find_vma(dst_mm, dst_start); 48926071cedSMike Rapoport if (!dst_vma) 49026071cedSMike Rapoport goto out_unlock; 4911c9e8defSMike Kravetz /* 49229ec9066SAndrea Arcangeli * Check the vma is registered in uffd, this is required to 49329ec9066SAndrea Arcangeli * enforce the VM_MAYWRITE check done at uffd registration 49429ec9066SAndrea Arcangeli * time. 495c1a4de99SAndrea Arcangeli */ 496c1a4de99SAndrea Arcangeli if (!dst_vma->vm_userfaultfd_ctx.ctx) 497b6ebaedbSAndrea Arcangeli goto out_unlock; 498c1a4de99SAndrea Arcangeli 49927d02568SMike Rapoport if (dst_start < dst_vma->vm_start || 50027d02568SMike Rapoport dst_start + len > dst_vma->vm_end) 50127d02568SMike Rapoport goto out_unlock; 50227d02568SMike Rapoport 50327d02568SMike Rapoport err = -EINVAL; 50427d02568SMike Rapoport /* 50527d02568SMike Rapoport * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but 50627d02568SMike Rapoport * it will overwrite vm_ops, so vma_is_anonymous must return false. 50727d02568SMike Rapoport */ 50827d02568SMike Rapoport if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) && 50927d02568SMike Rapoport dst_vma->vm_flags & VM_SHARED)) 51027d02568SMike Rapoport goto out_unlock; 51127d02568SMike Rapoport 51227d02568SMike Rapoport /* 51327d02568SMike Rapoport * If this is a HUGETLB vma, pass off to appropriate routine 51427d02568SMike Rapoport */ 51527d02568SMike Rapoport if (is_vm_hugetlb_page(dst_vma)) 51627d02568SMike Rapoport return __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start, 51727d02568SMike Rapoport src_start, len, zeropage); 51827d02568SMike Rapoport 51926071cedSMike Rapoport if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma)) 520b6ebaedbSAndrea Arcangeli goto out_unlock; 521c1a4de99SAndrea Arcangeli 522c1a4de99SAndrea Arcangeli /* 523c1a4de99SAndrea Arcangeli * Ensure the dst_vma has a anon_vma or this page 524c1a4de99SAndrea Arcangeli * would get a NULL anon_vma when moved in the 525c1a4de99SAndrea Arcangeli * dst_vma. 526c1a4de99SAndrea Arcangeli */ 527c1a4de99SAndrea Arcangeli err = -ENOMEM; 5285b51072eSAndrea Arcangeli if (!(dst_vma->vm_flags & VM_SHARED) && 5295b51072eSAndrea Arcangeli unlikely(anon_vma_prepare(dst_vma))) 530b6ebaedbSAndrea Arcangeli goto out_unlock; 531c1a4de99SAndrea Arcangeli 532b6ebaedbSAndrea Arcangeli while (src_addr < src_start + len) { 533c1a4de99SAndrea Arcangeli pmd_t dst_pmdval; 534b6ebaedbSAndrea Arcangeli 535c1a4de99SAndrea Arcangeli BUG_ON(dst_addr >= dst_start + len); 536b6ebaedbSAndrea Arcangeli 537c1a4de99SAndrea Arcangeli dst_pmd = mm_alloc_pmd(dst_mm, dst_addr); 538c1a4de99SAndrea Arcangeli if (unlikely(!dst_pmd)) { 539c1a4de99SAndrea Arcangeli err = -ENOMEM; 540c1a4de99SAndrea Arcangeli break; 541c1a4de99SAndrea Arcangeli } 542c1a4de99SAndrea Arcangeli 543c1a4de99SAndrea Arcangeli dst_pmdval = pmd_read_atomic(dst_pmd); 544c1a4de99SAndrea Arcangeli /* 545c1a4de99SAndrea Arcangeli * If the dst_pmd is mapped as THP don't 546c1a4de99SAndrea Arcangeli * override it and just be strict. 547c1a4de99SAndrea Arcangeli */ 548c1a4de99SAndrea Arcangeli if (unlikely(pmd_trans_huge(dst_pmdval))) { 549c1a4de99SAndrea Arcangeli err = -EEXIST; 550c1a4de99SAndrea Arcangeli break; 551c1a4de99SAndrea Arcangeli } 552c1a4de99SAndrea Arcangeli if (unlikely(pmd_none(dst_pmdval)) && 553*4cf58924SJoel Fernandes (Google) unlikely(__pte_alloc(dst_mm, dst_pmd))) { 554c1a4de99SAndrea Arcangeli err = -ENOMEM; 555c1a4de99SAndrea Arcangeli break; 556c1a4de99SAndrea Arcangeli } 557c1a4de99SAndrea Arcangeli /* If an huge pmd materialized from under us fail */ 558c1a4de99SAndrea Arcangeli if (unlikely(pmd_trans_huge(*dst_pmd))) { 559c1a4de99SAndrea Arcangeli err = -EFAULT; 560c1a4de99SAndrea Arcangeli break; 561c1a4de99SAndrea Arcangeli } 562c1a4de99SAndrea Arcangeli 563c1a4de99SAndrea Arcangeli BUG_ON(pmd_none(*dst_pmd)); 564c1a4de99SAndrea Arcangeli BUG_ON(pmd_trans_huge(*dst_pmd)); 565c1a4de99SAndrea Arcangeli 5663217d3c7SMike Rapoport err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, 5673217d3c7SMike Rapoport src_addr, &page, zeropage); 568c1a4de99SAndrea Arcangeli cond_resched(); 569c1a4de99SAndrea Arcangeli 5709e368259SAndrea Arcangeli if (unlikely(err == -ENOENT)) { 571b6ebaedbSAndrea Arcangeli void *page_kaddr; 572b6ebaedbSAndrea Arcangeli 573b6ebaedbSAndrea Arcangeli up_read(&dst_mm->mmap_sem); 574b6ebaedbSAndrea Arcangeli BUG_ON(!page); 575b6ebaedbSAndrea Arcangeli 576b6ebaedbSAndrea Arcangeli page_kaddr = kmap(page); 577b6ebaedbSAndrea Arcangeli err = copy_from_user(page_kaddr, 578b6ebaedbSAndrea Arcangeli (const void __user *) src_addr, 579b6ebaedbSAndrea Arcangeli PAGE_SIZE); 580b6ebaedbSAndrea Arcangeli kunmap(page); 581b6ebaedbSAndrea Arcangeli if (unlikely(err)) { 582b6ebaedbSAndrea Arcangeli err = -EFAULT; 583b6ebaedbSAndrea Arcangeli goto out; 584b6ebaedbSAndrea Arcangeli } 585b6ebaedbSAndrea Arcangeli goto retry; 586b6ebaedbSAndrea Arcangeli } else 587b6ebaedbSAndrea Arcangeli BUG_ON(page); 588b6ebaedbSAndrea Arcangeli 589c1a4de99SAndrea Arcangeli if (!err) { 590c1a4de99SAndrea Arcangeli dst_addr += PAGE_SIZE; 591c1a4de99SAndrea Arcangeli src_addr += PAGE_SIZE; 592c1a4de99SAndrea Arcangeli copied += PAGE_SIZE; 593c1a4de99SAndrea Arcangeli 594c1a4de99SAndrea Arcangeli if (fatal_signal_pending(current)) 595c1a4de99SAndrea Arcangeli err = -EINTR; 596c1a4de99SAndrea Arcangeli } 597c1a4de99SAndrea Arcangeli if (err) 598c1a4de99SAndrea Arcangeli break; 599c1a4de99SAndrea Arcangeli } 600c1a4de99SAndrea Arcangeli 601b6ebaedbSAndrea Arcangeli out_unlock: 602c1a4de99SAndrea Arcangeli up_read(&dst_mm->mmap_sem); 603b6ebaedbSAndrea Arcangeli out: 604b6ebaedbSAndrea Arcangeli if (page) 60509cbfeafSKirill A. Shutemov put_page(page); 606c1a4de99SAndrea Arcangeli BUG_ON(copied < 0); 607c1a4de99SAndrea Arcangeli BUG_ON(err > 0); 608c1a4de99SAndrea Arcangeli BUG_ON(!copied && !err); 609c1a4de99SAndrea Arcangeli return copied ? copied : err; 610c1a4de99SAndrea Arcangeli } 611c1a4de99SAndrea Arcangeli 612c1a4de99SAndrea Arcangeli ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, 613df2cc96eSMike Rapoport unsigned long src_start, unsigned long len, 614df2cc96eSMike Rapoport bool *mmap_changing) 615c1a4de99SAndrea Arcangeli { 616df2cc96eSMike Rapoport return __mcopy_atomic(dst_mm, dst_start, src_start, len, false, 617df2cc96eSMike Rapoport mmap_changing); 618c1a4de99SAndrea Arcangeli } 619c1a4de99SAndrea Arcangeli 620c1a4de99SAndrea Arcangeli ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start, 621df2cc96eSMike Rapoport unsigned long len, bool *mmap_changing) 622c1a4de99SAndrea Arcangeli { 623df2cc96eSMike Rapoport return __mcopy_atomic(dst_mm, start, 0, len, true, mmap_changing); 624c1a4de99SAndrea Arcangeli } 625