11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * mm/mremap.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * (C) Copyright 1996 Linus Torvalds 51da177e4SLinus Torvalds * 6046c6884SAlan Cox * Address space accounting code <alan@lxorguk.ukuu.org.uk> 71da177e4SLinus Torvalds * (C) Copyright 2002 Red Hat Inc, All Rights Reserved 81da177e4SLinus Torvalds */ 91da177e4SLinus Torvalds 101da177e4SLinus Torvalds #include <linux/mm.h> 111da177e4SLinus Torvalds #include <linux/hugetlb.h> 121da177e4SLinus Torvalds #include <linux/shm.h> 131ff82995SHugh Dickins #include <linux/ksm.h> 141da177e4SLinus Torvalds #include <linux/mman.h> 151da177e4SLinus Torvalds #include <linux/swap.h> 16c59ede7bSRandy.Dunlap #include <linux/capability.h> 171da177e4SLinus Torvalds #include <linux/fs.h> 186dec97dcSCyrill Gorcunov #include <linux/swapops.h> 191da177e4SLinus Torvalds #include <linux/highmem.h> 201da177e4SLinus Torvalds #include <linux/security.h> 211da177e4SLinus Torvalds #include <linux/syscalls.h> 22cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h> 23cf4aebc2SClark Williams #include <linux/sched/sysctl.h> 24*2581d202SPaul McQuade #include <linux/uaccess.h> 251da177e4SLinus Torvalds 261da177e4SLinus Torvalds #include <asm/cacheflush.h> 271da177e4SLinus Torvalds #include <asm/tlbflush.h> 281da177e4SLinus Torvalds 29ba470de4SRik van Riel #include "internal.h" 30ba470de4SRik van Riel 317be7a546SHugh Dickins static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr) 321da177e4SLinus Torvalds { 331da177e4SLinus Torvalds pgd_t *pgd; 341da177e4SLinus Torvalds pud_t *pud; 351da177e4SLinus Torvalds pmd_t *pmd; 361da177e4SLinus Torvalds 371da177e4SLinus Torvalds pgd = pgd_offset(mm, addr); 381da177e4SLinus Torvalds if (pgd_none_or_clear_bad(pgd)) 391da177e4SLinus Torvalds return NULL; 401da177e4SLinus Torvalds 411da177e4SLinus Torvalds pud = pud_offset(pgd, addr); 421da177e4SLinus Torvalds if (pud_none_or_clear_bad(pud)) 431da177e4SLinus Torvalds return NULL; 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds pmd = pmd_offset(pud, addr); 4637a1c49aSAndrea Arcangeli if (pmd_none(*pmd)) 471da177e4SLinus Torvalds return NULL; 481da177e4SLinus Torvalds 497be7a546SHugh Dickins return pmd; 501da177e4SLinus Torvalds } 511da177e4SLinus Torvalds 528ac1f832SAndrea Arcangeli static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma, 538ac1f832SAndrea Arcangeli unsigned long addr) 541da177e4SLinus Torvalds { 551da177e4SLinus Torvalds pgd_t *pgd; 561da177e4SLinus Torvalds pud_t *pud; 57c74df32cSHugh Dickins pmd_t *pmd; 581da177e4SLinus Torvalds 591da177e4SLinus Torvalds pgd = pgd_offset(mm, addr); 601da177e4SLinus Torvalds pud = pud_alloc(mm, pgd, addr); 611da177e4SLinus Torvalds if (!pud) 62c74df32cSHugh Dickins return NULL; 637be7a546SHugh Dickins 641da177e4SLinus Torvalds pmd = pmd_alloc(mm, pud, addr); 6557a8f0cdSHugh Dickins if (!pmd) 66c74df32cSHugh Dickins return NULL; 677be7a546SHugh Dickins 688ac1f832SAndrea Arcangeli VM_BUG_ON(pmd_trans_huge(*pmd)); 69c74df32cSHugh Dickins 707be7a546SHugh Dickins return pmd; 711da177e4SLinus Torvalds } 721da177e4SLinus Torvalds 736dec97dcSCyrill Gorcunov static pte_t move_soft_dirty_pte(pte_t pte) 746dec97dcSCyrill Gorcunov { 756dec97dcSCyrill Gorcunov /* 766dec97dcSCyrill Gorcunov * Set soft dirty bit so we can notice 776dec97dcSCyrill Gorcunov * in userspace the ptes were moved. 786dec97dcSCyrill Gorcunov */ 796dec97dcSCyrill Gorcunov #ifdef CONFIG_MEM_SOFT_DIRTY 806dec97dcSCyrill Gorcunov if (pte_present(pte)) 816dec97dcSCyrill Gorcunov pte = pte_mksoft_dirty(pte); 826dec97dcSCyrill Gorcunov else if (is_swap_pte(pte)) 836dec97dcSCyrill Gorcunov pte = pte_swp_mksoft_dirty(pte); 846dec97dcSCyrill Gorcunov else if (pte_file(pte)) 856dec97dcSCyrill Gorcunov pte = pte_file_mksoft_dirty(pte); 866dec97dcSCyrill Gorcunov #endif 876dec97dcSCyrill Gorcunov return pte; 886dec97dcSCyrill Gorcunov } 896dec97dcSCyrill Gorcunov 907be7a546SHugh Dickins static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, 917be7a546SHugh Dickins unsigned long old_addr, unsigned long old_end, 927be7a546SHugh Dickins struct vm_area_struct *new_vma, pmd_t *new_pmd, 9338a76013SMichel Lespinasse unsigned long new_addr, bool need_rmap_locks) 941da177e4SLinus Torvalds { 951da177e4SLinus Torvalds struct address_space *mapping = NULL; 9638a76013SMichel Lespinasse struct anon_vma *anon_vma = NULL; 971da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 987be7a546SHugh Dickins pte_t *old_pte, *new_pte, pte; 994c21e2f2SHugh Dickins spinlock_t *old_ptl, *new_ptl; 1001da177e4SLinus Torvalds 1011da177e4SLinus Torvalds /* 10238a76013SMichel Lespinasse * When need_rmap_locks is true, we take the i_mmap_mutex and anon_vma 10338a76013SMichel Lespinasse * locks to ensure that rmap will always observe either the old or the 10438a76013SMichel Lespinasse * new ptes. This is the easiest way to avoid races with 10538a76013SMichel Lespinasse * truncate_pagecache(), page migration, etc... 10638a76013SMichel Lespinasse * 10738a76013SMichel Lespinasse * When need_rmap_locks is false, we use other ways to avoid 10838a76013SMichel Lespinasse * such races: 10938a76013SMichel Lespinasse * 11038a76013SMichel Lespinasse * - During exec() shift_arg_pages(), we use a specially tagged vma 11138a76013SMichel Lespinasse * which rmap call sites look for using is_vma_temporary_stack(). 11238a76013SMichel Lespinasse * 11338a76013SMichel Lespinasse * - During mremap(), new_vma is often known to be placed after vma 11438a76013SMichel Lespinasse * in rmap traversal order. This ensures rmap will always observe 11538a76013SMichel Lespinasse * either the old pte, or the new pte, or both (the page table locks 11638a76013SMichel Lespinasse * serialize access to individual ptes, but only rmap traversal 11738a76013SMichel Lespinasse * order guarantees that we won't miss both the old and new ptes). 1181da177e4SLinus Torvalds */ 11938a76013SMichel Lespinasse if (need_rmap_locks) { 12038a76013SMichel Lespinasse if (vma->vm_file) { 1211da177e4SLinus Torvalds mapping = vma->vm_file->f_mapping; 1223d48ae45SPeter Zijlstra mutex_lock(&mapping->i_mmap_mutex); 1231da177e4SLinus Torvalds } 12438a76013SMichel Lespinasse if (vma->anon_vma) { 12538a76013SMichel Lespinasse anon_vma = vma->anon_vma; 1264fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 12738a76013SMichel Lespinasse } 12838a76013SMichel Lespinasse } 1291da177e4SLinus Torvalds 1304c21e2f2SHugh Dickins /* 1314c21e2f2SHugh Dickins * We don't have to worry about the ordering of src and dst 1324c21e2f2SHugh Dickins * pte locks because exclusive mmap_sem prevents deadlock. 1334c21e2f2SHugh Dickins */ 134c74df32cSHugh Dickins old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl); 135ece0e2b6SPeter Zijlstra new_pte = pte_offset_map(new_pmd, new_addr); 1364c21e2f2SHugh Dickins new_ptl = pte_lockptr(mm, new_pmd); 1374c21e2f2SHugh Dickins if (new_ptl != old_ptl) 138f20dc5f7SIngo Molnar spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 1396606c3e0SZachary Amsden arch_enter_lazy_mmu_mode(); 1408b1f3124SNick Piggin 1417be7a546SHugh Dickins for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE, 1427be7a546SHugh Dickins new_pte++, new_addr += PAGE_SIZE) { 1437be7a546SHugh Dickins if (pte_none(*old_pte)) 1447be7a546SHugh Dickins continue; 1457b6efc2bSAndrea Arcangeli pte = ptep_get_and_clear(mm, old_addr, old_pte); 1467be7a546SHugh Dickins pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr); 1476dec97dcSCyrill Gorcunov pte = move_soft_dirty_pte(pte); 1486dec97dcSCyrill Gorcunov set_pte_at(mm, new_addr, new_pte, pte); 1491da177e4SLinus Torvalds } 1507be7a546SHugh Dickins 1516606c3e0SZachary Amsden arch_leave_lazy_mmu_mode(); 1524c21e2f2SHugh Dickins if (new_ptl != old_ptl) 1534c21e2f2SHugh Dickins spin_unlock(new_ptl); 154ece0e2b6SPeter Zijlstra pte_unmap(new_pte - 1); 155c74df32cSHugh Dickins pte_unmap_unlock(old_pte - 1, old_ptl); 156108d6642SMichel Lespinasse if (anon_vma) 15708b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 1581da177e4SLinus Torvalds if (mapping) 1593d48ae45SPeter Zijlstra mutex_unlock(&mapping->i_mmap_mutex); 1601da177e4SLinus Torvalds } 1611da177e4SLinus Torvalds 1627be7a546SHugh Dickins #define LATENCY_LIMIT (64 * PAGE_SIZE) 1637be7a546SHugh Dickins 164b6a2fea3SOllie Wild unsigned long move_page_tables(struct vm_area_struct *vma, 1651da177e4SLinus Torvalds unsigned long old_addr, struct vm_area_struct *new_vma, 16638a76013SMichel Lespinasse unsigned long new_addr, unsigned long len, 16738a76013SMichel Lespinasse bool need_rmap_locks) 1681da177e4SLinus Torvalds { 1697be7a546SHugh Dickins unsigned long extent, next, old_end; 1707be7a546SHugh Dickins pmd_t *old_pmd, *new_pmd; 1717b6efc2bSAndrea Arcangeli bool need_flush = false; 1722ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 1732ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 1741da177e4SLinus Torvalds 1757be7a546SHugh Dickins old_end = old_addr + len; 1767be7a546SHugh Dickins flush_cache_range(vma, old_addr, old_end); 1771da177e4SLinus Torvalds 1782ec74c3eSSagi Grimberg mmun_start = old_addr; 1792ec74c3eSSagi Grimberg mmun_end = old_end; 1802ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end); 1817b6efc2bSAndrea Arcangeli 1827be7a546SHugh Dickins for (; old_addr < old_end; old_addr += extent, new_addr += extent) { 1831da177e4SLinus Torvalds cond_resched(); 1847be7a546SHugh Dickins next = (old_addr + PMD_SIZE) & PMD_MASK; 185ebed4846SAndrea Arcangeli /* even if next overflowed, extent below will be ok */ 1867be7a546SHugh Dickins extent = next - old_addr; 187ebed4846SAndrea Arcangeli if (extent > old_end - old_addr) 188ebed4846SAndrea Arcangeli extent = old_end - old_addr; 1897be7a546SHugh Dickins old_pmd = get_old_pmd(vma->vm_mm, old_addr); 1907be7a546SHugh Dickins if (!old_pmd) 1917be7a546SHugh Dickins continue; 1928ac1f832SAndrea Arcangeli new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr); 1937be7a546SHugh Dickins if (!new_pmd) 1947be7a546SHugh Dickins break; 19537a1c49aSAndrea Arcangeli if (pmd_trans_huge(*old_pmd)) { 19637a1c49aSAndrea Arcangeli int err = 0; 197dd18dbc2SKirill A. Shutemov if (extent == HPAGE_PMD_SIZE) { 19881d1b09cSSasha Levin VM_BUG_ON_VMA(vma->vm_file || !vma->anon_vma, 19981d1b09cSSasha Levin vma); 200dd18dbc2SKirill A. Shutemov /* See comment in move_ptes() */ 201dd18dbc2SKirill A. Shutemov if (need_rmap_locks) 202dd18dbc2SKirill A. Shutemov anon_vma_lock_write(vma->anon_vma); 20337a1c49aSAndrea Arcangeli err = move_huge_pmd(vma, new_vma, old_addr, 20437a1c49aSAndrea Arcangeli new_addr, old_end, 20537a1c49aSAndrea Arcangeli old_pmd, new_pmd); 206dd18dbc2SKirill A. Shutemov if (need_rmap_locks) 207dd18dbc2SKirill A. Shutemov anon_vma_unlock_write(vma->anon_vma); 208dd18dbc2SKirill A. Shutemov } 20937a1c49aSAndrea Arcangeli if (err > 0) { 21037a1c49aSAndrea Arcangeli need_flush = true; 21137a1c49aSAndrea Arcangeli continue; 21237a1c49aSAndrea Arcangeli } else if (!err) { 213e180377fSKirill A. Shutemov split_huge_page_pmd(vma, old_addr, old_pmd); 21437a1c49aSAndrea Arcangeli } 21537a1c49aSAndrea Arcangeli VM_BUG_ON(pmd_trans_huge(*old_pmd)); 21637a1c49aSAndrea Arcangeli } 21737a1c49aSAndrea Arcangeli if (pmd_none(*new_pmd) && __pte_alloc(new_vma->vm_mm, new_vma, 21837a1c49aSAndrea Arcangeli new_pmd, new_addr)) 21937a1c49aSAndrea Arcangeli break; 2207be7a546SHugh Dickins next = (new_addr + PMD_SIZE) & PMD_MASK; 2217be7a546SHugh Dickins if (extent > next - new_addr) 2227be7a546SHugh Dickins extent = next - new_addr; 2237be7a546SHugh Dickins if (extent > LATENCY_LIMIT) 2247be7a546SHugh Dickins extent = LATENCY_LIMIT; 2257be7a546SHugh Dickins move_ptes(vma, old_pmd, old_addr, old_addr + extent, 22638a76013SMichel Lespinasse new_vma, new_pmd, new_addr, need_rmap_locks); 2277b6efc2bSAndrea Arcangeli need_flush = true; 2281da177e4SLinus Torvalds } 2297b6efc2bSAndrea Arcangeli if (likely(need_flush)) 2307b6efc2bSAndrea Arcangeli flush_tlb_range(vma, old_end-len, old_addr); 2317b6efc2bSAndrea Arcangeli 2322ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end); 2337be7a546SHugh Dickins 2347be7a546SHugh Dickins return len + old_addr - old_end; /* how much done */ 2351da177e4SLinus Torvalds } 2361da177e4SLinus Torvalds 2371da177e4SLinus Torvalds static unsigned long move_vma(struct vm_area_struct *vma, 2381da177e4SLinus Torvalds unsigned long old_addr, unsigned long old_len, 23981909b84SMichel Lespinasse unsigned long new_len, unsigned long new_addr, bool *locked) 2401da177e4SLinus Torvalds { 2411da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 2421da177e4SLinus Torvalds struct vm_area_struct *new_vma; 2431da177e4SLinus Torvalds unsigned long vm_flags = vma->vm_flags; 2441da177e4SLinus Torvalds unsigned long new_pgoff; 2451da177e4SLinus Torvalds unsigned long moved_len; 2461da177e4SLinus Torvalds unsigned long excess = 0; 247365e9c87SHugh Dickins unsigned long hiwater_vm; 2481da177e4SLinus Torvalds int split = 0; 2497103ad32SHugh Dickins int err; 25038a76013SMichel Lespinasse bool need_rmap_locks; 2511da177e4SLinus Torvalds 2521da177e4SLinus Torvalds /* 2531da177e4SLinus Torvalds * We'd prefer to avoid failure later on in do_munmap: 2541da177e4SLinus Torvalds * which may split one vma into three before unmapping. 2551da177e4SLinus Torvalds */ 2561da177e4SLinus Torvalds if (mm->map_count >= sysctl_max_map_count - 3) 2571da177e4SLinus Torvalds return -ENOMEM; 2581da177e4SLinus Torvalds 2591ff82995SHugh Dickins /* 2601ff82995SHugh Dickins * Advise KSM to break any KSM pages in the area to be moved: 2611ff82995SHugh Dickins * it would be confusing if they were to turn up at the new 2621ff82995SHugh Dickins * location, where they happen to coincide with different KSM 2631ff82995SHugh Dickins * pages recently unmapped. But leave vma->vm_flags as it was, 2641ff82995SHugh Dickins * so KSM can come around to merge on vma and new_vma afterwards. 2651ff82995SHugh Dickins */ 2667103ad32SHugh Dickins err = ksm_madvise(vma, old_addr, old_addr + old_len, 2677103ad32SHugh Dickins MADV_UNMERGEABLE, &vm_flags); 2687103ad32SHugh Dickins if (err) 2697103ad32SHugh Dickins return err; 2701ff82995SHugh Dickins 2711da177e4SLinus Torvalds new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT); 27238a76013SMichel Lespinasse new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff, 27338a76013SMichel Lespinasse &need_rmap_locks); 2741da177e4SLinus Torvalds if (!new_vma) 2751da177e4SLinus Torvalds return -ENOMEM; 2761da177e4SLinus Torvalds 27738a76013SMichel Lespinasse moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len, 27838a76013SMichel Lespinasse need_rmap_locks); 2791da177e4SLinus Torvalds if (moved_len < old_len) { 2801da177e4SLinus Torvalds /* 2811da177e4SLinus Torvalds * On error, move entries back from new area to old, 2821da177e4SLinus Torvalds * which will succeed since page tables still there, 2831da177e4SLinus Torvalds * and then proceed to unmap new area instead of old. 2841da177e4SLinus Torvalds */ 28538a76013SMichel Lespinasse move_page_tables(new_vma, new_addr, vma, old_addr, moved_len, 28638a76013SMichel Lespinasse true); 2871da177e4SLinus Torvalds vma = new_vma; 2881da177e4SLinus Torvalds old_len = new_len; 2891da177e4SLinus Torvalds old_addr = new_addr; 2901da177e4SLinus Torvalds new_addr = -ENOMEM; 2911da177e4SLinus Torvalds } 2921da177e4SLinus Torvalds 2931da177e4SLinus Torvalds /* Conceal VM_ACCOUNT so old reservation is not undone */ 2941da177e4SLinus Torvalds if (vm_flags & VM_ACCOUNT) { 2951da177e4SLinus Torvalds vma->vm_flags &= ~VM_ACCOUNT; 2961da177e4SLinus Torvalds excess = vma->vm_end - vma->vm_start - old_len; 2971da177e4SLinus Torvalds if (old_addr > vma->vm_start && 2981da177e4SLinus Torvalds old_addr + old_len < vma->vm_end) 2991da177e4SLinus Torvalds split = 1; 3001da177e4SLinus Torvalds } 3011da177e4SLinus Torvalds 30271799062SKirill Korotaev /* 303365e9c87SHugh Dickins * If we failed to move page tables we still do total_vm increment 304365e9c87SHugh Dickins * since do_munmap() will decrement it by old_len == new_len. 305365e9c87SHugh Dickins * 306365e9c87SHugh Dickins * Since total_vm is about to be raised artificially high for a 307365e9c87SHugh Dickins * moment, we need to restore high watermark afterwards: if stats 308365e9c87SHugh Dickins * are taken meanwhile, total_vm and hiwater_vm appear too high. 309365e9c87SHugh Dickins * If this were a serious issue, we'd add a flag to do_munmap(). 31071799062SKirill Korotaev */ 311365e9c87SHugh Dickins hiwater_vm = mm->hiwater_vm; 312ab50b8edSHugh Dickins vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT); 31371799062SKirill Korotaev 3141da177e4SLinus Torvalds if (do_munmap(mm, old_addr, old_len) < 0) { 3151da177e4SLinus Torvalds /* OOM: unable to split vma, just get accounts right */ 3161da177e4SLinus Torvalds vm_unacct_memory(excess >> PAGE_SHIFT); 3171da177e4SLinus Torvalds excess = 0; 3181da177e4SLinus Torvalds } 319365e9c87SHugh Dickins mm->hiwater_vm = hiwater_vm; 3201da177e4SLinus Torvalds 3211da177e4SLinus Torvalds /* Restore VM_ACCOUNT if one or two pieces of vma left */ 3221da177e4SLinus Torvalds if (excess) { 3231da177e4SLinus Torvalds vma->vm_flags |= VM_ACCOUNT; 3241da177e4SLinus Torvalds if (split) 3251da177e4SLinus Torvalds vma->vm_next->vm_flags |= VM_ACCOUNT; 3261da177e4SLinus Torvalds } 3271da177e4SLinus Torvalds 3281da177e4SLinus Torvalds if (vm_flags & VM_LOCKED) { 3291da177e4SLinus Torvalds mm->locked_vm += new_len >> PAGE_SHIFT; 33081909b84SMichel Lespinasse *locked = true; 3311da177e4SLinus Torvalds } 3321da177e4SLinus Torvalds 3331da177e4SLinus Torvalds return new_addr; 3341da177e4SLinus Torvalds } 3351da177e4SLinus Torvalds 33654f5de70SAl Viro static struct vm_area_struct *vma_to_resize(unsigned long addr, 33754f5de70SAl Viro unsigned long old_len, unsigned long new_len, unsigned long *p) 33854f5de70SAl Viro { 33954f5de70SAl Viro struct mm_struct *mm = current->mm; 34054f5de70SAl Viro struct vm_area_struct *vma = find_vma(mm, addr); 34154f5de70SAl Viro 34254f5de70SAl Viro if (!vma || vma->vm_start > addr) 34354f5de70SAl Viro goto Efault; 34454f5de70SAl Viro 34554f5de70SAl Viro if (is_vm_hugetlb_page(vma)) 34654f5de70SAl Viro goto Einval; 34754f5de70SAl Viro 34854f5de70SAl Viro /* We can't remap across vm area boundaries */ 34954f5de70SAl Viro if (old_len > vma->vm_end - addr) 35054f5de70SAl Viro goto Efault; 35154f5de70SAl Viro 352982134baSLinus Torvalds /* Need to be careful about a growing mapping */ 353982134baSLinus Torvalds if (new_len > old_len) { 354982134baSLinus Torvalds unsigned long pgoff; 355982134baSLinus Torvalds 356982134baSLinus Torvalds if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) 35754f5de70SAl Viro goto Efault; 358982134baSLinus Torvalds pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; 359982134baSLinus Torvalds pgoff += vma->vm_pgoff; 360982134baSLinus Torvalds if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) 361982134baSLinus Torvalds goto Einval; 36254f5de70SAl Viro } 36354f5de70SAl Viro 36454f5de70SAl Viro if (vma->vm_flags & VM_LOCKED) { 36554f5de70SAl Viro unsigned long locked, lock_limit; 36654f5de70SAl Viro locked = mm->locked_vm << PAGE_SHIFT; 36759e99e5bSJiri Slaby lock_limit = rlimit(RLIMIT_MEMLOCK); 36854f5de70SAl Viro locked += new_len - old_len; 36954f5de70SAl Viro if (locked > lock_limit && !capable(CAP_IPC_LOCK)) 37054f5de70SAl Viro goto Eagain; 37154f5de70SAl Viro } 37254f5de70SAl Viro 37354f5de70SAl Viro if (!may_expand_vm(mm, (new_len - old_len) >> PAGE_SHIFT)) 37454f5de70SAl Viro goto Enomem; 37554f5de70SAl Viro 37654f5de70SAl Viro if (vma->vm_flags & VM_ACCOUNT) { 37754f5de70SAl Viro unsigned long charged = (new_len - old_len) >> PAGE_SHIFT; 378191c5424SAl Viro if (security_vm_enough_memory_mm(mm, charged)) 37954f5de70SAl Viro goto Efault; 38054f5de70SAl Viro *p = charged; 38154f5de70SAl Viro } 38254f5de70SAl Viro 38354f5de70SAl Viro return vma; 38454f5de70SAl Viro 38554f5de70SAl Viro Efault: /* very odd choice for most of the cases, but... */ 38654f5de70SAl Viro return ERR_PTR(-EFAULT); 38754f5de70SAl Viro Einval: 38854f5de70SAl Viro return ERR_PTR(-EINVAL); 38954f5de70SAl Viro Enomem: 39054f5de70SAl Viro return ERR_PTR(-ENOMEM); 39154f5de70SAl Viro Eagain: 39254f5de70SAl Viro return ERR_PTR(-EAGAIN); 39354f5de70SAl Viro } 39454f5de70SAl Viro 39581909b84SMichel Lespinasse static unsigned long mremap_to(unsigned long addr, unsigned long old_len, 39681909b84SMichel Lespinasse unsigned long new_addr, unsigned long new_len, bool *locked) 397ecc1a899SAl Viro { 398ecc1a899SAl Viro struct mm_struct *mm = current->mm; 399ecc1a899SAl Viro struct vm_area_struct *vma; 400ecc1a899SAl Viro unsigned long ret = -EINVAL; 401ecc1a899SAl Viro unsigned long charged = 0; 402097eed10SAl Viro unsigned long map_flags; 403ecc1a899SAl Viro 404ecc1a899SAl Viro if (new_addr & ~PAGE_MASK) 405ecc1a899SAl Viro goto out; 406ecc1a899SAl Viro 407ecc1a899SAl Viro if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len) 408ecc1a899SAl Viro goto out; 409ecc1a899SAl Viro 410ecc1a899SAl Viro /* Check if the location we're moving into overlaps the 411ecc1a899SAl Viro * old location at all, and fail if it does. 412ecc1a899SAl Viro */ 413ecc1a899SAl Viro if ((new_addr <= addr) && (new_addr+new_len) > addr) 414ecc1a899SAl Viro goto out; 415ecc1a899SAl Viro 416ecc1a899SAl Viro if ((addr <= new_addr) && (addr+old_len) > new_addr) 417ecc1a899SAl Viro goto out; 418ecc1a899SAl Viro 419ecc1a899SAl Viro ret = do_munmap(mm, new_addr, new_len); 420ecc1a899SAl Viro if (ret) 421ecc1a899SAl Viro goto out; 422ecc1a899SAl Viro 423ecc1a899SAl Viro if (old_len >= new_len) { 424ecc1a899SAl Viro ret = do_munmap(mm, addr+new_len, old_len - new_len); 425ecc1a899SAl Viro if (ret && old_len != new_len) 426ecc1a899SAl Viro goto out; 427ecc1a899SAl Viro old_len = new_len; 428ecc1a899SAl Viro } 429ecc1a899SAl Viro 430ecc1a899SAl Viro vma = vma_to_resize(addr, old_len, new_len, &charged); 431ecc1a899SAl Viro if (IS_ERR(vma)) { 432ecc1a899SAl Viro ret = PTR_ERR(vma); 433ecc1a899SAl Viro goto out; 434ecc1a899SAl Viro } 435ecc1a899SAl Viro 436097eed10SAl Viro map_flags = MAP_FIXED; 437097eed10SAl Viro if (vma->vm_flags & VM_MAYSHARE) 438097eed10SAl Viro map_flags |= MAP_SHARED; 4399206de95SAl Viro 440097eed10SAl Viro ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff + 441097eed10SAl Viro ((addr - vma->vm_start) >> PAGE_SHIFT), 442097eed10SAl Viro map_flags); 443ecc1a899SAl Viro if (ret & ~PAGE_MASK) 444097eed10SAl Viro goto out1; 445097eed10SAl Viro 44681909b84SMichel Lespinasse ret = move_vma(vma, addr, old_len, new_len, new_addr, locked); 447097eed10SAl Viro if (!(ret & ~PAGE_MASK)) 448097eed10SAl Viro goto out; 449097eed10SAl Viro out1: 450ecc1a899SAl Viro vm_unacct_memory(charged); 451ecc1a899SAl Viro 452ecc1a899SAl Viro out: 453ecc1a899SAl Viro return ret; 454ecc1a899SAl Viro } 455ecc1a899SAl Viro 4561a0ef85fSAl Viro static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) 4571a0ef85fSAl Viro { 458f106af4eSAl Viro unsigned long end = vma->vm_end + delta; 4599206de95SAl Viro if (end < vma->vm_end) /* overflow */ 4601a0ef85fSAl Viro return 0; 4619206de95SAl Viro if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */ 462f106af4eSAl Viro return 0; 463f106af4eSAl Viro if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, 464f106af4eSAl Viro 0, MAP_FIXED) & ~PAGE_MASK) 465f106af4eSAl Viro return 0; 4661a0ef85fSAl Viro return 1; 4671a0ef85fSAl Viro } 4681a0ef85fSAl Viro 4691da177e4SLinus Torvalds /* 4701da177e4SLinus Torvalds * Expand (or shrink) an existing mapping, potentially moving it at the 4711da177e4SLinus Torvalds * same time (controlled by the MREMAP_MAYMOVE flag and available VM space) 4721da177e4SLinus Torvalds * 4731da177e4SLinus Torvalds * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise 4741da177e4SLinus Torvalds * This option implies MREMAP_MAYMOVE. 4751da177e4SLinus Torvalds */ 47663a81db1SAl Viro SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, 47763a81db1SAl Viro unsigned long, new_len, unsigned long, flags, 47863a81db1SAl Viro unsigned long, new_addr) 4791da177e4SLinus Torvalds { 480d0de32d9SHugh Dickins struct mm_struct *mm = current->mm; 4811da177e4SLinus Torvalds struct vm_area_struct *vma; 4821da177e4SLinus Torvalds unsigned long ret = -EINVAL; 4831da177e4SLinus Torvalds unsigned long charged = 0; 48481909b84SMichel Lespinasse bool locked = false; 4851da177e4SLinus Torvalds 4861da177e4SLinus Torvalds if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE)) 4879a2458a6SRasmus Villemoes return ret; 4889a2458a6SRasmus Villemoes 4899a2458a6SRasmus Villemoes if (flags & MREMAP_FIXED && !(flags & MREMAP_MAYMOVE)) 4909a2458a6SRasmus Villemoes return ret; 4911da177e4SLinus Torvalds 4921da177e4SLinus Torvalds if (addr & ~PAGE_MASK) 4939a2458a6SRasmus Villemoes return ret; 4941da177e4SLinus Torvalds 4951da177e4SLinus Torvalds old_len = PAGE_ALIGN(old_len); 4961da177e4SLinus Torvalds new_len = PAGE_ALIGN(new_len); 4971da177e4SLinus Torvalds 4981da177e4SLinus Torvalds /* 4991da177e4SLinus Torvalds * We allow a zero old-len as a special case 5001da177e4SLinus Torvalds * for DOS-emu "duplicate shm area" thing. But 5011da177e4SLinus Torvalds * a zero new-len is nonsensical. 5021da177e4SLinus Torvalds */ 5031da177e4SLinus Torvalds if (!new_len) 5049a2458a6SRasmus Villemoes return ret; 5059a2458a6SRasmus Villemoes 5069a2458a6SRasmus Villemoes down_write(¤t->mm->mmap_sem); 5071da177e4SLinus Torvalds 5081da177e4SLinus Torvalds if (flags & MREMAP_FIXED) { 50981909b84SMichel Lespinasse ret = mremap_to(addr, old_len, new_addr, new_len, 51081909b84SMichel Lespinasse &locked); 5111da177e4SLinus Torvalds goto out; 5121da177e4SLinus Torvalds } 5131da177e4SLinus Torvalds 5141da177e4SLinus Torvalds /* 5151da177e4SLinus Torvalds * Always allow a shrinking remap: that just unmaps 5161da177e4SLinus Torvalds * the unnecessary pages.. 5171da177e4SLinus Torvalds * do_munmap does all the needed commit accounting 5181da177e4SLinus Torvalds */ 5191da177e4SLinus Torvalds if (old_len >= new_len) { 520d0de32d9SHugh Dickins ret = do_munmap(mm, addr+new_len, old_len - new_len); 5211da177e4SLinus Torvalds if (ret && old_len != new_len) 5221da177e4SLinus Torvalds goto out; 5231da177e4SLinus Torvalds ret = addr; 5241da177e4SLinus Torvalds goto out; 5251da177e4SLinus Torvalds } 5261da177e4SLinus Torvalds 5271da177e4SLinus Torvalds /* 528ecc1a899SAl Viro * Ok, we need to grow.. 5291da177e4SLinus Torvalds */ 53054f5de70SAl Viro vma = vma_to_resize(addr, old_len, new_len, &charged); 53154f5de70SAl Viro if (IS_ERR(vma)) { 53254f5de70SAl Viro ret = PTR_ERR(vma); 5331da177e4SLinus Torvalds goto out; 5341da177e4SLinus Torvalds } 5351da177e4SLinus Torvalds 5361da177e4SLinus Torvalds /* old_len exactly to the end of the area.. 5371da177e4SLinus Torvalds */ 538ecc1a899SAl Viro if (old_len == vma->vm_end - addr) { 5391da177e4SLinus Torvalds /* can we just expand the current mapping? */ 5401a0ef85fSAl Viro if (vma_expandable(vma, new_len - old_len)) { 5411da177e4SLinus Torvalds int pages = (new_len - old_len) >> PAGE_SHIFT; 5421da177e4SLinus Torvalds 5435beb4930SRik van Riel if (vma_adjust(vma, vma->vm_start, addr + new_len, 5445beb4930SRik van Riel vma->vm_pgoff, NULL)) { 5455beb4930SRik van Riel ret = -ENOMEM; 5465beb4930SRik van Riel goto out; 5475beb4930SRik van Riel } 5481da177e4SLinus Torvalds 549d0de32d9SHugh Dickins vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages); 5501da177e4SLinus Torvalds if (vma->vm_flags & VM_LOCKED) { 551d0de32d9SHugh Dickins mm->locked_vm += pages; 55281909b84SMichel Lespinasse locked = true; 55381909b84SMichel Lespinasse new_addr = addr; 5541da177e4SLinus Torvalds } 5551da177e4SLinus Torvalds ret = addr; 5561da177e4SLinus Torvalds goto out; 5571da177e4SLinus Torvalds } 5581da177e4SLinus Torvalds } 5591da177e4SLinus Torvalds 5601da177e4SLinus Torvalds /* 5611da177e4SLinus Torvalds * We weren't able to just expand or shrink the area, 5621da177e4SLinus Torvalds * we need to create a new one and move it.. 5631da177e4SLinus Torvalds */ 5641da177e4SLinus Torvalds ret = -ENOMEM; 5651da177e4SLinus Torvalds if (flags & MREMAP_MAYMOVE) { 5661da177e4SLinus Torvalds unsigned long map_flags = 0; 5671da177e4SLinus Torvalds if (vma->vm_flags & VM_MAYSHARE) 5681da177e4SLinus Torvalds map_flags |= MAP_SHARED; 5691da177e4SLinus Torvalds 5701da177e4SLinus Torvalds new_addr = get_unmapped_area(vma->vm_file, 0, new_len, 57193587414SAl Viro vma->vm_pgoff + 57293587414SAl Viro ((addr - vma->vm_start) >> PAGE_SHIFT), 57393587414SAl Viro map_flags); 574ed032189SEric Paris if (new_addr & ~PAGE_MASK) { 5751da177e4SLinus Torvalds ret = new_addr; 576ed032189SEric Paris goto out; 577ed032189SEric Paris } 578ed032189SEric Paris 57981909b84SMichel Lespinasse ret = move_vma(vma, addr, old_len, new_len, new_addr, &locked); 5801da177e4SLinus Torvalds } 5811da177e4SLinus Torvalds out: 5821da177e4SLinus Torvalds if (ret & ~PAGE_MASK) 5831da177e4SLinus Torvalds vm_unacct_memory(charged); 5841da177e4SLinus Torvalds up_write(¤t->mm->mmap_sem); 58581909b84SMichel Lespinasse if (locked && new_len > old_len) 58681909b84SMichel Lespinasse mm_populate(new_addr + old_len, new_len - old_len); 5871da177e4SLinus Torvalds return ret; 5881da177e4SLinus Torvalds } 589