11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * mm/mremap.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * (C) Copyright 1996 Linus Torvalds 51da177e4SLinus Torvalds * 6046c6884SAlan Cox * Address space accounting code <alan@lxorguk.ukuu.org.uk> 71da177e4SLinus Torvalds * (C) Copyright 2002 Red Hat Inc, All Rights Reserved 81da177e4SLinus Torvalds */ 91da177e4SLinus Torvalds 101da177e4SLinus Torvalds #include <linux/mm.h> 111da177e4SLinus Torvalds #include <linux/hugetlb.h> 121da177e4SLinus Torvalds #include <linux/shm.h> 131ff82995SHugh Dickins #include <linux/ksm.h> 141da177e4SLinus Torvalds #include <linux/mman.h> 151da177e4SLinus Torvalds #include <linux/swap.h> 16c59ede7bSRandy.Dunlap #include <linux/capability.h> 171da177e4SLinus Torvalds #include <linux/fs.h> 186dec97dcSCyrill Gorcunov #include <linux/swapops.h> 191da177e4SLinus Torvalds #include <linux/highmem.h> 201da177e4SLinus Torvalds #include <linux/security.h> 211da177e4SLinus Torvalds #include <linux/syscalls.h> 22cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h> 23cf4aebc2SClark Williams #include <linux/sched/sysctl.h> 242581d202SPaul McQuade #include <linux/uaccess.h> 254abad2caSLaurent Dufour #include <linux/mm-arch-hooks.h> 261da177e4SLinus Torvalds 271da177e4SLinus Torvalds #include <asm/cacheflush.h> 281da177e4SLinus Torvalds #include <asm/tlbflush.h> 291da177e4SLinus Torvalds 30ba470de4SRik van Riel #include "internal.h" 31ba470de4SRik van Riel 327be7a546SHugh Dickins static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr) 331da177e4SLinus Torvalds { 341da177e4SLinus Torvalds pgd_t *pgd; 351da177e4SLinus Torvalds pud_t *pud; 361da177e4SLinus Torvalds pmd_t *pmd; 371da177e4SLinus Torvalds 381da177e4SLinus Torvalds pgd = pgd_offset(mm, addr); 391da177e4SLinus Torvalds if (pgd_none_or_clear_bad(pgd)) 401da177e4SLinus Torvalds return NULL; 411da177e4SLinus Torvalds 421da177e4SLinus Torvalds pud = pud_offset(pgd, addr); 431da177e4SLinus Torvalds if (pud_none_or_clear_bad(pud)) 441da177e4SLinus Torvalds return NULL; 451da177e4SLinus Torvalds 461da177e4SLinus Torvalds pmd = pmd_offset(pud, addr); 4737a1c49aSAndrea Arcangeli if (pmd_none(*pmd)) 481da177e4SLinus Torvalds return NULL; 491da177e4SLinus Torvalds 507be7a546SHugh Dickins return pmd; 511da177e4SLinus Torvalds } 521da177e4SLinus Torvalds 538ac1f832SAndrea Arcangeli static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma, 548ac1f832SAndrea Arcangeli unsigned long addr) 551da177e4SLinus Torvalds { 561da177e4SLinus Torvalds pgd_t *pgd; 571da177e4SLinus Torvalds pud_t *pud; 58c74df32cSHugh Dickins pmd_t *pmd; 591da177e4SLinus Torvalds 601da177e4SLinus Torvalds pgd = pgd_offset(mm, addr); 611da177e4SLinus Torvalds pud = pud_alloc(mm, pgd, addr); 621da177e4SLinus Torvalds if (!pud) 63c74df32cSHugh Dickins return NULL; 647be7a546SHugh Dickins 651da177e4SLinus Torvalds pmd = pmd_alloc(mm, pud, addr); 6657a8f0cdSHugh Dickins if (!pmd) 67c74df32cSHugh Dickins return NULL; 687be7a546SHugh Dickins 698ac1f832SAndrea Arcangeli VM_BUG_ON(pmd_trans_huge(*pmd)); 70c74df32cSHugh Dickins 717be7a546SHugh Dickins return pmd; 721da177e4SLinus Torvalds } 731da177e4SLinus Torvalds 746dec97dcSCyrill Gorcunov static pte_t move_soft_dirty_pte(pte_t pte) 756dec97dcSCyrill Gorcunov { 766dec97dcSCyrill Gorcunov /* 776dec97dcSCyrill Gorcunov * Set soft dirty bit so we can notice 786dec97dcSCyrill Gorcunov * in userspace the ptes were moved. 796dec97dcSCyrill Gorcunov */ 806dec97dcSCyrill Gorcunov #ifdef CONFIG_MEM_SOFT_DIRTY 816dec97dcSCyrill Gorcunov if (pte_present(pte)) 826dec97dcSCyrill Gorcunov pte = pte_mksoft_dirty(pte); 836dec97dcSCyrill Gorcunov else if (is_swap_pte(pte)) 846dec97dcSCyrill Gorcunov pte = pte_swp_mksoft_dirty(pte); 856dec97dcSCyrill Gorcunov #endif 866dec97dcSCyrill Gorcunov return pte; 876dec97dcSCyrill Gorcunov } 886dec97dcSCyrill Gorcunov 897be7a546SHugh Dickins static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, 907be7a546SHugh Dickins unsigned long old_addr, unsigned long old_end, 917be7a546SHugh Dickins struct vm_area_struct *new_vma, pmd_t *new_pmd, 9238a76013SMichel Lespinasse unsigned long new_addr, bool need_rmap_locks) 931da177e4SLinus Torvalds { 941da177e4SLinus Torvalds struct address_space *mapping = NULL; 9538a76013SMichel Lespinasse struct anon_vma *anon_vma = NULL; 961da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 977be7a546SHugh Dickins pte_t *old_pte, *new_pte, pte; 984c21e2f2SHugh Dickins spinlock_t *old_ptl, *new_ptl; 991da177e4SLinus Torvalds 1001da177e4SLinus Torvalds /* 101c8c06efaSDavidlohr Bueso * When need_rmap_locks is true, we take the i_mmap_rwsem and anon_vma 10238a76013SMichel Lespinasse * locks to ensure that rmap will always observe either the old or the 10338a76013SMichel Lespinasse * new ptes. This is the easiest way to avoid races with 10438a76013SMichel Lespinasse * truncate_pagecache(), page migration, etc... 10538a76013SMichel Lespinasse * 10638a76013SMichel Lespinasse * When need_rmap_locks is false, we use other ways to avoid 10738a76013SMichel Lespinasse * such races: 10838a76013SMichel Lespinasse * 10938a76013SMichel Lespinasse * - During exec() shift_arg_pages(), we use a specially tagged vma 11038a76013SMichel Lespinasse * which rmap call sites look for using is_vma_temporary_stack(). 11138a76013SMichel Lespinasse * 11238a76013SMichel Lespinasse * - During mremap(), new_vma is often known to be placed after vma 11338a76013SMichel Lespinasse * in rmap traversal order. This ensures rmap will always observe 11438a76013SMichel Lespinasse * either the old pte, or the new pte, or both (the page table locks 11538a76013SMichel Lespinasse * serialize access to individual ptes, but only rmap traversal 11638a76013SMichel Lespinasse * order guarantees that we won't miss both the old and new ptes). 1171da177e4SLinus Torvalds */ 11838a76013SMichel Lespinasse if (need_rmap_locks) { 11938a76013SMichel Lespinasse if (vma->vm_file) { 1201da177e4SLinus Torvalds mapping = vma->vm_file->f_mapping; 12183cde9e8SDavidlohr Bueso i_mmap_lock_write(mapping); 1221da177e4SLinus Torvalds } 12338a76013SMichel Lespinasse if (vma->anon_vma) { 12438a76013SMichel Lespinasse anon_vma = vma->anon_vma; 1254fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 12638a76013SMichel Lespinasse } 12738a76013SMichel Lespinasse } 1281da177e4SLinus Torvalds 1294c21e2f2SHugh Dickins /* 1304c21e2f2SHugh Dickins * We don't have to worry about the ordering of src and dst 1314c21e2f2SHugh Dickins * pte locks because exclusive mmap_sem prevents deadlock. 1324c21e2f2SHugh Dickins */ 133c74df32cSHugh Dickins old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl); 134ece0e2b6SPeter Zijlstra new_pte = pte_offset_map(new_pmd, new_addr); 1354c21e2f2SHugh Dickins new_ptl = pte_lockptr(mm, new_pmd); 1364c21e2f2SHugh Dickins if (new_ptl != old_ptl) 137f20dc5f7SIngo Molnar spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 1386606c3e0SZachary Amsden arch_enter_lazy_mmu_mode(); 1398b1f3124SNick Piggin 1407be7a546SHugh Dickins for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE, 1417be7a546SHugh Dickins new_pte++, new_addr += PAGE_SIZE) { 1427be7a546SHugh Dickins if (pte_none(*old_pte)) 1437be7a546SHugh Dickins continue; 1447b6efc2bSAndrea Arcangeli pte = ptep_get_and_clear(mm, old_addr, old_pte); 1457be7a546SHugh Dickins pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr); 1466dec97dcSCyrill Gorcunov pte = move_soft_dirty_pte(pte); 1476dec97dcSCyrill Gorcunov set_pte_at(mm, new_addr, new_pte, pte); 1481da177e4SLinus Torvalds } 1497be7a546SHugh Dickins 1506606c3e0SZachary Amsden arch_leave_lazy_mmu_mode(); 1514c21e2f2SHugh Dickins if (new_ptl != old_ptl) 1524c21e2f2SHugh Dickins spin_unlock(new_ptl); 153ece0e2b6SPeter Zijlstra pte_unmap(new_pte - 1); 154c74df32cSHugh Dickins pte_unmap_unlock(old_pte - 1, old_ptl); 155108d6642SMichel Lespinasse if (anon_vma) 15608b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 1571da177e4SLinus Torvalds if (mapping) 15883cde9e8SDavidlohr Bueso i_mmap_unlock_write(mapping); 1591da177e4SLinus Torvalds } 1601da177e4SLinus Torvalds 1617be7a546SHugh Dickins #define LATENCY_LIMIT (64 * PAGE_SIZE) 1627be7a546SHugh Dickins 163b6a2fea3SOllie Wild unsigned long move_page_tables(struct vm_area_struct *vma, 1641da177e4SLinus Torvalds unsigned long old_addr, struct vm_area_struct *new_vma, 16538a76013SMichel Lespinasse unsigned long new_addr, unsigned long len, 16638a76013SMichel Lespinasse bool need_rmap_locks) 1671da177e4SLinus Torvalds { 1687be7a546SHugh Dickins unsigned long extent, next, old_end; 1697be7a546SHugh Dickins pmd_t *old_pmd, *new_pmd; 1707b6efc2bSAndrea Arcangeli bool need_flush = false; 1712ec74c3eSSagi Grimberg unsigned long mmun_start; /* For mmu_notifiers */ 1722ec74c3eSSagi Grimberg unsigned long mmun_end; /* For mmu_notifiers */ 1731da177e4SLinus Torvalds 1747be7a546SHugh Dickins old_end = old_addr + len; 1757be7a546SHugh Dickins flush_cache_range(vma, old_addr, old_end); 1761da177e4SLinus Torvalds 1772ec74c3eSSagi Grimberg mmun_start = old_addr; 1782ec74c3eSSagi Grimberg mmun_end = old_end; 1792ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end); 1807b6efc2bSAndrea Arcangeli 1817be7a546SHugh Dickins for (; old_addr < old_end; old_addr += extent, new_addr += extent) { 1821da177e4SLinus Torvalds cond_resched(); 1837be7a546SHugh Dickins next = (old_addr + PMD_SIZE) & PMD_MASK; 184ebed4846SAndrea Arcangeli /* even if next overflowed, extent below will be ok */ 1857be7a546SHugh Dickins extent = next - old_addr; 186ebed4846SAndrea Arcangeli if (extent > old_end - old_addr) 187ebed4846SAndrea Arcangeli extent = old_end - old_addr; 1887be7a546SHugh Dickins old_pmd = get_old_pmd(vma->vm_mm, old_addr); 1897be7a546SHugh Dickins if (!old_pmd) 1907be7a546SHugh Dickins continue; 1918ac1f832SAndrea Arcangeli new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr); 1927be7a546SHugh Dickins if (!new_pmd) 1937be7a546SHugh Dickins break; 19437a1c49aSAndrea Arcangeli if (pmd_trans_huge(*old_pmd)) { 19537a1c49aSAndrea Arcangeli int err = 0; 196dd18dbc2SKirill A. Shutemov if (extent == HPAGE_PMD_SIZE) { 19781d1b09cSSasha Levin VM_BUG_ON_VMA(vma->vm_file || !vma->anon_vma, 19881d1b09cSSasha Levin vma); 199dd18dbc2SKirill A. Shutemov /* See comment in move_ptes() */ 200dd18dbc2SKirill A. Shutemov if (need_rmap_locks) 201dd18dbc2SKirill A. Shutemov anon_vma_lock_write(vma->anon_vma); 20237a1c49aSAndrea Arcangeli err = move_huge_pmd(vma, new_vma, old_addr, 20337a1c49aSAndrea Arcangeli new_addr, old_end, 20437a1c49aSAndrea Arcangeli old_pmd, new_pmd); 205dd18dbc2SKirill A. Shutemov if (need_rmap_locks) 206dd18dbc2SKirill A. Shutemov anon_vma_unlock_write(vma->anon_vma); 207dd18dbc2SKirill A. Shutemov } 20837a1c49aSAndrea Arcangeli if (err > 0) { 20937a1c49aSAndrea Arcangeli need_flush = true; 21037a1c49aSAndrea Arcangeli continue; 21137a1c49aSAndrea Arcangeli } else if (!err) { 212e180377fSKirill A. Shutemov split_huge_page_pmd(vma, old_addr, old_pmd); 21337a1c49aSAndrea Arcangeli } 21437a1c49aSAndrea Arcangeli VM_BUG_ON(pmd_trans_huge(*old_pmd)); 21537a1c49aSAndrea Arcangeli } 21637a1c49aSAndrea Arcangeli if (pmd_none(*new_pmd) && __pte_alloc(new_vma->vm_mm, new_vma, 21737a1c49aSAndrea Arcangeli new_pmd, new_addr)) 21837a1c49aSAndrea Arcangeli break; 2197be7a546SHugh Dickins next = (new_addr + PMD_SIZE) & PMD_MASK; 2207be7a546SHugh Dickins if (extent > next - new_addr) 2217be7a546SHugh Dickins extent = next - new_addr; 2227be7a546SHugh Dickins if (extent > LATENCY_LIMIT) 2237be7a546SHugh Dickins extent = LATENCY_LIMIT; 2247be7a546SHugh Dickins move_ptes(vma, old_pmd, old_addr, old_addr + extent, 22538a76013SMichel Lespinasse new_vma, new_pmd, new_addr, need_rmap_locks); 2267b6efc2bSAndrea Arcangeli need_flush = true; 2271da177e4SLinus Torvalds } 2287b6efc2bSAndrea Arcangeli if (likely(need_flush)) 2297b6efc2bSAndrea Arcangeli flush_tlb_range(vma, old_end-len, old_addr); 2307b6efc2bSAndrea Arcangeli 2312ec74c3eSSagi Grimberg mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end); 2327be7a546SHugh Dickins 2337be7a546SHugh Dickins return len + old_addr - old_end; /* how much done */ 2341da177e4SLinus Torvalds } 2351da177e4SLinus Torvalds 2361da177e4SLinus Torvalds static unsigned long move_vma(struct vm_area_struct *vma, 2371da177e4SLinus Torvalds unsigned long old_addr, unsigned long old_len, 23881909b84SMichel Lespinasse unsigned long new_len, unsigned long new_addr, bool *locked) 2391da177e4SLinus Torvalds { 2401da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 2411da177e4SLinus Torvalds struct vm_area_struct *new_vma; 2421da177e4SLinus Torvalds unsigned long vm_flags = vma->vm_flags; 2431da177e4SLinus Torvalds unsigned long new_pgoff; 2441da177e4SLinus Torvalds unsigned long moved_len; 2451da177e4SLinus Torvalds unsigned long excess = 0; 246365e9c87SHugh Dickins unsigned long hiwater_vm; 2471da177e4SLinus Torvalds int split = 0; 2487103ad32SHugh Dickins int err; 24938a76013SMichel Lespinasse bool need_rmap_locks; 2501da177e4SLinus Torvalds 2511da177e4SLinus Torvalds /* 2521da177e4SLinus Torvalds * We'd prefer to avoid failure later on in do_munmap: 2531da177e4SLinus Torvalds * which may split one vma into three before unmapping. 2541da177e4SLinus Torvalds */ 2551da177e4SLinus Torvalds if (mm->map_count >= sysctl_max_map_count - 3) 2561da177e4SLinus Torvalds return -ENOMEM; 2571da177e4SLinus Torvalds 2581ff82995SHugh Dickins /* 2591ff82995SHugh Dickins * Advise KSM to break any KSM pages in the area to be moved: 2601ff82995SHugh Dickins * it would be confusing if they were to turn up at the new 2611ff82995SHugh Dickins * location, where they happen to coincide with different KSM 2621ff82995SHugh Dickins * pages recently unmapped. But leave vma->vm_flags as it was, 2631ff82995SHugh Dickins * so KSM can come around to merge on vma and new_vma afterwards. 2641ff82995SHugh Dickins */ 2657103ad32SHugh Dickins err = ksm_madvise(vma, old_addr, old_addr + old_len, 2667103ad32SHugh Dickins MADV_UNMERGEABLE, &vm_flags); 2677103ad32SHugh Dickins if (err) 2687103ad32SHugh Dickins return err; 2691ff82995SHugh Dickins 2701da177e4SLinus Torvalds new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT); 27138a76013SMichel Lespinasse new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff, 27238a76013SMichel Lespinasse &need_rmap_locks); 2731da177e4SLinus Torvalds if (!new_vma) 2741da177e4SLinus Torvalds return -ENOMEM; 2751da177e4SLinus Torvalds 27638a76013SMichel Lespinasse moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len, 27738a76013SMichel Lespinasse need_rmap_locks); 2781da177e4SLinus Torvalds if (moved_len < old_len) { 279df1eab30SOleg Nesterov err = -ENOMEM; 2805477e70aSOleg Nesterov } else if (vma->vm_ops && vma->vm_ops->mremap) { 2815477e70aSOleg Nesterov err = vma->vm_ops->mremap(new_vma); 282df1eab30SOleg Nesterov } 283df1eab30SOleg Nesterov 284df1eab30SOleg Nesterov if (unlikely(err)) { 2851da177e4SLinus Torvalds /* 2861da177e4SLinus Torvalds * On error, move entries back from new area to old, 2871da177e4SLinus Torvalds * which will succeed since page tables still there, 2881da177e4SLinus Torvalds * and then proceed to unmap new area instead of old. 2891da177e4SLinus Torvalds */ 29038a76013SMichel Lespinasse move_page_tables(new_vma, new_addr, vma, old_addr, moved_len, 29138a76013SMichel Lespinasse true); 2921da177e4SLinus Torvalds vma = new_vma; 2931da177e4SLinus Torvalds old_len = new_len; 2941da177e4SLinus Torvalds old_addr = new_addr; 295df1eab30SOleg Nesterov new_addr = err; 2964abad2caSLaurent Dufour } else { 2974abad2caSLaurent Dufour arch_remap(mm, old_addr, old_addr + old_len, 2984abad2caSLaurent Dufour new_addr, new_addr + new_len); 2994abad2caSLaurent Dufour } 3001da177e4SLinus Torvalds 3011da177e4SLinus Torvalds /* Conceal VM_ACCOUNT so old reservation is not undone */ 3021da177e4SLinus Torvalds if (vm_flags & VM_ACCOUNT) { 3031da177e4SLinus Torvalds vma->vm_flags &= ~VM_ACCOUNT; 3041da177e4SLinus Torvalds excess = vma->vm_end - vma->vm_start - old_len; 3051da177e4SLinus Torvalds if (old_addr > vma->vm_start && 3061da177e4SLinus Torvalds old_addr + old_len < vma->vm_end) 3071da177e4SLinus Torvalds split = 1; 3081da177e4SLinus Torvalds } 3091da177e4SLinus Torvalds 31071799062SKirill Korotaev /* 311365e9c87SHugh Dickins * If we failed to move page tables we still do total_vm increment 312365e9c87SHugh Dickins * since do_munmap() will decrement it by old_len == new_len. 313365e9c87SHugh Dickins * 314365e9c87SHugh Dickins * Since total_vm is about to be raised artificially high for a 315365e9c87SHugh Dickins * moment, we need to restore high watermark afterwards: if stats 316365e9c87SHugh Dickins * are taken meanwhile, total_vm and hiwater_vm appear too high. 317365e9c87SHugh Dickins * If this were a serious issue, we'd add a flag to do_munmap(). 31871799062SKirill Korotaev */ 319365e9c87SHugh Dickins hiwater_vm = mm->hiwater_vm; 320ab50b8edSHugh Dickins vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT); 32171799062SKirill Korotaev 3221da177e4SLinus Torvalds if (do_munmap(mm, old_addr, old_len) < 0) { 3231da177e4SLinus Torvalds /* OOM: unable to split vma, just get accounts right */ 3241da177e4SLinus Torvalds vm_unacct_memory(excess >> PAGE_SHIFT); 3251da177e4SLinus Torvalds excess = 0; 3261da177e4SLinus Torvalds } 327365e9c87SHugh Dickins mm->hiwater_vm = hiwater_vm; 3281da177e4SLinus Torvalds 3291da177e4SLinus Torvalds /* Restore VM_ACCOUNT if one or two pieces of vma left */ 3301da177e4SLinus Torvalds if (excess) { 3311da177e4SLinus Torvalds vma->vm_flags |= VM_ACCOUNT; 3321da177e4SLinus Torvalds if (split) 3331da177e4SLinus Torvalds vma->vm_next->vm_flags |= VM_ACCOUNT; 3341da177e4SLinus Torvalds } 3351da177e4SLinus Torvalds 3361da177e4SLinus Torvalds if (vm_flags & VM_LOCKED) { 3371da177e4SLinus Torvalds mm->locked_vm += new_len >> PAGE_SHIFT; 33881909b84SMichel Lespinasse *locked = true; 3391da177e4SLinus Torvalds } 3401da177e4SLinus Torvalds 3411da177e4SLinus Torvalds return new_addr; 3421da177e4SLinus Torvalds } 3431da177e4SLinus Torvalds 34454f5de70SAl Viro static struct vm_area_struct *vma_to_resize(unsigned long addr, 34554f5de70SAl Viro unsigned long old_len, unsigned long new_len, unsigned long *p) 34654f5de70SAl Viro { 34754f5de70SAl Viro struct mm_struct *mm = current->mm; 34854f5de70SAl Viro struct vm_area_struct *vma = find_vma(mm, addr); 3491d391686SOleg Nesterov unsigned long pgoff; 35054f5de70SAl Viro 35154f5de70SAl Viro if (!vma || vma->vm_start > addr) 3526cd57613SDerek return ERR_PTR(-EFAULT); 35354f5de70SAl Viro 35454f5de70SAl Viro if (is_vm_hugetlb_page(vma)) 3556cd57613SDerek return ERR_PTR(-EINVAL); 35654f5de70SAl Viro 35754f5de70SAl Viro /* We can't remap across vm area boundaries */ 35854f5de70SAl Viro if (old_len > vma->vm_end - addr) 3596cd57613SDerek return ERR_PTR(-EFAULT); 36054f5de70SAl Viro 3611d391686SOleg Nesterov if (new_len == old_len) 3621d391686SOleg Nesterov return vma; 363982134baSLinus Torvalds 3641d391686SOleg Nesterov /* Need to be careful about a growing mapping */ 365982134baSLinus Torvalds pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; 366982134baSLinus Torvalds pgoff += vma->vm_pgoff; 367982134baSLinus Torvalds if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) 3686cd57613SDerek return ERR_PTR(-EINVAL); 3691d391686SOleg Nesterov 3701d391686SOleg Nesterov if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) 3711d391686SOleg Nesterov return ERR_PTR(-EFAULT); 37254f5de70SAl Viro 37354f5de70SAl Viro if (vma->vm_flags & VM_LOCKED) { 37454f5de70SAl Viro unsigned long locked, lock_limit; 37554f5de70SAl Viro locked = mm->locked_vm << PAGE_SHIFT; 37659e99e5bSJiri Slaby lock_limit = rlimit(RLIMIT_MEMLOCK); 37754f5de70SAl Viro locked += new_len - old_len; 37854f5de70SAl Viro if (locked > lock_limit && !capable(CAP_IPC_LOCK)) 3796cd57613SDerek return ERR_PTR(-EAGAIN); 38054f5de70SAl Viro } 38154f5de70SAl Viro 38254f5de70SAl Viro if (!may_expand_vm(mm, (new_len - old_len) >> PAGE_SHIFT)) 3836cd57613SDerek return ERR_PTR(-ENOMEM); 38454f5de70SAl Viro 38554f5de70SAl Viro if (vma->vm_flags & VM_ACCOUNT) { 38654f5de70SAl Viro unsigned long charged = (new_len - old_len) >> PAGE_SHIFT; 387191c5424SAl Viro if (security_vm_enough_memory_mm(mm, charged)) 3886cd57613SDerek return ERR_PTR(-ENOMEM); 38954f5de70SAl Viro *p = charged; 39054f5de70SAl Viro } 39154f5de70SAl Viro 39254f5de70SAl Viro return vma; 39354f5de70SAl Viro } 39454f5de70SAl Viro 39581909b84SMichel Lespinasse static unsigned long mremap_to(unsigned long addr, unsigned long old_len, 39681909b84SMichel Lespinasse unsigned long new_addr, unsigned long new_len, bool *locked) 397ecc1a899SAl Viro { 398ecc1a899SAl Viro struct mm_struct *mm = current->mm; 399ecc1a899SAl Viro struct vm_area_struct *vma; 400ecc1a899SAl Viro unsigned long ret = -EINVAL; 401ecc1a899SAl Viro unsigned long charged = 0; 402097eed10SAl Viro unsigned long map_flags; 403ecc1a899SAl Viro 404*f19cb115SAlexander Kuleshov if (offset_in_page(new_addr)) 405ecc1a899SAl Viro goto out; 406ecc1a899SAl Viro 407ecc1a899SAl Viro if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len) 408ecc1a899SAl Viro goto out; 409ecc1a899SAl Viro 4109943242cSOleg Nesterov /* Ensure the old/new locations do not overlap */ 4119943242cSOleg Nesterov if (addr + old_len > new_addr && new_addr + new_len > addr) 412ecc1a899SAl Viro goto out; 413ecc1a899SAl Viro 414ecc1a899SAl Viro ret = do_munmap(mm, new_addr, new_len); 415ecc1a899SAl Viro if (ret) 416ecc1a899SAl Viro goto out; 417ecc1a899SAl Viro 418ecc1a899SAl Viro if (old_len >= new_len) { 419ecc1a899SAl Viro ret = do_munmap(mm, addr+new_len, old_len - new_len); 420ecc1a899SAl Viro if (ret && old_len != new_len) 421ecc1a899SAl Viro goto out; 422ecc1a899SAl Viro old_len = new_len; 423ecc1a899SAl Viro } 424ecc1a899SAl Viro 425ecc1a899SAl Viro vma = vma_to_resize(addr, old_len, new_len, &charged); 426ecc1a899SAl Viro if (IS_ERR(vma)) { 427ecc1a899SAl Viro ret = PTR_ERR(vma); 428ecc1a899SAl Viro goto out; 429ecc1a899SAl Viro } 430ecc1a899SAl Viro 431097eed10SAl Viro map_flags = MAP_FIXED; 432097eed10SAl Viro if (vma->vm_flags & VM_MAYSHARE) 433097eed10SAl Viro map_flags |= MAP_SHARED; 4349206de95SAl Viro 435097eed10SAl Viro ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff + 436097eed10SAl Viro ((addr - vma->vm_start) >> PAGE_SHIFT), 437097eed10SAl Viro map_flags); 438*f19cb115SAlexander Kuleshov if (offset_in_page(ret)) 439097eed10SAl Viro goto out1; 440097eed10SAl Viro 44181909b84SMichel Lespinasse ret = move_vma(vma, addr, old_len, new_len, new_addr, locked); 442*f19cb115SAlexander Kuleshov if (!(offset_in_page(ret))) 443097eed10SAl Viro goto out; 444097eed10SAl Viro out1: 445ecc1a899SAl Viro vm_unacct_memory(charged); 446ecc1a899SAl Viro 447ecc1a899SAl Viro out: 448ecc1a899SAl Viro return ret; 449ecc1a899SAl Viro } 450ecc1a899SAl Viro 4511a0ef85fSAl Viro static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) 4521a0ef85fSAl Viro { 453f106af4eSAl Viro unsigned long end = vma->vm_end + delta; 4549206de95SAl Viro if (end < vma->vm_end) /* overflow */ 4551a0ef85fSAl Viro return 0; 4569206de95SAl Viro if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */ 457f106af4eSAl Viro return 0; 458f106af4eSAl Viro if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, 459f106af4eSAl Viro 0, MAP_FIXED) & ~PAGE_MASK) 460f106af4eSAl Viro return 0; 4611a0ef85fSAl Viro return 1; 4621a0ef85fSAl Viro } 4631a0ef85fSAl Viro 4641da177e4SLinus Torvalds /* 4651da177e4SLinus Torvalds * Expand (or shrink) an existing mapping, potentially moving it at the 4661da177e4SLinus Torvalds * same time (controlled by the MREMAP_MAYMOVE flag and available VM space) 4671da177e4SLinus Torvalds * 4681da177e4SLinus Torvalds * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise 4691da177e4SLinus Torvalds * This option implies MREMAP_MAYMOVE. 4701da177e4SLinus Torvalds */ 47163a81db1SAl Viro SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, 47263a81db1SAl Viro unsigned long, new_len, unsigned long, flags, 47363a81db1SAl Viro unsigned long, new_addr) 4741da177e4SLinus Torvalds { 475d0de32d9SHugh Dickins struct mm_struct *mm = current->mm; 4761da177e4SLinus Torvalds struct vm_area_struct *vma; 4771da177e4SLinus Torvalds unsigned long ret = -EINVAL; 4781da177e4SLinus Torvalds unsigned long charged = 0; 47981909b84SMichel Lespinasse bool locked = false; 4801da177e4SLinus Torvalds 4811da177e4SLinus Torvalds if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE)) 4829a2458a6SRasmus Villemoes return ret; 4839a2458a6SRasmus Villemoes 4849a2458a6SRasmus Villemoes if (flags & MREMAP_FIXED && !(flags & MREMAP_MAYMOVE)) 4859a2458a6SRasmus Villemoes return ret; 4861da177e4SLinus Torvalds 487*f19cb115SAlexander Kuleshov if (offset_in_page(addr)) 4889a2458a6SRasmus Villemoes return ret; 4891da177e4SLinus Torvalds 4901da177e4SLinus Torvalds old_len = PAGE_ALIGN(old_len); 4911da177e4SLinus Torvalds new_len = PAGE_ALIGN(new_len); 4921da177e4SLinus Torvalds 4931da177e4SLinus Torvalds /* 4941da177e4SLinus Torvalds * We allow a zero old-len as a special case 4951da177e4SLinus Torvalds * for DOS-emu "duplicate shm area" thing. But 4961da177e4SLinus Torvalds * a zero new-len is nonsensical. 4971da177e4SLinus Torvalds */ 4981da177e4SLinus Torvalds if (!new_len) 4999a2458a6SRasmus Villemoes return ret; 5009a2458a6SRasmus Villemoes 5019a2458a6SRasmus Villemoes down_write(¤t->mm->mmap_sem); 5021da177e4SLinus Torvalds 5031da177e4SLinus Torvalds if (flags & MREMAP_FIXED) { 50481909b84SMichel Lespinasse ret = mremap_to(addr, old_len, new_addr, new_len, 50581909b84SMichel Lespinasse &locked); 5061da177e4SLinus Torvalds goto out; 5071da177e4SLinus Torvalds } 5081da177e4SLinus Torvalds 5091da177e4SLinus Torvalds /* 5101da177e4SLinus Torvalds * Always allow a shrinking remap: that just unmaps 5111da177e4SLinus Torvalds * the unnecessary pages.. 5121da177e4SLinus Torvalds * do_munmap does all the needed commit accounting 5131da177e4SLinus Torvalds */ 5141da177e4SLinus Torvalds if (old_len >= new_len) { 515d0de32d9SHugh Dickins ret = do_munmap(mm, addr+new_len, old_len - new_len); 5161da177e4SLinus Torvalds if (ret && old_len != new_len) 5171da177e4SLinus Torvalds goto out; 5181da177e4SLinus Torvalds ret = addr; 5191da177e4SLinus Torvalds goto out; 5201da177e4SLinus Torvalds } 5211da177e4SLinus Torvalds 5221da177e4SLinus Torvalds /* 523ecc1a899SAl Viro * Ok, we need to grow.. 5241da177e4SLinus Torvalds */ 52554f5de70SAl Viro vma = vma_to_resize(addr, old_len, new_len, &charged); 52654f5de70SAl Viro if (IS_ERR(vma)) { 52754f5de70SAl Viro ret = PTR_ERR(vma); 5281da177e4SLinus Torvalds goto out; 5291da177e4SLinus Torvalds } 5301da177e4SLinus Torvalds 5311da177e4SLinus Torvalds /* old_len exactly to the end of the area.. 5321da177e4SLinus Torvalds */ 533ecc1a899SAl Viro if (old_len == vma->vm_end - addr) { 5341da177e4SLinus Torvalds /* can we just expand the current mapping? */ 5351a0ef85fSAl Viro if (vma_expandable(vma, new_len - old_len)) { 5361da177e4SLinus Torvalds int pages = (new_len - old_len) >> PAGE_SHIFT; 5371da177e4SLinus Torvalds 5385beb4930SRik van Riel if (vma_adjust(vma, vma->vm_start, addr + new_len, 5395beb4930SRik van Riel vma->vm_pgoff, NULL)) { 5405beb4930SRik van Riel ret = -ENOMEM; 5415beb4930SRik van Riel goto out; 5425beb4930SRik van Riel } 5431da177e4SLinus Torvalds 544d0de32d9SHugh Dickins vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages); 5451da177e4SLinus Torvalds if (vma->vm_flags & VM_LOCKED) { 546d0de32d9SHugh Dickins mm->locked_vm += pages; 54781909b84SMichel Lespinasse locked = true; 54881909b84SMichel Lespinasse new_addr = addr; 5491da177e4SLinus Torvalds } 5501da177e4SLinus Torvalds ret = addr; 5511da177e4SLinus Torvalds goto out; 5521da177e4SLinus Torvalds } 5531da177e4SLinus Torvalds } 5541da177e4SLinus Torvalds 5551da177e4SLinus Torvalds /* 5561da177e4SLinus Torvalds * We weren't able to just expand or shrink the area, 5571da177e4SLinus Torvalds * we need to create a new one and move it.. 5581da177e4SLinus Torvalds */ 5591da177e4SLinus Torvalds ret = -ENOMEM; 5601da177e4SLinus Torvalds if (flags & MREMAP_MAYMOVE) { 5611da177e4SLinus Torvalds unsigned long map_flags = 0; 5621da177e4SLinus Torvalds if (vma->vm_flags & VM_MAYSHARE) 5631da177e4SLinus Torvalds map_flags |= MAP_SHARED; 5641da177e4SLinus Torvalds 5651da177e4SLinus Torvalds new_addr = get_unmapped_area(vma->vm_file, 0, new_len, 56693587414SAl Viro vma->vm_pgoff + 56793587414SAl Viro ((addr - vma->vm_start) >> PAGE_SHIFT), 56893587414SAl Viro map_flags); 569*f19cb115SAlexander Kuleshov if (offset_in_page(new_addr)) { 5701da177e4SLinus Torvalds ret = new_addr; 571ed032189SEric Paris goto out; 572ed032189SEric Paris } 573ed032189SEric Paris 57481909b84SMichel Lespinasse ret = move_vma(vma, addr, old_len, new_len, new_addr, &locked); 5751da177e4SLinus Torvalds } 5761da177e4SLinus Torvalds out: 577*f19cb115SAlexander Kuleshov if (offset_in_page(ret)) { 5781da177e4SLinus Torvalds vm_unacct_memory(charged); 579d456fb9eSOleg Nesterov locked = 0; 580d456fb9eSOleg Nesterov } 5811da177e4SLinus Torvalds up_write(¤t->mm->mmap_sem); 58281909b84SMichel Lespinasse if (locked && new_len > old_len) 58381909b84SMichel Lespinasse mm_populate(new_addr + old_len, new_len - old_len); 5841da177e4SLinus Torvalds return ret; 5851da177e4SLinus Torvalds } 586