1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * mm/mmap.c 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Written by obz. 61da177e4SLinus Torvalds * 7046c6884SAlan Cox * Address space accounting code <alan@lxorguk.ukuu.org.uk> 81da177e4SLinus Torvalds */ 91da177e4SLinus Torvalds 10b1de0d13SMitchel Humpherys #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 11b1de0d13SMitchel Humpherys 12e8420a8eSCyril Hrubis #include <linux/kernel.h> 131da177e4SLinus Torvalds #include <linux/slab.h> 144af3c9ccSAlexey Dobriyan #include <linux/backing-dev.h> 151da177e4SLinus Torvalds #include <linux/mm.h> 1617fca131SArnd Bergmann #include <linux/mm_inline.h> 17615d6e87SDavidlohr Bueso #include <linux/vmacache.h> 181da177e4SLinus Torvalds #include <linux/shm.h> 191da177e4SLinus Torvalds #include <linux/mman.h> 201da177e4SLinus Torvalds #include <linux/pagemap.h> 211da177e4SLinus Torvalds #include <linux/swap.h> 221da177e4SLinus Torvalds #include <linux/syscalls.h> 23c59ede7bSRandy.Dunlap #include <linux/capability.h> 241da177e4SLinus Torvalds #include <linux/init.h> 251da177e4SLinus Torvalds #include <linux/file.h> 261da177e4SLinus Torvalds #include <linux/fs.h> 271da177e4SLinus Torvalds #include <linux/personality.h> 281da177e4SLinus Torvalds #include <linux/security.h> 291da177e4SLinus Torvalds #include <linux/hugetlb.h> 30c01d5b30SHugh Dickins #include <linux/shmem_fs.h> 311da177e4SLinus Torvalds #include <linux/profile.h> 32b95f1b31SPaul Gortmaker #include <linux/export.h> 331da177e4SLinus Torvalds #include <linux/mount.h> 341da177e4SLinus Torvalds #include <linux/mempolicy.h> 351da177e4SLinus Torvalds #include <linux/rmap.h> 36cddb8a5cSAndrea Arcangeli #include <linux/mmu_notifier.h> 3782f71ae4SKonstantin Khlebnikov #include <linux/mmdebug.h> 38cdd6c482SIngo Molnar #include <linux/perf_event.h> 39120a795dSAl Viro #include <linux/audit.h> 40b15d00b6SAndrea Arcangeli #include <linux/khugepaged.h> 412b144498SSrikar Dronamraju #include <linux/uprobes.h> 421640879aSAndrew Shewmaker #include <linux/notifier.h> 431640879aSAndrew Shewmaker #include <linux/memory.h> 44b1de0d13SMitchel Humpherys #include <linux/printk.h> 4519a809afSAndrea Arcangeli #include <linux/userfaultfd_k.h> 46d977d56cSKonstantin Khlebnikov #include <linux/moduleparam.h> 4762b5f7d0SDave Hansen #include <linux/pkeys.h> 4821292580SAndrea Arcangeli #include <linux/oom.h> 4904f5866eSAndrea Arcangeli #include <linux/sched/mm.h> 501da177e4SLinus Torvalds 517c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 521da177e4SLinus Torvalds #include <asm/cacheflush.h> 531da177e4SLinus Torvalds #include <asm/tlb.h> 54d6dd61c8SJeremy Fitzhardinge #include <asm/mmu_context.h> 551da177e4SLinus Torvalds 56df529cabSJaewon Kim #define CREATE_TRACE_POINTS 57df529cabSJaewon Kim #include <trace/events/mmap.h> 58df529cabSJaewon Kim 5942b77728SJan Beulich #include "internal.h" 6042b77728SJan Beulich 613a459756SKirill Korotaev #ifndef arch_mmap_check 623a459756SKirill Korotaev #define arch_mmap_check(addr, len, flags) (0) 633a459756SKirill Korotaev #endif 643a459756SKirill Korotaev 65d07e2259SDaniel Cashman #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS 66d07e2259SDaniel Cashman const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN; 67d07e2259SDaniel Cashman const int mmap_rnd_bits_max = CONFIG_ARCH_MMAP_RND_BITS_MAX; 68d07e2259SDaniel Cashman int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS; 69d07e2259SDaniel Cashman #endif 70d07e2259SDaniel Cashman #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS 71d07e2259SDaniel Cashman const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN; 72d07e2259SDaniel Cashman const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX; 73d07e2259SDaniel Cashman int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS; 74d07e2259SDaniel Cashman #endif 75d07e2259SDaniel Cashman 76f4fcd558SKonstantin Khlebnikov static bool ignore_rlimit_data; 77d977d56cSKonstantin Khlebnikov core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); 78d07e2259SDaniel Cashman 79e0da382cSHugh Dickins static void unmap_region(struct mm_struct *mm, 80e0da382cSHugh Dickins struct vm_area_struct *vma, struct vm_area_struct *prev, 81e0da382cSHugh Dickins unsigned long start, unsigned long end); 82e0da382cSHugh Dickins 8364e45507SPeter Feiner static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) 8464e45507SPeter Feiner { 8564e45507SPeter Feiner return pgprot_modify(oldprot, vm_get_page_prot(vm_flags)); 8664e45507SPeter Feiner } 8764e45507SPeter Feiner 8864e45507SPeter Feiner /* Update vma->vm_page_prot to reflect vma->vm_flags. */ 8964e45507SPeter Feiner void vma_set_page_prot(struct vm_area_struct *vma) 9064e45507SPeter Feiner { 9164e45507SPeter Feiner unsigned long vm_flags = vma->vm_flags; 926d2329f8SAndrea Arcangeli pgprot_t vm_page_prot; 9364e45507SPeter Feiner 946d2329f8SAndrea Arcangeli vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags); 956d2329f8SAndrea Arcangeli if (vma_wants_writenotify(vma, vm_page_prot)) { 9664e45507SPeter Feiner vm_flags &= ~VM_SHARED; 976d2329f8SAndrea Arcangeli vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags); 9864e45507SPeter Feiner } 99c1e8d7c6SMichel Lespinasse /* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */ 1006d2329f8SAndrea Arcangeli WRITE_ONCE(vma->vm_page_prot, vm_page_prot); 10164e45507SPeter Feiner } 10264e45507SPeter Feiner 1031da177e4SLinus Torvalds /* 104c8c06efaSDavidlohr Bueso * Requires inode->i_mapping->i_mmap_rwsem 1051da177e4SLinus Torvalds */ 1061da177e4SLinus Torvalds static void __remove_shared_vm_struct(struct vm_area_struct *vma, 1071da177e4SLinus Torvalds struct file *file, struct address_space *mapping) 1081da177e4SLinus Torvalds { 1091da177e4SLinus Torvalds if (vma->vm_flags & VM_SHARED) 1104bb5f5d9SDavid Herrmann mapping_unmap_writable(mapping); 1111da177e4SLinus Torvalds 1121da177e4SLinus Torvalds flush_dcache_mmap_lock(mapping); 1136b2dbba8SMichel Lespinasse vma_interval_tree_remove(vma, &mapping->i_mmap); 1141da177e4SLinus Torvalds flush_dcache_mmap_unlock(mapping); 1151da177e4SLinus Torvalds } 1161da177e4SLinus Torvalds 1171da177e4SLinus Torvalds /* 1186b2dbba8SMichel Lespinasse * Unlink a file-based vm structure from its interval tree, to hide 119a8fb5618SHugh Dickins * vma from rmap and vmtruncate before freeing its page tables. 1201da177e4SLinus Torvalds */ 121a8fb5618SHugh Dickins void unlink_file_vma(struct vm_area_struct *vma) 1221da177e4SLinus Torvalds { 1231da177e4SLinus Torvalds struct file *file = vma->vm_file; 1241da177e4SLinus Torvalds 1251da177e4SLinus Torvalds if (file) { 1261da177e4SLinus Torvalds struct address_space *mapping = file->f_mapping; 12783cde9e8SDavidlohr Bueso i_mmap_lock_write(mapping); 1281da177e4SLinus Torvalds __remove_shared_vm_struct(vma, file, mapping); 12983cde9e8SDavidlohr Bueso i_mmap_unlock_write(mapping); 1301da177e4SLinus Torvalds } 131a8fb5618SHugh Dickins } 132a8fb5618SHugh Dickins 133a8fb5618SHugh Dickins /* 134a8fb5618SHugh Dickins * Close a vm structure and free it, returning the next. 135a8fb5618SHugh Dickins */ 136a8fb5618SHugh Dickins static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) 137a8fb5618SHugh Dickins { 138a8fb5618SHugh Dickins struct vm_area_struct *next = vma->vm_next; 139a8fb5618SHugh Dickins 140a8fb5618SHugh Dickins might_sleep(); 1411da177e4SLinus Torvalds if (vma->vm_ops && vma->vm_ops->close) 1421da177e4SLinus Torvalds vma->vm_ops->close(vma); 143e9714acfSKonstantin Khlebnikov if (vma->vm_file) 144a8fb5618SHugh Dickins fput(vma->vm_file); 145f0be3d32SLee Schermerhorn mpol_put(vma_policy(vma)); 1463928d4f5SLinus Torvalds vm_area_free(vma); 147a8fb5618SHugh Dickins return next; 1481da177e4SLinus Torvalds } 1491da177e4SLinus Torvalds 150bb177a73SMichal Hocko static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags, 151bb177a73SMichal Hocko struct list_head *uf); 1526a6160a7SHeiko Carstens SYSCALL_DEFINE1(brk, unsigned long, brk) 1531da177e4SLinus Torvalds { 1549bc8039eSYang Shi unsigned long newbrk, oldbrk, origbrk; 1551da177e4SLinus Torvalds struct mm_struct *mm = current->mm; 1561be7107fSHugh Dickins struct vm_area_struct *next; 157a5b4592cSJiri Kosina unsigned long min_brk; 158128557ffSMichel Lespinasse bool populate; 1599bc8039eSYang Shi bool downgraded = false; 160897ab3e0SMike Rapoport LIST_HEAD(uf); 1611da177e4SLinus Torvalds 162d8ed45c5SMichel Lespinasse if (mmap_write_lock_killable(mm)) 163dc0ef0dfSMichal Hocko return -EINTR; 1641da177e4SLinus Torvalds 1659bc8039eSYang Shi origbrk = mm->brk; 1669bc8039eSYang Shi 167a5b4592cSJiri Kosina #ifdef CONFIG_COMPAT_BRK 1685520e894SJiri Kosina /* 1695520e894SJiri Kosina * CONFIG_COMPAT_BRK can still be overridden by setting 1705520e894SJiri Kosina * randomize_va_space to 2, which will still cause mm->start_brk 1715520e894SJiri Kosina * to be arbitrarily shifted 1725520e894SJiri Kosina */ 1734471a675SJiri Kosina if (current->brk_randomized) 1745520e894SJiri Kosina min_brk = mm->start_brk; 1755520e894SJiri Kosina else 1765520e894SJiri Kosina min_brk = mm->end_data; 177a5b4592cSJiri Kosina #else 178a5b4592cSJiri Kosina min_brk = mm->start_brk; 179a5b4592cSJiri Kosina #endif 180a5b4592cSJiri Kosina if (brk < min_brk) 1811da177e4SLinus Torvalds goto out; 1821e624196SRam Gupta 1831e624196SRam Gupta /* 1841e624196SRam Gupta * Check against rlimit here. If this check is done later after the test 1851e624196SRam Gupta * of oldbrk with newbrk then it can escape the test and let the data 1861e624196SRam Gupta * segment grow beyond its set limit the in case where the limit is 1871e624196SRam Gupta * not page aligned -Ram Gupta 1881e624196SRam Gupta */ 1898764b338SCyrill Gorcunov if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk, 1908764b338SCyrill Gorcunov mm->end_data, mm->start_data)) 1911e624196SRam Gupta goto out; 1921e624196SRam Gupta 1931da177e4SLinus Torvalds newbrk = PAGE_ALIGN(brk); 1941da177e4SLinus Torvalds oldbrk = PAGE_ALIGN(mm->brk); 1959bc8039eSYang Shi if (oldbrk == newbrk) { 1969bc8039eSYang Shi mm->brk = brk; 1979bc8039eSYang Shi goto success; 1989bc8039eSYang Shi } 1991da177e4SLinus Torvalds 2009bc8039eSYang Shi /* 2019bc8039eSYang Shi * Always allow shrinking brk. 202c1e8d7c6SMichel Lespinasse * __do_munmap() may downgrade mmap_lock to read. 2039bc8039eSYang Shi */ 2041da177e4SLinus Torvalds if (brk <= mm->brk) { 2059bc8039eSYang Shi int ret; 2069bc8039eSYang Shi 2079bc8039eSYang Shi /* 208c1e8d7c6SMichel Lespinasse * mm->brk must to be protected by write mmap_lock so update it 209c1e8d7c6SMichel Lespinasse * before downgrading mmap_lock. When __do_munmap() fails, 2109bc8039eSYang Shi * mm->brk will be restored from origbrk. 2119bc8039eSYang Shi */ 2129bc8039eSYang Shi mm->brk = brk; 2139bc8039eSYang Shi ret = __do_munmap(mm, newbrk, oldbrk-newbrk, &uf, true); 2149bc8039eSYang Shi if (ret < 0) { 2159bc8039eSYang Shi mm->brk = origbrk; 2161da177e4SLinus Torvalds goto out; 2179bc8039eSYang Shi } else if (ret == 1) { 2189bc8039eSYang Shi downgraded = true; 2199bc8039eSYang Shi } 2209bc8039eSYang Shi goto success; 2211da177e4SLinus Torvalds } 2221da177e4SLinus Torvalds 2231da177e4SLinus Torvalds /* Check against existing mmap mappings. */ 2241be7107fSHugh Dickins next = find_vma(mm, oldbrk); 2251be7107fSHugh Dickins if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) 2261da177e4SLinus Torvalds goto out; 2271da177e4SLinus Torvalds 2281da177e4SLinus Torvalds /* Ok, looks good - let it rip. */ 229bb177a73SMichal Hocko if (do_brk_flags(oldbrk, newbrk-oldbrk, 0, &uf) < 0) 2301da177e4SLinus Torvalds goto out; 2311da177e4SLinus Torvalds mm->brk = brk; 2329bc8039eSYang Shi 2339bc8039eSYang Shi success: 234128557ffSMichel Lespinasse populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0; 2359bc8039eSYang Shi if (downgraded) 236d8ed45c5SMichel Lespinasse mmap_read_unlock(mm); 2379bc8039eSYang Shi else 238d8ed45c5SMichel Lespinasse mmap_write_unlock(mm); 239897ab3e0SMike Rapoport userfaultfd_unmap_complete(mm, &uf); 240128557ffSMichel Lespinasse if (populate) 241128557ffSMichel Lespinasse mm_populate(oldbrk, newbrk - oldbrk); 242128557ffSMichel Lespinasse return brk; 243128557ffSMichel Lespinasse 2441da177e4SLinus Torvalds out: 245d8ed45c5SMichel Lespinasse mmap_write_unlock(mm); 246b7204006SAdrian Huang return origbrk; 2471da177e4SLinus Torvalds } 2481da177e4SLinus Torvalds 249d4af56c5SLiam R. Howlett #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 250d4af56c5SLiam R. Howlett extern void mt_validate(struct maple_tree *mt); 251d4af56c5SLiam R. Howlett extern void mt_dump(const struct maple_tree *mt); 2521da177e4SLinus Torvalds 253d4af56c5SLiam R. Howlett /* Validate the maple tree */ 254d4af56c5SLiam R. Howlett static void validate_mm_mt(struct mm_struct *mm) 255d4af56c5SLiam R. Howlett { 256d4af56c5SLiam R. Howlett struct maple_tree *mt = &mm->mm_mt; 257d4af56c5SLiam R. Howlett struct vm_area_struct *vma_mt, *vma = mm->mmap; 258d4af56c5SLiam R. Howlett 259d4af56c5SLiam R. Howlett MA_STATE(mas, mt, 0, 0); 260d4af56c5SLiam R. Howlett 261d4af56c5SLiam R. Howlett mt_validate(&mm->mm_mt); 262d4af56c5SLiam R. Howlett mas_for_each(&mas, vma_mt, ULONG_MAX) { 263d4af56c5SLiam R. Howlett if (xa_is_zero(vma_mt)) 264d4af56c5SLiam R. Howlett continue; 265d4af56c5SLiam R. Howlett 266d4af56c5SLiam R. Howlett if (!vma) 267d4af56c5SLiam R. Howlett break; 268d4af56c5SLiam R. Howlett 269d4af56c5SLiam R. Howlett if ((vma != vma_mt) || 270d4af56c5SLiam R. Howlett (vma->vm_start != vma_mt->vm_start) || 271d4af56c5SLiam R. Howlett (vma->vm_end != vma_mt->vm_end) || 272d4af56c5SLiam R. Howlett (vma->vm_start != mas.index) || 273d4af56c5SLiam R. Howlett (vma->vm_end - 1 != mas.last)) { 274d4af56c5SLiam R. Howlett pr_emerg("issue in %s\n", current->comm); 275d4af56c5SLiam R. Howlett dump_stack(); 276d4af56c5SLiam R. Howlett dump_vma(vma_mt); 277524e00b3SLiam R. Howlett pr_emerg("and vm_next\n"); 278d4af56c5SLiam R. Howlett dump_vma(vma->vm_next); 279d4af56c5SLiam R. Howlett pr_emerg("mt piv: %p %lu - %lu\n", vma_mt, 280d4af56c5SLiam R. Howlett mas.index, mas.last); 281d4af56c5SLiam R. Howlett pr_emerg("mt vma: %p %lu - %lu\n", vma_mt, 282d4af56c5SLiam R. Howlett vma_mt->vm_start, vma_mt->vm_end); 283524e00b3SLiam R. Howlett if (vma->vm_prev) { 284524e00b3SLiam R. Howlett pr_emerg("ll prev: %p %lu - %lu\n", 285524e00b3SLiam R. Howlett vma->vm_prev, vma->vm_prev->vm_start, 286524e00b3SLiam R. Howlett vma->vm_prev->vm_end); 287524e00b3SLiam R. Howlett } 288524e00b3SLiam R. Howlett pr_emerg("ll vma: %p %lu - %lu\n", vma, 289d4af56c5SLiam R. Howlett vma->vm_start, vma->vm_end); 290524e00b3SLiam R. Howlett if (vma->vm_next) { 291524e00b3SLiam R. Howlett pr_emerg("ll next: %p %lu - %lu\n", 292524e00b3SLiam R. Howlett vma->vm_next, vma->vm_next->vm_start, 293524e00b3SLiam R. Howlett vma->vm_next->vm_end); 294524e00b3SLiam R. Howlett } 295d4af56c5SLiam R. Howlett 296d4af56c5SLiam R. Howlett mt_dump(mas.tree); 297d4af56c5SLiam R. Howlett if (vma_mt->vm_end != mas.last + 1) { 298d4af56c5SLiam R. Howlett pr_err("vma: %p vma_mt %lu-%lu\tmt %lu-%lu\n", 299d4af56c5SLiam R. Howlett mm, vma_mt->vm_start, vma_mt->vm_end, 300d4af56c5SLiam R. Howlett mas.index, mas.last); 301d4af56c5SLiam R. Howlett mt_dump(mas.tree); 302d4af56c5SLiam R. Howlett } 303d4af56c5SLiam R. Howlett VM_BUG_ON_MM(vma_mt->vm_end != mas.last + 1, mm); 304d4af56c5SLiam R. Howlett if (vma_mt->vm_start != mas.index) { 305d4af56c5SLiam R. Howlett pr_err("vma: %p vma_mt %p %lu - %lu doesn't match\n", 306d4af56c5SLiam R. Howlett mm, vma_mt, vma_mt->vm_start, vma_mt->vm_end); 307d4af56c5SLiam R. Howlett mt_dump(mas.tree); 308d4af56c5SLiam R. Howlett } 309d4af56c5SLiam R. Howlett VM_BUG_ON_MM(vma_mt->vm_start != mas.index, mm); 310d4af56c5SLiam R. Howlett } 311d4af56c5SLiam R. Howlett VM_BUG_ON(vma != vma_mt); 312d4af56c5SLiam R. Howlett vma = vma->vm_next; 313d4af56c5SLiam R. Howlett 314d4af56c5SLiam R. Howlett } 315d4af56c5SLiam R. Howlett VM_BUG_ON(vma); 316d4af56c5SLiam R. Howlett } 3171da177e4SLinus Torvalds 318eafd4dc4SRashika Kheria static void validate_mm(struct mm_struct *mm) 3191da177e4SLinus Torvalds { 3201da177e4SLinus Torvalds int bug = 0; 3211da177e4SLinus Torvalds int i = 0; 3225a0768f6SMichel Lespinasse unsigned long highest_address = 0; 323ed8ea815SMichel Lespinasse struct vm_area_struct *vma = mm->mmap; 324ff26f70fSAndrew Morton 325524e00b3SLiam R. Howlett validate_mm_mt(mm); 326524e00b3SLiam R. Howlett 327ed8ea815SMichel Lespinasse while (vma) { 328524e00b3SLiam R. Howlett #ifdef CONFIG_DEBUG_VM_RB 32912352d3cSKonstantin Khlebnikov struct anon_vma *anon_vma = vma->anon_vma; 330ed8ea815SMichel Lespinasse struct anon_vma_chain *avc; 331ff26f70fSAndrew Morton 33212352d3cSKonstantin Khlebnikov if (anon_vma) { 33312352d3cSKonstantin Khlebnikov anon_vma_lock_read(anon_vma); 334ed8ea815SMichel Lespinasse list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 335ed8ea815SMichel Lespinasse anon_vma_interval_tree_verify(avc); 33612352d3cSKonstantin Khlebnikov anon_vma_unlock_read(anon_vma); 33712352d3cSKonstantin Khlebnikov } 338524e00b3SLiam R. Howlett #endif 33912352d3cSKonstantin Khlebnikov 3401be7107fSHugh Dickins highest_address = vm_end_gap(vma); 341ed8ea815SMichel Lespinasse vma = vma->vm_next; 3421da177e4SLinus Torvalds i++; 3431da177e4SLinus Torvalds } 3445a0768f6SMichel Lespinasse if (i != mm->map_count) { 3458542bdfcSSasha Levin pr_emerg("map_count %d vm_next %d\n", mm->map_count, i); 3465a0768f6SMichel Lespinasse bug = 1; 3475a0768f6SMichel Lespinasse } 3485a0768f6SMichel Lespinasse if (highest_address != mm->highest_vm_end) { 3498542bdfcSSasha Levin pr_emerg("mm->highest_vm_end %lx, found %lx\n", 3505a0768f6SMichel Lespinasse mm->highest_vm_end, highest_address); 3515a0768f6SMichel Lespinasse bug = 1; 3525a0768f6SMichel Lespinasse } 35396dad67fSSasha Levin VM_BUG_ON_MM(bug, mm); 3541da177e4SLinus Torvalds } 355524e00b3SLiam R. Howlett 356524e00b3SLiam R. Howlett #else /* !CONFIG_DEBUG_VM_MAPLE_TREE */ 357d4af56c5SLiam R. Howlett #define validate_mm_mt(root) do { } while (0) 3581da177e4SLinus Torvalds #define validate_mm(mm) do { } while (0) 359524e00b3SLiam R. Howlett #endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ 360d3737187SMichel Lespinasse 361bf181b9fSMichel Lespinasse /* 362bf181b9fSMichel Lespinasse * vma has some anon_vma assigned, and is already inserted on that 363bf181b9fSMichel Lespinasse * anon_vma's interval trees. 364bf181b9fSMichel Lespinasse * 365bf181b9fSMichel Lespinasse * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the 366bf181b9fSMichel Lespinasse * vma must be removed from the anon_vma's interval trees using 367bf181b9fSMichel Lespinasse * anon_vma_interval_tree_pre_update_vma(). 368bf181b9fSMichel Lespinasse * 369bf181b9fSMichel Lespinasse * After the update, the vma will be reinserted using 370bf181b9fSMichel Lespinasse * anon_vma_interval_tree_post_update_vma(). 371bf181b9fSMichel Lespinasse * 372c1e8d7c6SMichel Lespinasse * The entire update must be protected by exclusive mmap_lock and by 373bf181b9fSMichel Lespinasse * the root anon_vma's mutex. 374bf181b9fSMichel Lespinasse */ 375bf181b9fSMichel Lespinasse static inline void 376bf181b9fSMichel Lespinasse anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) 377bf181b9fSMichel Lespinasse { 378bf181b9fSMichel Lespinasse struct anon_vma_chain *avc; 379bf181b9fSMichel Lespinasse 380bf181b9fSMichel Lespinasse list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 381bf181b9fSMichel Lespinasse anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); 382bf181b9fSMichel Lespinasse } 383bf181b9fSMichel Lespinasse 384bf181b9fSMichel Lespinasse static inline void 385bf181b9fSMichel Lespinasse anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) 386bf181b9fSMichel Lespinasse { 387bf181b9fSMichel Lespinasse struct anon_vma_chain *avc; 388bf181b9fSMichel Lespinasse 389bf181b9fSMichel Lespinasse list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 390bf181b9fSMichel Lespinasse anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); 391bf181b9fSMichel Lespinasse } 392bf181b9fSMichel Lespinasse 393524e00b3SLiam R. Howlett /* 394524e00b3SLiam R. Howlett * range_has_overlap() - Check the @start - @end range for overlapping VMAs and 395524e00b3SLiam R. Howlett * sets up a pointer to the previous VMA 396524e00b3SLiam R. Howlett * @mm: the mm struct 397524e00b3SLiam R. Howlett * @start: the start address of the range 398524e00b3SLiam R. Howlett * @end: the end address of the range 399524e00b3SLiam R. Howlett * @pprev: the pointer to the pointer of the previous VMA 400524e00b3SLiam R. Howlett * 401524e00b3SLiam R. Howlett * Returns: True if there is an overlapping VMA, false otherwise 402524e00b3SLiam R. Howlett */ 403524e00b3SLiam R. Howlett static inline 404524e00b3SLiam R. Howlett bool range_has_overlap(struct mm_struct *mm, unsigned long start, 405524e00b3SLiam R. Howlett unsigned long end, struct vm_area_struct **pprev) 4061da177e4SLinus Torvalds { 407524e00b3SLiam R. Howlett struct vm_area_struct *existing; 4081da177e4SLinus Torvalds 409524e00b3SLiam R. Howlett MA_STATE(mas, &mm->mm_mt, start, start); 410524e00b3SLiam R. Howlett existing = mas_find(&mas, end - 1); 411524e00b3SLiam R. Howlett *pprev = mas_prev(&mas, 0); 412524e00b3SLiam R. Howlett return existing ? true : false; 4131da177e4SLinus Torvalds } 4141da177e4SLinus Torvalds 4153903b55aSLiam R. Howlett /* 416f39af059SMatthew Wilcox (Oracle) * __vma_next() - Get the next VMA. 4173903b55aSLiam R. Howlett * @mm: The mm_struct. 4183903b55aSLiam R. Howlett * @vma: The current vma. 4193903b55aSLiam R. Howlett * 4203903b55aSLiam R. Howlett * If @vma is NULL, return the first vma in the mm. 4213903b55aSLiam R. Howlett * 4223903b55aSLiam R. Howlett * Returns: The next VMA after @vma. 4233903b55aSLiam R. Howlett */ 424f39af059SMatthew Wilcox (Oracle) static inline struct vm_area_struct *__vma_next(struct mm_struct *mm, 4253903b55aSLiam R. Howlett struct vm_area_struct *vma) 4263903b55aSLiam R. Howlett { 4273903b55aSLiam R. Howlett if (!vma) 4283903b55aSLiam R. Howlett return mm->mmap; 4293903b55aSLiam R. Howlett 4303903b55aSLiam R. Howlett return vma->vm_next; 4313903b55aSLiam R. Howlett } 432fb8090b6SLiam R. Howlett 433fb8090b6SLiam R. Howlett /* 434fb8090b6SLiam R. Howlett * munmap_vma_range() - munmap VMAs that overlap a range. 435fb8090b6SLiam R. Howlett * @mm: The mm struct 436fb8090b6SLiam R. Howlett * @start: The start of the range. 437fb8090b6SLiam R. Howlett * @len: The length of the range. 438fb8090b6SLiam R. Howlett * @pprev: pointer to the pointer that will be set to previous vm_area_struct 439fb8090b6SLiam R. Howlett * 440fb8090b6SLiam R. Howlett * Find all the vm_area_struct that overlap from @start to 441fb8090b6SLiam R. Howlett * @end and munmap them. Set @pprev to the previous vm_area_struct. 442fb8090b6SLiam R. Howlett * 443fb8090b6SLiam R. Howlett * Returns: -ENOMEM on munmap failure or 0 on success. 444fb8090b6SLiam R. Howlett */ 445fb8090b6SLiam R. Howlett static inline int 446fb8090b6SLiam R. Howlett munmap_vma_range(struct mm_struct *mm, unsigned long start, unsigned long len, 447524e00b3SLiam R. Howlett struct vm_area_struct **pprev, struct list_head *uf) 448fb8090b6SLiam R. Howlett { 449524e00b3SLiam R. Howlett while (range_has_overlap(mm, start, start + len, pprev)) 450fb8090b6SLiam R. Howlett if (do_munmap(mm, start, len, uf)) 451fb8090b6SLiam R. Howlett return -ENOMEM; 452fb8090b6SLiam R. Howlett return 0; 453fb8090b6SLiam R. Howlett } 4542e3af1dbSMatthew Wilcox (Oracle) 455e8420a8eSCyril Hrubis static unsigned long count_vma_pages_range(struct mm_struct *mm, 456e8420a8eSCyril Hrubis unsigned long addr, unsigned long end) 457e8420a8eSCyril Hrubis { 4582e3af1dbSMatthew Wilcox (Oracle) VMA_ITERATOR(vmi, mm, addr); 459e8420a8eSCyril Hrubis struct vm_area_struct *vma; 4602e3af1dbSMatthew Wilcox (Oracle) unsigned long nr_pages = 0; 461e8420a8eSCyril Hrubis 4622e3af1dbSMatthew Wilcox (Oracle) for_each_vma_range(vmi, vma, end) { 4632e3af1dbSMatthew Wilcox (Oracle) unsigned long vm_start = max(addr, vma->vm_start); 4642e3af1dbSMatthew Wilcox (Oracle) unsigned long vm_end = min(end, vma->vm_end); 465e8420a8eSCyril Hrubis 4662e3af1dbSMatthew Wilcox (Oracle) nr_pages += PHYS_PFN(vm_end - vm_start); 467e8420a8eSCyril Hrubis } 468e8420a8eSCyril Hrubis 469e8420a8eSCyril Hrubis return nr_pages; 470e8420a8eSCyril Hrubis } 471e8420a8eSCyril Hrubis 472cb8f488cSDenys Vlasenko static void __vma_link_file(struct vm_area_struct *vma) 4731da177e4SLinus Torvalds { 4741da177e4SLinus Torvalds struct file *file; 4751da177e4SLinus Torvalds 4761da177e4SLinus Torvalds file = vma->vm_file; 4771da177e4SLinus Torvalds if (file) { 4781da177e4SLinus Torvalds struct address_space *mapping = file->f_mapping; 4791da177e4SLinus Torvalds 4801da177e4SLinus Torvalds if (vma->vm_flags & VM_SHARED) 481cf508b58SMiaohe Lin mapping_allow_writable(mapping); 4821da177e4SLinus Torvalds 4831da177e4SLinus Torvalds flush_dcache_mmap_lock(mapping); 4846b2dbba8SMichel Lespinasse vma_interval_tree_insert(vma, &mapping->i_mmap); 4851da177e4SLinus Torvalds flush_dcache_mmap_unlock(mapping); 4861da177e4SLinus Torvalds } 4871da177e4SLinus Torvalds } 4881da177e4SLinus Torvalds 489d4af56c5SLiam R. Howlett /* 490d4af56c5SLiam R. Howlett * vma_mas_store() - Store a VMA in the maple tree. 491d4af56c5SLiam R. Howlett * @vma: The vm_area_struct 492d4af56c5SLiam R. Howlett * @mas: The maple state 493d4af56c5SLiam R. Howlett * 494d4af56c5SLiam R. Howlett * Efficient way to store a VMA in the maple tree when the @mas has already 495d4af56c5SLiam R. Howlett * walked to the correct location. 496d4af56c5SLiam R. Howlett * 497d4af56c5SLiam R. Howlett * Note: the end address is inclusive in the maple tree. 498d4af56c5SLiam R. Howlett */ 499d4af56c5SLiam R. Howlett void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas) 500d4af56c5SLiam R. Howlett { 501d4af56c5SLiam R. Howlett trace_vma_store(mas->tree, vma); 502d4af56c5SLiam R. Howlett mas_set_range(mas, vma->vm_start, vma->vm_end - 1); 503d4af56c5SLiam R. Howlett mas_store_prealloc(mas, vma); 504d4af56c5SLiam R. Howlett } 505d4af56c5SLiam R. Howlett 506d4af56c5SLiam R. Howlett /* 507d4af56c5SLiam R. Howlett * vma_mas_remove() - Remove a VMA from the maple tree. 508d4af56c5SLiam R. Howlett * @vma: The vm_area_struct 509d4af56c5SLiam R. Howlett * @mas: The maple state 510d4af56c5SLiam R. Howlett * 511d4af56c5SLiam R. Howlett * Efficient way to remove a VMA from the maple tree when the @mas has already 512d4af56c5SLiam R. Howlett * been established and points to the correct location. 513d4af56c5SLiam R. Howlett * Note: the end address is inclusive in the maple tree. 514d4af56c5SLiam R. Howlett */ 515d4af56c5SLiam R. Howlett void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas) 516d4af56c5SLiam R. Howlett { 517d4af56c5SLiam R. Howlett trace_vma_mas_szero(mas->tree, vma->vm_start, vma->vm_end - 1); 518d4af56c5SLiam R. Howlett mas->index = vma->vm_start; 519d4af56c5SLiam R. Howlett mas->last = vma->vm_end - 1; 520d4af56c5SLiam R. Howlett mas_store_prealloc(mas, NULL); 521d4af56c5SLiam R. Howlett } 522d4af56c5SLiam R. Howlett 523d4af56c5SLiam R. Howlett /* 524d4af56c5SLiam R. Howlett * vma_mas_szero() - Set a given range to zero. Used when modifying a 525d4af56c5SLiam R. Howlett * vm_area_struct start or end. 526d4af56c5SLiam R. Howlett * 527d4af56c5SLiam R. Howlett * @mm: The struct_mm 528d4af56c5SLiam R. Howlett * @start: The start address to zero 529d4af56c5SLiam R. Howlett * @end: The end address to zero. 530d4af56c5SLiam R. Howlett */ 531d4af56c5SLiam R. Howlett static inline void vma_mas_szero(struct ma_state *mas, unsigned long start, 532d4af56c5SLiam R. Howlett unsigned long end) 533d4af56c5SLiam R. Howlett { 534d4af56c5SLiam R. Howlett trace_vma_mas_szero(mas->tree, start, end - 1); 535d4af56c5SLiam R. Howlett mas_set_range(mas, start, end - 1); 536d4af56c5SLiam R. Howlett mas_store_prealloc(mas, NULL); 537d4af56c5SLiam R. Howlett } 538d4af56c5SLiam R. Howlett 539d4af56c5SLiam R. Howlett static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma, 540524e00b3SLiam R. Howlett struct vm_area_struct *prev) 5411da177e4SLinus Torvalds { 542d4af56c5SLiam R. Howlett MA_STATE(mas, &mm->mm_mt, 0, 0); 5431da177e4SLinus Torvalds struct address_space *mapping = NULL; 5441da177e4SLinus Torvalds 545d4af56c5SLiam R. Howlett if (mas_preallocate(&mas, vma, GFP_KERNEL)) 546d4af56c5SLiam R. Howlett return -ENOMEM; 547d4af56c5SLiam R. Howlett 54864ac4940SHuang Shijie if (vma->vm_file) { 5491da177e4SLinus Torvalds mapping = vma->vm_file->f_mapping; 55083cde9e8SDavidlohr Bueso i_mmap_lock_write(mapping); 55164ac4940SHuang Shijie } 5521da177e4SLinus Torvalds 553d4af56c5SLiam R. Howlett vma_mas_store(vma, &mas); 554524e00b3SLiam R. Howlett __vma_link_list(mm, vma, prev); 5551da177e4SLinus Torvalds __vma_link_file(vma); 5561da177e4SLinus Torvalds 5571da177e4SLinus Torvalds if (mapping) 55883cde9e8SDavidlohr Bueso i_mmap_unlock_write(mapping); 5591da177e4SLinus Torvalds 5601da177e4SLinus Torvalds mm->map_count++; 5611da177e4SLinus Torvalds validate_mm(mm); 562d4af56c5SLiam R. Howlett return 0; 5631da177e4SLinus Torvalds } 5641da177e4SLinus Torvalds 5651da177e4SLinus Torvalds /* 56688f6b4c3SKautuk Consul * Helper for vma_adjust() in the split_vma insert case: insert a vma into the 567524e00b3SLiam R. Howlett * mm's list and the mm tree. It has already been inserted into the interval tree. 5681da177e4SLinus Torvalds */ 569d4af56c5SLiam R. Howlett static void __insert_vm_struct(struct mm_struct *mm, struct ma_state *mas, 570*3b0e81a1SLiam R. Howlett struct vm_area_struct *vma, unsigned long location) 5711da177e4SLinus Torvalds { 5726597d783SHugh Dickins struct vm_area_struct *prev; 5731da177e4SLinus Torvalds 574*3b0e81a1SLiam R. Howlett mas_set(mas, location); 575524e00b3SLiam R. Howlett prev = mas_prev(mas, 0); 576d4af56c5SLiam R. Howlett vma_mas_store(vma, mas); 577d4af56c5SLiam R. Howlett __vma_link_list(mm, vma, prev); 5781da177e4SLinus Torvalds mm->map_count++; 5791da177e4SLinus Torvalds } 5801da177e4SLinus Torvalds 5811da177e4SLinus Torvalds /* 5821da177e4SLinus Torvalds * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that 5831da177e4SLinus Torvalds * is already present in an i_mmap tree without adjusting the tree. 5841da177e4SLinus Torvalds * The following helper function should be used when such adjustments 5851da177e4SLinus Torvalds * are necessary. The "insert" vma (if any) is to be inserted 5861da177e4SLinus Torvalds * before we drop the necessary locks. 5871da177e4SLinus Torvalds */ 588e86f15eeSAndrea Arcangeli int __vma_adjust(struct vm_area_struct *vma, unsigned long start, 589e86f15eeSAndrea Arcangeli unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert, 590e86f15eeSAndrea Arcangeli struct vm_area_struct *expand) 5911da177e4SLinus Torvalds { 5921da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 593524e00b3SLiam R. Howlett struct vm_area_struct *next_next, *next = find_vma(mm, vma->vm_end); 594524e00b3SLiam R. Howlett struct vm_area_struct *orig_vma = vma; 5951da177e4SLinus Torvalds struct address_space *mapping = NULL; 596f808c13fSDavidlohr Bueso struct rb_root_cached *root = NULL; 597012f1800SRik van Riel struct anon_vma *anon_vma = NULL; 5981da177e4SLinus Torvalds struct file *file = vma->vm_file; 599524e00b3SLiam R. Howlett bool vma_changed = false; 6001da177e4SLinus Torvalds long adjust_next = 0; 6011da177e4SLinus Torvalds int remove_next = 0; 602d4af56c5SLiam R. Howlett MA_STATE(mas, &mm->mm_mt, 0, 0); 603734537c9SKirill A. Shutemov struct vm_area_struct *exporter = NULL, *importer = NULL; 604*3b0e81a1SLiam R. Howlett unsigned long ll_prev = vma->vm_start; /* linked list prev. */ 605287d97acSLinus Torvalds 606d4af56c5SLiam R. Howlett if (next && !insert) { 6071da177e4SLinus Torvalds if (end >= next->vm_end) { 6081da177e4SLinus Torvalds /* 6091da177e4SLinus Torvalds * vma expands, overlapping all the next, and 6101da177e4SLinus Torvalds * perhaps the one after too (mprotect case 6). 61186d12e47SAndrea Arcangeli * The only other cases that gets here are 612e86f15eeSAndrea Arcangeli * case 1, case 7 and case 8. 613e86f15eeSAndrea Arcangeli */ 614e86f15eeSAndrea Arcangeli if (next == expand) { 615e86f15eeSAndrea Arcangeli /* 616e86f15eeSAndrea Arcangeli * The only case where we don't expand "vma" 617e86f15eeSAndrea Arcangeli * and we expand "next" instead is case 8. 618e86f15eeSAndrea Arcangeli */ 619e86f15eeSAndrea Arcangeli VM_WARN_ON(end != next->vm_end); 620e86f15eeSAndrea Arcangeli /* 621e86f15eeSAndrea Arcangeli * remove_next == 3 means we're 622e86f15eeSAndrea Arcangeli * removing "vma" and that to do so we 623e86f15eeSAndrea Arcangeli * swapped "vma" and "next". 624e86f15eeSAndrea Arcangeli */ 625e86f15eeSAndrea Arcangeli remove_next = 3; 626e86f15eeSAndrea Arcangeli VM_WARN_ON(file != next->vm_file); 627e86f15eeSAndrea Arcangeli swap(vma, next); 628e86f15eeSAndrea Arcangeli } else { 629e86f15eeSAndrea Arcangeli VM_WARN_ON(expand != vma); 630e86f15eeSAndrea Arcangeli /* 631e86f15eeSAndrea Arcangeli * case 1, 6, 7, remove_next == 2 is case 6, 632e86f15eeSAndrea Arcangeli * remove_next == 1 is case 1 or 7. 6331da177e4SLinus Torvalds */ 634734537c9SKirill A. Shutemov remove_next = 1 + (end > next->vm_end); 635d4af56c5SLiam R. Howlett if (remove_next == 2) 636d4af56c5SLiam R. Howlett next_next = find_vma(mm, next->vm_end); 637d4af56c5SLiam R. Howlett 638e86f15eeSAndrea Arcangeli VM_WARN_ON(remove_next == 2 && 639e86f15eeSAndrea Arcangeli end != next->vm_next->vm_end); 640e86f15eeSAndrea Arcangeli } 641e86f15eeSAndrea Arcangeli 642287d97acSLinus Torvalds exporter = next; 6431da177e4SLinus Torvalds importer = vma; 644734537c9SKirill A. Shutemov 645734537c9SKirill A. Shutemov /* 646734537c9SKirill A. Shutemov * If next doesn't have anon_vma, import from vma after 647734537c9SKirill A. Shutemov * next, if the vma overlaps with it. 648734537c9SKirill A. Shutemov */ 64997a42cd4SAndrea Arcangeli if (remove_next == 2 && !next->anon_vma) 650734537c9SKirill A. Shutemov exporter = next->vm_next; 651734537c9SKirill A. Shutemov 6521da177e4SLinus Torvalds } else if (end > next->vm_start) { 6531da177e4SLinus Torvalds /* 6541da177e4SLinus Torvalds * vma expands, overlapping part of the next: 6551da177e4SLinus Torvalds * mprotect case 5 shifting the boundary up. 6561da177e4SLinus Torvalds */ 657f9d86a60SWei Yang adjust_next = (end - next->vm_start); 658287d97acSLinus Torvalds exporter = next; 6591da177e4SLinus Torvalds importer = vma; 660e86f15eeSAndrea Arcangeli VM_WARN_ON(expand != importer); 6611da177e4SLinus Torvalds } else if (end < vma->vm_end) { 6621da177e4SLinus Torvalds /* 6631da177e4SLinus Torvalds * vma shrinks, and !insert tells it's not 6641da177e4SLinus Torvalds * split_vma inserting another: so it must be 6651da177e4SLinus Torvalds * mprotect case 4 shifting the boundary down. 6661da177e4SLinus Torvalds */ 667f9d86a60SWei Yang adjust_next = -(vma->vm_end - end); 668287d97acSLinus Torvalds exporter = vma; 6691da177e4SLinus Torvalds importer = next; 670e86f15eeSAndrea Arcangeli VM_WARN_ON(expand != importer); 6711da177e4SLinus Torvalds } 6721da177e4SLinus Torvalds 6735beb4930SRik van Riel /* 6745beb4930SRik van Riel * Easily overlooked: when mprotect shifts the boundary, 6755beb4930SRik van Riel * make sure the expanding vma has anon_vma set if the 6765beb4930SRik van Riel * shrinking vma had, to cover any anon pages imported. 6775beb4930SRik van Riel */ 678287d97acSLinus Torvalds if (exporter && exporter->anon_vma && !importer->anon_vma) { 679c4ea95d7SDaniel Forrest int error; 680c4ea95d7SDaniel Forrest 681287d97acSLinus Torvalds importer->anon_vma = exporter->anon_vma; 682b800c91aSKonstantin Khlebnikov error = anon_vma_clone(importer, exporter); 6833fe89b3eSLeon Yu if (error) 684b800c91aSKonstantin Khlebnikov return error; 685b800c91aSKonstantin Khlebnikov } 6865beb4930SRik van Riel } 68737f9f559SKirill A. Shutemov 688d4af56c5SLiam R. Howlett if (mas_preallocate(&mas, vma, GFP_KERNEL)) 689d4af56c5SLiam R. Howlett return -ENOMEM; 690d4af56c5SLiam R. Howlett 691d4af56c5SLiam R. Howlett vma_adjust_trans_huge(orig_vma, start, end, adjust_next); 6921da177e4SLinus Torvalds if (file) { 6931da177e4SLinus Torvalds mapping = file->f_mapping; 6941da177e4SLinus Torvalds root = &mapping->i_mmap; 695cbc91f71SSrikar Dronamraju uprobe_munmap(vma, vma->vm_start, vma->vm_end); 696682968e0SSrikar Dronamraju 697682968e0SSrikar Dronamraju if (adjust_next) 69827ba0644SKirill A. Shutemov uprobe_munmap(next, next->vm_start, next->vm_end); 699682968e0SSrikar Dronamraju 70083cde9e8SDavidlohr Bueso i_mmap_lock_write(mapping); 7011da177e4SLinus Torvalds if (insert) { 7021da177e4SLinus Torvalds /* 7036b2dbba8SMichel Lespinasse * Put into interval tree now, so instantiated pages 7041da177e4SLinus Torvalds * are visible to arm/parisc __flush_dcache_page 7051da177e4SLinus Torvalds * throughout; but we cannot insert into address 7061da177e4SLinus Torvalds * space until vma start or end is updated. 7071da177e4SLinus Torvalds */ 7081da177e4SLinus Torvalds __vma_link_file(insert); 7091da177e4SLinus Torvalds } 7101da177e4SLinus Torvalds } 7111da177e4SLinus Torvalds 712012f1800SRik van Riel anon_vma = vma->anon_vma; 713bf181b9fSMichel Lespinasse if (!anon_vma && adjust_next) 714bf181b9fSMichel Lespinasse anon_vma = next->anon_vma; 715bf181b9fSMichel Lespinasse if (anon_vma) { 716e86f15eeSAndrea Arcangeli VM_WARN_ON(adjust_next && next->anon_vma && 717e86f15eeSAndrea Arcangeli anon_vma != next->anon_vma); 7184fc3f1d6SIngo Molnar anon_vma_lock_write(anon_vma); 719bf181b9fSMichel Lespinasse anon_vma_interval_tree_pre_update_vma(vma); 720bf181b9fSMichel Lespinasse if (adjust_next) 721bf181b9fSMichel Lespinasse anon_vma_interval_tree_pre_update_vma(next); 722bf181b9fSMichel Lespinasse } 723012f1800SRik van Riel 7240fc48a6eSWei Yang if (file) { 7251da177e4SLinus Torvalds flush_dcache_mmap_lock(mapping); 7266b2dbba8SMichel Lespinasse vma_interval_tree_remove(vma, root); 7271da177e4SLinus Torvalds if (adjust_next) 7286b2dbba8SMichel Lespinasse vma_interval_tree_remove(next, root); 7291da177e4SLinus Torvalds } 7301da177e4SLinus Torvalds 731d3737187SMichel Lespinasse if (start != vma->vm_start) { 732*3b0e81a1SLiam R. Howlett if ((vma->vm_start < start) && 733*3b0e81a1SLiam R. Howlett (!insert || (insert->vm_end != start))) { 734524e00b3SLiam R. Howlett vma_mas_szero(&mas, vma->vm_start, start); 735*3b0e81a1SLiam R. Howlett VM_WARN_ON(insert && insert->vm_start > vma->vm_start); 736*3b0e81a1SLiam R. Howlett } else { 737524e00b3SLiam R. Howlett vma_changed = true; 738*3b0e81a1SLiam R. Howlett } 7391da177e4SLinus Torvalds vma->vm_start = start; 740d3737187SMichel Lespinasse } 741d3737187SMichel Lespinasse if (end != vma->vm_end) { 742*3b0e81a1SLiam R. Howlett if (vma->vm_end > end) { 743*3b0e81a1SLiam R. Howlett if (!insert || (insert->vm_start != end)) { 744524e00b3SLiam R. Howlett vma_mas_szero(&mas, end, vma->vm_end); 745*3b0e81a1SLiam R. Howlett VM_WARN_ON(insert && 746*3b0e81a1SLiam R. Howlett insert->vm_end < vma->vm_end); 747*3b0e81a1SLiam R. Howlett } else if (insert->vm_start == end) { 748*3b0e81a1SLiam R. Howlett ll_prev = vma->vm_end; 749*3b0e81a1SLiam R. Howlett } 750*3b0e81a1SLiam R. Howlett } else { 751524e00b3SLiam R. Howlett vma_changed = true; 752*3b0e81a1SLiam R. Howlett } 7531da177e4SLinus Torvalds vma->vm_end = end; 754524e00b3SLiam R. Howlett if (!next) 755524e00b3SLiam R. Howlett mm->highest_vm_end = vm_end_gap(vma); 756d3737187SMichel Lespinasse } 757d4af56c5SLiam R. Howlett 758524e00b3SLiam R. Howlett if (vma_changed) 759d4af56c5SLiam R. Howlett vma_mas_store(vma, &mas); 760d4af56c5SLiam R. Howlett 7611da177e4SLinus Torvalds vma->vm_pgoff = pgoff; 7621da177e4SLinus Torvalds if (adjust_next) { 763f9d86a60SWei Yang next->vm_start += adjust_next; 764f9d86a60SWei Yang next->vm_pgoff += adjust_next >> PAGE_SHIFT; 765d4af56c5SLiam R. Howlett vma_mas_store(next, &mas); 7661da177e4SLinus Torvalds } 7671da177e4SLinus Torvalds 7680fc48a6eSWei Yang if (file) { 7691da177e4SLinus Torvalds if (adjust_next) 7706b2dbba8SMichel Lespinasse vma_interval_tree_insert(next, root); 7716b2dbba8SMichel Lespinasse vma_interval_tree_insert(vma, root); 7721da177e4SLinus Torvalds flush_dcache_mmap_unlock(mapping); 7731da177e4SLinus Torvalds } 7741da177e4SLinus Torvalds 7751da177e4SLinus Torvalds if (remove_next) { 7761da177e4SLinus Torvalds /* 7771da177e4SLinus Torvalds * vma_merge has merged next into vma, and needs 7781da177e4SLinus Torvalds * us to remove next before dropping the locks. 779d4af56c5SLiam R. Howlett * Since we have expanded over this vma, the maple tree will 780d4af56c5SLiam R. Howlett * have overwritten by storing the value 7811da177e4SLinus Torvalds */ 782524e00b3SLiam R. Howlett __vma_unlink_list(mm, next); 783d4af56c5SLiam R. Howlett if (remove_next == 2) 784524e00b3SLiam R. Howlett __vma_unlink_list(mm, next_next); 785524e00b3SLiam R. Howlett /* Kill the cache */ 786524e00b3SLiam R. Howlett vmacache_invalidate(mm); 787524e00b3SLiam R. Howlett 788d4af56c5SLiam R. Howlett if (file) { 7891da177e4SLinus Torvalds __remove_shared_vm_struct(next, file, mapping); 790d4af56c5SLiam R. Howlett if (remove_next == 2) 791d4af56c5SLiam R. Howlett __remove_shared_vm_struct(next_next, file, mapping); 792d4af56c5SLiam R. Howlett } 7931da177e4SLinus Torvalds } else if (insert) { 7941da177e4SLinus Torvalds /* 7951da177e4SLinus Torvalds * split_vma has split insert from vma, and needs 7961da177e4SLinus Torvalds * us to insert it before dropping the locks 7971da177e4SLinus Torvalds * (it may either follow vma or precede it). 7981da177e4SLinus Torvalds */ 799*3b0e81a1SLiam R. Howlett __insert_vm_struct(mm, &mas, insert, ll_prev); 8001da177e4SLinus Torvalds } 8011da177e4SLinus Torvalds 802bf181b9fSMichel Lespinasse if (anon_vma) { 803bf181b9fSMichel Lespinasse anon_vma_interval_tree_post_update_vma(vma); 804bf181b9fSMichel Lespinasse if (adjust_next) 805bf181b9fSMichel Lespinasse anon_vma_interval_tree_post_update_vma(next); 80608b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 807bf181b9fSMichel Lespinasse } 8081da177e4SLinus Torvalds 8090fc48a6eSWei Yang if (file) { 810808fbdbeSWei Yang i_mmap_unlock_write(mapping); 8117b2d81d4SIngo Molnar uprobe_mmap(vma); 8122b144498SSrikar Dronamraju 8132b144498SSrikar Dronamraju if (adjust_next) 8147b2d81d4SIngo Molnar uprobe_mmap(next); 8152b144498SSrikar Dronamraju } 8162b144498SSrikar Dronamraju 8171da177e4SLinus Torvalds if (remove_next) { 818d4af56c5SLiam R. Howlett again: 819925d1c40SMatt Helsley if (file) { 820cbc91f71SSrikar Dronamraju uprobe_munmap(next, next->vm_start, next->vm_end); 8211da177e4SLinus Torvalds fput(file); 822925d1c40SMatt Helsley } 8235beb4930SRik van Riel if (next->anon_vma) 8245beb4930SRik van Riel anon_vma_merge(vma, next); 8251da177e4SLinus Torvalds mm->map_count--; 8263964acd0SOleg Nesterov mpol_put(vma_policy(next)); 827524e00b3SLiam R. Howlett if (remove_next != 2) 828524e00b3SLiam R. Howlett BUG_ON(vma->vm_end < next->vm_end); 8293928d4f5SLinus Torvalds vm_area_free(next); 830524e00b3SLiam R. Howlett 8311da177e4SLinus Torvalds /* 8321da177e4SLinus Torvalds * In mprotect's case 6 (see comments on vma_merge), 8331da177e4SLinus Torvalds * we must remove another next too. It would clutter 8341da177e4SLinus Torvalds * up the code too much to do both in one go. 8351da177e4SLinus Torvalds */ 836e86f15eeSAndrea Arcangeli if (remove_next != 3) { 837e86f15eeSAndrea Arcangeli /* 838e86f15eeSAndrea Arcangeli * If "next" was removed and vma->vm_end was 839e86f15eeSAndrea Arcangeli * expanded (up) over it, in turn 840e86f15eeSAndrea Arcangeli * "next->vm_prev->vm_end" changed and the 841e86f15eeSAndrea Arcangeli * "vma->vm_next" gap must be updated. 842e86f15eeSAndrea Arcangeli */ 843d4af56c5SLiam R. Howlett next = next_next; 844e86f15eeSAndrea Arcangeli } else { 845e86f15eeSAndrea Arcangeli /* 846e86f15eeSAndrea Arcangeli * For the scope of the comment "next" and 847e86f15eeSAndrea Arcangeli * "vma" considered pre-swap(): if "vma" was 848e86f15eeSAndrea Arcangeli * removed, next->vm_start was expanded (down) 849e86f15eeSAndrea Arcangeli * over it and the "next" gap must be updated. 850e86f15eeSAndrea Arcangeli * Because of the swap() the post-swap() "vma" 851e86f15eeSAndrea Arcangeli * actually points to pre-swap() "next" 852e86f15eeSAndrea Arcangeli * (post-swap() "next" as opposed is now a 853e86f15eeSAndrea Arcangeli * dangling pointer). 854e86f15eeSAndrea Arcangeli */ 855e86f15eeSAndrea Arcangeli next = vma; 856e86f15eeSAndrea Arcangeli } 857734537c9SKirill A. Shutemov if (remove_next == 2) { 858734537c9SKirill A. Shutemov remove_next = 1; 8591da177e4SLinus Torvalds goto again; 860524e00b3SLiam R. Howlett } else if (!next) { 861fb8c41e9SAndrea Arcangeli /* 862fb8c41e9SAndrea Arcangeli * If remove_next == 2 we obviously can't 863fb8c41e9SAndrea Arcangeli * reach this path. 864fb8c41e9SAndrea Arcangeli * 865fb8c41e9SAndrea Arcangeli * If remove_next == 3 we can't reach this 866fb8c41e9SAndrea Arcangeli * path because pre-swap() next is always not 867fb8c41e9SAndrea Arcangeli * NULL. pre-swap() "next" is not being 868fb8c41e9SAndrea Arcangeli * removed and its next->vm_end is not altered 869fb8c41e9SAndrea Arcangeli * (and furthermore "end" already matches 870fb8c41e9SAndrea Arcangeli * next->vm_end in remove_next == 3). 871fb8c41e9SAndrea Arcangeli * 872fb8c41e9SAndrea Arcangeli * We reach this only in the remove_next == 1 873fb8c41e9SAndrea Arcangeli * case if the "next" vma that was removed was 874fb8c41e9SAndrea Arcangeli * the highest vma of the mm. However in such 875fb8c41e9SAndrea Arcangeli * case next->vm_end == "end" and the extended 876fb8c41e9SAndrea Arcangeli * "vma" has vma->vm_end == next->vm_end so 877fb8c41e9SAndrea Arcangeli * mm->highest_vm_end doesn't need any update 878fb8c41e9SAndrea Arcangeli * in remove_next == 1 case. 879fb8c41e9SAndrea Arcangeli */ 8801be7107fSHugh Dickins VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma)); 881fb8c41e9SAndrea Arcangeli } 8821da177e4SLinus Torvalds } 8832b144498SSrikar Dronamraju if (insert && file) 8847b2d81d4SIngo Molnar uprobe_mmap(insert); 8851da177e4SLinus Torvalds 886*3b0e81a1SLiam R. Howlett mas_destroy(&mas); 8871da177e4SLinus Torvalds validate_mm(mm); 8885beb4930SRik van Riel return 0; 8891da177e4SLinus Torvalds } 8901da177e4SLinus Torvalds 8911da177e4SLinus Torvalds /* 8921da177e4SLinus Torvalds * If the vma has a ->close operation then the driver probably needs to release 8931da177e4SLinus Torvalds * per-vma resources, so we don't attempt to merge those. 8941da177e4SLinus Torvalds */ 8951da177e4SLinus Torvalds static inline int is_mergeable_vma(struct vm_area_struct *vma, 89619a809afSAndrea Arcangeli struct file *file, unsigned long vm_flags, 8979a10064fSColin Cross struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 8985c26f6acSSuren Baghdasaryan struct anon_vma_name *anon_name) 8991da177e4SLinus Torvalds { 90034228d47SCyrill Gorcunov /* 90134228d47SCyrill Gorcunov * VM_SOFTDIRTY should not prevent from VMA merging, if we 90234228d47SCyrill Gorcunov * match the flags but dirty bit -- the caller should mark 90334228d47SCyrill Gorcunov * merged VMA as dirty. If dirty bit won't be excluded from 9048bb4e7a2SWei Yang * comparison, we increase pressure on the memory system forcing 90534228d47SCyrill Gorcunov * the kernel to generate new VMAs when old one could be 90634228d47SCyrill Gorcunov * extended instead. 90734228d47SCyrill Gorcunov */ 90834228d47SCyrill Gorcunov if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY) 9091da177e4SLinus Torvalds return 0; 9101da177e4SLinus Torvalds if (vma->vm_file != file) 9111da177e4SLinus Torvalds return 0; 9121da177e4SLinus Torvalds if (vma->vm_ops && vma->vm_ops->close) 9131da177e4SLinus Torvalds return 0; 91419a809afSAndrea Arcangeli if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx)) 91519a809afSAndrea Arcangeli return 0; 9165c26f6acSSuren Baghdasaryan if (!anon_vma_name_eq(anon_vma_name(vma), anon_name)) 9179a10064fSColin Cross return 0; 9181da177e4SLinus Torvalds return 1; 9191da177e4SLinus Torvalds } 9201da177e4SLinus Torvalds 9211da177e4SLinus Torvalds static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1, 922965f55deSShaohua Li struct anon_vma *anon_vma2, 923965f55deSShaohua Li struct vm_area_struct *vma) 9241da177e4SLinus Torvalds { 925965f55deSShaohua Li /* 926965f55deSShaohua Li * The list_is_singular() test is to avoid merging VMA cloned from 927965f55deSShaohua Li * parents. This can improve scalability caused by anon_vma lock. 928965f55deSShaohua Li */ 929965f55deSShaohua Li if ((!anon_vma1 || !anon_vma2) && (!vma || 930965f55deSShaohua Li list_is_singular(&vma->anon_vma_chain))) 931965f55deSShaohua Li return 1; 932965f55deSShaohua Li return anon_vma1 == anon_vma2; 9331da177e4SLinus Torvalds } 9341da177e4SLinus Torvalds 9351da177e4SLinus Torvalds /* 9361da177e4SLinus Torvalds * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 9371da177e4SLinus Torvalds * in front of (at a lower virtual address and file offset than) the vma. 9381da177e4SLinus Torvalds * 9391da177e4SLinus Torvalds * We cannot merge two vmas if they have differently assigned (non-NULL) 9401da177e4SLinus Torvalds * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 9411da177e4SLinus Torvalds * 9421da177e4SLinus Torvalds * We don't check here for the merged mmap wrapping around the end of pagecache 94345e55300SPeter Collingbourne * indices (16TB on ia32) because do_mmap() does not permit mmap's which 9441da177e4SLinus Torvalds * wrap, nor mmaps which cover the final page at index -1UL. 9451da177e4SLinus Torvalds */ 9461da177e4SLinus Torvalds static int 9471da177e4SLinus Torvalds can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, 94819a809afSAndrea Arcangeli struct anon_vma *anon_vma, struct file *file, 94919a809afSAndrea Arcangeli pgoff_t vm_pgoff, 9509a10064fSColin Cross struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 9515c26f6acSSuren Baghdasaryan struct anon_vma_name *anon_name) 9521da177e4SLinus Torvalds { 9539a10064fSColin Cross if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) && 954965f55deSShaohua Li is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 9551da177e4SLinus Torvalds if (vma->vm_pgoff == vm_pgoff) 9561da177e4SLinus Torvalds return 1; 9571da177e4SLinus Torvalds } 9581da177e4SLinus Torvalds return 0; 9591da177e4SLinus Torvalds } 9601da177e4SLinus Torvalds 9611da177e4SLinus Torvalds /* 9621da177e4SLinus Torvalds * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 9631da177e4SLinus Torvalds * beyond (at a higher virtual address and file offset than) the vma. 9641da177e4SLinus Torvalds * 9651da177e4SLinus Torvalds * We cannot merge two vmas if they have differently assigned (non-NULL) 9661da177e4SLinus Torvalds * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 9671da177e4SLinus Torvalds */ 9681da177e4SLinus Torvalds static int 9691da177e4SLinus Torvalds can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, 97019a809afSAndrea Arcangeli struct anon_vma *anon_vma, struct file *file, 97119a809afSAndrea Arcangeli pgoff_t vm_pgoff, 9729a10064fSColin Cross struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 9735c26f6acSSuren Baghdasaryan struct anon_vma_name *anon_name) 9741da177e4SLinus Torvalds { 9759a10064fSColin Cross if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) && 976965f55deSShaohua Li is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 9771da177e4SLinus Torvalds pgoff_t vm_pglen; 978d6e93217SLibin vm_pglen = vma_pages(vma); 9791da177e4SLinus Torvalds if (vma->vm_pgoff + vm_pglen == vm_pgoff) 9801da177e4SLinus Torvalds return 1; 9811da177e4SLinus Torvalds } 9821da177e4SLinus Torvalds return 0; 9831da177e4SLinus Torvalds } 9841da177e4SLinus Torvalds 9851da177e4SLinus Torvalds /* 9869a10064fSColin Cross * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name), 9879a10064fSColin Cross * figure out whether that can be merged with its predecessor or its 9889a10064fSColin Cross * successor. Or both (it neatly fills a hole). 9891da177e4SLinus Torvalds * 9901da177e4SLinus Torvalds * In most cases - when called for mmap, brk or mremap - [addr,end) is 9911da177e4SLinus Torvalds * certain not to be mapped by the time vma_merge is called; but when 9921da177e4SLinus Torvalds * called for mprotect, it is certain to be already mapped (either at 9931da177e4SLinus Torvalds * an offset within prev, or at the start of next), and the flags of 9941da177e4SLinus Torvalds * this area are about to be changed to vm_flags - and the no-change 9951da177e4SLinus Torvalds * case has already been eliminated. 9961da177e4SLinus Torvalds * 9971da177e4SLinus Torvalds * The following mprotect cases have to be considered, where AAAA is 9981da177e4SLinus Torvalds * the area passed down from mprotect_fixup, never extending beyond one 9991da177e4SLinus Torvalds * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after: 10001da177e4SLinus Torvalds * 10015d42ab29SWei Yang * AAAA AAAA AAAA 10025d42ab29SWei Yang * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN 10035d42ab29SWei Yang * cannot merge might become might become 10045d42ab29SWei Yang * PPNNNNNNNNNN PPPPPPPPPPNN 10055d42ab29SWei Yang * mmap, brk or case 4 below case 5 below 10065d42ab29SWei Yang * mremap move: 10075d42ab29SWei Yang * AAAA AAAA 10085d42ab29SWei Yang * PPPP NNNN PPPPNNNNXXXX 10095d42ab29SWei Yang * might become might become 10105d42ab29SWei Yang * PPPPPPPPPPPP 1 or PPPPPPPPPPPP 6 or 10115d42ab29SWei Yang * PPPPPPPPNNNN 2 or PPPPPPPPXXXX 7 or 10125d42ab29SWei Yang * PPPPNNNNNNNN 3 PPPPXXXXXXXX 8 10131da177e4SLinus Torvalds * 10148bb4e7a2SWei Yang * It is important for case 8 that the vma NNNN overlapping the 1015e86f15eeSAndrea Arcangeli * region AAAA is never going to extended over XXXX. Instead XXXX must 1016e86f15eeSAndrea Arcangeli * be extended in region AAAA and NNNN must be removed. This way in 1017e86f15eeSAndrea Arcangeli * all cases where vma_merge succeeds, the moment vma_adjust drops the 1018e86f15eeSAndrea Arcangeli * rmap_locks, the properties of the merged vma will be already 1019e86f15eeSAndrea Arcangeli * correct for the whole merged range. Some of those properties like 1020e86f15eeSAndrea Arcangeli * vm_page_prot/vm_flags may be accessed by rmap_walks and they must 1021e86f15eeSAndrea Arcangeli * be correct for the whole merged range immediately after the 1022e86f15eeSAndrea Arcangeli * rmap_locks are released. Otherwise if XXXX would be removed and 1023e86f15eeSAndrea Arcangeli * NNNN would be extended over the XXXX range, remove_migration_ptes 1024e86f15eeSAndrea Arcangeli * or other rmap walkers (if working on addresses beyond the "end" 1025e86f15eeSAndrea Arcangeli * parameter) may establish ptes with the wrong permissions of NNNN 1026e86f15eeSAndrea Arcangeli * instead of the right permissions of XXXX. 10271da177e4SLinus Torvalds */ 10281da177e4SLinus Torvalds struct vm_area_struct *vma_merge(struct mm_struct *mm, 10291da177e4SLinus Torvalds struct vm_area_struct *prev, unsigned long addr, 10301da177e4SLinus Torvalds unsigned long end, unsigned long vm_flags, 10311da177e4SLinus Torvalds struct anon_vma *anon_vma, struct file *file, 103219a809afSAndrea Arcangeli pgoff_t pgoff, struct mempolicy *policy, 10339a10064fSColin Cross struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 10345c26f6acSSuren Baghdasaryan struct anon_vma_name *anon_name) 10351da177e4SLinus Torvalds { 10361da177e4SLinus Torvalds pgoff_t pglen = (end - addr) >> PAGE_SHIFT; 10371da177e4SLinus Torvalds struct vm_area_struct *area, *next; 10385beb4930SRik van Riel int err; 10391da177e4SLinus Torvalds 10401da177e4SLinus Torvalds /* 10411da177e4SLinus Torvalds * We later require that vma->vm_flags == vm_flags, 10421da177e4SLinus Torvalds * so this tests vma->vm_flags & VM_SPECIAL, too. 10431da177e4SLinus Torvalds */ 10441da177e4SLinus Torvalds if (vm_flags & VM_SPECIAL) 10451da177e4SLinus Torvalds return NULL; 10461da177e4SLinus Torvalds 1047f39af059SMatthew Wilcox (Oracle) next = __vma_next(mm, prev); 10481da177e4SLinus Torvalds area = next; 1049e86f15eeSAndrea Arcangeli if (area && area->vm_end == end) /* cases 6, 7, 8 */ 10501da177e4SLinus Torvalds next = next->vm_next; 10511da177e4SLinus Torvalds 1052e86f15eeSAndrea Arcangeli /* verify some invariant that must be enforced by the caller */ 1053e86f15eeSAndrea Arcangeli VM_WARN_ON(prev && addr <= prev->vm_start); 1054e86f15eeSAndrea Arcangeli VM_WARN_ON(area && end > area->vm_end); 1055e86f15eeSAndrea Arcangeli VM_WARN_ON(addr >= end); 1056e86f15eeSAndrea Arcangeli 10571da177e4SLinus Torvalds /* 10581da177e4SLinus Torvalds * Can it merge with the predecessor? 10591da177e4SLinus Torvalds */ 10601da177e4SLinus Torvalds if (prev && prev->vm_end == addr && 10611da177e4SLinus Torvalds mpol_equal(vma_policy(prev), policy) && 10621da177e4SLinus Torvalds can_vma_merge_after(prev, vm_flags, 106319a809afSAndrea Arcangeli anon_vma, file, pgoff, 10649a10064fSColin Cross vm_userfaultfd_ctx, anon_name)) { 10651da177e4SLinus Torvalds /* 10661da177e4SLinus Torvalds * OK, it can. Can we now merge in the successor as well? 10671da177e4SLinus Torvalds */ 10681da177e4SLinus Torvalds if (next && end == next->vm_start && 10691da177e4SLinus Torvalds mpol_equal(policy, vma_policy(next)) && 10701da177e4SLinus Torvalds can_vma_merge_before(next, vm_flags, 107119a809afSAndrea Arcangeli anon_vma, file, 107219a809afSAndrea Arcangeli pgoff+pglen, 10739a10064fSColin Cross vm_userfaultfd_ctx, anon_name) && 10741da177e4SLinus Torvalds is_mergeable_anon_vma(prev->anon_vma, 1075965f55deSShaohua Li next->anon_vma, NULL)) { 10761da177e4SLinus Torvalds /* cases 1, 6 */ 1077e86f15eeSAndrea Arcangeli err = __vma_adjust(prev, prev->vm_start, 1078e86f15eeSAndrea Arcangeli next->vm_end, prev->vm_pgoff, NULL, 1079e86f15eeSAndrea Arcangeli prev); 10801da177e4SLinus Torvalds } else /* cases 2, 5, 7 */ 1081e86f15eeSAndrea Arcangeli err = __vma_adjust(prev, prev->vm_start, 1082e86f15eeSAndrea Arcangeli end, prev->vm_pgoff, NULL, prev); 10835beb4930SRik van Riel if (err) 10845beb4930SRik van Riel return NULL; 1085c791576cSYang Shi khugepaged_enter_vma(prev, vm_flags); 10861da177e4SLinus Torvalds return prev; 10871da177e4SLinus Torvalds } 10881da177e4SLinus Torvalds 10891da177e4SLinus Torvalds /* 10901da177e4SLinus Torvalds * Can this new request be merged in front of next? 10911da177e4SLinus Torvalds */ 10921da177e4SLinus Torvalds if (next && end == next->vm_start && 10931da177e4SLinus Torvalds mpol_equal(policy, vma_policy(next)) && 10941da177e4SLinus Torvalds can_vma_merge_before(next, vm_flags, 109519a809afSAndrea Arcangeli anon_vma, file, pgoff+pglen, 10969a10064fSColin Cross vm_userfaultfd_ctx, anon_name)) { 10971da177e4SLinus Torvalds if (prev && addr < prev->vm_end) /* case 4 */ 1098e86f15eeSAndrea Arcangeli err = __vma_adjust(prev, prev->vm_start, 1099e86f15eeSAndrea Arcangeli addr, prev->vm_pgoff, NULL, next); 1100e86f15eeSAndrea Arcangeli else { /* cases 3, 8 */ 1101e86f15eeSAndrea Arcangeli err = __vma_adjust(area, addr, next->vm_end, 1102e86f15eeSAndrea Arcangeli next->vm_pgoff - pglen, NULL, next); 1103e86f15eeSAndrea Arcangeli /* 1104e86f15eeSAndrea Arcangeli * In case 3 area is already equal to next and 1105e86f15eeSAndrea Arcangeli * this is a noop, but in case 8 "area" has 1106e86f15eeSAndrea Arcangeli * been removed and next was expanded over it. 1107e86f15eeSAndrea Arcangeli */ 1108e86f15eeSAndrea Arcangeli area = next; 1109e86f15eeSAndrea Arcangeli } 11105beb4930SRik van Riel if (err) 11115beb4930SRik van Riel return NULL; 1112c791576cSYang Shi khugepaged_enter_vma(area, vm_flags); 11131da177e4SLinus Torvalds return area; 11141da177e4SLinus Torvalds } 11151da177e4SLinus Torvalds 11161da177e4SLinus Torvalds return NULL; 11171da177e4SLinus Torvalds } 11181da177e4SLinus Torvalds 11191da177e4SLinus Torvalds /* 1120b4f315b4SEthon Paul * Rough compatibility check to quickly see if it's even worth looking 1121d0e9fe17SLinus Torvalds * at sharing an anon_vma. 1122d0e9fe17SLinus Torvalds * 1123d0e9fe17SLinus Torvalds * They need to have the same vm_file, and the flags can only differ 1124d0e9fe17SLinus Torvalds * in things that mprotect may change. 1125d0e9fe17SLinus Torvalds * 1126d0e9fe17SLinus Torvalds * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that 1127d0e9fe17SLinus Torvalds * we can merge the two vma's. For example, we refuse to merge a vma if 1128d0e9fe17SLinus Torvalds * there is a vm_ops->close() function, because that indicates that the 1129d0e9fe17SLinus Torvalds * driver is doing some kind of reference counting. But that doesn't 1130d0e9fe17SLinus Torvalds * really matter for the anon_vma sharing case. 1131d0e9fe17SLinus Torvalds */ 1132d0e9fe17SLinus Torvalds static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b) 1133d0e9fe17SLinus Torvalds { 1134d0e9fe17SLinus Torvalds return a->vm_end == b->vm_start && 1135d0e9fe17SLinus Torvalds mpol_equal(vma_policy(a), vma_policy(b)) && 1136d0e9fe17SLinus Torvalds a->vm_file == b->vm_file && 11376cb4d9a2SAnshuman Khandual !((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) && 1138d0e9fe17SLinus Torvalds b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); 1139d0e9fe17SLinus Torvalds } 1140d0e9fe17SLinus Torvalds 1141d0e9fe17SLinus Torvalds /* 1142d0e9fe17SLinus Torvalds * Do some basic sanity checking to see if we can re-use the anon_vma 1143d0e9fe17SLinus Torvalds * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be 1144d0e9fe17SLinus Torvalds * the same as 'old', the other will be the new one that is trying 1145d0e9fe17SLinus Torvalds * to share the anon_vma. 1146d0e9fe17SLinus Torvalds * 11475b449489SFlorian Rommel * NOTE! This runs with mmap_lock held for reading, so it is possible that 1148d0e9fe17SLinus Torvalds * the anon_vma of 'old' is concurrently in the process of being set up 1149d0e9fe17SLinus Torvalds * by another page fault trying to merge _that_. But that's ok: if it 1150d0e9fe17SLinus Torvalds * is being set up, that automatically means that it will be a singleton 1151d0e9fe17SLinus Torvalds * acceptable for merging, so we can do all of this optimistically. But 11524db0c3c2SJason Low * we do that READ_ONCE() to make sure that we never re-load the pointer. 1153d0e9fe17SLinus Torvalds * 1154d0e9fe17SLinus Torvalds * IOW: that the "list_is_singular()" test on the anon_vma_chain only 1155d0e9fe17SLinus Torvalds * matters for the 'stable anon_vma' case (ie the thing we want to avoid 1156d0e9fe17SLinus Torvalds * is to return an anon_vma that is "complex" due to having gone through 1157d0e9fe17SLinus Torvalds * a fork). 1158d0e9fe17SLinus Torvalds * 1159d0e9fe17SLinus Torvalds * We also make sure that the two vma's are compatible (adjacent, 1160d0e9fe17SLinus Torvalds * and with the same memory policies). That's all stable, even with just 11615b449489SFlorian Rommel * a read lock on the mmap_lock. 1162d0e9fe17SLinus Torvalds */ 1163d0e9fe17SLinus Torvalds static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b) 1164d0e9fe17SLinus Torvalds { 1165d0e9fe17SLinus Torvalds if (anon_vma_compatible(a, b)) { 11664db0c3c2SJason Low struct anon_vma *anon_vma = READ_ONCE(old->anon_vma); 1167d0e9fe17SLinus Torvalds 1168d0e9fe17SLinus Torvalds if (anon_vma && list_is_singular(&old->anon_vma_chain)) 1169d0e9fe17SLinus Torvalds return anon_vma; 1170d0e9fe17SLinus Torvalds } 1171d0e9fe17SLinus Torvalds return NULL; 1172d0e9fe17SLinus Torvalds } 1173d0e9fe17SLinus Torvalds 1174d0e9fe17SLinus Torvalds /* 11751da177e4SLinus Torvalds * find_mergeable_anon_vma is used by anon_vma_prepare, to check 11761da177e4SLinus Torvalds * neighbouring vmas for a suitable anon_vma, before it goes off 11771da177e4SLinus Torvalds * to allocate a new anon_vma. It checks because a repetitive 11781da177e4SLinus Torvalds * sequence of mprotects and faults may otherwise lead to distinct 11791da177e4SLinus Torvalds * anon_vmas being allocated, preventing vma merge in subsequent 11801da177e4SLinus Torvalds * mprotect. 11811da177e4SLinus Torvalds */ 11821da177e4SLinus Torvalds struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) 11831da177e4SLinus Torvalds { 1184a67c8caaSMiaohe Lin struct anon_vma *anon_vma = NULL; 11851da177e4SLinus Torvalds 1186a67c8caaSMiaohe Lin /* Try next first. */ 1187a67c8caaSMiaohe Lin if (vma->vm_next) { 1188a67c8caaSMiaohe Lin anon_vma = reusable_anon_vma(vma->vm_next, vma, vma->vm_next); 1189d0e9fe17SLinus Torvalds if (anon_vma) 1190d0e9fe17SLinus Torvalds return anon_vma; 1191a67c8caaSMiaohe Lin } 11921da177e4SLinus Torvalds 1193a67c8caaSMiaohe Lin /* Try prev next. */ 1194a67c8caaSMiaohe Lin if (vma->vm_prev) 1195a67c8caaSMiaohe Lin anon_vma = reusable_anon_vma(vma->vm_prev, vma->vm_prev, vma); 1196a67c8caaSMiaohe Lin 11971da177e4SLinus Torvalds /* 1198a67c8caaSMiaohe Lin * We might reach here with anon_vma == NULL if we can't find 1199a67c8caaSMiaohe Lin * any reusable anon_vma. 12001da177e4SLinus Torvalds * There's no absolute need to look only at touching neighbours: 12011da177e4SLinus Torvalds * we could search further afield for "compatible" anon_vmas. 12021da177e4SLinus Torvalds * But it would probably just be a waste of time searching, 12031da177e4SLinus Torvalds * or lead to too many vmas hanging off the same anon_vma. 12041da177e4SLinus Torvalds * We're trying to allow mprotect remerging later on, 12051da177e4SLinus Torvalds * not trying to minimize memory used for anon_vmas. 12061da177e4SLinus Torvalds */ 1207a67c8caaSMiaohe Lin return anon_vma; 12081da177e4SLinus Torvalds } 12091da177e4SLinus Torvalds 12101da177e4SLinus Torvalds /* 121140401530SAl Viro * If a hint addr is less than mmap_min_addr change hint to be as 121240401530SAl Viro * low as possible but still greater than mmap_min_addr 121340401530SAl Viro */ 121440401530SAl Viro static inline unsigned long round_hint_to_min(unsigned long hint) 121540401530SAl Viro { 121640401530SAl Viro hint &= PAGE_MASK; 121740401530SAl Viro if (((void *)hint != NULL) && 121840401530SAl Viro (hint < mmap_min_addr)) 121940401530SAl Viro return PAGE_ALIGN(mmap_min_addr); 122040401530SAl Viro return hint; 122140401530SAl Viro } 122240401530SAl Viro 12236aeb2542SMike Rapoport int mlock_future_check(struct mm_struct *mm, unsigned long flags, 1224363ee17fSDavidlohr Bueso unsigned long len) 1225363ee17fSDavidlohr Bueso { 1226363ee17fSDavidlohr Bueso unsigned long locked, lock_limit; 1227363ee17fSDavidlohr Bueso 1228363ee17fSDavidlohr Bueso /* mlock MCL_FUTURE? */ 1229363ee17fSDavidlohr Bueso if (flags & VM_LOCKED) { 1230363ee17fSDavidlohr Bueso locked = len >> PAGE_SHIFT; 1231363ee17fSDavidlohr Bueso locked += mm->locked_vm; 1232363ee17fSDavidlohr Bueso lock_limit = rlimit(RLIMIT_MEMLOCK); 1233363ee17fSDavidlohr Bueso lock_limit >>= PAGE_SHIFT; 1234363ee17fSDavidlohr Bueso if (locked > lock_limit && !capable(CAP_IPC_LOCK)) 1235363ee17fSDavidlohr Bueso return -EAGAIN; 1236363ee17fSDavidlohr Bueso } 1237363ee17fSDavidlohr Bueso return 0; 1238363ee17fSDavidlohr Bueso } 1239363ee17fSDavidlohr Bueso 1240be83bbf8SLinus Torvalds static inline u64 file_mmap_size_max(struct file *file, struct inode *inode) 1241be83bbf8SLinus Torvalds { 1242be83bbf8SLinus Torvalds if (S_ISREG(inode->i_mode)) 1243423913adSLinus Torvalds return MAX_LFS_FILESIZE; 1244be83bbf8SLinus Torvalds 1245be83bbf8SLinus Torvalds if (S_ISBLK(inode->i_mode)) 1246be83bbf8SLinus Torvalds return MAX_LFS_FILESIZE; 1247be83bbf8SLinus Torvalds 124876f34950SIvan Khoronzhuk if (S_ISSOCK(inode->i_mode)) 124976f34950SIvan Khoronzhuk return MAX_LFS_FILESIZE; 125076f34950SIvan Khoronzhuk 1251be83bbf8SLinus Torvalds /* Special "we do even unsigned file positions" case */ 1252be83bbf8SLinus Torvalds if (file->f_mode & FMODE_UNSIGNED_OFFSET) 1253be83bbf8SLinus Torvalds return 0; 1254be83bbf8SLinus Torvalds 1255be83bbf8SLinus Torvalds /* Yes, random drivers might want more. But I'm tired of buggy drivers */ 1256be83bbf8SLinus Torvalds return ULONG_MAX; 1257be83bbf8SLinus Torvalds } 1258be83bbf8SLinus Torvalds 1259be83bbf8SLinus Torvalds static inline bool file_mmap_ok(struct file *file, struct inode *inode, 1260be83bbf8SLinus Torvalds unsigned long pgoff, unsigned long len) 1261be83bbf8SLinus Torvalds { 1262be83bbf8SLinus Torvalds u64 maxsize = file_mmap_size_max(file, inode); 1263be83bbf8SLinus Torvalds 1264be83bbf8SLinus Torvalds if (maxsize && len > maxsize) 1265be83bbf8SLinus Torvalds return false; 1266be83bbf8SLinus Torvalds maxsize -= len; 1267be83bbf8SLinus Torvalds if (pgoff > maxsize >> PAGE_SHIFT) 1268be83bbf8SLinus Torvalds return false; 1269be83bbf8SLinus Torvalds return true; 1270be83bbf8SLinus Torvalds } 1271be83bbf8SLinus Torvalds 127240401530SAl Viro /* 12733e4e28c5SMichel Lespinasse * The caller must write-lock current->mm->mmap_lock. 12741da177e4SLinus Torvalds */ 12751fcfd8dbSOleg Nesterov unsigned long do_mmap(struct file *file, unsigned long addr, 12761da177e4SLinus Torvalds unsigned long len, unsigned long prot, 127745e55300SPeter Collingbourne unsigned long flags, unsigned long pgoff, 127845e55300SPeter Collingbourne unsigned long *populate, struct list_head *uf) 12791da177e4SLinus Torvalds { 12801da177e4SLinus Torvalds struct mm_struct *mm = current->mm; 128145e55300SPeter Collingbourne vm_flags_t vm_flags; 128262b5f7d0SDave Hansen int pkey = 0; 12831da177e4SLinus Torvalds 1284524e00b3SLiam R. Howlett validate_mm(mm); 128541badc15SMichel Lespinasse *populate = 0; 1286bebeb3d6SMichel Lespinasse 1287e37609bbSPiotr Kwapulinski if (!len) 1288e37609bbSPiotr Kwapulinski return -EINVAL; 1289e37609bbSPiotr Kwapulinski 12901da177e4SLinus Torvalds /* 12911da177e4SLinus Torvalds * Does the application expect PROT_READ to imply PROT_EXEC? 12921da177e4SLinus Torvalds * 12931da177e4SLinus Torvalds * (the exception is when the underlying filesystem is noexec 12941da177e4SLinus Torvalds * mounted, in which case we dont add PROT_EXEC.) 12951da177e4SLinus Torvalds */ 12961da177e4SLinus Torvalds if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) 129790f8572bSEric W. Biederman if (!(file && path_noexec(&file->f_path))) 12981da177e4SLinus Torvalds prot |= PROT_EXEC; 12991da177e4SLinus Torvalds 1300a4ff8e86SMichal Hocko /* force arch specific MAP_FIXED handling in get_unmapped_area */ 1301a4ff8e86SMichal Hocko if (flags & MAP_FIXED_NOREPLACE) 1302a4ff8e86SMichal Hocko flags |= MAP_FIXED; 1303a4ff8e86SMichal Hocko 13047cd94146SEric Paris if (!(flags & MAP_FIXED)) 13057cd94146SEric Paris addr = round_hint_to_min(addr); 13067cd94146SEric Paris 13071da177e4SLinus Torvalds /* Careful about overflows.. */ 13081da177e4SLinus Torvalds len = PAGE_ALIGN(len); 13099206de95SAl Viro if (!len) 13101da177e4SLinus Torvalds return -ENOMEM; 13111da177e4SLinus Torvalds 13121da177e4SLinus Torvalds /* offset overflow? */ 13131da177e4SLinus Torvalds if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) 13141da177e4SLinus Torvalds return -EOVERFLOW; 13151da177e4SLinus Torvalds 13161da177e4SLinus Torvalds /* Too many mappings? */ 13171da177e4SLinus Torvalds if (mm->map_count > sysctl_max_map_count) 13181da177e4SLinus Torvalds return -ENOMEM; 13191da177e4SLinus Torvalds 13201da177e4SLinus Torvalds /* Obtain the address to map to. we verify (or select) it and ensure 13211da177e4SLinus Torvalds * that it represents a valid section of the address space. 13221da177e4SLinus Torvalds */ 13231da177e4SLinus Torvalds addr = get_unmapped_area(file, addr, len, pgoff, flags); 1324ff68dac6SGaowei Pu if (IS_ERR_VALUE(addr)) 13251da177e4SLinus Torvalds return addr; 13261da177e4SLinus Torvalds 1327a4ff8e86SMichal Hocko if (flags & MAP_FIXED_NOREPLACE) { 132835e43c5fSLiam Howlett if (find_vma_intersection(mm, addr, addr + len)) 1329a4ff8e86SMichal Hocko return -EEXIST; 1330a4ff8e86SMichal Hocko } 1331a4ff8e86SMichal Hocko 133262b5f7d0SDave Hansen if (prot == PROT_EXEC) { 133362b5f7d0SDave Hansen pkey = execute_only_pkey(mm); 133462b5f7d0SDave Hansen if (pkey < 0) 133562b5f7d0SDave Hansen pkey = 0; 133662b5f7d0SDave Hansen } 133762b5f7d0SDave Hansen 13381da177e4SLinus Torvalds /* Do simple checking here so the lower-level routines won't have 13391da177e4SLinus Torvalds * to. we assume access permissions have been handled by the open 13401da177e4SLinus Torvalds * of the memory object, so we don't do any here. 13411da177e4SLinus Torvalds */ 134245e55300SPeter Collingbourne vm_flags = calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) | 13431da177e4SLinus Torvalds mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; 13441da177e4SLinus Torvalds 1345cdf7b341SHuang Shijie if (flags & MAP_LOCKED) 13461da177e4SLinus Torvalds if (!can_do_mlock()) 13471da177e4SLinus Torvalds return -EPERM; 1348ba470de4SRik van Riel 1349363ee17fSDavidlohr Bueso if (mlock_future_check(mm, vm_flags, len)) 13501da177e4SLinus Torvalds return -EAGAIN; 13511da177e4SLinus Torvalds 13521da177e4SLinus Torvalds if (file) { 1353077bf22bSOleg Nesterov struct inode *inode = file_inode(file); 13541c972597SDan Williams unsigned long flags_mask; 13551c972597SDan Williams 1356be83bbf8SLinus Torvalds if (!file_mmap_ok(file, inode, pgoff, len)) 1357be83bbf8SLinus Torvalds return -EOVERFLOW; 1358be83bbf8SLinus Torvalds 13591c972597SDan Williams flags_mask = LEGACY_MAP_MASK | file->f_op->mmap_supported_flags; 1360077bf22bSOleg Nesterov 13611da177e4SLinus Torvalds switch (flags & MAP_TYPE) { 13621da177e4SLinus Torvalds case MAP_SHARED: 13631c972597SDan Williams /* 13641c972597SDan Williams * Force use of MAP_SHARED_VALIDATE with non-legacy 13651c972597SDan Williams * flags. E.g. MAP_SYNC is dangerous to use with 13661c972597SDan Williams * MAP_SHARED as you don't know which consistency model 13671c972597SDan Williams * you will get. We silently ignore unsupported flags 13681c972597SDan Williams * with MAP_SHARED to preserve backward compatibility. 13691c972597SDan Williams */ 13701c972597SDan Williams flags &= LEGACY_MAP_MASK; 1371e4a9bc58SJoe Perches fallthrough; 13721c972597SDan Williams case MAP_SHARED_VALIDATE: 13731c972597SDan Williams if (flags & ~flags_mask) 13741c972597SDan Williams return -EOPNOTSUPP; 1375dc617f29SDarrick J. Wong if (prot & PROT_WRITE) { 1376dc617f29SDarrick J. Wong if (!(file->f_mode & FMODE_WRITE)) 13771da177e4SLinus Torvalds return -EACCES; 1378dc617f29SDarrick J. Wong if (IS_SWAPFILE(file->f_mapping->host)) 1379dc617f29SDarrick J. Wong return -ETXTBSY; 1380dc617f29SDarrick J. Wong } 13811da177e4SLinus Torvalds 13821da177e4SLinus Torvalds /* 13831da177e4SLinus Torvalds * Make sure we don't allow writing to an append-only 13841da177e4SLinus Torvalds * file.. 13851da177e4SLinus Torvalds */ 13861da177e4SLinus Torvalds if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE)) 13871da177e4SLinus Torvalds return -EACCES; 13881da177e4SLinus Torvalds 13891da177e4SLinus Torvalds vm_flags |= VM_SHARED | VM_MAYSHARE; 13901da177e4SLinus Torvalds if (!(file->f_mode & FMODE_WRITE)) 13911da177e4SLinus Torvalds vm_flags &= ~(VM_MAYWRITE | VM_SHARED); 1392e4a9bc58SJoe Perches fallthrough; 13931da177e4SLinus Torvalds case MAP_PRIVATE: 13941da177e4SLinus Torvalds if (!(file->f_mode & FMODE_READ)) 13951da177e4SLinus Torvalds return -EACCES; 139690f8572bSEric W. Biederman if (path_noexec(&file->f_path)) { 139780c5606cSLinus Torvalds if (vm_flags & VM_EXEC) 139880c5606cSLinus Torvalds return -EPERM; 139980c5606cSLinus Torvalds vm_flags &= ~VM_MAYEXEC; 140080c5606cSLinus Torvalds } 140180c5606cSLinus Torvalds 140272c2d531SAl Viro if (!file->f_op->mmap) 140380c5606cSLinus Torvalds return -ENODEV; 1404b2c56e4fSOleg Nesterov if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) 1405b2c56e4fSOleg Nesterov return -EINVAL; 14061da177e4SLinus Torvalds break; 14071da177e4SLinus Torvalds 14081da177e4SLinus Torvalds default: 14091da177e4SLinus Torvalds return -EINVAL; 14101da177e4SLinus Torvalds } 14111da177e4SLinus Torvalds } else { 14121da177e4SLinus Torvalds switch (flags & MAP_TYPE) { 14131da177e4SLinus Torvalds case MAP_SHARED: 1414b2c56e4fSOleg Nesterov if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) 1415b2c56e4fSOleg Nesterov return -EINVAL; 1416ce363942STejun Heo /* 1417ce363942STejun Heo * Ignore pgoff. 1418ce363942STejun Heo */ 1419ce363942STejun Heo pgoff = 0; 14201da177e4SLinus Torvalds vm_flags |= VM_SHARED | VM_MAYSHARE; 14211da177e4SLinus Torvalds break; 14221da177e4SLinus Torvalds case MAP_PRIVATE: 14231da177e4SLinus Torvalds /* 14241da177e4SLinus Torvalds * Set pgoff according to addr for anon_vma. 14251da177e4SLinus Torvalds */ 14261da177e4SLinus Torvalds pgoff = addr >> PAGE_SHIFT; 14271da177e4SLinus Torvalds break; 14281da177e4SLinus Torvalds default: 14291da177e4SLinus Torvalds return -EINVAL; 14301da177e4SLinus Torvalds } 14311da177e4SLinus Torvalds } 14321da177e4SLinus Torvalds 1433c22c0d63SMichel Lespinasse /* 1434c22c0d63SMichel Lespinasse * Set 'VM_NORESERVE' if we should not account for the 1435c22c0d63SMichel Lespinasse * memory use of this mapping. 1436c22c0d63SMichel Lespinasse */ 1437c22c0d63SMichel Lespinasse if (flags & MAP_NORESERVE) { 1438c22c0d63SMichel Lespinasse /* We honor MAP_NORESERVE if allowed to overcommit */ 1439c22c0d63SMichel Lespinasse if (sysctl_overcommit_memory != OVERCOMMIT_NEVER) 1440c22c0d63SMichel Lespinasse vm_flags |= VM_NORESERVE; 1441c22c0d63SMichel Lespinasse 1442c22c0d63SMichel Lespinasse /* hugetlb applies strict overcommit unless MAP_NORESERVE */ 1443c22c0d63SMichel Lespinasse if (file && is_file_hugepages(file)) 1444c22c0d63SMichel Lespinasse vm_flags |= VM_NORESERVE; 1445c22c0d63SMichel Lespinasse } 1446c22c0d63SMichel Lespinasse 1447897ab3e0SMike Rapoport addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); 144809a9f1d2SMichel Lespinasse if (!IS_ERR_VALUE(addr) && 144909a9f1d2SMichel Lespinasse ((vm_flags & VM_LOCKED) || 145009a9f1d2SMichel Lespinasse (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) 145141badc15SMichel Lespinasse *populate = len; 1452bebeb3d6SMichel Lespinasse return addr; 14530165ab44SMiklos Szeredi } 14546be5ceb0SLinus Torvalds 1455a90f590aSDominik Brodowski unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, 1456a90f590aSDominik Brodowski unsigned long prot, unsigned long flags, 1457a90f590aSDominik Brodowski unsigned long fd, unsigned long pgoff) 145866f0dc48SHugh Dickins { 145966f0dc48SHugh Dickins struct file *file = NULL; 14601e3ee14bSChen Gang unsigned long retval; 146166f0dc48SHugh Dickins 146266f0dc48SHugh Dickins if (!(flags & MAP_ANONYMOUS)) { 1463120a795dSAl Viro audit_mmap_fd(fd, flags); 146466f0dc48SHugh Dickins file = fget(fd); 146566f0dc48SHugh Dickins if (!file) 14661e3ee14bSChen Gang return -EBADF; 14677bba8f0eSZhen Lei if (is_file_hugepages(file)) { 1468af73e4d9SNaoya Horiguchi len = ALIGN(len, huge_page_size(hstate_file(file))); 14697bba8f0eSZhen Lei } else if (unlikely(flags & MAP_HUGETLB)) { 1470493af578SJörn Engel retval = -EINVAL; 1471493af578SJörn Engel goto out_fput; 14727bba8f0eSZhen Lei } 147366f0dc48SHugh Dickins } else if (flags & MAP_HUGETLB) { 1474c103a4dcSAndrew Morton struct hstate *hs; 1475af73e4d9SNaoya Horiguchi 147620ac2893SAnshuman Khandual hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); 1477091d0d55SLi Zefan if (!hs) 1478091d0d55SLi Zefan return -EINVAL; 1479091d0d55SLi Zefan 1480091d0d55SLi Zefan len = ALIGN(len, huge_page_size(hs)); 148166f0dc48SHugh Dickins /* 148266f0dc48SHugh Dickins * VM_NORESERVE is used because the reservations will be 148366f0dc48SHugh Dickins * taken when vm_ops->mmap() is called 148466f0dc48SHugh Dickins */ 1485af73e4d9SNaoya Horiguchi file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, 148642d7395fSAndi Kleen VM_NORESERVE, 148783c1fd76Szhangyiru HUGETLB_ANONHUGE_INODE, 148842d7395fSAndi Kleen (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK); 148966f0dc48SHugh Dickins if (IS_ERR(file)) 149066f0dc48SHugh Dickins return PTR_ERR(file); 149166f0dc48SHugh Dickins } 149266f0dc48SHugh Dickins 14939fbeb5abSMichal Hocko retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); 1494493af578SJörn Engel out_fput: 149566f0dc48SHugh Dickins if (file) 149666f0dc48SHugh Dickins fput(file); 149766f0dc48SHugh Dickins return retval; 149866f0dc48SHugh Dickins } 149966f0dc48SHugh Dickins 1500a90f590aSDominik Brodowski SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, 1501a90f590aSDominik Brodowski unsigned long, prot, unsigned long, flags, 1502a90f590aSDominik Brodowski unsigned long, fd, unsigned long, pgoff) 1503a90f590aSDominik Brodowski { 1504a90f590aSDominik Brodowski return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff); 1505a90f590aSDominik Brodowski } 1506a90f590aSDominik Brodowski 1507a4679373SChristoph Hellwig #ifdef __ARCH_WANT_SYS_OLD_MMAP 1508a4679373SChristoph Hellwig struct mmap_arg_struct { 1509a4679373SChristoph Hellwig unsigned long addr; 1510a4679373SChristoph Hellwig unsigned long len; 1511a4679373SChristoph Hellwig unsigned long prot; 1512a4679373SChristoph Hellwig unsigned long flags; 1513a4679373SChristoph Hellwig unsigned long fd; 1514a4679373SChristoph Hellwig unsigned long offset; 1515a4679373SChristoph Hellwig }; 1516a4679373SChristoph Hellwig 1517a4679373SChristoph Hellwig SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) 1518a4679373SChristoph Hellwig { 1519a4679373SChristoph Hellwig struct mmap_arg_struct a; 1520a4679373SChristoph Hellwig 1521a4679373SChristoph Hellwig if (copy_from_user(&a, arg, sizeof(a))) 1522a4679373SChristoph Hellwig return -EFAULT; 1523de1741a1SAlexander Kuleshov if (offset_in_page(a.offset)) 1524a4679373SChristoph Hellwig return -EINVAL; 1525a4679373SChristoph Hellwig 1526a90f590aSDominik Brodowski return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, 1527a4679373SChristoph Hellwig a.offset >> PAGE_SHIFT); 1528a4679373SChristoph Hellwig } 1529a4679373SChristoph Hellwig #endif /* __ARCH_WANT_SYS_OLD_MMAP */ 1530a4679373SChristoph Hellwig 15314e950f6fSAlexey Dobriyan /* 15328bb4e7a2SWei Yang * Some shared mappings will want the pages marked read-only 15334e950f6fSAlexey Dobriyan * to track write events. If so, we'll downgrade vm_page_prot 15344e950f6fSAlexey Dobriyan * to the private version (using protection_map[] without the 15354e950f6fSAlexey Dobriyan * VM_SHARED bit). 15364e950f6fSAlexey Dobriyan */ 15376d2329f8SAndrea Arcangeli int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) 15384e950f6fSAlexey Dobriyan { 1539ca16d140SKOSAKI Motohiro vm_flags_t vm_flags = vma->vm_flags; 15408a04446aSKirill A. Shutemov const struct vm_operations_struct *vm_ops = vma->vm_ops; 15414e950f6fSAlexey Dobriyan 15424e950f6fSAlexey Dobriyan /* If it was private or non-writable, the write bit is already clear */ 15434e950f6fSAlexey Dobriyan if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED))) 15444e950f6fSAlexey Dobriyan return 0; 15454e950f6fSAlexey Dobriyan 15464e950f6fSAlexey Dobriyan /* The backer wishes to know when pages are first written to? */ 15478a04446aSKirill A. Shutemov if (vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite)) 15484e950f6fSAlexey Dobriyan return 1; 15494e950f6fSAlexey Dobriyan 155064e45507SPeter Feiner /* The open routine did something to the protections that pgprot_modify 155164e45507SPeter Feiner * won't preserve? */ 15526d2329f8SAndrea Arcangeli if (pgprot_val(vm_page_prot) != 15536d2329f8SAndrea Arcangeli pgprot_val(vm_pgprot_modify(vm_page_prot, vm_flags))) 15544e950f6fSAlexey Dobriyan return 0; 15554e950f6fSAlexey Dobriyan 1556f96f7a40SDavid Hildenbrand /* 1557f96f7a40SDavid Hildenbrand * Do we need to track softdirty? hugetlb does not support softdirty 1558f96f7a40SDavid Hildenbrand * tracking yet. 1559f96f7a40SDavid Hildenbrand */ 1560f96f7a40SDavid Hildenbrand if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma)) 156164e45507SPeter Feiner return 1; 156264e45507SPeter Feiner 15634e950f6fSAlexey Dobriyan /* Specialty mapping? */ 15644b6e1e37SKonstantin Khlebnikov if (vm_flags & VM_PFNMAP) 15654e950f6fSAlexey Dobriyan return 0; 15664e950f6fSAlexey Dobriyan 15674e950f6fSAlexey Dobriyan /* Can the mapping track the dirty pages? */ 15684e950f6fSAlexey Dobriyan return vma->vm_file && vma->vm_file->f_mapping && 1569f56753acSChristoph Hellwig mapping_can_writeback(vma->vm_file->f_mapping); 15704e950f6fSAlexey Dobriyan } 15714e950f6fSAlexey Dobriyan 1572fc8744adSLinus Torvalds /* 1573fc8744adSLinus Torvalds * We account for memory if it's a private writeable mapping, 15745a6fe125SMel Gorman * not hugepages and VM_NORESERVE wasn't set. 1575fc8744adSLinus Torvalds */ 1576ca16d140SKOSAKI Motohiro static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags) 1577fc8744adSLinus Torvalds { 15785a6fe125SMel Gorman /* 15795a6fe125SMel Gorman * hugetlb has its own accounting separate from the core VM 15805a6fe125SMel Gorman * VM_HUGETLB may not be set yet so we cannot check for that flag. 15815a6fe125SMel Gorman */ 15825a6fe125SMel Gorman if (file && is_file_hugepages(file)) 15835a6fe125SMel Gorman return 0; 15845a6fe125SMel Gorman 1585fc8744adSLinus Torvalds return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE; 1586fc8744adSLinus Torvalds } 1587fc8744adSLinus Torvalds 15880165ab44SMiklos Szeredi unsigned long mmap_region(struct file *file, unsigned long addr, 1589897ab3e0SMike Rapoport unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, 1590897ab3e0SMike Rapoport struct list_head *uf) 15910165ab44SMiklos Szeredi { 15920165ab44SMiklos Szeredi struct mm_struct *mm = current->mm; 1593d70cec89SMiaohe Lin struct vm_area_struct *vma, *prev, *merge; 15940165ab44SMiklos Szeredi int error; 15950165ab44SMiklos Szeredi unsigned long charged = 0; 15960165ab44SMiklos Szeredi 1597e8420a8eSCyril Hrubis /* Check against address space limit. */ 159884638335SKonstantin Khlebnikov if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { 1599e8420a8eSCyril Hrubis unsigned long nr_pages; 1600e8420a8eSCyril Hrubis 1601e8420a8eSCyril Hrubis /* 1602e8420a8eSCyril Hrubis * MAP_FIXED may remove pages of mappings that intersects with 1603e8420a8eSCyril Hrubis * requested mapping. Account for the pages it would unmap. 1604e8420a8eSCyril Hrubis */ 1605e8420a8eSCyril Hrubis nr_pages = count_vma_pages_range(mm, addr, addr + len); 1606e8420a8eSCyril Hrubis 160784638335SKonstantin Khlebnikov if (!may_expand_vm(mm, vm_flags, 160884638335SKonstantin Khlebnikov (len >> PAGE_SHIFT) - nr_pages)) 1609e8420a8eSCyril Hrubis return -ENOMEM; 1610e8420a8eSCyril Hrubis } 1611e8420a8eSCyril Hrubis 1612524e00b3SLiam R. Howlett /* Clear old maps, set up prev and uf */ 1613524e00b3SLiam R. Howlett if (munmap_vma_range(mm, addr, len, &prev, uf)) 16141da177e4SLinus Torvalds return -ENOMEM; 1615fc8744adSLinus Torvalds /* 16161da177e4SLinus Torvalds * Private writable mapping: check memory availability 16171da177e4SLinus Torvalds */ 16185a6fe125SMel Gorman if (accountable_mapping(file, vm_flags)) { 16191da177e4SLinus Torvalds charged = len >> PAGE_SHIFT; 1620191c5424SAl Viro if (security_vm_enough_memory_mm(mm, charged)) 16211da177e4SLinus Torvalds return -ENOMEM; 16221da177e4SLinus Torvalds vm_flags |= VM_ACCOUNT; 16231da177e4SLinus Torvalds } 16241da177e4SLinus Torvalds 16251da177e4SLinus Torvalds /* 1626de33c8dbSLinus Torvalds * Can we just expand an old mapping? 16271da177e4SLinus Torvalds */ 162819a809afSAndrea Arcangeli vma = vma_merge(mm, prev, addr, addr + len, vm_flags, 16299a10064fSColin Cross NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); 1630ba470de4SRik van Riel if (vma) 16311da177e4SLinus Torvalds goto out; 16321da177e4SLinus Torvalds 16331da177e4SLinus Torvalds /* 16341da177e4SLinus Torvalds * Determine the object being mapped and call the appropriate 16351da177e4SLinus Torvalds * specific mapper. the address has already been validated, but 16361da177e4SLinus Torvalds * not unmapped, but the maps are removed from the list. 16371da177e4SLinus Torvalds */ 1638490fc053SLinus Torvalds vma = vm_area_alloc(mm); 16391da177e4SLinus Torvalds if (!vma) { 16401da177e4SLinus Torvalds error = -ENOMEM; 16411da177e4SLinus Torvalds goto unacct_error; 16421da177e4SLinus Torvalds } 16431da177e4SLinus Torvalds 16441da177e4SLinus Torvalds vma->vm_start = addr; 16451da177e4SLinus Torvalds vma->vm_end = addr + len; 16461da177e4SLinus Torvalds vma->vm_flags = vm_flags; 16473ed75eb8SColy Li vma->vm_page_prot = vm_get_page_prot(vm_flags); 16481da177e4SLinus Torvalds vma->vm_pgoff = pgoff; 16491da177e4SLinus Torvalds 16501da177e4SLinus Torvalds if (file) { 16514bb5f5d9SDavid Herrmann if (vm_flags & VM_SHARED) { 16524bb5f5d9SDavid Herrmann error = mapping_map_writable(file->f_mapping); 16534bb5f5d9SDavid Herrmann if (error) 16548d0920bdSDavid Hildenbrand goto free_vma; 16554bb5f5d9SDavid Herrmann } 16564bb5f5d9SDavid Herrmann 1657cb0942b8SAl Viro vma->vm_file = get_file(file); 1658f74ac015SMiklos Szeredi error = call_mmap(file, vma); 16591da177e4SLinus Torvalds if (error) 16601da177e4SLinus Torvalds goto unmap_and_free_vma; 16611da177e4SLinus Torvalds 1662309d08d9SLiu Zixian /* Can addr have changed?? 1663309d08d9SLiu Zixian * 1664309d08d9SLiu Zixian * Answer: Yes, several device drivers can do it in their 1665309d08d9SLiu Zixian * f_op->mmap method. -DaveM 1666309d08d9SLiu Zixian * Bug: If addr is changed, prev, rb_link, rb_parent should 1667309d08d9SLiu Zixian * be updated for vma_link() 1668309d08d9SLiu Zixian */ 1669309d08d9SLiu Zixian WARN_ON_ONCE(addr != vma->vm_start); 1670309d08d9SLiu Zixian 1671309d08d9SLiu Zixian addr = vma->vm_start; 1672309d08d9SLiu Zixian 1673d70cec89SMiaohe Lin /* If vm_flags changed after call_mmap(), we should try merge vma again 1674d70cec89SMiaohe Lin * as we may succeed this time. 1675d70cec89SMiaohe Lin */ 1676d70cec89SMiaohe Lin if (unlikely(vm_flags != vma->vm_flags && prev)) { 1677d70cec89SMiaohe Lin merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, 16789a10064fSColin Cross NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); 1679d70cec89SMiaohe Lin if (merge) { 1680bc4fe4cdSMiaohe Lin /* ->mmap() can change vma->vm_file and fput the original file. So 1681bc4fe4cdSMiaohe Lin * fput the vma->vm_file here or we would add an extra fput for file 1682bc4fe4cdSMiaohe Lin * and cause general protection fault ultimately. 1683bc4fe4cdSMiaohe Lin */ 1684bc4fe4cdSMiaohe Lin fput(vma->vm_file); 1685d70cec89SMiaohe Lin vm_area_free(vma); 1686d70cec89SMiaohe Lin vma = merge; 1687309d08d9SLiu Zixian /* Update vm_flags to pick up the change. */ 1688d70cec89SMiaohe Lin vm_flags = vma->vm_flags; 1689d70cec89SMiaohe Lin goto unmap_writable; 1690d70cec89SMiaohe Lin } 1691d70cec89SMiaohe Lin } 1692d70cec89SMiaohe Lin 16931da177e4SLinus Torvalds vm_flags = vma->vm_flags; 1694f8dbf0a7SHuang Shijie } else if (vm_flags & VM_SHARED) { 1695f8dbf0a7SHuang Shijie error = shmem_zero_setup(vma); 1696f8dbf0a7SHuang Shijie if (error) 1697f8dbf0a7SHuang Shijie goto free_vma; 1698bfd40eafSKirill A. Shutemov } else { 1699bfd40eafSKirill A. Shutemov vma_set_anonymous(vma); 1700f8dbf0a7SHuang Shijie } 17011da177e4SLinus Torvalds 1702c462ac28SCatalin Marinas /* Allow architectures to sanity-check the vm_flags */ 1703c462ac28SCatalin Marinas if (!arch_validate_flags(vma->vm_flags)) { 1704c462ac28SCatalin Marinas error = -EINVAL; 1705c462ac28SCatalin Marinas if (file) 1706c462ac28SCatalin Marinas goto unmap_and_free_vma; 1707c462ac28SCatalin Marinas else 1708c462ac28SCatalin Marinas goto free_vma; 1709c462ac28SCatalin Marinas } 1710c462ac28SCatalin Marinas 1711524e00b3SLiam R. Howlett if (vma_link(mm, vma, prev)) { 1712d4af56c5SLiam R. Howlett error = -ENOMEM; 1713d4af56c5SLiam R. Howlett if (file) 1714d4af56c5SLiam R. Howlett goto unmap_and_free_vma; 1715d4af56c5SLiam R. Howlett else 1716d4af56c5SLiam R. Howlett goto free_vma; 1717d4af56c5SLiam R. Howlett } 1718613bec09SYang Shi 1719613bec09SYang Shi /* 1720613bec09SYang Shi * vma_merge() calls khugepaged_enter_vma() either, the below 1721613bec09SYang Shi * call covers the non-merge case. 1722613bec09SYang Shi */ 1723613bec09SYang Shi khugepaged_enter_vma(vma, vma->vm_flags); 1724613bec09SYang Shi 17254d3d5b41SOleg Nesterov /* Once vma denies write, undo our temporary denial count */ 1726d70cec89SMiaohe Lin unmap_writable: 17278d0920bdSDavid Hildenbrand if (file && vm_flags & VM_SHARED) 17284bb5f5d9SDavid Herrmann mapping_unmap_writable(file->f_mapping); 1729e8686772SOleg Nesterov file = vma->vm_file; 17301da177e4SLinus Torvalds out: 1731cdd6c482SIngo Molnar perf_event_mmap(vma); 17320a4a9391SPeter Zijlstra 173384638335SKonstantin Khlebnikov vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); 17341da177e4SLinus Torvalds if (vm_flags & VM_LOCKED) { 1735e1fb4a08SDave Jiang if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || 1736e1fb4a08SDave Jiang is_vm_hugetlb_page(vma) || 1737e1fb4a08SDave Jiang vma == get_gate_vma(current->mm)) 1738de60f5f1SEric B Munson vma->vm_flags &= VM_LOCKED_CLEAR_MASK; 1739e1fb4a08SDave Jiang else 1740e1fb4a08SDave Jiang mm->locked_vm += (len >> PAGE_SHIFT); 1741bebeb3d6SMichel Lespinasse } 17422b144498SSrikar Dronamraju 1743c7a3a88cSOleg Nesterov if (file) 1744c7a3a88cSOleg Nesterov uprobe_mmap(vma); 17452b144498SSrikar Dronamraju 1746d9104d1cSCyrill Gorcunov /* 1747d9104d1cSCyrill Gorcunov * New (or expanded) vma always get soft dirty status. 1748d9104d1cSCyrill Gorcunov * Otherwise user-space soft-dirty page tracker won't 1749d9104d1cSCyrill Gorcunov * be able to distinguish situation when vma area unmapped, 1750d9104d1cSCyrill Gorcunov * then new mapped in-place (which must be aimed as 1751d9104d1cSCyrill Gorcunov * a completely new data area). 1752d9104d1cSCyrill Gorcunov */ 1753d9104d1cSCyrill Gorcunov vma->vm_flags |= VM_SOFTDIRTY; 1754d9104d1cSCyrill Gorcunov 175564e45507SPeter Feiner vma_set_page_prot(vma); 175664e45507SPeter Feiner 17571da177e4SLinus Torvalds return addr; 17581da177e4SLinus Torvalds 17591da177e4SLinus Torvalds unmap_and_free_vma: 17601527f926SChristian König fput(vma->vm_file); 17611da177e4SLinus Torvalds vma->vm_file = NULL; 17621da177e4SLinus Torvalds 17631da177e4SLinus Torvalds /* Undo any partial mapping done by a device driver. */ 1764e0da382cSHugh Dickins unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); 17654bb5f5d9SDavid Herrmann if (vm_flags & VM_SHARED) 17664bb5f5d9SDavid Herrmann mapping_unmap_writable(file->f_mapping); 17671da177e4SLinus Torvalds free_vma: 17683928d4f5SLinus Torvalds vm_area_free(vma); 17691da177e4SLinus Torvalds unacct_error: 17701da177e4SLinus Torvalds if (charged) 17711da177e4SLinus Torvalds vm_unacct_memory(charged); 17721da177e4SLinus Torvalds return error; 17731da177e4SLinus Torvalds } 17741da177e4SLinus Torvalds 17753499a131SLiam R. Howlett /** 17763499a131SLiam R. Howlett * unmapped_area() - Find an area between the low_limit and the high_limit with 17773499a131SLiam R. Howlett * the correct alignment and offset, all from @info. Note: current->mm is used 17783499a131SLiam R. Howlett * for the search. 17793499a131SLiam R. Howlett * 17803499a131SLiam R. Howlett * @info: The unmapped area information including the range (low_limit - 17813499a131SLiam R. Howlett * hight_limit), the alignment offset and mask. 17823499a131SLiam R. Howlett * 17833499a131SLiam R. Howlett * Return: A memory address or -ENOMEM. 17843499a131SLiam R. Howlett */ 1785baceaf1cSJaewon Kim static unsigned long unmapped_area(struct vm_unmapped_area_info *info) 1786db4fbfb9SMichel Lespinasse { 17873499a131SLiam R. Howlett unsigned long length, gap; 1788db4fbfb9SMichel Lespinasse 17893499a131SLiam R. Howlett MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); 1790db4fbfb9SMichel Lespinasse 1791db4fbfb9SMichel Lespinasse /* Adjust search length to account for worst case alignment overhead */ 1792db4fbfb9SMichel Lespinasse length = info->length + info->align_mask; 1793db4fbfb9SMichel Lespinasse if (length < info->length) 1794db4fbfb9SMichel Lespinasse return -ENOMEM; 1795db4fbfb9SMichel Lespinasse 17963499a131SLiam R. Howlett if (mas_empty_area(&mas, info->low_limit, info->high_limit - 1, 17973499a131SLiam R. Howlett length)) 17983499a131SLiam R. Howlett return -ENOMEM; 17993499a131SLiam R. Howlett 1800d4af56c5SLiam R. Howlett gap = mas.index; 1801d4af56c5SLiam R. Howlett gap += (info->align_offset - gap) & info->align_mask; 18023499a131SLiam R. Howlett return gap; 1803db4fbfb9SMichel Lespinasse } 1804db4fbfb9SMichel Lespinasse 18053499a131SLiam R. Howlett /** 18063499a131SLiam R. Howlett * unmapped_area_topdown() - Find an area between the low_limit and the 18073499a131SLiam R. Howlett * high_limit with * the correct alignment and offset at the highest available 18083499a131SLiam R. Howlett * address, all from @info. Note: current->mm is used for the search. 18093499a131SLiam R. Howlett * 18103499a131SLiam R. Howlett * @info: The unmapped area information including the range (low_limit - 18113499a131SLiam R. Howlett * hight_limit), the alignment offset and mask. 18123499a131SLiam R. Howlett * 18133499a131SLiam R. Howlett * Return: A memory address or -ENOMEM. 18143499a131SLiam R. Howlett */ 1815baceaf1cSJaewon Kim static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) 1816db4fbfb9SMichel Lespinasse { 18173499a131SLiam R. Howlett unsigned long length, gap; 1818d4af56c5SLiam R. Howlett 18193499a131SLiam R. Howlett MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); 1820db4fbfb9SMichel Lespinasse /* Adjust search length to account for worst case alignment overhead */ 1821db4fbfb9SMichel Lespinasse length = info->length + info->align_mask; 1822db4fbfb9SMichel Lespinasse if (length < info->length) 1823db4fbfb9SMichel Lespinasse return -ENOMEM; 1824db4fbfb9SMichel Lespinasse 18253499a131SLiam R. Howlett if (mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1, 18263499a131SLiam R. Howlett length)) 18273499a131SLiam R. Howlett return -ENOMEM; 18283499a131SLiam R. Howlett 1829d4af56c5SLiam R. Howlett gap = mas.last + 1 - info->length; 1830d4af56c5SLiam R. Howlett gap -= (gap - info->align_offset) & info->align_mask; 18313499a131SLiam R. Howlett return gap; 1832db4fbfb9SMichel Lespinasse } 1833db4fbfb9SMichel Lespinasse 1834baceaf1cSJaewon Kim /* 1835baceaf1cSJaewon Kim * Search for an unmapped address range. 1836baceaf1cSJaewon Kim * 1837baceaf1cSJaewon Kim * We are looking for a range that: 1838baceaf1cSJaewon Kim * - does not intersect with any VMA; 1839baceaf1cSJaewon Kim * - is contained within the [low_limit, high_limit) interval; 1840baceaf1cSJaewon Kim * - is at least the desired size. 1841baceaf1cSJaewon Kim * - satisfies (begin_addr & align_mask) == (align_offset & align_mask) 1842baceaf1cSJaewon Kim */ 1843baceaf1cSJaewon Kim unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) 1844baceaf1cSJaewon Kim { 1845df529cabSJaewon Kim unsigned long addr; 1846df529cabSJaewon Kim 1847baceaf1cSJaewon Kim if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) 1848df529cabSJaewon Kim addr = unmapped_area_topdown(info); 1849baceaf1cSJaewon Kim else 1850df529cabSJaewon Kim addr = unmapped_area(info); 1851df529cabSJaewon Kim 1852df529cabSJaewon Kim trace_vm_unmapped_area(addr, info); 1853df529cabSJaewon Kim return addr; 1854baceaf1cSJaewon Kim } 1855f6795053SSteve Capper 18561da177e4SLinus Torvalds /* Get an address range which is currently unmapped. 18571da177e4SLinus Torvalds * For shmat() with addr=0. 18581da177e4SLinus Torvalds * 18591da177e4SLinus Torvalds * Ugly calling convention alert: 18601da177e4SLinus Torvalds * Return value with the low bits set means error value, 18611da177e4SLinus Torvalds * ie 18621da177e4SLinus Torvalds * if (ret & ~PAGE_MASK) 18631da177e4SLinus Torvalds * error = ret; 18641da177e4SLinus Torvalds * 18651da177e4SLinus Torvalds * This function "knows" that -ENOMEM has the bits set. 18661da177e4SLinus Torvalds */ 18671da177e4SLinus Torvalds unsigned long 18684b439e25SChristophe Leroy generic_get_unmapped_area(struct file *filp, unsigned long addr, 18694b439e25SChristophe Leroy unsigned long len, unsigned long pgoff, 18704b439e25SChristophe Leroy unsigned long flags) 18711da177e4SLinus Torvalds { 18721da177e4SLinus Torvalds struct mm_struct *mm = current->mm; 18731be7107fSHugh Dickins struct vm_area_struct *vma, *prev; 1874db4fbfb9SMichel Lespinasse struct vm_unmapped_area_info info; 18752cb4de08SChristophe Leroy const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); 18761da177e4SLinus Torvalds 1877f6795053SSteve Capper if (len > mmap_end - mmap_min_addr) 18781da177e4SLinus Torvalds return -ENOMEM; 18791da177e4SLinus Torvalds 188006abdfb4SBenjamin Herrenschmidt if (flags & MAP_FIXED) 188106abdfb4SBenjamin Herrenschmidt return addr; 188206abdfb4SBenjamin Herrenschmidt 18831da177e4SLinus Torvalds if (addr) { 18841da177e4SLinus Torvalds addr = PAGE_ALIGN(addr); 18851be7107fSHugh Dickins vma = find_vma_prev(mm, addr, &prev); 1886f6795053SSteve Capper if (mmap_end - len >= addr && addr >= mmap_min_addr && 18871be7107fSHugh Dickins (!vma || addr + len <= vm_start_gap(vma)) && 18881be7107fSHugh Dickins (!prev || addr >= vm_end_gap(prev))) 18891da177e4SLinus Torvalds return addr; 18901da177e4SLinus Torvalds } 18911da177e4SLinus Torvalds 1892db4fbfb9SMichel Lespinasse info.flags = 0; 1893db4fbfb9SMichel Lespinasse info.length = len; 18944e99b021SHeiko Carstens info.low_limit = mm->mmap_base; 1895f6795053SSteve Capper info.high_limit = mmap_end; 1896db4fbfb9SMichel Lespinasse info.align_mask = 0; 189709ef5283SJaewon Kim info.align_offset = 0; 1898db4fbfb9SMichel Lespinasse return vm_unmapped_area(&info); 18991da177e4SLinus Torvalds } 19004b439e25SChristophe Leroy 19014b439e25SChristophe Leroy #ifndef HAVE_ARCH_UNMAPPED_AREA 19024b439e25SChristophe Leroy unsigned long 19034b439e25SChristophe Leroy arch_get_unmapped_area(struct file *filp, unsigned long addr, 19044b439e25SChristophe Leroy unsigned long len, unsigned long pgoff, 19054b439e25SChristophe Leroy unsigned long flags) 19064b439e25SChristophe Leroy { 19074b439e25SChristophe Leroy return generic_get_unmapped_area(filp, addr, len, pgoff, flags); 19084b439e25SChristophe Leroy } 19091da177e4SLinus Torvalds #endif 19101da177e4SLinus Torvalds 19111da177e4SLinus Torvalds /* 19121da177e4SLinus Torvalds * This mmap-allocator allocates new areas top-down from below the 19131da177e4SLinus Torvalds * stack's low limit (the base): 19141da177e4SLinus Torvalds */ 19151da177e4SLinus Torvalds unsigned long 19164b439e25SChristophe Leroy generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 191743cca0b1SYang Fan unsigned long len, unsigned long pgoff, 191843cca0b1SYang Fan unsigned long flags) 19191da177e4SLinus Torvalds { 19201be7107fSHugh Dickins struct vm_area_struct *vma, *prev; 19211da177e4SLinus Torvalds struct mm_struct *mm = current->mm; 1922db4fbfb9SMichel Lespinasse struct vm_unmapped_area_info info; 19232cb4de08SChristophe Leroy const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); 19241da177e4SLinus Torvalds 19251da177e4SLinus Torvalds /* requested length too big for entire address space */ 1926f6795053SSteve Capper if (len > mmap_end - mmap_min_addr) 19271da177e4SLinus Torvalds return -ENOMEM; 19281da177e4SLinus Torvalds 192906abdfb4SBenjamin Herrenschmidt if (flags & MAP_FIXED) 193006abdfb4SBenjamin Herrenschmidt return addr; 193106abdfb4SBenjamin Herrenschmidt 19321da177e4SLinus Torvalds /* requesting a specific address */ 19331da177e4SLinus Torvalds if (addr) { 19341da177e4SLinus Torvalds addr = PAGE_ALIGN(addr); 19351be7107fSHugh Dickins vma = find_vma_prev(mm, addr, &prev); 1936f6795053SSteve Capper if (mmap_end - len >= addr && addr >= mmap_min_addr && 19371be7107fSHugh Dickins (!vma || addr + len <= vm_start_gap(vma)) && 19381be7107fSHugh Dickins (!prev || addr >= vm_end_gap(prev))) 19391da177e4SLinus Torvalds return addr; 19401da177e4SLinus Torvalds } 19411da177e4SLinus Torvalds 1942db4fbfb9SMichel Lespinasse info.flags = VM_UNMAPPED_AREA_TOPDOWN; 1943db4fbfb9SMichel Lespinasse info.length = len; 19442afc745fSAkira Takeuchi info.low_limit = max(PAGE_SIZE, mmap_min_addr); 1945f6795053SSteve Capper info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); 1946db4fbfb9SMichel Lespinasse info.align_mask = 0; 194709ef5283SJaewon Kim info.align_offset = 0; 1948db4fbfb9SMichel Lespinasse addr = vm_unmapped_area(&info); 1949b716ad95SXiao Guangrong 19501da177e4SLinus Torvalds /* 19511da177e4SLinus Torvalds * A failed mmap() very likely causes application failure, 19521da177e4SLinus Torvalds * so fall back to the bottom-up function here. This scenario 19531da177e4SLinus Torvalds * can happen with large stack limits and large mmap() 19541da177e4SLinus Torvalds * allocations. 19551da177e4SLinus Torvalds */ 1956de1741a1SAlexander Kuleshov if (offset_in_page(addr)) { 1957db4fbfb9SMichel Lespinasse VM_BUG_ON(addr != -ENOMEM); 1958db4fbfb9SMichel Lespinasse info.flags = 0; 1959db4fbfb9SMichel Lespinasse info.low_limit = TASK_UNMAPPED_BASE; 1960f6795053SSteve Capper info.high_limit = mmap_end; 1961db4fbfb9SMichel Lespinasse addr = vm_unmapped_area(&info); 1962db4fbfb9SMichel Lespinasse } 19631da177e4SLinus Torvalds 19641da177e4SLinus Torvalds return addr; 19651da177e4SLinus Torvalds } 19664b439e25SChristophe Leroy 19674b439e25SChristophe Leroy #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN 19684b439e25SChristophe Leroy unsigned long 19694b439e25SChristophe Leroy arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 19704b439e25SChristophe Leroy unsigned long len, unsigned long pgoff, 19714b439e25SChristophe Leroy unsigned long flags) 19724b439e25SChristophe Leroy { 19734b439e25SChristophe Leroy return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags); 19744b439e25SChristophe Leroy } 19751da177e4SLinus Torvalds #endif 19761da177e4SLinus Torvalds 19771da177e4SLinus Torvalds unsigned long 19781da177e4SLinus Torvalds get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 19791da177e4SLinus Torvalds unsigned long pgoff, unsigned long flags) 19801da177e4SLinus Torvalds { 198106abdfb4SBenjamin Herrenschmidt unsigned long (*get_area)(struct file *, unsigned long, 198206abdfb4SBenjamin Herrenschmidt unsigned long, unsigned long, unsigned long); 198307ab67c8SLinus Torvalds 19849206de95SAl Viro unsigned long error = arch_mmap_check(addr, len, flags); 19859206de95SAl Viro if (error) 19869206de95SAl Viro return error; 19879206de95SAl Viro 19889206de95SAl Viro /* Careful about overflows.. */ 19899206de95SAl Viro if (len > TASK_SIZE) 19909206de95SAl Viro return -ENOMEM; 19919206de95SAl Viro 199207ab67c8SLinus Torvalds get_area = current->mm->get_unmapped_area; 1993c01d5b30SHugh Dickins if (file) { 1994c01d5b30SHugh Dickins if (file->f_op->get_unmapped_area) 199507ab67c8SLinus Torvalds get_area = file->f_op->get_unmapped_area; 1996c01d5b30SHugh Dickins } else if (flags & MAP_SHARED) { 1997c01d5b30SHugh Dickins /* 1998c01d5b30SHugh Dickins * mmap_region() will call shmem_zero_setup() to create a file, 1999c01d5b30SHugh Dickins * so use shmem's get_unmapped_area in case it can be huge. 200045e55300SPeter Collingbourne * do_mmap() will clear pgoff, so match alignment. 2001c01d5b30SHugh Dickins */ 2002c01d5b30SHugh Dickins pgoff = 0; 2003c01d5b30SHugh Dickins get_area = shmem_get_unmapped_area; 2004f35b5d7dSRik van Riel } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 2005f35b5d7dSRik van Riel /* Ensures that larger anonymous mappings are THP aligned. */ 2006f35b5d7dSRik van Riel get_area = thp_get_unmapped_area; 2007c01d5b30SHugh Dickins } 2008c01d5b30SHugh Dickins 200907ab67c8SLinus Torvalds addr = get_area(file, addr, len, pgoff, flags); 201007ab67c8SLinus Torvalds if (IS_ERR_VALUE(addr)) 201107ab67c8SLinus Torvalds return addr; 201207ab67c8SLinus Torvalds 20131da177e4SLinus Torvalds if (addr > TASK_SIZE - len) 20141da177e4SLinus Torvalds return -ENOMEM; 2015de1741a1SAlexander Kuleshov if (offset_in_page(addr)) 20161da177e4SLinus Torvalds return -EINVAL; 201706abdfb4SBenjamin Herrenschmidt 20189ac4ed4bSAl Viro error = security_mmap_addr(addr); 20199ac4ed4bSAl Viro return error ? error : addr; 20201da177e4SLinus Torvalds } 20211da177e4SLinus Torvalds 20221da177e4SLinus Torvalds EXPORT_SYMBOL(get_unmapped_area); 20231da177e4SLinus Torvalds 2024be8432e7SLiam R. Howlett /** 2025be8432e7SLiam R. Howlett * find_vma() - Find the VMA for a given address, or the next VMA. 2026be8432e7SLiam R. Howlett * @mm: The mm_struct to check 2027be8432e7SLiam R. Howlett * @addr: The address 2028be8432e7SLiam R. Howlett * 2029be8432e7SLiam R. Howlett * Returns: The VMA associated with addr, or the next VMA. 2030be8432e7SLiam R. Howlett * May return %NULL in the case of no VMA at addr or above. 2031be8432e7SLiam R. Howlett */ 20321da177e4SLinus Torvalds struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) 20331da177e4SLinus Torvalds { 2034615d6e87SDavidlohr Bueso struct vm_area_struct *vma; 2035be8432e7SLiam R. Howlett unsigned long index = addr; 20361da177e4SLinus Torvalds 20375b78ed24SLuigi Rizzo mmap_assert_locked(mm); 20381da177e4SLinus Torvalds /* Check the cache first. */ 2039615d6e87SDavidlohr Bueso vma = vmacache_find(mm, addr); 2040615d6e87SDavidlohr Bueso if (likely(vma)) 2041615d6e87SDavidlohr Bueso return vma; 20421da177e4SLinus Torvalds 2043be8432e7SLiam R. Howlett vma = mt_find(&mm->mm_mt, &index, ULONG_MAX); 20441da177e4SLinus Torvalds if (vma) 2045615d6e87SDavidlohr Bueso vmacache_update(addr, vma); 20461da177e4SLinus Torvalds return vma; 20471da177e4SLinus Torvalds } 20481da177e4SLinus Torvalds EXPORT_SYMBOL(find_vma); 20491da177e4SLinus Torvalds 20507fdbd37dSLiam R. Howlett /** 20517fdbd37dSLiam R. Howlett * find_vma_prev() - Find the VMA for a given address, or the next vma and 20527fdbd37dSLiam R. Howlett * set %pprev to the previous VMA, if any. 20537fdbd37dSLiam R. Howlett * @mm: The mm_struct to check 20547fdbd37dSLiam R. Howlett * @addr: The address 20557fdbd37dSLiam R. Howlett * @pprev: The pointer to set to the previous VMA 20567fdbd37dSLiam R. Howlett * 20577fdbd37dSLiam R. Howlett * Note that RCU lock is missing here since the external mmap_lock() is used 20587fdbd37dSLiam R. Howlett * instead. 20597fdbd37dSLiam R. Howlett * 20607fdbd37dSLiam R. Howlett * Returns: The VMA associated with @addr, or the next vma. 20617fdbd37dSLiam R. Howlett * May return %NULL in the case of no vma at addr or above. 20626bd4837dSKOSAKI Motohiro */ 20631da177e4SLinus Torvalds struct vm_area_struct * 20641da177e4SLinus Torvalds find_vma_prev(struct mm_struct *mm, unsigned long addr, 20651da177e4SLinus Torvalds struct vm_area_struct **pprev) 20661da177e4SLinus Torvalds { 20676bd4837dSKOSAKI Motohiro struct vm_area_struct *vma; 20687fdbd37dSLiam R. Howlett MA_STATE(mas, &mm->mm_mt, addr, addr); 20691da177e4SLinus Torvalds 20707fdbd37dSLiam R. Howlett vma = mas_walk(&mas); 20717fdbd37dSLiam R. Howlett *pprev = mas_prev(&mas, 0); 20727fdbd37dSLiam R. Howlett if (!vma) 20737fdbd37dSLiam R. Howlett vma = mas_next(&mas, ULONG_MAX); 20746bd4837dSKOSAKI Motohiro return vma; 20751da177e4SLinus Torvalds } 20761da177e4SLinus Torvalds 20771da177e4SLinus Torvalds /* 20781da177e4SLinus Torvalds * Verify that the stack growth is acceptable and 20791da177e4SLinus Torvalds * update accounting. This is shared with both the 20801da177e4SLinus Torvalds * grow-up and grow-down cases. 20811da177e4SLinus Torvalds */ 20821be7107fSHugh Dickins static int acct_stack_growth(struct vm_area_struct *vma, 20831be7107fSHugh Dickins unsigned long size, unsigned long grow) 20841da177e4SLinus Torvalds { 20851da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 20861be7107fSHugh Dickins unsigned long new_start; 20871da177e4SLinus Torvalds 20881da177e4SLinus Torvalds /* address space limit tests */ 208984638335SKonstantin Khlebnikov if (!may_expand_vm(mm, vma->vm_flags, grow)) 20901da177e4SLinus Torvalds return -ENOMEM; 20911da177e4SLinus Torvalds 20921da177e4SLinus Torvalds /* Stack limit test */ 209324c79d8eSKrzysztof Opasiak if (size > rlimit(RLIMIT_STACK)) 20941da177e4SLinus Torvalds return -ENOMEM; 20951da177e4SLinus Torvalds 20961da177e4SLinus Torvalds /* mlock limit tests */ 2097c5d8a364SMiaohe Lin if (mlock_future_check(mm, vma->vm_flags, grow << PAGE_SHIFT)) 20981da177e4SLinus Torvalds return -ENOMEM; 20991da177e4SLinus Torvalds 21000d59a01bSAdam Litke /* Check to ensure the stack will not grow into a hugetlb-only region */ 21010d59a01bSAdam Litke new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start : 21020d59a01bSAdam Litke vma->vm_end - size; 21030d59a01bSAdam Litke if (is_hugepage_only_range(vma->vm_mm, new_start, size)) 21040d59a01bSAdam Litke return -EFAULT; 21050d59a01bSAdam Litke 21061da177e4SLinus Torvalds /* 21071da177e4SLinus Torvalds * Overcommit.. This must be the final test, as it will 21081da177e4SLinus Torvalds * update security statistics. 21091da177e4SLinus Torvalds */ 211005fa199dSHugh Dickins if (security_vm_enough_memory_mm(mm, grow)) 21111da177e4SLinus Torvalds return -ENOMEM; 21121da177e4SLinus Torvalds 21131da177e4SLinus Torvalds return 0; 21141da177e4SLinus Torvalds } 21151da177e4SLinus Torvalds 211646dea3d0SHugh Dickins #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64) 21171da177e4SLinus Torvalds /* 211846dea3d0SHugh Dickins * PA-RISC uses this for its stack; IA64 for its Register Backing Store. 211946dea3d0SHugh Dickins * vma is the last one with address > vma->vm_end. Have to extend vma. 21201da177e4SLinus Torvalds */ 212146dea3d0SHugh Dickins int expand_upwards(struct vm_area_struct *vma, unsigned long address) 21221da177e4SLinus Torvalds { 212309357814SOleg Nesterov struct mm_struct *mm = vma->vm_mm; 21241be7107fSHugh Dickins struct vm_area_struct *next; 21251be7107fSHugh Dickins unsigned long gap_addr; 212612352d3cSKonstantin Khlebnikov int error = 0; 2127d4af56c5SLiam R. Howlett MA_STATE(mas, &mm->mm_mt, 0, 0); 21281da177e4SLinus Torvalds 21291da177e4SLinus Torvalds if (!(vma->vm_flags & VM_GROWSUP)) 21301da177e4SLinus Torvalds return -EFAULT; 21311da177e4SLinus Torvalds 2132bd726c90SHelge Deller /* Guard against exceeding limits of the address space. */ 21331be7107fSHugh Dickins address &= PAGE_MASK; 213437511fb5SHelge Deller if (address >= (TASK_SIZE & PAGE_MASK)) 213512352d3cSKonstantin Khlebnikov return -ENOMEM; 2136bd726c90SHelge Deller address += PAGE_SIZE; 213712352d3cSKonstantin Khlebnikov 21381be7107fSHugh Dickins /* Enforce stack_guard_gap */ 21391be7107fSHugh Dickins gap_addr = address + stack_guard_gap; 2140bd726c90SHelge Deller 2141bd726c90SHelge Deller /* Guard against overflow */ 2142bd726c90SHelge Deller if (gap_addr < address || gap_addr > TASK_SIZE) 2143bd726c90SHelge Deller gap_addr = TASK_SIZE; 2144bd726c90SHelge Deller 21451be7107fSHugh Dickins next = vma->vm_next; 21463122e80eSAnshuman Khandual if (next && next->vm_start < gap_addr && vma_is_accessible(next)) { 21471be7107fSHugh Dickins if (!(next->vm_flags & VM_GROWSUP)) 21481be7107fSHugh Dickins return -ENOMEM; 21491be7107fSHugh Dickins /* Check that both stack segments have the same anon_vma? */ 21501be7107fSHugh Dickins } 21511be7107fSHugh Dickins 2152d4af56c5SLiam R. Howlett if (mas_preallocate(&mas, vma, GFP_KERNEL)) 21531da177e4SLinus Torvalds return -ENOMEM; 21541da177e4SLinus Torvalds 2155d4af56c5SLiam R. Howlett /* We must make sure the anon_vma is allocated. */ 2156d4af56c5SLiam R. Howlett if (unlikely(anon_vma_prepare(vma))) { 2157d4af56c5SLiam R. Howlett mas_destroy(&mas); 2158d4af56c5SLiam R. Howlett return -ENOMEM; 2159d4af56c5SLiam R. Howlett } 2160d4af56c5SLiam R. Howlett 21611da177e4SLinus Torvalds /* 21621da177e4SLinus Torvalds * vma->vm_start/vm_end cannot change under us because the caller 2163c1e8d7c6SMichel Lespinasse * is required to hold the mmap_lock in read mode. We need the 21641da177e4SLinus Torvalds * anon_vma lock to serialize against concurrent expand_stacks. 21651da177e4SLinus Torvalds */ 216612352d3cSKonstantin Khlebnikov anon_vma_lock_write(vma->anon_vma); 21671da177e4SLinus Torvalds 21681da177e4SLinus Torvalds /* Somebody else might have raced and expanded it already */ 21691da177e4SLinus Torvalds if (address > vma->vm_end) { 21701da177e4SLinus Torvalds unsigned long size, grow; 21711da177e4SLinus Torvalds 21721da177e4SLinus Torvalds size = address - vma->vm_start; 21731da177e4SLinus Torvalds grow = (address - vma->vm_end) >> PAGE_SHIFT; 21741da177e4SLinus Torvalds 217542c36f63SHugh Dickins error = -ENOMEM; 217642c36f63SHugh Dickins if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { 21771da177e4SLinus Torvalds error = acct_stack_growth(vma, size, grow); 21783af9e859SEric B Munson if (!error) { 21794128997bSMichel Lespinasse /* 2180524e00b3SLiam R. Howlett * We only hold a shared mmap_lock lock here, so 2181524e00b3SLiam R. Howlett * we need to protect against concurrent vma 2182524e00b3SLiam R. Howlett * expansions. anon_vma_lock_write() doesn't 2183524e00b3SLiam R. Howlett * help here, as we don't guarantee that all 2184524e00b3SLiam R. Howlett * growable vmas in a mm share the same root 2185524e00b3SLiam R. Howlett * anon vma. So, we reuse mm->page_table_lock 2186524e00b3SLiam R. Howlett * to guard against concurrent vma expansions. 21874128997bSMichel Lespinasse */ 218809357814SOleg Nesterov spin_lock(&mm->page_table_lock); 218987e8827bSOleg Nesterov if (vma->vm_flags & VM_LOCKED) 219009357814SOleg Nesterov mm->locked_vm += grow; 219184638335SKonstantin Khlebnikov vm_stat_account(mm, vma->vm_flags, grow); 2192bf181b9fSMichel Lespinasse anon_vma_interval_tree_pre_update_vma(vma); 21931da177e4SLinus Torvalds vma->vm_end = address; 2194d4af56c5SLiam R. Howlett /* Overwrite old entry in mtree. */ 2195d4af56c5SLiam R. Howlett vma_mas_store(vma, &mas); 2196bf181b9fSMichel Lespinasse anon_vma_interval_tree_post_update_vma(vma); 2197524e00b3SLiam R. Howlett if (!vma->vm_next) 21981be7107fSHugh Dickins mm->highest_vm_end = vm_end_gap(vma); 219909357814SOleg Nesterov spin_unlock(&mm->page_table_lock); 22004128997bSMichel Lespinasse 22013af9e859SEric B Munson perf_event_mmap(vma); 22023af9e859SEric B Munson } 22031da177e4SLinus Torvalds } 220442c36f63SHugh Dickins } 220512352d3cSKonstantin Khlebnikov anon_vma_unlock_write(vma->anon_vma); 2206c791576cSYang Shi khugepaged_enter_vma(vma, vma->vm_flags); 2207d4af56c5SLiam R. Howlett mas_destroy(&mas); 22081da177e4SLinus Torvalds return error; 22091da177e4SLinus Torvalds } 221046dea3d0SHugh Dickins #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */ 221146dea3d0SHugh Dickins 22121da177e4SLinus Torvalds /* 22131da177e4SLinus Torvalds * vma is the first one with address < vma->vm_start. Have to extend vma. 22141da177e4SLinus Torvalds */ 2215524e00b3SLiam R. Howlett int expand_downwards(struct vm_area_struct *vma, unsigned long address) 22161da177e4SLinus Torvalds { 221709357814SOleg Nesterov struct mm_struct *mm = vma->vm_mm; 22181be7107fSHugh Dickins struct vm_area_struct *prev; 22190a1d5299SJann Horn int error = 0; 2220d4af56c5SLiam R. Howlett MA_STATE(mas, &mm->mm_mt, 0, 0); 22211da177e4SLinus Torvalds 22228869477aSEric Paris address &= PAGE_MASK; 22230a1d5299SJann Horn if (address < mmap_min_addr) 22240a1d5299SJann Horn return -EPERM; 22258869477aSEric Paris 22261be7107fSHugh Dickins /* Enforce stack_guard_gap */ 22271be7107fSHugh Dickins prev = vma->vm_prev; 22281be7107fSHugh Dickins /* Check that both stack segments have the same anon_vma? */ 222932e4e6d5SOleg Nesterov if (prev && !(prev->vm_flags & VM_GROWSDOWN) && 22303122e80eSAnshuman Khandual vma_is_accessible(prev)) { 223132e4e6d5SOleg Nesterov if (address - prev->vm_end < stack_guard_gap) 223232e4e6d5SOleg Nesterov return -ENOMEM; 22331be7107fSHugh Dickins } 22341be7107fSHugh Dickins 2235d4af56c5SLiam R. Howlett if (mas_preallocate(&mas, vma, GFP_KERNEL)) 223612352d3cSKonstantin Khlebnikov return -ENOMEM; 22371da177e4SLinus Torvalds 2238d4af56c5SLiam R. Howlett /* We must make sure the anon_vma is allocated. */ 2239d4af56c5SLiam R. Howlett if (unlikely(anon_vma_prepare(vma))) { 2240d4af56c5SLiam R. Howlett mas_destroy(&mas); 2241d4af56c5SLiam R. Howlett return -ENOMEM; 2242d4af56c5SLiam R. Howlett } 2243d4af56c5SLiam R. Howlett 22441da177e4SLinus Torvalds /* 22451da177e4SLinus Torvalds * vma->vm_start/vm_end cannot change under us because the caller 2246c1e8d7c6SMichel Lespinasse * is required to hold the mmap_lock in read mode. We need the 22471da177e4SLinus Torvalds * anon_vma lock to serialize against concurrent expand_stacks. 22481da177e4SLinus Torvalds */ 224912352d3cSKonstantin Khlebnikov anon_vma_lock_write(vma->anon_vma); 22501da177e4SLinus Torvalds 22511da177e4SLinus Torvalds /* Somebody else might have raced and expanded it already */ 22521da177e4SLinus Torvalds if (address < vma->vm_start) { 22531da177e4SLinus Torvalds unsigned long size, grow; 22541da177e4SLinus Torvalds 22551da177e4SLinus Torvalds size = vma->vm_end - address; 22561da177e4SLinus Torvalds grow = (vma->vm_start - address) >> PAGE_SHIFT; 22571da177e4SLinus Torvalds 2258a626ca6aSLinus Torvalds error = -ENOMEM; 2259a626ca6aSLinus Torvalds if (grow <= vma->vm_pgoff) { 22601da177e4SLinus Torvalds error = acct_stack_growth(vma, size, grow); 22611da177e4SLinus Torvalds if (!error) { 22624128997bSMichel Lespinasse /* 2263524e00b3SLiam R. Howlett * We only hold a shared mmap_lock lock here, so 2264524e00b3SLiam R. Howlett * we need to protect against concurrent vma 2265524e00b3SLiam R. Howlett * expansions. anon_vma_lock_write() doesn't 2266524e00b3SLiam R. Howlett * help here, as we don't guarantee that all 2267524e00b3SLiam R. Howlett * growable vmas in a mm share the same root 2268524e00b3SLiam R. Howlett * anon vma. So, we reuse mm->page_table_lock 2269524e00b3SLiam R. Howlett * to guard against concurrent vma expansions. 22704128997bSMichel Lespinasse */ 227109357814SOleg Nesterov spin_lock(&mm->page_table_lock); 227287e8827bSOleg Nesterov if (vma->vm_flags & VM_LOCKED) 227309357814SOleg Nesterov mm->locked_vm += grow; 227484638335SKonstantin Khlebnikov vm_stat_account(mm, vma->vm_flags, grow); 2275bf181b9fSMichel Lespinasse anon_vma_interval_tree_pre_update_vma(vma); 22761da177e4SLinus Torvalds vma->vm_start = address; 22771da177e4SLinus Torvalds vma->vm_pgoff -= grow; 2278d4af56c5SLiam R. Howlett /* Overwrite old entry in mtree. */ 2279d4af56c5SLiam R. Howlett vma_mas_store(vma, &mas); 2280bf181b9fSMichel Lespinasse anon_vma_interval_tree_post_update_vma(vma); 228109357814SOleg Nesterov spin_unlock(&mm->page_table_lock); 22824128997bSMichel Lespinasse 22833af9e859SEric B Munson perf_event_mmap(vma); 22841da177e4SLinus Torvalds } 22851da177e4SLinus Torvalds } 2286a626ca6aSLinus Torvalds } 228712352d3cSKonstantin Khlebnikov anon_vma_unlock_write(vma->anon_vma); 2288c791576cSYang Shi khugepaged_enter_vma(vma, vma->vm_flags); 2289d4af56c5SLiam R. Howlett mas_destroy(&mas); 22901da177e4SLinus Torvalds return error; 22911da177e4SLinus Torvalds } 22921da177e4SLinus Torvalds 22931be7107fSHugh Dickins /* enforced gap between the expanding stack and other mappings. */ 22941be7107fSHugh Dickins unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT; 22951be7107fSHugh Dickins 22961be7107fSHugh Dickins static int __init cmdline_parse_stack_guard_gap(char *p) 22971be7107fSHugh Dickins { 22981be7107fSHugh Dickins unsigned long val; 22991be7107fSHugh Dickins char *endptr; 23001be7107fSHugh Dickins 23011be7107fSHugh Dickins val = simple_strtoul(p, &endptr, 10); 23021be7107fSHugh Dickins if (!*endptr) 23031be7107fSHugh Dickins stack_guard_gap = val << PAGE_SHIFT; 23041be7107fSHugh Dickins 2305e6d09493SRandy Dunlap return 1; 23061be7107fSHugh Dickins } 23071be7107fSHugh Dickins __setup("stack_guard_gap=", cmdline_parse_stack_guard_gap); 23081be7107fSHugh Dickins 2309b6a2fea3SOllie Wild #ifdef CONFIG_STACK_GROWSUP 2310b6a2fea3SOllie Wild int expand_stack(struct vm_area_struct *vma, unsigned long address) 2311b6a2fea3SOllie Wild { 2312b6a2fea3SOllie Wild return expand_upwards(vma, address); 2313b6a2fea3SOllie Wild } 2314b6a2fea3SOllie Wild 2315b6a2fea3SOllie Wild struct vm_area_struct * 2316b6a2fea3SOllie Wild find_extend_vma(struct mm_struct *mm, unsigned long addr) 2317b6a2fea3SOllie Wild { 2318b6a2fea3SOllie Wild struct vm_area_struct *vma, *prev; 2319b6a2fea3SOllie Wild 2320b6a2fea3SOllie Wild addr &= PAGE_MASK; 2321b6a2fea3SOllie Wild vma = find_vma_prev(mm, addr, &prev); 2322b6a2fea3SOllie Wild if (vma && (vma->vm_start <= addr)) 2323b6a2fea3SOllie Wild return vma; 23244d45e75aSJann Horn if (!prev || expand_stack(prev, addr)) 2325b6a2fea3SOllie Wild return NULL; 2326cea10a19SMichel Lespinasse if (prev->vm_flags & VM_LOCKED) 2327fc05f566SKirill A. Shutemov populate_vma_page_range(prev, addr, prev->vm_end, NULL); 2328b6a2fea3SOllie Wild return prev; 2329b6a2fea3SOllie Wild } 2330b6a2fea3SOllie Wild #else 2331b6a2fea3SOllie Wild int expand_stack(struct vm_area_struct *vma, unsigned long address) 2332b6a2fea3SOllie Wild { 2333b6a2fea3SOllie Wild return expand_downwards(vma, address); 2334b6a2fea3SOllie Wild } 2335b6a2fea3SOllie Wild 23361da177e4SLinus Torvalds struct vm_area_struct * 23371da177e4SLinus Torvalds find_extend_vma(struct mm_struct *mm, unsigned long addr) 23381da177e4SLinus Torvalds { 23391da177e4SLinus Torvalds struct vm_area_struct *vma; 23401da177e4SLinus Torvalds unsigned long start; 23411da177e4SLinus Torvalds 23421da177e4SLinus Torvalds addr &= PAGE_MASK; 23431da177e4SLinus Torvalds vma = find_vma(mm, addr); 23441da177e4SLinus Torvalds if (!vma) 23451da177e4SLinus Torvalds return NULL; 23461da177e4SLinus Torvalds if (vma->vm_start <= addr) 23471da177e4SLinus Torvalds return vma; 23481da177e4SLinus Torvalds if (!(vma->vm_flags & VM_GROWSDOWN)) 23491da177e4SLinus Torvalds return NULL; 23501da177e4SLinus Torvalds start = vma->vm_start; 23511da177e4SLinus Torvalds if (expand_stack(vma, addr)) 23521da177e4SLinus Torvalds return NULL; 2353cea10a19SMichel Lespinasse if (vma->vm_flags & VM_LOCKED) 2354fc05f566SKirill A. Shutemov populate_vma_page_range(vma, addr, start, NULL); 23551da177e4SLinus Torvalds return vma; 23561da177e4SLinus Torvalds } 23571da177e4SLinus Torvalds #endif 23581da177e4SLinus Torvalds 2359e1d6d01aSJesse Barnes EXPORT_SYMBOL_GPL(find_extend_vma); 2360e1d6d01aSJesse Barnes 23612c0b3814SHugh Dickins /* 23622c0b3814SHugh Dickins * Ok - we have the memory areas we should free on the vma list, 23632c0b3814SHugh Dickins * so release them, and do the vma updates. 23641da177e4SLinus Torvalds * 23652c0b3814SHugh Dickins * Called with the mm semaphore held. 23661da177e4SLinus Torvalds */ 23672c0b3814SHugh Dickins static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) 23681da177e4SLinus Torvalds { 23694f74d2c8SLinus Torvalds unsigned long nr_accounted = 0; 23704f74d2c8SLinus Torvalds 2371365e9c87SHugh Dickins /* Update high watermark before we lower total_vm */ 2372365e9c87SHugh Dickins update_hiwater_vm(mm); 23732c0b3814SHugh Dickins do { 2374ab50b8edSHugh Dickins long nrpages = vma_pages(vma); 23751da177e4SLinus Torvalds 23764f74d2c8SLinus Torvalds if (vma->vm_flags & VM_ACCOUNT) 23774f74d2c8SLinus Torvalds nr_accounted += nrpages; 237884638335SKonstantin Khlebnikov vm_stat_account(mm, vma->vm_flags, -nrpages); 2379a8fb5618SHugh Dickins vma = remove_vma(vma); 2380146425a3SHugh Dickins } while (vma); 23814f74d2c8SLinus Torvalds vm_unacct_memory(nr_accounted); 23821da177e4SLinus Torvalds validate_mm(mm); 23831da177e4SLinus Torvalds } 23841da177e4SLinus Torvalds 23851da177e4SLinus Torvalds /* 23861da177e4SLinus Torvalds * Get rid of page table information in the indicated region. 23871da177e4SLinus Torvalds * 2388f10df686SPaolo 'Blaisorblade' Giarrusso * Called with the mm semaphore held. 23891da177e4SLinus Torvalds */ 23901da177e4SLinus Torvalds static void unmap_region(struct mm_struct *mm, 2391e0da382cSHugh Dickins struct vm_area_struct *vma, struct vm_area_struct *prev, 2392e0da382cSHugh Dickins unsigned long start, unsigned long end) 23931da177e4SLinus Torvalds { 2394f39af059SMatthew Wilcox (Oracle) struct vm_area_struct *next = __vma_next(mm, prev); 2395d16dfc55SPeter Zijlstra struct mmu_gather tlb; 23961da177e4SLinus Torvalds 23971da177e4SLinus Torvalds lru_add_drain(); 2398a72afd87SWill Deacon tlb_gather_mmu(&tlb, mm); 2399365e9c87SHugh Dickins update_hiwater_rss(mm); 24004f74d2c8SLinus Torvalds unmap_vmas(&tlb, vma, start, end); 2401d16dfc55SPeter Zijlstra free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, 24026ee8630eSHugh Dickins next ? next->vm_start : USER_PGTABLES_CEILING); 2403ae8eba8bSWill Deacon tlb_finish_mmu(&tlb); 24041da177e4SLinus Torvalds } 24051da177e4SLinus Torvalds 24061da177e4SLinus Torvalds /* 24071da177e4SLinus Torvalds * Create a list of vma's touched by the unmap, removing them from the mm's 24081da177e4SLinus Torvalds * vma list as we go.. 24091da177e4SLinus Torvalds */ 2410246c320aSKirill A. Shutemov static bool 2411d4af56c5SLiam R. Howlett detach_vmas_to_be_unmapped(struct mm_struct *mm, struct ma_state *mas, 2412d4af56c5SLiam R. Howlett struct vm_area_struct *vma, struct vm_area_struct *prev, 2413d4af56c5SLiam R. Howlett unsigned long end) 24141da177e4SLinus Torvalds { 24151da177e4SLinus Torvalds struct vm_area_struct **insertion_point; 24161da177e4SLinus Torvalds struct vm_area_struct *tail_vma = NULL; 24171da177e4SLinus Torvalds 24181da177e4SLinus Torvalds insertion_point = (prev ? &prev->vm_next : &mm->mmap); 2419297c5eeeSLinus Torvalds vma->vm_prev = NULL; 2420524e00b3SLiam R. Howlett vma_mas_szero(mas, vma->vm_start, end); 24211da177e4SLinus Torvalds do { 2422a213e5cfSHugh Dickins if (vma->vm_flags & VM_LOCKED) 2423a213e5cfSHugh Dickins mm->locked_vm -= vma_pages(vma); 24241da177e4SLinus Torvalds mm->map_count--; 24251da177e4SLinus Torvalds tail_vma = vma; 24261da177e4SLinus Torvalds vma = vma->vm_next; 24271da177e4SLinus Torvalds } while (vma && vma->vm_start < end); 24281da177e4SLinus Torvalds *insertion_point = vma; 2429524e00b3SLiam R. Howlett if (vma) 2430297c5eeeSLinus Torvalds vma->vm_prev = prev; 2431524e00b3SLiam R. Howlett else 24321be7107fSHugh Dickins mm->highest_vm_end = prev ? vm_end_gap(prev) : 0; 24331da177e4SLinus Torvalds tail_vma->vm_next = NULL; 2434615d6e87SDavidlohr Bueso 2435615d6e87SDavidlohr Bueso /* Kill the cache */ 2436615d6e87SDavidlohr Bueso vmacache_invalidate(mm); 2437246c320aSKirill A. Shutemov 2438246c320aSKirill A. Shutemov /* 2439246c320aSKirill A. Shutemov * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or 2440246c320aSKirill A. Shutemov * VM_GROWSUP VMA. Such VMAs can change their size under 2441246c320aSKirill A. Shutemov * down_read(mmap_lock) and collide with the VMA we are about to unmap. 2442246c320aSKirill A. Shutemov */ 2443246c320aSKirill A. Shutemov if (vma && (vma->vm_flags & VM_GROWSDOWN)) 2444246c320aSKirill A. Shutemov return false; 2445246c320aSKirill A. Shutemov if (prev && (prev->vm_flags & VM_GROWSUP)) 2446246c320aSKirill A. Shutemov return false; 2447246c320aSKirill A. Shutemov return true; 24481da177e4SLinus Torvalds } 24491da177e4SLinus Torvalds 24501da177e4SLinus Torvalds /* 2451def5efe0SDavid Rientjes * __split_vma() bypasses sysctl_max_map_count checking. We use this where it 2452def5efe0SDavid Rientjes * has already been checked or doesn't make sense to fail. 24531da177e4SLinus Torvalds */ 2454def5efe0SDavid Rientjes int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, 24551da177e4SLinus Torvalds unsigned long addr, int new_below) 24561da177e4SLinus Torvalds { 24571da177e4SLinus Torvalds struct vm_area_struct *new; 2458e3975891SChen Gang int err; 2459d4af56c5SLiam R. Howlett validate_mm_mt(mm); 24601da177e4SLinus Torvalds 2461dd3b614fSDmitry Safonov if (vma->vm_ops && vma->vm_ops->may_split) { 2462dd3b614fSDmitry Safonov err = vma->vm_ops->may_split(vma, addr); 246331383c68SDan Williams if (err) 246431383c68SDan Williams return err; 246531383c68SDan Williams } 24661da177e4SLinus Torvalds 24673928d4f5SLinus Torvalds new = vm_area_dup(vma); 24681da177e4SLinus Torvalds if (!new) 2469e3975891SChen Gang return -ENOMEM; 24701da177e4SLinus Torvalds 24711da177e4SLinus Torvalds if (new_below) 24721da177e4SLinus Torvalds new->vm_end = addr; 24731da177e4SLinus Torvalds else { 24741da177e4SLinus Torvalds new->vm_start = addr; 24751da177e4SLinus Torvalds new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); 24761da177e4SLinus Torvalds } 24771da177e4SLinus Torvalds 2478ef0855d3SOleg Nesterov err = vma_dup_policy(vma, new); 2479ef0855d3SOleg Nesterov if (err) 24805beb4930SRik van Riel goto out_free_vma; 24811da177e4SLinus Torvalds 2482c4ea95d7SDaniel Forrest err = anon_vma_clone(new, vma); 2483c4ea95d7SDaniel Forrest if (err) 24845beb4930SRik van Riel goto out_free_mpol; 24855beb4930SRik van Riel 2486e9714acfSKonstantin Khlebnikov if (new->vm_file) 24871da177e4SLinus Torvalds get_file(new->vm_file); 24881da177e4SLinus Torvalds 24891da177e4SLinus Torvalds if (new->vm_ops && new->vm_ops->open) 24901da177e4SLinus Torvalds new->vm_ops->open(new); 24911da177e4SLinus Torvalds 24921da177e4SLinus Torvalds if (new_below) 24935beb4930SRik van Riel err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff + 24941da177e4SLinus Torvalds ((addr - new->vm_start) >> PAGE_SHIFT), new); 24951da177e4SLinus Torvalds else 24965beb4930SRik van Riel err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new); 24971da177e4SLinus Torvalds 24985beb4930SRik van Riel /* Success. */ 24995beb4930SRik van Riel if (!err) 25001da177e4SLinus Torvalds return 0; 25015beb4930SRik van Riel 2502d4af56c5SLiam R. Howlett /* Avoid vm accounting in close() operation */ 2503d4af56c5SLiam R. Howlett new->vm_start = new->vm_end; 2504d4af56c5SLiam R. Howlett new->vm_pgoff = 0; 25055beb4930SRik van Riel /* Clean everything up if vma_adjust failed. */ 250658927533SRik van Riel if (new->vm_ops && new->vm_ops->close) 25075beb4930SRik van Riel new->vm_ops->close(new); 2508e9714acfSKonstantin Khlebnikov if (new->vm_file) 25095beb4930SRik van Riel fput(new->vm_file); 25102aeadc30SAndrea Arcangeli unlink_anon_vmas(new); 25115beb4930SRik van Riel out_free_mpol: 2512ef0855d3SOleg Nesterov mpol_put(vma_policy(new)); 25135beb4930SRik van Riel out_free_vma: 25143928d4f5SLinus Torvalds vm_area_free(new); 2515d4af56c5SLiam R. Howlett validate_mm_mt(mm); 25165beb4930SRik van Riel return err; 25171da177e4SLinus Torvalds } 25181da177e4SLinus Torvalds 2519659ace58SKOSAKI Motohiro /* 2520659ace58SKOSAKI Motohiro * Split a vma into two pieces at address 'addr', a new vma is allocated 2521659ace58SKOSAKI Motohiro * either for the first part or the tail. 2522659ace58SKOSAKI Motohiro */ 2523659ace58SKOSAKI Motohiro int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, 2524659ace58SKOSAKI Motohiro unsigned long addr, int new_below) 2525659ace58SKOSAKI Motohiro { 2526659ace58SKOSAKI Motohiro if (mm->map_count >= sysctl_max_map_count) 2527659ace58SKOSAKI Motohiro return -ENOMEM; 2528659ace58SKOSAKI Motohiro 2529659ace58SKOSAKI Motohiro return __split_vma(mm, vma, addr, new_below); 2530659ace58SKOSAKI Motohiro } 2531659ace58SKOSAKI Motohiro 25321da177e4SLinus Torvalds /* Munmap is split into 2 main parts -- this part which finds 25331da177e4SLinus Torvalds * what needs doing, and the areas themselves, which do the 25341da177e4SLinus Torvalds * work. This now handles partial unmappings. 25351da177e4SLinus Torvalds * Jeremy Fitzhardinge <jeremy@goop.org> 25361da177e4SLinus Torvalds */ 253785a06835SYang Shi int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, 2538dd2283f2SYang Shi struct list_head *uf, bool downgrade) 25391da177e4SLinus Torvalds { 25401da177e4SLinus Torvalds unsigned long end; 2541146425a3SHugh Dickins struct vm_area_struct *vma, *prev, *last; 2542d4af56c5SLiam R. Howlett int error = -ENOMEM; 2543d4af56c5SLiam R. Howlett MA_STATE(mas, &mm->mm_mt, 0, 0); 25441da177e4SLinus Torvalds 2545de1741a1SAlexander Kuleshov if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) 25461da177e4SLinus Torvalds return -EINVAL; 25471da177e4SLinus Torvalds 2548cc71aba3Svishnu.ps len = PAGE_ALIGN(len); 25495a28fc94SDave Hansen end = start + len; 2550cc71aba3Svishnu.ps if (len == 0) 25511da177e4SLinus Torvalds return -EINVAL; 25521da177e4SLinus Torvalds 2553524e00b3SLiam R. Howlett /* arch_unmap() might do unmaps itself. */ 25545a28fc94SDave Hansen arch_unmap(mm, start, end); 25555a28fc94SDave Hansen 255678d9cf60SGonzalo Matias Juarez Tello /* Find the first overlapping VMA where start < vma->vm_end */ 255778d9cf60SGonzalo Matias Juarez Tello vma = find_vma_intersection(mm, start, end); 2558146425a3SHugh Dickins if (!vma) 25591da177e4SLinus Torvalds return 0; 2560d4af56c5SLiam R. Howlett 2561d4af56c5SLiam R. Howlett if (mas_preallocate(&mas, vma, GFP_KERNEL)) 2562d4af56c5SLiam R. Howlett return -ENOMEM; 25639be34c9dSLinus Torvalds prev = vma->vm_prev; 2564524e00b3SLiam R. Howlett /* we have start < vma->vm_end */ 2565524e00b3SLiam R. Howlett 2566524e00b3SLiam R. Howlett /* if it doesn't overlap, we have nothing.. */ 2567524e00b3SLiam R. Howlett if (vma->vm_start >= end) 2568524e00b3SLiam R. Howlett return 0; 25691da177e4SLinus Torvalds 25701da177e4SLinus Torvalds /* 25711da177e4SLinus Torvalds * If we need to split any vma, do it now to save pain later. 25721da177e4SLinus Torvalds * 25731da177e4SLinus Torvalds * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially 25741da177e4SLinus Torvalds * unmapped vm_area_struct will remain in use: so lower split_vma 25751da177e4SLinus Torvalds * places tmp vma above, and higher split_vma places tmp vma below. 25761da177e4SLinus Torvalds */ 2577146425a3SHugh Dickins if (start > vma->vm_start) { 2578659ace58SKOSAKI Motohiro 2579659ace58SKOSAKI Motohiro /* 2580659ace58SKOSAKI Motohiro * Make sure that map_count on return from munmap() will 2581659ace58SKOSAKI Motohiro * not exceed its limit; but let map_count go just above 2582659ace58SKOSAKI Motohiro * its limit temporarily, to help free resources as expected. 2583659ace58SKOSAKI Motohiro */ 2584659ace58SKOSAKI Motohiro if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count) 2585d4af56c5SLiam R. Howlett goto map_count_exceeded; 2586659ace58SKOSAKI Motohiro 2587659ace58SKOSAKI Motohiro error = __split_vma(mm, vma, start, 0); 25881da177e4SLinus Torvalds if (error) 2589d4af56c5SLiam R. Howlett goto split_failed; 2590146425a3SHugh Dickins prev = vma; 25911da177e4SLinus Torvalds } 25921da177e4SLinus Torvalds 25931da177e4SLinus Torvalds /* Does it split the last one? */ 25941da177e4SLinus Torvalds last = find_vma(mm, end); 25951da177e4SLinus Torvalds if (last && end > last->vm_start) { 2596d4af56c5SLiam R. Howlett error = __split_vma(mm, last, end, 1); 25971da177e4SLinus Torvalds if (error) 2598d4af56c5SLiam R. Howlett goto split_failed; 25991da177e4SLinus Torvalds } 2600f39af059SMatthew Wilcox (Oracle) vma = __vma_next(mm, prev); 26011da177e4SLinus Torvalds 26022376dd7cSAndrea Arcangeli if (unlikely(uf)) { 26032376dd7cSAndrea Arcangeli /* 26042376dd7cSAndrea Arcangeli * If userfaultfd_unmap_prep returns an error the vmas 2605f0953a1bSIngo Molnar * will remain split, but userland will get a 26062376dd7cSAndrea Arcangeli * highly unexpected error anyway. This is no 26072376dd7cSAndrea Arcangeli * different than the case where the first of the two 26082376dd7cSAndrea Arcangeli * __split_vma fails, but we don't undo the first 26092376dd7cSAndrea Arcangeli * split, despite we could. This is unlikely enough 26102376dd7cSAndrea Arcangeli * failure that it's not worth optimizing it for. 26112376dd7cSAndrea Arcangeli */ 2612d4af56c5SLiam R. Howlett error = userfaultfd_unmap_prep(vma, start, end, uf); 26132376dd7cSAndrea Arcangeli if (error) 2614d4af56c5SLiam R. Howlett goto userfaultfd_error; 26152376dd7cSAndrea Arcangeli } 26162376dd7cSAndrea Arcangeli 2617dd2283f2SYang Shi /* Detach vmas from rbtree */ 2618d4af56c5SLiam R. Howlett if (!detach_vmas_to_be_unmapped(mm, &mas, vma, prev, end)) 2619246c320aSKirill A. Shutemov downgrade = false; 26201da177e4SLinus Torvalds 2621dd2283f2SYang Shi if (downgrade) 2622d8ed45c5SMichel Lespinasse mmap_write_downgrade(mm); 2623dd2283f2SYang Shi 2624dd2283f2SYang Shi unmap_region(mm, vma, prev, start, end); 2625dd2283f2SYang Shi 26261da177e4SLinus Torvalds /* Fix up all other VM information */ 26272c0b3814SHugh Dickins remove_vma_list(mm, vma); 26281da177e4SLinus Torvalds 2629524e00b3SLiam R. Howlett 2630524e00b3SLiam R. Howlett validate_mm(mm); 2631dd2283f2SYang Shi return downgrade ? 1 : 0; 2632d4af56c5SLiam R. Howlett 2633d4af56c5SLiam R. Howlett map_count_exceeded: 2634d4af56c5SLiam R. Howlett split_failed: 2635d4af56c5SLiam R. Howlett userfaultfd_error: 2636d4af56c5SLiam R. Howlett mas_destroy(&mas); 2637d4af56c5SLiam R. Howlett return error; 26381da177e4SLinus Torvalds } 26391da177e4SLinus Torvalds 2640dd2283f2SYang Shi int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, 2641dd2283f2SYang Shi struct list_head *uf) 2642dd2283f2SYang Shi { 2643dd2283f2SYang Shi return __do_munmap(mm, start, len, uf, false); 2644dd2283f2SYang Shi } 2645dd2283f2SYang Shi 2646dd2283f2SYang Shi static int __vm_munmap(unsigned long start, size_t len, bool downgrade) 2647a46ef99dSLinus Torvalds { 2648a46ef99dSLinus Torvalds int ret; 2649bfce281cSAl Viro struct mm_struct *mm = current->mm; 2650897ab3e0SMike Rapoport LIST_HEAD(uf); 2651a46ef99dSLinus Torvalds 2652d8ed45c5SMichel Lespinasse if (mmap_write_lock_killable(mm)) 2653ae798783SMichal Hocko return -EINTR; 2654ae798783SMichal Hocko 2655dd2283f2SYang Shi ret = __do_munmap(mm, start, len, &uf, downgrade); 2656dd2283f2SYang Shi /* 2657c1e8d7c6SMichel Lespinasse * Returning 1 indicates mmap_lock is downgraded. 2658dd2283f2SYang Shi * But 1 is not legal return value of vm_munmap() and munmap(), reset 2659dd2283f2SYang Shi * it to 0 before return. 2660dd2283f2SYang Shi */ 2661dd2283f2SYang Shi if (ret == 1) { 2662d8ed45c5SMichel Lespinasse mmap_read_unlock(mm); 2663dd2283f2SYang Shi ret = 0; 2664dd2283f2SYang Shi } else 2665d8ed45c5SMichel Lespinasse mmap_write_unlock(mm); 2666dd2283f2SYang Shi 2667897ab3e0SMike Rapoport userfaultfd_unmap_complete(mm, &uf); 2668a46ef99dSLinus Torvalds return ret; 2669a46ef99dSLinus Torvalds } 2670dd2283f2SYang Shi 2671dd2283f2SYang Shi int vm_munmap(unsigned long start, size_t len) 2672dd2283f2SYang Shi { 2673dd2283f2SYang Shi return __vm_munmap(start, len, false); 2674dd2283f2SYang Shi } 2675a46ef99dSLinus Torvalds EXPORT_SYMBOL(vm_munmap); 2676a46ef99dSLinus Torvalds 26776a6160a7SHeiko Carstens SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) 26781da177e4SLinus Torvalds { 2679ce18d171SCatalin Marinas addr = untagged_addr(addr); 2680dd2283f2SYang Shi return __vm_munmap(addr, len, true); 26811da177e4SLinus Torvalds } 26821da177e4SLinus Torvalds 2683c8d78c18SKirill A. Shutemov 2684c8d78c18SKirill A. Shutemov /* 2685c8d78c18SKirill A. Shutemov * Emulation of deprecated remap_file_pages() syscall. 2686c8d78c18SKirill A. Shutemov */ 2687c8d78c18SKirill A. Shutemov SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, 2688c8d78c18SKirill A. Shutemov unsigned long, prot, unsigned long, pgoff, unsigned long, flags) 2689c8d78c18SKirill A. Shutemov { 2690c8d78c18SKirill A. Shutemov 2691c8d78c18SKirill A. Shutemov struct mm_struct *mm = current->mm; 2692c8d78c18SKirill A. Shutemov struct vm_area_struct *vma; 2693c8d78c18SKirill A. Shutemov unsigned long populate = 0; 2694c8d78c18SKirill A. Shutemov unsigned long ret = -EINVAL; 2695c8d78c18SKirill A. Shutemov struct file *file; 2696c8d78c18SKirill A. Shutemov 2697ee65728eSMike Rapoport pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/mm/remap_file_pages.rst.\n", 2698c8d78c18SKirill A. Shutemov current->comm, current->pid); 2699c8d78c18SKirill A. Shutemov 2700c8d78c18SKirill A. Shutemov if (prot) 2701c8d78c18SKirill A. Shutemov return ret; 2702c8d78c18SKirill A. Shutemov start = start & PAGE_MASK; 2703c8d78c18SKirill A. Shutemov size = size & PAGE_MASK; 2704c8d78c18SKirill A. Shutemov 2705c8d78c18SKirill A. Shutemov if (start + size <= start) 2706c8d78c18SKirill A. Shutemov return ret; 2707c8d78c18SKirill A. Shutemov 2708c8d78c18SKirill A. Shutemov /* Does pgoff wrap? */ 2709c8d78c18SKirill A. Shutemov if (pgoff + (size >> PAGE_SHIFT) < pgoff) 2710c8d78c18SKirill A. Shutemov return ret; 2711c8d78c18SKirill A. Shutemov 2712d8ed45c5SMichel Lespinasse if (mmap_write_lock_killable(mm)) 2713dc0ef0dfSMichal Hocko return -EINTR; 2714dc0ef0dfSMichal Hocko 27159b593cb2SLiam R. Howlett vma = vma_lookup(mm, start); 2716c8d78c18SKirill A. Shutemov 2717c8d78c18SKirill A. Shutemov if (!vma || !(vma->vm_flags & VM_SHARED)) 2718c8d78c18SKirill A. Shutemov goto out; 2719c8d78c18SKirill A. Shutemov 272048f7df32SKirill A. Shutemov if (start + size > vma->vm_end) { 272148f7df32SKirill A. Shutemov struct vm_area_struct *next; 272248f7df32SKirill A. Shutemov 272348f7df32SKirill A. Shutemov for (next = vma->vm_next; next; next = next->vm_next) { 272448f7df32SKirill A. Shutemov /* hole between vmas ? */ 272548f7df32SKirill A. Shutemov if (next->vm_start != next->vm_prev->vm_end) 272648f7df32SKirill A. Shutemov goto out; 272748f7df32SKirill A. Shutemov 272848f7df32SKirill A. Shutemov if (next->vm_file != vma->vm_file) 272948f7df32SKirill A. Shutemov goto out; 273048f7df32SKirill A. Shutemov 273148f7df32SKirill A. Shutemov if (next->vm_flags != vma->vm_flags) 273248f7df32SKirill A. Shutemov goto out; 273348f7df32SKirill A. Shutemov 273448f7df32SKirill A. Shutemov if (start + size <= next->vm_end) 273548f7df32SKirill A. Shutemov break; 273648f7df32SKirill A. Shutemov } 273748f7df32SKirill A. Shutemov 273848f7df32SKirill A. Shutemov if (!next) 2739c8d78c18SKirill A. Shutemov goto out; 2740c8d78c18SKirill A. Shutemov } 2741c8d78c18SKirill A. Shutemov 2742c8d78c18SKirill A. Shutemov prot |= vma->vm_flags & VM_READ ? PROT_READ : 0; 2743c8d78c18SKirill A. Shutemov prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0; 2744c8d78c18SKirill A. Shutemov prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0; 2745c8d78c18SKirill A. Shutemov 2746c8d78c18SKirill A. Shutemov flags &= MAP_NONBLOCK; 2747c8d78c18SKirill A. Shutemov flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE; 2748fce000b1SLiam Howlett if (vma->vm_flags & VM_LOCKED) 2749c8d78c18SKirill A. Shutemov flags |= MAP_LOCKED; 275048f7df32SKirill A. Shutemov 2751c8d78c18SKirill A. Shutemov file = get_file(vma->vm_file); 275245e55300SPeter Collingbourne ret = do_mmap(vma->vm_file, start, size, 2753897ab3e0SMike Rapoport prot, flags, pgoff, &populate, NULL); 2754c8d78c18SKirill A. Shutemov fput(file); 2755c8d78c18SKirill A. Shutemov out: 2756d8ed45c5SMichel Lespinasse mmap_write_unlock(mm); 2757c8d78c18SKirill A. Shutemov if (populate) 2758c8d78c18SKirill A. Shutemov mm_populate(ret, populate); 2759c8d78c18SKirill A. Shutemov if (!IS_ERR_VALUE(ret)) 2760c8d78c18SKirill A. Shutemov ret = 0; 2761c8d78c18SKirill A. Shutemov return ret; 2762c8d78c18SKirill A. Shutemov } 2763c8d78c18SKirill A. Shutemov 27641da177e4SLinus Torvalds /* 27651da177e4SLinus Torvalds * this is really a simplified "do_mmap". it only handles 27661da177e4SLinus Torvalds * anonymous maps. eventually we may be able to do some 27671da177e4SLinus Torvalds * brk-specific accounting here. 27681da177e4SLinus Torvalds */ 2769524e00b3SLiam R. Howlett static int do_brk_flags(unsigned long addr, unsigned long len, 2770524e00b3SLiam R. Howlett unsigned long flags, struct list_head *uf) 27711da177e4SLinus Torvalds { 27721da177e4SLinus Torvalds struct mm_struct *mm = current->mm; 27731da177e4SLinus Torvalds struct vm_area_struct *vma, *prev; 27741da177e4SLinus Torvalds pgoff_t pgoff = addr >> PAGE_SHIFT; 27753a459756SKirill Korotaev int error; 2776ff68dac6SGaowei Pu unsigned long mapped_addr; 2777d4af56c5SLiam R. Howlett validate_mm_mt(mm); 27781da177e4SLinus Torvalds 277916e72e9bSDenys Vlasenko /* Until we need other flags, refuse anything except VM_EXEC. */ 278016e72e9bSDenys Vlasenko if ((flags & (~VM_EXEC)) != 0) 278116e72e9bSDenys Vlasenko return -EINVAL; 278216e72e9bSDenys Vlasenko flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; 27833a459756SKirill Korotaev 2784ff68dac6SGaowei Pu mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); 2785ff68dac6SGaowei Pu if (IS_ERR_VALUE(mapped_addr)) 2786ff68dac6SGaowei Pu return mapped_addr; 27873a459756SKirill Korotaev 2788363ee17fSDavidlohr Bueso error = mlock_future_check(mm, mm->def_flags, len); 2789363ee17fSDavidlohr Bueso if (error) 2790363ee17fSDavidlohr Bueso return error; 27911da177e4SLinus Torvalds 2792524e00b3SLiam R. Howlett /* Clear old maps, set up prev and uf */ 2793524e00b3SLiam R. Howlett if (munmap_vma_range(mm, addr, len, &prev, uf)) 27941da177e4SLinus Torvalds return -ENOMEM; 27951da177e4SLinus Torvalds 27961da177e4SLinus Torvalds /* Check against address space limits *after* clearing old maps... */ 279784638335SKonstantin Khlebnikov if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT)) 27981da177e4SLinus Torvalds return -ENOMEM; 27991da177e4SLinus Torvalds 28001da177e4SLinus Torvalds if (mm->map_count > sysctl_max_map_count) 28011da177e4SLinus Torvalds return -ENOMEM; 28021da177e4SLinus Torvalds 2803191c5424SAl Viro if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) 28041da177e4SLinus Torvalds return -ENOMEM; 28051da177e4SLinus Torvalds 28061da177e4SLinus Torvalds /* Can we just expand an old private anonymous mapping? */ 2807ba470de4SRik van Riel vma = vma_merge(mm, prev, addr, addr + len, flags, 28089a10064fSColin Cross NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); 2809ba470de4SRik van Riel if (vma) 28101da177e4SLinus Torvalds goto out; 28111da177e4SLinus Torvalds 28121da177e4SLinus Torvalds /* 28131da177e4SLinus Torvalds * create a vma struct for an anonymous mapping 28141da177e4SLinus Torvalds */ 2815490fc053SLinus Torvalds vma = vm_area_alloc(mm); 28161da177e4SLinus Torvalds if (!vma) { 28171da177e4SLinus Torvalds vm_unacct_memory(len >> PAGE_SHIFT); 28181da177e4SLinus Torvalds return -ENOMEM; 28191da177e4SLinus Torvalds } 28201da177e4SLinus Torvalds 2821bfd40eafSKirill A. Shutemov vma_set_anonymous(vma); 28221da177e4SLinus Torvalds vma->vm_start = addr; 28231da177e4SLinus Torvalds vma->vm_end = addr + len; 28241da177e4SLinus Torvalds vma->vm_pgoff = pgoff; 28251da177e4SLinus Torvalds vma->vm_flags = flags; 28263ed75eb8SColy Li vma->vm_page_prot = vm_get_page_prot(flags); 2827524e00b3SLiam R. Howlett if (vma_link(mm, vma, prev)) 2828d4af56c5SLiam R. Howlett goto no_vma_link; 2829d4af56c5SLiam R. Howlett 28301da177e4SLinus Torvalds out: 28313af9e859SEric B Munson perf_event_mmap(vma); 28321da177e4SLinus Torvalds mm->total_vm += len >> PAGE_SHIFT; 283384638335SKonstantin Khlebnikov mm->data_vm += len >> PAGE_SHIFT; 2834128557ffSMichel Lespinasse if (flags & VM_LOCKED) 2835ba470de4SRik van Riel mm->locked_vm += (len >> PAGE_SHIFT); 2836d9104d1cSCyrill Gorcunov vma->vm_flags |= VM_SOFTDIRTY; 2837d4af56c5SLiam R. Howlett validate_mm_mt(mm); 28385d22fc25SLinus Torvalds return 0; 2839d4af56c5SLiam R. Howlett 2840d4af56c5SLiam R. Howlett no_vma_link: 2841d4af56c5SLiam R. Howlett vm_area_free(vma); 2842d4af56c5SLiam R. Howlett return -ENOMEM; 28431da177e4SLinus Torvalds } 28441da177e4SLinus Torvalds 2845bb177a73SMichal Hocko int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) 2846e4eb1ff6SLinus Torvalds { 2847e4eb1ff6SLinus Torvalds struct mm_struct *mm = current->mm; 2848bb177a73SMichal Hocko unsigned long len; 28495d22fc25SLinus Torvalds int ret; 2850128557ffSMichel Lespinasse bool populate; 2851897ab3e0SMike Rapoport LIST_HEAD(uf); 2852e4eb1ff6SLinus Torvalds 2853bb177a73SMichal Hocko len = PAGE_ALIGN(request); 2854bb177a73SMichal Hocko if (len < request) 2855bb177a73SMichal Hocko return -ENOMEM; 2856bb177a73SMichal Hocko if (!len) 2857bb177a73SMichal Hocko return 0; 2858bb177a73SMichal Hocko 2859d8ed45c5SMichel Lespinasse if (mmap_write_lock_killable(mm)) 28602d6c9282SMichal Hocko return -EINTR; 28612d6c9282SMichal Hocko 2862897ab3e0SMike Rapoport ret = do_brk_flags(addr, len, flags, &uf); 2863128557ffSMichel Lespinasse populate = ((mm->def_flags & VM_LOCKED) != 0); 2864d8ed45c5SMichel Lespinasse mmap_write_unlock(mm); 2865897ab3e0SMike Rapoport userfaultfd_unmap_complete(mm, &uf); 28665d22fc25SLinus Torvalds if (populate && !ret) 2867128557ffSMichel Lespinasse mm_populate(addr, len); 2868e4eb1ff6SLinus Torvalds return ret; 2869e4eb1ff6SLinus Torvalds } 287016e72e9bSDenys Vlasenko EXPORT_SYMBOL(vm_brk_flags); 287116e72e9bSDenys Vlasenko 287216e72e9bSDenys Vlasenko int vm_brk(unsigned long addr, unsigned long len) 287316e72e9bSDenys Vlasenko { 287416e72e9bSDenys Vlasenko return vm_brk_flags(addr, len, 0); 287516e72e9bSDenys Vlasenko } 2876e4eb1ff6SLinus Torvalds EXPORT_SYMBOL(vm_brk); 28771da177e4SLinus Torvalds 28781da177e4SLinus Torvalds /* Release all mmaps. */ 28791da177e4SLinus Torvalds void exit_mmap(struct mm_struct *mm) 28801da177e4SLinus Torvalds { 2881d16dfc55SPeter Zijlstra struct mmu_gather tlb; 2882ba470de4SRik van Riel struct vm_area_struct *vma; 28831da177e4SLinus Torvalds unsigned long nr_accounted = 0; 28841da177e4SLinus Torvalds 2885d6dd61c8SJeremy Fitzhardinge /* mm's last user has gone, and its about to be pulled down */ 2886cddb8a5cSAndrea Arcangeli mmu_notifier_release(mm); 2887d6dd61c8SJeremy Fitzhardinge 288827ae357fSDavid Rientjes if (unlikely(mm_is_oom_victim(mm))) { 288927ae357fSDavid Rientjes /* 289027ae357fSDavid Rientjes * Manually reap the mm to free as much memory as possible. 289127ae357fSDavid Rientjes * Then, as the oom reaper does, set MMF_OOM_SKIP to disregard 2892c1e8d7c6SMichel Lespinasse * this mm from further consideration. Taking mm->mmap_lock for 289327ae357fSDavid Rientjes * write after setting MMF_OOM_SKIP will guarantee that the oom 2894c1e8d7c6SMichel Lespinasse * reaper will not run on this mm again after mmap_lock is 289527ae357fSDavid Rientjes * dropped. 289627ae357fSDavid Rientjes * 2897c1e8d7c6SMichel Lespinasse * Nothing can be holding mm->mmap_lock here and the above call 289827ae357fSDavid Rientjes * to mmu_notifier_release(mm) ensures mmu notifier callbacks in 289927ae357fSDavid Rientjes * __oom_reap_task_mm() will not block. 290027ae357fSDavid Rientjes */ 290193065ac7SMichal Hocko (void)__oom_reap_task_mm(mm); 290227ae357fSDavid Rientjes set_bit(MMF_OOM_SKIP, &mm->flags); 290327ae357fSDavid Rientjes } 290427ae357fSDavid Rientjes 290564591e86SSuren Baghdasaryan mmap_write_lock(mm); 29069480c53eSJeremy Fitzhardinge arch_exit_mmap(mm); 29079480c53eSJeremy Fitzhardinge 2908ba470de4SRik van Riel vma = mm->mmap; 290964591e86SSuren Baghdasaryan if (!vma) { 291064591e86SSuren Baghdasaryan /* Can happen if dup_mmap() received an OOM */ 291164591e86SSuren Baghdasaryan mmap_write_unlock(mm); 29129480c53eSJeremy Fitzhardinge return; 291364591e86SSuren Baghdasaryan } 29149480c53eSJeremy Fitzhardinge 29151da177e4SLinus Torvalds lru_add_drain(); 29161da177e4SLinus Torvalds flush_cache_mm(mm); 2917d8b45053SWill Deacon tlb_gather_mmu_fullmm(&tlb, mm); 2918901608d9SOleg Nesterov /* update_hiwater_rss(mm) here? but nobody should be looking */ 2919e0da382cSHugh Dickins /* Use -1 here to ensure all VMAs in the mm are unmapped */ 29204f74d2c8SLinus Torvalds unmap_vmas(&tlb, vma, 0, -1); 29216ee8630eSHugh Dickins free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); 2922ae8eba8bSWill Deacon tlb_finish_mmu(&tlb); 29231da177e4SLinus Torvalds 292464591e86SSuren Baghdasaryan /* Walk the list again, actually closing and freeing it. */ 29254f74d2c8SLinus Torvalds while (vma) { 29264f74d2c8SLinus Torvalds if (vma->vm_flags & VM_ACCOUNT) 29274f74d2c8SLinus Torvalds nr_accounted += vma_pages(vma); 2928a8fb5618SHugh Dickins vma = remove_vma(vma); 29290a3b3c25SPaul E. McKenney cond_resched(); 29304f74d2c8SLinus Torvalds } 2931d4af56c5SLiam R. Howlett 2932d4af56c5SLiam R. Howlett trace_exit_mmap(mm); 2933d4af56c5SLiam R. Howlett __mt_destroy(&mm->mm_mt); 2934f798a1d4SSuren Baghdasaryan mm->mmap = NULL; 293564591e86SSuren Baghdasaryan mmap_write_unlock(mm); 29364f74d2c8SLinus Torvalds vm_unacct_memory(nr_accounted); 29371da177e4SLinus Torvalds } 29381da177e4SLinus Torvalds 29391da177e4SLinus Torvalds /* Insert vm structure into process list sorted by address 29401da177e4SLinus Torvalds * and into the inode's i_mmap tree. If vm_file is non-NULL 2941c8c06efaSDavidlohr Bueso * then i_mmap_rwsem is taken here. 29421da177e4SLinus Torvalds */ 29431da177e4SLinus Torvalds int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) 29441da177e4SLinus Torvalds { 29456597d783SHugh Dickins struct vm_area_struct *prev; 2946d4af56c5SLiam R. Howlett unsigned long charged = vma_pages(vma); 29471da177e4SLinus Torvalds 2948d4af56c5SLiam R. Howlett 2949524e00b3SLiam R. Howlett if (range_has_overlap(mm, vma->vm_start, vma->vm_end, &prev)) 2950c9d13f5fSChen Gang return -ENOMEM; 2951d4af56c5SLiam R. Howlett 2952c9d13f5fSChen Gang if ((vma->vm_flags & VM_ACCOUNT) && 2953d4af56c5SLiam R. Howlett security_vm_enough_memory_mm(mm, charged)) 2954c9d13f5fSChen Gang return -ENOMEM; 2955c9d13f5fSChen Gang 29561da177e4SLinus Torvalds /* 29571da177e4SLinus Torvalds * The vm_pgoff of a purely anonymous vma should be irrelevant 29581da177e4SLinus Torvalds * until its first write fault, when page's anon_vma and index 29591da177e4SLinus Torvalds * are set. But now set the vm_pgoff it will almost certainly 29601da177e4SLinus Torvalds * end up with (unless mremap moves it elsewhere before that 29611da177e4SLinus Torvalds * first wfault), so /proc/pid/maps tells a consistent story. 29621da177e4SLinus Torvalds * 29631da177e4SLinus Torvalds * By setting it to reflect the virtual start address of the 29641da177e4SLinus Torvalds * vma, merges and splits can happen in a seamless way, just 29651da177e4SLinus Torvalds * using the existing file pgoff checks and manipulations. 29668332326eSLiao Pingfang * Similarly in do_mmap and in do_brk_flags. 29671da177e4SLinus Torvalds */ 29688a9cc3b5SOleg Nesterov if (vma_is_anonymous(vma)) { 29691da177e4SLinus Torvalds BUG_ON(vma->anon_vma); 29701da177e4SLinus Torvalds vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; 29711da177e4SLinus Torvalds } 29722b144498SSrikar Dronamraju 2973524e00b3SLiam R. Howlett if (vma_link(mm, vma, prev)) { 2974d4af56c5SLiam R. Howlett vm_unacct_memory(charged); 2975d4af56c5SLiam R. Howlett return -ENOMEM; 2976d4af56c5SLiam R. Howlett } 2977d4af56c5SLiam R. Howlett 29781da177e4SLinus Torvalds return 0; 29791da177e4SLinus Torvalds } 29801da177e4SLinus Torvalds 29811da177e4SLinus Torvalds /* 29821da177e4SLinus Torvalds * Copy the vma structure to a new location in the same mm, 29831da177e4SLinus Torvalds * prior to moving page table entries, to effect an mremap move. 29841da177e4SLinus Torvalds */ 29851da177e4SLinus Torvalds struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, 298638a76013SMichel Lespinasse unsigned long addr, unsigned long len, pgoff_t pgoff, 298738a76013SMichel Lespinasse bool *need_rmap_locks) 29881da177e4SLinus Torvalds { 29891da177e4SLinus Torvalds struct vm_area_struct *vma = *vmap; 29901da177e4SLinus Torvalds unsigned long vma_start = vma->vm_start; 29911da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 29921da177e4SLinus Torvalds struct vm_area_struct *new_vma, *prev; 2993948f017bSAndrea Arcangeli bool faulted_in_anon_vma = true; 29941da177e4SLinus Torvalds 2995d4af56c5SLiam R. Howlett validate_mm_mt(mm); 29961da177e4SLinus Torvalds /* 29971da177e4SLinus Torvalds * If anonymous vma has not yet been faulted, update new pgoff 29981da177e4SLinus Torvalds * to match new location, to increase its chance of merging. 29991da177e4SLinus Torvalds */ 3000ce75799bSOleg Nesterov if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { 30011da177e4SLinus Torvalds pgoff = addr >> PAGE_SHIFT; 3002948f017bSAndrea Arcangeli faulted_in_anon_vma = false; 3003948f017bSAndrea Arcangeli } 30041da177e4SLinus Torvalds 3005524e00b3SLiam R. Howlett if (range_has_overlap(mm, addr, addr + len, &prev)) 30066597d783SHugh Dickins return NULL; /* should never get here */ 3007524e00b3SLiam R. Howlett 30081da177e4SLinus Torvalds new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, 300919a809afSAndrea Arcangeli vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), 30105c26f6acSSuren Baghdasaryan vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 30111da177e4SLinus Torvalds if (new_vma) { 30121da177e4SLinus Torvalds /* 30131da177e4SLinus Torvalds * Source vma may have been merged into new_vma 30141da177e4SLinus Torvalds */ 3015948f017bSAndrea Arcangeli if (unlikely(vma_start >= new_vma->vm_start && 3016948f017bSAndrea Arcangeli vma_start < new_vma->vm_end)) { 3017948f017bSAndrea Arcangeli /* 3018948f017bSAndrea Arcangeli * The only way we can get a vma_merge with 3019948f017bSAndrea Arcangeli * self during an mremap is if the vma hasn't 3020948f017bSAndrea Arcangeli * been faulted in yet and we were allowed to 3021948f017bSAndrea Arcangeli * reset the dst vma->vm_pgoff to the 3022948f017bSAndrea Arcangeli * destination address of the mremap to allow 3023948f017bSAndrea Arcangeli * the merge to happen. mremap must change the 3024948f017bSAndrea Arcangeli * vm_pgoff linearity between src and dst vmas 3025948f017bSAndrea Arcangeli * (in turn preventing a vma_merge) to be 3026948f017bSAndrea Arcangeli * safe. It is only safe to keep the vm_pgoff 3027948f017bSAndrea Arcangeli * linear if there are no pages mapped yet. 3028948f017bSAndrea Arcangeli */ 302981d1b09cSSasha Levin VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); 303038a76013SMichel Lespinasse *vmap = vma = new_vma; 3031108d6642SMichel Lespinasse } 303238a76013SMichel Lespinasse *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff); 30331da177e4SLinus Torvalds } else { 30343928d4f5SLinus Torvalds new_vma = vm_area_dup(vma); 3035e3975891SChen Gang if (!new_vma) 3036e3975891SChen Gang goto out; 30371da177e4SLinus Torvalds new_vma->vm_start = addr; 30381da177e4SLinus Torvalds new_vma->vm_end = addr + len; 30391da177e4SLinus Torvalds new_vma->vm_pgoff = pgoff; 3040ef0855d3SOleg Nesterov if (vma_dup_policy(vma, new_vma)) 3041523d4e20SMichel Lespinasse goto out_free_vma; 3042523d4e20SMichel Lespinasse if (anon_vma_clone(new_vma, vma)) 3043523d4e20SMichel Lespinasse goto out_free_mempol; 3044e9714acfSKonstantin Khlebnikov if (new_vma->vm_file) 30451da177e4SLinus Torvalds get_file(new_vma->vm_file); 30461da177e4SLinus Torvalds if (new_vma->vm_ops && new_vma->vm_ops->open) 30471da177e4SLinus Torvalds new_vma->vm_ops->open(new_vma); 3048524e00b3SLiam R. Howlett if (vma_link(mm, new_vma, prev)) 3049524e00b3SLiam R. Howlett goto out_vma_link; 305038a76013SMichel Lespinasse *need_rmap_locks = false; 30511da177e4SLinus Torvalds } 3052d4af56c5SLiam R. Howlett validate_mm_mt(mm); 30531da177e4SLinus Torvalds return new_vma; 30545beb4930SRik van Riel 3055524e00b3SLiam R. Howlett out_vma_link: 3056524e00b3SLiam R. Howlett if (new_vma->vm_ops && new_vma->vm_ops->close) 3057524e00b3SLiam R. Howlett new_vma->vm_ops->close(new_vma); 30585beb4930SRik van Riel out_free_mempol: 3059ef0855d3SOleg Nesterov mpol_put(vma_policy(new_vma)); 30605beb4930SRik van Riel out_free_vma: 30613928d4f5SLinus Torvalds vm_area_free(new_vma); 3062e3975891SChen Gang out: 3063d4af56c5SLiam R. Howlett validate_mm_mt(mm); 30645beb4930SRik van Riel return NULL; 30651da177e4SLinus Torvalds } 3066119f657cSakpm@osdl.org 3067119f657cSakpm@osdl.org /* 3068119f657cSakpm@osdl.org * Return true if the calling process may expand its vm space by the passed 3069119f657cSakpm@osdl.org * number of pages 3070119f657cSakpm@osdl.org */ 307184638335SKonstantin Khlebnikov bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages) 3072119f657cSakpm@osdl.org { 307384638335SKonstantin Khlebnikov if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT) 307484638335SKonstantin Khlebnikov return false; 3075119f657cSakpm@osdl.org 3076d977d56cSKonstantin Khlebnikov if (is_data_mapping(flags) && 3077d977d56cSKonstantin Khlebnikov mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) { 3078f4fcd558SKonstantin Khlebnikov /* Workaround for Valgrind */ 3079f4fcd558SKonstantin Khlebnikov if (rlimit(RLIMIT_DATA) == 0 && 3080f4fcd558SKonstantin Khlebnikov mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) 3081f4fcd558SKonstantin Khlebnikov return true; 308257a7702bSDavid Woodhouse 308357a7702bSDavid Woodhouse pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n", 3084d977d56cSKonstantin Khlebnikov current->comm, current->pid, 3085d977d56cSKonstantin Khlebnikov (mm->data_vm + npages) << PAGE_SHIFT, 308657a7702bSDavid Woodhouse rlimit(RLIMIT_DATA), 308757a7702bSDavid Woodhouse ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data"); 308857a7702bSDavid Woodhouse 308957a7702bSDavid Woodhouse if (!ignore_rlimit_data) 3090d977d56cSKonstantin Khlebnikov return false; 3091d977d56cSKonstantin Khlebnikov } 3092119f657cSakpm@osdl.org 309384638335SKonstantin Khlebnikov return true; 309484638335SKonstantin Khlebnikov } 309584638335SKonstantin Khlebnikov 309684638335SKonstantin Khlebnikov void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages) 309784638335SKonstantin Khlebnikov { 30987866076bSPeng Liu WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm)+npages); 309984638335SKonstantin Khlebnikov 3100d977d56cSKonstantin Khlebnikov if (is_exec_mapping(flags)) 310184638335SKonstantin Khlebnikov mm->exec_vm += npages; 3102d977d56cSKonstantin Khlebnikov else if (is_stack_mapping(flags)) 310384638335SKonstantin Khlebnikov mm->stack_vm += npages; 3104d977d56cSKonstantin Khlebnikov else if (is_data_mapping(flags)) 310584638335SKonstantin Khlebnikov mm->data_vm += npages; 3106119f657cSakpm@osdl.org } 3107fa5dc22fSRoland McGrath 3108b3ec9f33SSouptick Joarder static vm_fault_t special_mapping_fault(struct vm_fault *vmf); 3109a62c34bdSAndy Lutomirski 3110a62c34bdSAndy Lutomirski /* 3111a62c34bdSAndy Lutomirski * Having a close hook prevents vma merging regardless of flags. 3112a62c34bdSAndy Lutomirski */ 3113a62c34bdSAndy Lutomirski static void special_mapping_close(struct vm_area_struct *vma) 3114a62c34bdSAndy Lutomirski { 3115a62c34bdSAndy Lutomirski } 3116a62c34bdSAndy Lutomirski 3117a62c34bdSAndy Lutomirski static const char *special_mapping_name(struct vm_area_struct *vma) 3118a62c34bdSAndy Lutomirski { 3119a62c34bdSAndy Lutomirski return ((struct vm_special_mapping *)vma->vm_private_data)->name; 3120a62c34bdSAndy Lutomirski } 3121a62c34bdSAndy Lutomirski 312214d07113SBrian Geffon static int special_mapping_mremap(struct vm_area_struct *new_vma) 3123b059a453SDmitry Safonov { 3124b059a453SDmitry Safonov struct vm_special_mapping *sm = new_vma->vm_private_data; 3125b059a453SDmitry Safonov 3126280e87e9SDmitry Safonov if (WARN_ON_ONCE(current->mm != new_vma->vm_mm)) 3127280e87e9SDmitry Safonov return -EFAULT; 3128280e87e9SDmitry Safonov 3129b059a453SDmitry Safonov if (sm->mremap) 3130b059a453SDmitry Safonov return sm->mremap(sm, new_vma); 3131280e87e9SDmitry Safonov 3132b059a453SDmitry Safonov return 0; 3133b059a453SDmitry Safonov } 3134b059a453SDmitry Safonov 3135871402e0SDmitry Safonov static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr) 3136871402e0SDmitry Safonov { 3137871402e0SDmitry Safonov /* 3138871402e0SDmitry Safonov * Forbid splitting special mappings - kernel has expectations over 3139871402e0SDmitry Safonov * the number of pages in mapping. Together with VM_DONTEXPAND 3140871402e0SDmitry Safonov * the size of vma should stay the same over the special mapping's 3141871402e0SDmitry Safonov * lifetime. 3142871402e0SDmitry Safonov */ 3143871402e0SDmitry Safonov return -EINVAL; 3144871402e0SDmitry Safonov } 3145871402e0SDmitry Safonov 3146a62c34bdSAndy Lutomirski static const struct vm_operations_struct special_mapping_vmops = { 3147a62c34bdSAndy Lutomirski .close = special_mapping_close, 3148a62c34bdSAndy Lutomirski .fault = special_mapping_fault, 3149b059a453SDmitry Safonov .mremap = special_mapping_mremap, 3150a62c34bdSAndy Lutomirski .name = special_mapping_name, 3151af34ebebSDmitry Safonov /* vDSO code relies that VVAR can't be accessed remotely */ 3152af34ebebSDmitry Safonov .access = NULL, 3153871402e0SDmitry Safonov .may_split = special_mapping_split, 3154a62c34bdSAndy Lutomirski }; 3155a62c34bdSAndy Lutomirski 3156a62c34bdSAndy Lutomirski static const struct vm_operations_struct legacy_special_mapping_vmops = { 3157a62c34bdSAndy Lutomirski .close = special_mapping_close, 3158a62c34bdSAndy Lutomirski .fault = special_mapping_fault, 3159a62c34bdSAndy Lutomirski }; 3160fa5dc22fSRoland McGrath 3161b3ec9f33SSouptick Joarder static vm_fault_t special_mapping_fault(struct vm_fault *vmf) 3162fa5dc22fSRoland McGrath { 316311bac800SDave Jiang struct vm_area_struct *vma = vmf->vma; 3164b1d0e4f5SNick Piggin pgoff_t pgoff; 3165fa5dc22fSRoland McGrath struct page **pages; 3166fa5dc22fSRoland McGrath 3167f872f540SAndy Lutomirski if (vma->vm_ops == &legacy_special_mapping_vmops) { 3168a62c34bdSAndy Lutomirski pages = vma->vm_private_data; 3169f872f540SAndy Lutomirski } else { 3170f872f540SAndy Lutomirski struct vm_special_mapping *sm = vma->vm_private_data; 3171f872f540SAndy Lutomirski 3172f872f540SAndy Lutomirski if (sm->fault) 317311bac800SDave Jiang return sm->fault(sm, vmf->vma, vmf); 3174f872f540SAndy Lutomirski 3175f872f540SAndy Lutomirski pages = sm->pages; 3176f872f540SAndy Lutomirski } 3177a62c34bdSAndy Lutomirski 31788a9cc3b5SOleg Nesterov for (pgoff = vmf->pgoff; pgoff && *pages; ++pages) 3179b1d0e4f5SNick Piggin pgoff--; 3180fa5dc22fSRoland McGrath 3181fa5dc22fSRoland McGrath if (*pages) { 3182fa5dc22fSRoland McGrath struct page *page = *pages; 3183fa5dc22fSRoland McGrath get_page(page); 3184b1d0e4f5SNick Piggin vmf->page = page; 3185b1d0e4f5SNick Piggin return 0; 3186fa5dc22fSRoland McGrath } 3187fa5dc22fSRoland McGrath 3188b1d0e4f5SNick Piggin return VM_FAULT_SIGBUS; 3189fa5dc22fSRoland McGrath } 3190fa5dc22fSRoland McGrath 3191a62c34bdSAndy Lutomirski static struct vm_area_struct *__install_special_mapping( 3192a62c34bdSAndy Lutomirski struct mm_struct *mm, 3193fa5dc22fSRoland McGrath unsigned long addr, unsigned long len, 319427f28b97SChen Gang unsigned long vm_flags, void *priv, 319527f28b97SChen Gang const struct vm_operations_struct *ops) 3196fa5dc22fSRoland McGrath { 3197462e635eSTavis Ormandy int ret; 3198fa5dc22fSRoland McGrath struct vm_area_struct *vma; 3199fa5dc22fSRoland McGrath 3200d4af56c5SLiam R. Howlett validate_mm_mt(mm); 3201490fc053SLinus Torvalds vma = vm_area_alloc(mm); 3202fa5dc22fSRoland McGrath if (unlikely(vma == NULL)) 32033935ed6aSStefani Seibold return ERR_PTR(-ENOMEM); 3204fa5dc22fSRoland McGrath 3205fa5dc22fSRoland McGrath vma->vm_start = addr; 3206fa5dc22fSRoland McGrath vma->vm_end = addr + len; 3207fa5dc22fSRoland McGrath 3208d9104d1cSCyrill Gorcunov vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY; 32091fc09228SHugh Dickins vma->vm_flags &= VM_LOCKED_CLEAR_MASK; 32103ed75eb8SColy Li vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); 3211fa5dc22fSRoland McGrath 3212a62c34bdSAndy Lutomirski vma->vm_ops = ops; 3213a62c34bdSAndy Lutomirski vma->vm_private_data = priv; 3214fa5dc22fSRoland McGrath 3215462e635eSTavis Ormandy ret = insert_vm_struct(mm, vma); 3216462e635eSTavis Ormandy if (ret) 3217462e635eSTavis Ormandy goto out; 3218fa5dc22fSRoland McGrath 321984638335SKonstantin Khlebnikov vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); 3220fa5dc22fSRoland McGrath 3221cdd6c482SIngo Molnar perf_event_mmap(vma); 3222089dd79dSPeter Zijlstra 3223d4af56c5SLiam R. Howlett validate_mm_mt(mm); 32243935ed6aSStefani Seibold return vma; 3225462e635eSTavis Ormandy 3226462e635eSTavis Ormandy out: 32273928d4f5SLinus Torvalds vm_area_free(vma); 3228d4af56c5SLiam R. Howlett validate_mm_mt(mm); 32293935ed6aSStefani Seibold return ERR_PTR(ret); 32303935ed6aSStefani Seibold } 32313935ed6aSStefani Seibold 32322eefd878SDmitry Safonov bool vma_is_special_mapping(const struct vm_area_struct *vma, 32332eefd878SDmitry Safonov const struct vm_special_mapping *sm) 32342eefd878SDmitry Safonov { 32352eefd878SDmitry Safonov return vma->vm_private_data == sm && 32362eefd878SDmitry Safonov (vma->vm_ops == &special_mapping_vmops || 32372eefd878SDmitry Safonov vma->vm_ops == &legacy_special_mapping_vmops); 32382eefd878SDmitry Safonov } 32392eefd878SDmitry Safonov 3240a62c34bdSAndy Lutomirski /* 3241c1e8d7c6SMichel Lespinasse * Called with mm->mmap_lock held for writing. 3242a62c34bdSAndy Lutomirski * Insert a new vma covering the given region, with the given flags. 3243a62c34bdSAndy Lutomirski * Its pages are supplied by the given array of struct page *. 3244a62c34bdSAndy Lutomirski * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated. 3245a62c34bdSAndy Lutomirski * The region past the last page supplied will always produce SIGBUS. 3246a62c34bdSAndy Lutomirski * The array pointer and the pages it points to are assumed to stay alive 3247a62c34bdSAndy Lutomirski * for as long as this mapping might exist. 3248a62c34bdSAndy Lutomirski */ 3249a62c34bdSAndy Lutomirski struct vm_area_struct *_install_special_mapping( 3250a62c34bdSAndy Lutomirski struct mm_struct *mm, 3251a62c34bdSAndy Lutomirski unsigned long addr, unsigned long len, 3252a62c34bdSAndy Lutomirski unsigned long vm_flags, const struct vm_special_mapping *spec) 3253a62c34bdSAndy Lutomirski { 325427f28b97SChen Gang return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec, 325527f28b97SChen Gang &special_mapping_vmops); 3256a62c34bdSAndy Lutomirski } 3257a62c34bdSAndy Lutomirski 32583935ed6aSStefani Seibold int install_special_mapping(struct mm_struct *mm, 32593935ed6aSStefani Seibold unsigned long addr, unsigned long len, 32603935ed6aSStefani Seibold unsigned long vm_flags, struct page **pages) 32613935ed6aSStefani Seibold { 3262a62c34bdSAndy Lutomirski struct vm_area_struct *vma = __install_special_mapping( 326327f28b97SChen Gang mm, addr, len, vm_flags, (void *)pages, 326427f28b97SChen Gang &legacy_special_mapping_vmops); 32653935ed6aSStefani Seibold 326614bd5b45SDuan Jiong return PTR_ERR_OR_ZERO(vma); 3267fa5dc22fSRoland McGrath } 32687906d00cSAndrea Arcangeli 32697906d00cSAndrea Arcangeli static DEFINE_MUTEX(mm_all_locks_mutex); 32707906d00cSAndrea Arcangeli 3271454ed842SPeter Zijlstra static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma) 32727906d00cSAndrea Arcangeli { 3273f808c13fSDavidlohr Bueso if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 32747906d00cSAndrea Arcangeli /* 32757906d00cSAndrea Arcangeli * The LSB of head.next can't change from under us 32767906d00cSAndrea Arcangeli * because we hold the mm_all_locks_mutex. 32777906d00cSAndrea Arcangeli */ 3278da1c55f1SMichel Lespinasse down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock); 32797906d00cSAndrea Arcangeli /* 32807906d00cSAndrea Arcangeli * We can safely modify head.next after taking the 32815a505085SIngo Molnar * anon_vma->root->rwsem. If some other vma in this mm shares 32827906d00cSAndrea Arcangeli * the same anon_vma we won't take it again. 32837906d00cSAndrea Arcangeli * 32847906d00cSAndrea Arcangeli * No need of atomic instructions here, head.next 32857906d00cSAndrea Arcangeli * can't change from under us thanks to the 32865a505085SIngo Molnar * anon_vma->root->rwsem. 32877906d00cSAndrea Arcangeli */ 32887906d00cSAndrea Arcangeli if (__test_and_set_bit(0, (unsigned long *) 3289f808c13fSDavidlohr Bueso &anon_vma->root->rb_root.rb_root.rb_node)) 32907906d00cSAndrea Arcangeli BUG(); 32917906d00cSAndrea Arcangeli } 32927906d00cSAndrea Arcangeli } 32937906d00cSAndrea Arcangeli 3294454ed842SPeter Zijlstra static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) 32957906d00cSAndrea Arcangeli { 32967906d00cSAndrea Arcangeli if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 32977906d00cSAndrea Arcangeli /* 32987906d00cSAndrea Arcangeli * AS_MM_ALL_LOCKS can't change from under us because 32997906d00cSAndrea Arcangeli * we hold the mm_all_locks_mutex. 33007906d00cSAndrea Arcangeli * 33017906d00cSAndrea Arcangeli * Operations on ->flags have to be atomic because 33027906d00cSAndrea Arcangeli * even if AS_MM_ALL_LOCKS is stable thanks to the 33037906d00cSAndrea Arcangeli * mm_all_locks_mutex, there may be other cpus 33047906d00cSAndrea Arcangeli * changing other bitflags in parallel to us. 33057906d00cSAndrea Arcangeli */ 33067906d00cSAndrea Arcangeli if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags)) 33077906d00cSAndrea Arcangeli BUG(); 3308da1c55f1SMichel Lespinasse down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock); 33097906d00cSAndrea Arcangeli } 33107906d00cSAndrea Arcangeli } 33117906d00cSAndrea Arcangeli 33127906d00cSAndrea Arcangeli /* 33137906d00cSAndrea Arcangeli * This operation locks against the VM for all pte/vma/mm related 33147906d00cSAndrea Arcangeli * operations that could ever happen on a certain mm. This includes 33157906d00cSAndrea Arcangeli * vmtruncate, try_to_unmap, and all page faults. 33167906d00cSAndrea Arcangeli * 3317c1e8d7c6SMichel Lespinasse * The caller must take the mmap_lock in write mode before calling 33187906d00cSAndrea Arcangeli * mm_take_all_locks(). The caller isn't allowed to release the 3319c1e8d7c6SMichel Lespinasse * mmap_lock until mm_drop_all_locks() returns. 33207906d00cSAndrea Arcangeli * 3321c1e8d7c6SMichel Lespinasse * mmap_lock in write mode is required in order to block all operations 33227906d00cSAndrea Arcangeli * that could modify pagetables and free pages without need of 332327ba0644SKirill A. Shutemov * altering the vma layout. It's also needed in write mode to avoid new 33247906d00cSAndrea Arcangeli * anon_vmas to be associated with existing vmas. 33257906d00cSAndrea Arcangeli * 33267906d00cSAndrea Arcangeli * A single task can't take more than one mm_take_all_locks() in a row 33277906d00cSAndrea Arcangeli * or it would deadlock. 33287906d00cSAndrea Arcangeli * 3329bf181b9fSMichel Lespinasse * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in 33307906d00cSAndrea Arcangeli * mapping->flags avoid to take the same lock twice, if more than one 33317906d00cSAndrea Arcangeli * vma in this mm is backed by the same anon_vma or address_space. 33327906d00cSAndrea Arcangeli * 333388f306b6SKirill A. Shutemov * We take locks in following order, accordingly to comment at beginning 333488f306b6SKirill A. Shutemov * of mm/rmap.c: 333588f306b6SKirill A. Shutemov * - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for 333688f306b6SKirill A. Shutemov * hugetlb mapping); 333788f306b6SKirill A. Shutemov * - all i_mmap_rwsem locks; 333888f306b6SKirill A. Shutemov * - all anon_vma->rwseml 333988f306b6SKirill A. Shutemov * 334088f306b6SKirill A. Shutemov * We can take all locks within these types randomly because the VM code 334188f306b6SKirill A. Shutemov * doesn't nest them and we protected from parallel mm_take_all_locks() by 334288f306b6SKirill A. Shutemov * mm_all_locks_mutex. 33437906d00cSAndrea Arcangeli * 33447906d00cSAndrea Arcangeli * mm_take_all_locks() and mm_drop_all_locks are expensive operations 33457906d00cSAndrea Arcangeli * that may have to take thousand of locks. 33467906d00cSAndrea Arcangeli * 33477906d00cSAndrea Arcangeli * mm_take_all_locks() can fail if it's interrupted by signals. 33487906d00cSAndrea Arcangeli */ 33497906d00cSAndrea Arcangeli int mm_take_all_locks(struct mm_struct *mm) 33507906d00cSAndrea Arcangeli { 33517906d00cSAndrea Arcangeli struct vm_area_struct *vma; 33525beb4930SRik van Riel struct anon_vma_chain *avc; 33537906d00cSAndrea Arcangeli 3354325bca1fSRolf Eike Beer mmap_assert_write_locked(mm); 33557906d00cSAndrea Arcangeli 33567906d00cSAndrea Arcangeli mutex_lock(&mm_all_locks_mutex); 33577906d00cSAndrea Arcangeli 33587906d00cSAndrea Arcangeli for (vma = mm->mmap; vma; vma = vma->vm_next) { 33597906d00cSAndrea Arcangeli if (signal_pending(current)) 33607906d00cSAndrea Arcangeli goto out_unlock; 336188f306b6SKirill A. Shutemov if (vma->vm_file && vma->vm_file->f_mapping && 336288f306b6SKirill A. Shutemov is_vm_hugetlb_page(vma)) 336388f306b6SKirill A. Shutemov vm_lock_mapping(mm, vma->vm_file->f_mapping); 336488f306b6SKirill A. Shutemov } 336588f306b6SKirill A. Shutemov 336688f306b6SKirill A. Shutemov for (vma = mm->mmap; vma; vma = vma->vm_next) { 336788f306b6SKirill A. Shutemov if (signal_pending(current)) 336888f306b6SKirill A. Shutemov goto out_unlock; 336988f306b6SKirill A. Shutemov if (vma->vm_file && vma->vm_file->f_mapping && 337088f306b6SKirill A. Shutemov !is_vm_hugetlb_page(vma)) 3371454ed842SPeter Zijlstra vm_lock_mapping(mm, vma->vm_file->f_mapping); 33727906d00cSAndrea Arcangeli } 33737cd5a02fSPeter Zijlstra 33747cd5a02fSPeter Zijlstra for (vma = mm->mmap; vma; vma = vma->vm_next) { 33757cd5a02fSPeter Zijlstra if (signal_pending(current)) 33767cd5a02fSPeter Zijlstra goto out_unlock; 33777cd5a02fSPeter Zijlstra if (vma->anon_vma) 33785beb4930SRik van Riel list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 33795beb4930SRik van Riel vm_lock_anon_vma(mm, avc->anon_vma); 33807cd5a02fSPeter Zijlstra } 33817cd5a02fSPeter Zijlstra 3382584cff54SKautuk Consul return 0; 33837906d00cSAndrea Arcangeli 33847906d00cSAndrea Arcangeli out_unlock: 33857906d00cSAndrea Arcangeli mm_drop_all_locks(mm); 3386584cff54SKautuk Consul return -EINTR; 33877906d00cSAndrea Arcangeli } 33887906d00cSAndrea Arcangeli 33897906d00cSAndrea Arcangeli static void vm_unlock_anon_vma(struct anon_vma *anon_vma) 33907906d00cSAndrea Arcangeli { 3391f808c13fSDavidlohr Bueso if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 33927906d00cSAndrea Arcangeli /* 33937906d00cSAndrea Arcangeli * The LSB of head.next can't change to 0 from under 33947906d00cSAndrea Arcangeli * us because we hold the mm_all_locks_mutex. 33957906d00cSAndrea Arcangeli * 33967906d00cSAndrea Arcangeli * We must however clear the bitflag before unlocking 3397bf181b9fSMichel Lespinasse * the vma so the users using the anon_vma->rb_root will 33987906d00cSAndrea Arcangeli * never see our bitflag. 33997906d00cSAndrea Arcangeli * 34007906d00cSAndrea Arcangeli * No need of atomic instructions here, head.next 34017906d00cSAndrea Arcangeli * can't change from under us until we release the 34025a505085SIngo Molnar * anon_vma->root->rwsem. 34037906d00cSAndrea Arcangeli */ 34047906d00cSAndrea Arcangeli if (!__test_and_clear_bit(0, (unsigned long *) 3405f808c13fSDavidlohr Bueso &anon_vma->root->rb_root.rb_root.rb_node)) 34067906d00cSAndrea Arcangeli BUG(); 340708b52706SKonstantin Khlebnikov anon_vma_unlock_write(anon_vma); 34087906d00cSAndrea Arcangeli } 34097906d00cSAndrea Arcangeli } 34107906d00cSAndrea Arcangeli 34117906d00cSAndrea Arcangeli static void vm_unlock_mapping(struct address_space *mapping) 34127906d00cSAndrea Arcangeli { 34137906d00cSAndrea Arcangeli if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 34147906d00cSAndrea Arcangeli /* 34157906d00cSAndrea Arcangeli * AS_MM_ALL_LOCKS can't change to 0 from under us 34167906d00cSAndrea Arcangeli * because we hold the mm_all_locks_mutex. 34177906d00cSAndrea Arcangeli */ 341883cde9e8SDavidlohr Bueso i_mmap_unlock_write(mapping); 34197906d00cSAndrea Arcangeli if (!test_and_clear_bit(AS_MM_ALL_LOCKS, 34207906d00cSAndrea Arcangeli &mapping->flags)) 34217906d00cSAndrea Arcangeli BUG(); 34227906d00cSAndrea Arcangeli } 34237906d00cSAndrea Arcangeli } 34247906d00cSAndrea Arcangeli 34257906d00cSAndrea Arcangeli /* 3426c1e8d7c6SMichel Lespinasse * The mmap_lock cannot be released by the caller until 34277906d00cSAndrea Arcangeli * mm_drop_all_locks() returns. 34287906d00cSAndrea Arcangeli */ 34297906d00cSAndrea Arcangeli void mm_drop_all_locks(struct mm_struct *mm) 34307906d00cSAndrea Arcangeli { 34317906d00cSAndrea Arcangeli struct vm_area_struct *vma; 34325beb4930SRik van Riel struct anon_vma_chain *avc; 34337906d00cSAndrea Arcangeli 3434325bca1fSRolf Eike Beer mmap_assert_write_locked(mm); 34357906d00cSAndrea Arcangeli BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); 34367906d00cSAndrea Arcangeli 34377906d00cSAndrea Arcangeli for (vma = mm->mmap; vma; vma = vma->vm_next) { 34387906d00cSAndrea Arcangeli if (vma->anon_vma) 34395beb4930SRik van Riel list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 34405beb4930SRik van Riel vm_unlock_anon_vma(avc->anon_vma); 34417906d00cSAndrea Arcangeli if (vma->vm_file && vma->vm_file->f_mapping) 34427906d00cSAndrea Arcangeli vm_unlock_mapping(vma->vm_file->f_mapping); 34437906d00cSAndrea Arcangeli } 34447906d00cSAndrea Arcangeli 34457906d00cSAndrea Arcangeli mutex_unlock(&mm_all_locks_mutex); 34467906d00cSAndrea Arcangeli } 34478feae131SDavid Howells 34488feae131SDavid Howells /* 34493edf41d8Sseokhoon.yoon * initialise the percpu counter for VM 34508feae131SDavid Howells */ 34518feae131SDavid Howells void __init mmap_init(void) 34528feae131SDavid Howells { 345300a62ce9SKOSAKI Motohiro int ret; 345400a62ce9SKOSAKI Motohiro 3455908c7f19STejun Heo ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL); 345600a62ce9SKOSAKI Motohiro VM_BUG_ON(ret); 34578feae131SDavid Howells } 3458c9b1d098SAndrew Shewmaker 3459c9b1d098SAndrew Shewmaker /* 3460c9b1d098SAndrew Shewmaker * Initialise sysctl_user_reserve_kbytes. 3461c9b1d098SAndrew Shewmaker * 3462c9b1d098SAndrew Shewmaker * This is intended to prevent a user from starting a single memory hogging 3463c9b1d098SAndrew Shewmaker * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER 3464c9b1d098SAndrew Shewmaker * mode. 3465c9b1d098SAndrew Shewmaker * 3466c9b1d098SAndrew Shewmaker * The default value is min(3% of free memory, 128MB) 3467c9b1d098SAndrew Shewmaker * 128MB is enough to recover with sshd/login, bash, and top/kill. 3468c9b1d098SAndrew Shewmaker */ 34691640879aSAndrew Shewmaker static int init_user_reserve(void) 3470c9b1d098SAndrew Shewmaker { 3471c9b1d098SAndrew Shewmaker unsigned long free_kbytes; 3472c9b1d098SAndrew Shewmaker 3473c41f012aSMichal Hocko free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); 3474c9b1d098SAndrew Shewmaker 3475c9b1d098SAndrew Shewmaker sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17); 3476c9b1d098SAndrew Shewmaker return 0; 3477c9b1d098SAndrew Shewmaker } 3478a64fb3cdSPaul Gortmaker subsys_initcall(init_user_reserve); 34794eeab4f5SAndrew Shewmaker 34804eeab4f5SAndrew Shewmaker /* 34814eeab4f5SAndrew Shewmaker * Initialise sysctl_admin_reserve_kbytes. 34824eeab4f5SAndrew Shewmaker * 34834eeab4f5SAndrew Shewmaker * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin 34844eeab4f5SAndrew Shewmaker * to log in and kill a memory hogging process. 34854eeab4f5SAndrew Shewmaker * 34864eeab4f5SAndrew Shewmaker * Systems with more than 256MB will reserve 8MB, enough to recover 34874eeab4f5SAndrew Shewmaker * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will 34884eeab4f5SAndrew Shewmaker * only reserve 3% of free pages by default. 34894eeab4f5SAndrew Shewmaker */ 34901640879aSAndrew Shewmaker static int init_admin_reserve(void) 34914eeab4f5SAndrew Shewmaker { 34924eeab4f5SAndrew Shewmaker unsigned long free_kbytes; 34934eeab4f5SAndrew Shewmaker 3494c41f012aSMichal Hocko free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); 34954eeab4f5SAndrew Shewmaker 34964eeab4f5SAndrew Shewmaker sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13); 34974eeab4f5SAndrew Shewmaker return 0; 34984eeab4f5SAndrew Shewmaker } 3499a64fb3cdSPaul Gortmaker subsys_initcall(init_admin_reserve); 35001640879aSAndrew Shewmaker 35011640879aSAndrew Shewmaker /* 35021640879aSAndrew Shewmaker * Reinititalise user and admin reserves if memory is added or removed. 35031640879aSAndrew Shewmaker * 35041640879aSAndrew Shewmaker * The default user reserve max is 128MB, and the default max for the 35051640879aSAndrew Shewmaker * admin reserve is 8MB. These are usually, but not always, enough to 35061640879aSAndrew Shewmaker * enable recovery from a memory hogging process using login/sshd, a shell, 35071640879aSAndrew Shewmaker * and tools like top. It may make sense to increase or even disable the 35081640879aSAndrew Shewmaker * reserve depending on the existence of swap or variations in the recovery 35091640879aSAndrew Shewmaker * tools. So, the admin may have changed them. 35101640879aSAndrew Shewmaker * 35111640879aSAndrew Shewmaker * If memory is added and the reserves have been eliminated or increased above 35121640879aSAndrew Shewmaker * the default max, then we'll trust the admin. 35131640879aSAndrew Shewmaker * 35141640879aSAndrew Shewmaker * If memory is removed and there isn't enough free memory, then we 35151640879aSAndrew Shewmaker * need to reset the reserves. 35161640879aSAndrew Shewmaker * 35171640879aSAndrew Shewmaker * Otherwise keep the reserve set by the admin. 35181640879aSAndrew Shewmaker */ 35191640879aSAndrew Shewmaker static int reserve_mem_notifier(struct notifier_block *nb, 35201640879aSAndrew Shewmaker unsigned long action, void *data) 35211640879aSAndrew Shewmaker { 35221640879aSAndrew Shewmaker unsigned long tmp, free_kbytes; 35231640879aSAndrew Shewmaker 35241640879aSAndrew Shewmaker switch (action) { 35251640879aSAndrew Shewmaker case MEM_ONLINE: 35261640879aSAndrew Shewmaker /* Default max is 128MB. Leave alone if modified by operator. */ 35271640879aSAndrew Shewmaker tmp = sysctl_user_reserve_kbytes; 35281640879aSAndrew Shewmaker if (0 < tmp && tmp < (1UL << 17)) 35291640879aSAndrew Shewmaker init_user_reserve(); 35301640879aSAndrew Shewmaker 35311640879aSAndrew Shewmaker /* Default max is 8MB. Leave alone if modified by operator. */ 35321640879aSAndrew Shewmaker tmp = sysctl_admin_reserve_kbytes; 35331640879aSAndrew Shewmaker if (0 < tmp && tmp < (1UL << 13)) 35341640879aSAndrew Shewmaker init_admin_reserve(); 35351640879aSAndrew Shewmaker 35361640879aSAndrew Shewmaker break; 35371640879aSAndrew Shewmaker case MEM_OFFLINE: 3538c41f012aSMichal Hocko free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); 35391640879aSAndrew Shewmaker 35401640879aSAndrew Shewmaker if (sysctl_user_reserve_kbytes > free_kbytes) { 35411640879aSAndrew Shewmaker init_user_reserve(); 35421640879aSAndrew Shewmaker pr_info("vm.user_reserve_kbytes reset to %lu\n", 35431640879aSAndrew Shewmaker sysctl_user_reserve_kbytes); 35441640879aSAndrew Shewmaker } 35451640879aSAndrew Shewmaker 35461640879aSAndrew Shewmaker if (sysctl_admin_reserve_kbytes > free_kbytes) { 35471640879aSAndrew Shewmaker init_admin_reserve(); 35481640879aSAndrew Shewmaker pr_info("vm.admin_reserve_kbytes reset to %lu\n", 35491640879aSAndrew Shewmaker sysctl_admin_reserve_kbytes); 35501640879aSAndrew Shewmaker } 35511640879aSAndrew Shewmaker break; 35521640879aSAndrew Shewmaker default: 35531640879aSAndrew Shewmaker break; 35541640879aSAndrew Shewmaker } 35551640879aSAndrew Shewmaker return NOTIFY_OK; 35561640879aSAndrew Shewmaker } 35571640879aSAndrew Shewmaker 35581640879aSAndrew Shewmaker static struct notifier_block reserve_mem_nb = { 35591640879aSAndrew Shewmaker .notifier_call = reserve_mem_notifier, 35601640879aSAndrew Shewmaker }; 35611640879aSAndrew Shewmaker 35621640879aSAndrew Shewmaker static int __meminit init_reserve_notifier(void) 35631640879aSAndrew Shewmaker { 35641640879aSAndrew Shewmaker if (register_hotmemory_notifier(&reserve_mem_nb)) 3565b1de0d13SMitchel Humpherys pr_err("Failed registering memory add/remove notifier for admin reserve\n"); 35661640879aSAndrew Shewmaker 35671640879aSAndrew Shewmaker return 0; 35681640879aSAndrew Shewmaker } 3569a64fb3cdSPaul Gortmaker subsys_initcall(init_reserve_notifier); 3570