1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * mm/mremap.c 4 * 5 * (C) Copyright 1996 Linus Torvalds 6 * 7 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 8 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved 9 */ 10 11 #include <linux/mm.h> 12 #include <linux/hugetlb.h> 13 #include <linux/shm.h> 14 #include <linux/ksm.h> 15 #include <linux/mman.h> 16 #include <linux/swap.h> 17 #include <linux/capability.h> 18 #include <linux/fs.h> 19 #include <linux/swapops.h> 20 #include <linux/highmem.h> 21 #include <linux/security.h> 22 #include <linux/syscalls.h> 23 #include <linux/mmu_notifier.h> 24 #include <linux/uaccess.h> 25 #include <linux/mm-arch-hooks.h> 26 #include <linux/userfaultfd_k.h> 27 28 #include <asm/cacheflush.h> 29 #include <asm/tlbflush.h> 30 31 #include "internal.h" 32 33 static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr) 34 { 35 pgd_t *pgd; 36 p4d_t *p4d; 37 pud_t *pud; 38 pmd_t *pmd; 39 40 pgd = pgd_offset(mm, addr); 41 if (pgd_none_or_clear_bad(pgd)) 42 return NULL; 43 44 p4d = p4d_offset(pgd, addr); 45 if (p4d_none_or_clear_bad(p4d)) 46 return NULL; 47 48 pud = pud_offset(p4d, addr); 49 if (pud_none_or_clear_bad(pud)) 50 return NULL; 51 52 pmd = pmd_offset(pud, addr); 53 if (pmd_none(*pmd)) 54 return NULL; 55 56 return pmd; 57 } 58 59 static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma, 60 unsigned long addr) 61 { 62 pgd_t *pgd; 63 p4d_t *p4d; 64 pud_t *pud; 65 pmd_t *pmd; 66 67 pgd = pgd_offset(mm, addr); 68 p4d = p4d_alloc(mm, pgd, addr); 69 if (!p4d) 70 return NULL; 71 pud = pud_alloc(mm, p4d, addr); 72 if (!pud) 73 return NULL; 74 75 pmd = pmd_alloc(mm, pud, addr); 76 if (!pmd) 77 return NULL; 78 79 VM_BUG_ON(pmd_trans_huge(*pmd)); 80 81 return pmd; 82 } 83 84 static void take_rmap_locks(struct vm_area_struct *vma) 85 { 86 if (vma->vm_file) 87 i_mmap_lock_write(vma->vm_file->f_mapping); 88 if (vma->anon_vma) 89 anon_vma_lock_write(vma->anon_vma); 90 } 91 92 static void drop_rmap_locks(struct vm_area_struct *vma) 93 { 94 if (vma->anon_vma) 95 anon_vma_unlock_write(vma->anon_vma); 96 if (vma->vm_file) 97 i_mmap_unlock_write(vma->vm_file->f_mapping); 98 } 99 100 static pte_t move_soft_dirty_pte(pte_t pte) 101 { 102 /* 103 * Set soft dirty bit so we can notice 104 * in userspace the ptes were moved. 105 */ 106 #ifdef CONFIG_MEM_SOFT_DIRTY 107 if (pte_present(pte)) 108 pte = pte_mksoft_dirty(pte); 109 else if (is_swap_pte(pte)) 110 pte = pte_swp_mksoft_dirty(pte); 111 #endif 112 return pte; 113 } 114 115 static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, 116 unsigned long old_addr, unsigned long old_end, 117 struct vm_area_struct *new_vma, pmd_t *new_pmd, 118 unsigned long new_addr, bool need_rmap_locks, bool *need_flush) 119 { 120 struct mm_struct *mm = vma->vm_mm; 121 pte_t *old_pte, *new_pte, pte; 122 spinlock_t *old_ptl, *new_ptl; 123 bool force_flush = false; 124 unsigned long len = old_end - old_addr; 125 126 /* 127 * When need_rmap_locks is true, we take the i_mmap_rwsem and anon_vma 128 * locks to ensure that rmap will always observe either the old or the 129 * new ptes. This is the easiest way to avoid races with 130 * truncate_pagecache(), page migration, etc... 131 * 132 * When need_rmap_locks is false, we use other ways to avoid 133 * such races: 134 * 135 * - During exec() shift_arg_pages(), we use a specially tagged vma 136 * which rmap call sites look for using is_vma_temporary_stack(). 137 * 138 * - During mremap(), new_vma is often known to be placed after vma 139 * in rmap traversal order. This ensures rmap will always observe 140 * either the old pte, or the new pte, or both (the page table locks 141 * serialize access to individual ptes, but only rmap traversal 142 * order guarantees that we won't miss both the old and new ptes). 143 */ 144 if (need_rmap_locks) 145 take_rmap_locks(vma); 146 147 /* 148 * We don't have to worry about the ordering of src and dst 149 * pte locks because exclusive mmap_sem prevents deadlock. 150 */ 151 old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl); 152 new_pte = pte_offset_map(new_pmd, new_addr); 153 new_ptl = pte_lockptr(mm, new_pmd); 154 if (new_ptl != old_ptl) 155 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 156 flush_tlb_batched_pending(vma->vm_mm); 157 arch_enter_lazy_mmu_mode(); 158 159 for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE, 160 new_pte++, new_addr += PAGE_SIZE) { 161 if (pte_none(*old_pte)) 162 continue; 163 164 pte = ptep_get_and_clear(mm, old_addr, old_pte); 165 /* 166 * If we are remapping a dirty PTE, make sure 167 * to flush TLB before we drop the PTL for the 168 * old PTE or we may race with page_mkclean(). 169 * 170 * This check has to be done after we removed the 171 * old PTE from page tables or another thread may 172 * dirty it after the check and before the removal. 173 */ 174 if (pte_present(pte) && pte_dirty(pte)) 175 force_flush = true; 176 pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr); 177 pte = move_soft_dirty_pte(pte); 178 set_pte_at(mm, new_addr, new_pte, pte); 179 } 180 181 arch_leave_lazy_mmu_mode(); 182 if (new_ptl != old_ptl) 183 spin_unlock(new_ptl); 184 pte_unmap(new_pte - 1); 185 if (force_flush) 186 flush_tlb_range(vma, old_end - len, old_end); 187 else 188 *need_flush = true; 189 pte_unmap_unlock(old_pte - 1, old_ptl); 190 if (need_rmap_locks) 191 drop_rmap_locks(vma); 192 } 193 194 unsigned long move_page_tables(struct vm_area_struct *vma, 195 unsigned long old_addr, struct vm_area_struct *new_vma, 196 unsigned long new_addr, unsigned long len, 197 bool need_rmap_locks) 198 { 199 unsigned long extent, next, old_end; 200 pmd_t *old_pmd, *new_pmd; 201 bool need_flush = false; 202 unsigned long mmun_start; /* For mmu_notifiers */ 203 unsigned long mmun_end; /* For mmu_notifiers */ 204 205 old_end = old_addr + len; 206 flush_cache_range(vma, old_addr, old_end); 207 208 mmun_start = old_addr; 209 mmun_end = old_end; 210 mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end); 211 212 for (; old_addr < old_end; old_addr += extent, new_addr += extent) { 213 cond_resched(); 214 next = (old_addr + PMD_SIZE) & PMD_MASK; 215 /* even if next overflowed, extent below will be ok */ 216 extent = next - old_addr; 217 if (extent > old_end - old_addr) 218 extent = old_end - old_addr; 219 old_pmd = get_old_pmd(vma->vm_mm, old_addr); 220 if (!old_pmd) 221 continue; 222 new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr); 223 if (!new_pmd) 224 break; 225 if (is_swap_pmd(*old_pmd) || pmd_trans_huge(*old_pmd)) { 226 if (extent == HPAGE_PMD_SIZE) { 227 bool moved; 228 /* See comment in move_ptes() */ 229 if (need_rmap_locks) 230 take_rmap_locks(vma); 231 moved = move_huge_pmd(vma, old_addr, new_addr, 232 old_end, old_pmd, new_pmd, 233 &need_flush); 234 if (need_rmap_locks) 235 drop_rmap_locks(vma); 236 if (moved) 237 continue; 238 } 239 split_huge_pmd(vma, old_pmd, old_addr); 240 if (pmd_trans_unstable(old_pmd)) 241 continue; 242 } 243 if (pte_alloc(new_vma->vm_mm, new_pmd, new_addr)) 244 break; 245 next = (new_addr + PMD_SIZE) & PMD_MASK; 246 if (extent > next - new_addr) 247 extent = next - new_addr; 248 move_ptes(vma, old_pmd, old_addr, old_addr + extent, new_vma, 249 new_pmd, new_addr, need_rmap_locks, &need_flush); 250 } 251 if (need_flush) 252 flush_tlb_range(vma, old_end-len, old_addr); 253 254 mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end); 255 256 return len + old_addr - old_end; /* how much done */ 257 } 258 259 static unsigned long move_vma(struct vm_area_struct *vma, 260 unsigned long old_addr, unsigned long old_len, 261 unsigned long new_len, unsigned long new_addr, 262 bool *locked, struct vm_userfaultfd_ctx *uf, 263 struct list_head *uf_unmap) 264 { 265 struct mm_struct *mm = vma->vm_mm; 266 struct vm_area_struct *new_vma; 267 unsigned long vm_flags = vma->vm_flags; 268 unsigned long new_pgoff; 269 unsigned long moved_len; 270 unsigned long excess = 0; 271 unsigned long hiwater_vm; 272 int split = 0; 273 int err; 274 bool need_rmap_locks; 275 276 /* 277 * We'd prefer to avoid failure later on in do_munmap: 278 * which may split one vma into three before unmapping. 279 */ 280 if (mm->map_count >= sysctl_max_map_count - 3) 281 return -ENOMEM; 282 283 /* 284 * Advise KSM to break any KSM pages in the area to be moved: 285 * it would be confusing if they were to turn up at the new 286 * location, where they happen to coincide with different KSM 287 * pages recently unmapped. But leave vma->vm_flags as it was, 288 * so KSM can come around to merge on vma and new_vma afterwards. 289 */ 290 err = ksm_madvise(vma, old_addr, old_addr + old_len, 291 MADV_UNMERGEABLE, &vm_flags); 292 if (err) 293 return err; 294 295 new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT); 296 new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff, 297 &need_rmap_locks); 298 if (!new_vma) 299 return -ENOMEM; 300 301 moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len, 302 need_rmap_locks); 303 if (moved_len < old_len) { 304 err = -ENOMEM; 305 } else if (vma->vm_ops && vma->vm_ops->mremap) { 306 err = vma->vm_ops->mremap(new_vma); 307 } 308 309 if (unlikely(err)) { 310 /* 311 * On error, move entries back from new area to old, 312 * which will succeed since page tables still there, 313 * and then proceed to unmap new area instead of old. 314 */ 315 move_page_tables(new_vma, new_addr, vma, old_addr, moved_len, 316 true); 317 vma = new_vma; 318 old_len = new_len; 319 old_addr = new_addr; 320 new_addr = err; 321 } else { 322 mremap_userfaultfd_prep(new_vma, uf); 323 arch_remap(mm, old_addr, old_addr + old_len, 324 new_addr, new_addr + new_len); 325 } 326 327 /* Conceal VM_ACCOUNT so old reservation is not undone */ 328 if (vm_flags & VM_ACCOUNT) { 329 vma->vm_flags &= ~VM_ACCOUNT; 330 excess = vma->vm_end - vma->vm_start - old_len; 331 if (old_addr > vma->vm_start && 332 old_addr + old_len < vma->vm_end) 333 split = 1; 334 } 335 336 /* 337 * If we failed to move page tables we still do total_vm increment 338 * since do_munmap() will decrement it by old_len == new_len. 339 * 340 * Since total_vm is about to be raised artificially high for a 341 * moment, we need to restore high watermark afterwards: if stats 342 * are taken meanwhile, total_vm and hiwater_vm appear too high. 343 * If this were a serious issue, we'd add a flag to do_munmap(). 344 */ 345 hiwater_vm = mm->hiwater_vm; 346 vm_stat_account(mm, vma->vm_flags, new_len >> PAGE_SHIFT); 347 348 /* Tell pfnmap has moved from this vma */ 349 if (unlikely(vma->vm_flags & VM_PFNMAP)) 350 untrack_pfn_moved(vma); 351 352 if (do_munmap(mm, old_addr, old_len, uf_unmap) < 0) { 353 /* OOM: unable to split vma, just get accounts right */ 354 vm_unacct_memory(excess >> PAGE_SHIFT); 355 excess = 0; 356 } 357 mm->hiwater_vm = hiwater_vm; 358 359 /* Restore VM_ACCOUNT if one or two pieces of vma left */ 360 if (excess) { 361 vma->vm_flags |= VM_ACCOUNT; 362 if (split) 363 vma->vm_next->vm_flags |= VM_ACCOUNT; 364 } 365 366 if (vm_flags & VM_LOCKED) { 367 mm->locked_vm += new_len >> PAGE_SHIFT; 368 *locked = true; 369 } 370 371 return new_addr; 372 } 373 374 static struct vm_area_struct *vma_to_resize(unsigned long addr, 375 unsigned long old_len, unsigned long new_len, unsigned long *p) 376 { 377 struct mm_struct *mm = current->mm; 378 struct vm_area_struct *vma = find_vma(mm, addr); 379 unsigned long pgoff; 380 381 if (!vma || vma->vm_start > addr) 382 return ERR_PTR(-EFAULT); 383 384 /* 385 * !old_len is a special case where an attempt is made to 'duplicate' 386 * a mapping. This makes no sense for private mappings as it will 387 * instead create a fresh/new mapping unrelated to the original. This 388 * is contrary to the basic idea of mremap which creates new mappings 389 * based on the original. There are no known use cases for this 390 * behavior. As a result, fail such attempts. 391 */ 392 if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) { 393 pr_warn_once("%s (%d): attempted to duplicate a private mapping with mremap. This is not supported.\n", current->comm, current->pid); 394 return ERR_PTR(-EINVAL); 395 } 396 397 if (is_vm_hugetlb_page(vma)) 398 return ERR_PTR(-EINVAL); 399 400 /* We can't remap across vm area boundaries */ 401 if (old_len > vma->vm_end - addr) 402 return ERR_PTR(-EFAULT); 403 404 if (new_len == old_len) 405 return vma; 406 407 /* Need to be careful about a growing mapping */ 408 pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; 409 pgoff += vma->vm_pgoff; 410 if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) 411 return ERR_PTR(-EINVAL); 412 413 if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) 414 return ERR_PTR(-EFAULT); 415 416 if (vma->vm_flags & VM_LOCKED) { 417 unsigned long locked, lock_limit; 418 locked = mm->locked_vm << PAGE_SHIFT; 419 lock_limit = rlimit(RLIMIT_MEMLOCK); 420 locked += new_len - old_len; 421 if (locked > lock_limit && !capable(CAP_IPC_LOCK)) 422 return ERR_PTR(-EAGAIN); 423 } 424 425 if (!may_expand_vm(mm, vma->vm_flags, 426 (new_len - old_len) >> PAGE_SHIFT)) 427 return ERR_PTR(-ENOMEM); 428 429 if (vma->vm_flags & VM_ACCOUNT) { 430 unsigned long charged = (new_len - old_len) >> PAGE_SHIFT; 431 if (security_vm_enough_memory_mm(mm, charged)) 432 return ERR_PTR(-ENOMEM); 433 *p = charged; 434 } 435 436 return vma; 437 } 438 439 static unsigned long mremap_to(unsigned long addr, unsigned long old_len, 440 unsigned long new_addr, unsigned long new_len, bool *locked, 441 struct vm_userfaultfd_ctx *uf, 442 struct list_head *uf_unmap_early, 443 struct list_head *uf_unmap) 444 { 445 struct mm_struct *mm = current->mm; 446 struct vm_area_struct *vma; 447 unsigned long ret = -EINVAL; 448 unsigned long charged = 0; 449 unsigned long map_flags; 450 451 if (offset_in_page(new_addr)) 452 goto out; 453 454 if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len) 455 goto out; 456 457 /* Ensure the old/new locations do not overlap */ 458 if (addr + old_len > new_addr && new_addr + new_len > addr) 459 goto out; 460 461 ret = do_munmap(mm, new_addr, new_len, uf_unmap_early); 462 if (ret) 463 goto out; 464 465 if (old_len >= new_len) { 466 ret = do_munmap(mm, addr+new_len, old_len - new_len, uf_unmap); 467 if (ret && old_len != new_len) 468 goto out; 469 old_len = new_len; 470 } 471 472 vma = vma_to_resize(addr, old_len, new_len, &charged); 473 if (IS_ERR(vma)) { 474 ret = PTR_ERR(vma); 475 goto out; 476 } 477 478 map_flags = MAP_FIXED; 479 if (vma->vm_flags & VM_MAYSHARE) 480 map_flags |= MAP_SHARED; 481 482 ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff + 483 ((addr - vma->vm_start) >> PAGE_SHIFT), 484 map_flags); 485 if (offset_in_page(ret)) 486 goto out1; 487 488 ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, uf, 489 uf_unmap); 490 if (!(offset_in_page(ret))) 491 goto out; 492 out1: 493 vm_unacct_memory(charged); 494 495 out: 496 return ret; 497 } 498 499 static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) 500 { 501 unsigned long end = vma->vm_end + delta; 502 if (end < vma->vm_end) /* overflow */ 503 return 0; 504 if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */ 505 return 0; 506 if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, 507 0, MAP_FIXED) & ~PAGE_MASK) 508 return 0; 509 return 1; 510 } 511 512 /* 513 * Expand (or shrink) an existing mapping, potentially moving it at the 514 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space) 515 * 516 * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise 517 * This option implies MREMAP_MAYMOVE. 518 */ 519 SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, 520 unsigned long, new_len, unsigned long, flags, 521 unsigned long, new_addr) 522 { 523 struct mm_struct *mm = current->mm; 524 struct vm_area_struct *vma; 525 unsigned long ret = -EINVAL; 526 unsigned long charged = 0; 527 bool locked = false; 528 struct vm_userfaultfd_ctx uf = NULL_VM_UFFD_CTX; 529 LIST_HEAD(uf_unmap_early); 530 LIST_HEAD(uf_unmap); 531 532 if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE)) 533 return ret; 534 535 if (flags & MREMAP_FIXED && !(flags & MREMAP_MAYMOVE)) 536 return ret; 537 538 if (offset_in_page(addr)) 539 return ret; 540 541 old_len = PAGE_ALIGN(old_len); 542 new_len = PAGE_ALIGN(new_len); 543 544 /* 545 * We allow a zero old-len as a special case 546 * for DOS-emu "duplicate shm area" thing. But 547 * a zero new-len is nonsensical. 548 */ 549 if (!new_len) 550 return ret; 551 552 if (down_write_killable(¤t->mm->mmap_sem)) 553 return -EINTR; 554 555 if (flags & MREMAP_FIXED) { 556 ret = mremap_to(addr, old_len, new_addr, new_len, 557 &locked, &uf, &uf_unmap_early, &uf_unmap); 558 goto out; 559 } 560 561 /* 562 * Always allow a shrinking remap: that just unmaps 563 * the unnecessary pages.. 564 * do_munmap does all the needed commit accounting 565 */ 566 if (old_len >= new_len) { 567 ret = do_munmap(mm, addr+new_len, old_len - new_len, &uf_unmap); 568 if (ret && old_len != new_len) 569 goto out; 570 ret = addr; 571 goto out; 572 } 573 574 /* 575 * Ok, we need to grow.. 576 */ 577 vma = vma_to_resize(addr, old_len, new_len, &charged); 578 if (IS_ERR(vma)) { 579 ret = PTR_ERR(vma); 580 goto out; 581 } 582 583 /* old_len exactly to the end of the area.. 584 */ 585 if (old_len == vma->vm_end - addr) { 586 /* can we just expand the current mapping? */ 587 if (vma_expandable(vma, new_len - old_len)) { 588 int pages = (new_len - old_len) >> PAGE_SHIFT; 589 590 if (vma_adjust(vma, vma->vm_start, addr + new_len, 591 vma->vm_pgoff, NULL)) { 592 ret = -ENOMEM; 593 goto out; 594 } 595 596 vm_stat_account(mm, vma->vm_flags, pages); 597 if (vma->vm_flags & VM_LOCKED) { 598 mm->locked_vm += pages; 599 locked = true; 600 new_addr = addr; 601 } 602 ret = addr; 603 goto out; 604 } 605 } 606 607 /* 608 * We weren't able to just expand or shrink the area, 609 * we need to create a new one and move it.. 610 */ 611 ret = -ENOMEM; 612 if (flags & MREMAP_MAYMOVE) { 613 unsigned long map_flags = 0; 614 if (vma->vm_flags & VM_MAYSHARE) 615 map_flags |= MAP_SHARED; 616 617 new_addr = get_unmapped_area(vma->vm_file, 0, new_len, 618 vma->vm_pgoff + 619 ((addr - vma->vm_start) >> PAGE_SHIFT), 620 map_flags); 621 if (offset_in_page(new_addr)) { 622 ret = new_addr; 623 goto out; 624 } 625 626 ret = move_vma(vma, addr, old_len, new_len, new_addr, 627 &locked, &uf, &uf_unmap); 628 } 629 out: 630 if (offset_in_page(ret)) { 631 vm_unacct_memory(charged); 632 locked = 0; 633 } 634 up_write(¤t->mm->mmap_sem); 635 if (locked && new_len > old_len) 636 mm_populate(new_addr + old_len, new_len - old_len); 637 userfaultfd_unmap_complete(mm, &uf_unmap_early); 638 mremap_userfaultfd_complete(&uf, addr, new_addr, old_len); 639 userfaultfd_unmap_complete(mm, &uf_unmap); 640 return ret; 641 } 642