1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 /* 4 * VMA-specific functions. 5 */ 6 7 #include "vma_internal.h" 8 #include "vma.h" 9 10 /* 11 * If the vma has a ->close operation then the driver probably needs to release 12 * per-vma resources, so we don't attempt to merge those if the caller indicates 13 * the current vma may be removed as part of the merge. 14 */ 15 static inline bool is_mergeable_vma(struct vm_area_struct *vma, 16 struct file *file, unsigned long vm_flags, 17 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 18 struct anon_vma_name *anon_name, bool may_remove_vma) 19 { 20 /* 21 * VM_SOFTDIRTY should not prevent from VMA merging, if we 22 * match the flags but dirty bit -- the caller should mark 23 * merged VMA as dirty. If dirty bit won't be excluded from 24 * comparison, we increase pressure on the memory system forcing 25 * the kernel to generate new VMAs when old one could be 26 * extended instead. 27 */ 28 if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY) 29 return false; 30 if (vma->vm_file != file) 31 return false; 32 if (may_remove_vma && vma->vm_ops && vma->vm_ops->close) 33 return false; 34 if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx)) 35 return false; 36 if (!anon_vma_name_eq(anon_vma_name(vma), anon_name)) 37 return false; 38 return true; 39 } 40 41 static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1, 42 struct anon_vma *anon_vma2, struct vm_area_struct *vma) 43 { 44 /* 45 * The list_is_singular() test is to avoid merging VMA cloned from 46 * parents. This can improve scalability caused by anon_vma lock. 47 */ 48 if ((!anon_vma1 || !anon_vma2) && (!vma || 49 list_is_singular(&vma->anon_vma_chain))) 50 return true; 51 return anon_vma1 == anon_vma2; 52 } 53 54 /* 55 * init_multi_vma_prep() - Initializer for struct vma_prepare 56 * @vp: The vma_prepare struct 57 * @vma: The vma that will be altered once locked 58 * @next: The next vma if it is to be adjusted 59 * @remove: The first vma to be removed 60 * @remove2: The second vma to be removed 61 */ 62 static void init_multi_vma_prep(struct vma_prepare *vp, 63 struct vm_area_struct *vma, 64 struct vm_area_struct *next, 65 struct vm_area_struct *remove, 66 struct vm_area_struct *remove2) 67 { 68 memset(vp, 0, sizeof(struct vma_prepare)); 69 vp->vma = vma; 70 vp->anon_vma = vma->anon_vma; 71 vp->remove = remove; 72 vp->remove2 = remove2; 73 vp->adj_next = next; 74 if (!vp->anon_vma && next) 75 vp->anon_vma = next->anon_vma; 76 77 vp->file = vma->vm_file; 78 if (vp->file) 79 vp->mapping = vma->vm_file->f_mapping; 80 81 } 82 83 /* 84 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 85 * in front of (at a lower virtual address and file offset than) the vma. 86 * 87 * We cannot merge two vmas if they have differently assigned (non-NULL) 88 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 89 * 90 * We don't check here for the merged mmap wrapping around the end of pagecache 91 * indices (16TB on ia32) because do_mmap() does not permit mmap's which 92 * wrap, nor mmaps which cover the final page at index -1UL. 93 * 94 * We assume the vma may be removed as part of the merge. 95 */ 96 bool 97 can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, 98 struct anon_vma *anon_vma, struct file *file, 99 pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 100 struct anon_vma_name *anon_name) 101 { 102 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) && 103 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 104 if (vma->vm_pgoff == vm_pgoff) 105 return true; 106 } 107 return false; 108 } 109 110 /* 111 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 112 * beyond (at a higher virtual address and file offset than) the vma. 113 * 114 * We cannot merge two vmas if they have differently assigned (non-NULL) 115 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 116 * 117 * We assume that vma is not removed as part of the merge. 118 */ 119 bool 120 can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, 121 struct anon_vma *anon_vma, struct file *file, 122 pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 123 struct anon_vma_name *anon_name) 124 { 125 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) && 126 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 127 pgoff_t vm_pglen; 128 129 vm_pglen = vma_pages(vma); 130 if (vma->vm_pgoff + vm_pglen == vm_pgoff) 131 return true; 132 } 133 return false; 134 } 135 136 /* 137 * Close a vm structure and free it. 138 */ 139 void remove_vma(struct vm_area_struct *vma, bool unreachable) 140 { 141 might_sleep(); 142 if (vma->vm_ops && vma->vm_ops->close) 143 vma->vm_ops->close(vma); 144 if (vma->vm_file) 145 fput(vma->vm_file); 146 mpol_put(vma_policy(vma)); 147 if (unreachable) 148 __vm_area_free(vma); 149 else 150 vm_area_free(vma); 151 } 152 153 /* 154 * Get rid of page table information in the indicated region. 155 * 156 * Called with the mm semaphore held. 157 */ 158 void unmap_region(struct mm_struct *mm, struct ma_state *mas, 159 struct vm_area_struct *vma, struct vm_area_struct *prev, 160 struct vm_area_struct *next, unsigned long start, 161 unsigned long end, unsigned long tree_end, bool mm_wr_locked) 162 { 163 struct mmu_gather tlb; 164 unsigned long mt_start = mas->index; 165 166 lru_add_drain(); 167 tlb_gather_mmu(&tlb, mm); 168 update_hiwater_rss(mm); 169 unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked); 170 mas_set(mas, mt_start); 171 free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, 172 next ? next->vm_start : USER_PGTABLES_CEILING, 173 mm_wr_locked); 174 tlb_finish_mmu(&tlb); 175 } 176 177 /* 178 * __split_vma() bypasses sysctl_max_map_count checking. We use this where it 179 * has already been checked or doesn't make sense to fail. 180 * VMA Iterator will point to the original VMA. 181 */ 182 static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, 183 unsigned long addr, int new_below) 184 { 185 struct vma_prepare vp; 186 struct vm_area_struct *new; 187 int err; 188 189 WARN_ON(vma->vm_start >= addr); 190 WARN_ON(vma->vm_end <= addr); 191 192 if (vma->vm_ops && vma->vm_ops->may_split) { 193 err = vma->vm_ops->may_split(vma, addr); 194 if (err) 195 return err; 196 } 197 198 new = vm_area_dup(vma); 199 if (!new) 200 return -ENOMEM; 201 202 if (new_below) { 203 new->vm_end = addr; 204 } else { 205 new->vm_start = addr; 206 new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); 207 } 208 209 err = -ENOMEM; 210 vma_iter_config(vmi, new->vm_start, new->vm_end); 211 if (vma_iter_prealloc(vmi, new)) 212 goto out_free_vma; 213 214 err = vma_dup_policy(vma, new); 215 if (err) 216 goto out_free_vmi; 217 218 err = anon_vma_clone(new, vma); 219 if (err) 220 goto out_free_mpol; 221 222 if (new->vm_file) 223 get_file(new->vm_file); 224 225 if (new->vm_ops && new->vm_ops->open) 226 new->vm_ops->open(new); 227 228 vma_start_write(vma); 229 vma_start_write(new); 230 231 init_vma_prep(&vp, vma); 232 vp.insert = new; 233 vma_prepare(&vp); 234 vma_adjust_trans_huge(vma, vma->vm_start, addr, 0); 235 236 if (new_below) { 237 vma->vm_start = addr; 238 vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; 239 } else { 240 vma->vm_end = addr; 241 } 242 243 /* vma_complete stores the new vma */ 244 vma_complete(&vp, vmi, vma->vm_mm); 245 246 /* Success. */ 247 if (new_below) 248 vma_next(vmi); 249 else 250 vma_prev(vmi); 251 252 return 0; 253 254 out_free_mpol: 255 mpol_put(vma_policy(new)); 256 out_free_vmi: 257 vma_iter_free(vmi); 258 out_free_vma: 259 vm_area_free(new); 260 return err; 261 } 262 263 /* 264 * Split a vma into two pieces at address 'addr', a new vma is allocated 265 * either for the first part or the tail. 266 */ 267 static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, 268 unsigned long addr, int new_below) 269 { 270 if (vma->vm_mm->map_count >= sysctl_max_map_count) 271 return -ENOMEM; 272 273 return __split_vma(vmi, vma, addr, new_below); 274 } 275 276 /* 277 * Ok - we have the memory areas we should free on a maple tree so release them, 278 * and do the vma updates. 279 * 280 * Called with the mm semaphore held. 281 */ 282 static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas) 283 { 284 unsigned long nr_accounted = 0; 285 struct vm_area_struct *vma; 286 287 /* Update high watermark before we lower total_vm */ 288 update_hiwater_vm(mm); 289 mas_for_each(mas, vma, ULONG_MAX) { 290 long nrpages = vma_pages(vma); 291 292 if (vma->vm_flags & VM_ACCOUNT) 293 nr_accounted += nrpages; 294 vm_stat_account(mm, vma->vm_flags, -nrpages); 295 remove_vma(vma, false); 296 } 297 vm_unacct_memory(nr_accounted); 298 } 299 300 /* 301 * init_vma_prep() - Initializer wrapper for vma_prepare struct 302 * @vp: The vma_prepare struct 303 * @vma: The vma that will be altered once locked 304 */ 305 void init_vma_prep(struct vma_prepare *vp, 306 struct vm_area_struct *vma) 307 { 308 init_multi_vma_prep(vp, vma, NULL, NULL, NULL); 309 } 310 311 /* 312 * Requires inode->i_mapping->i_mmap_rwsem 313 */ 314 static void __remove_shared_vm_struct(struct vm_area_struct *vma, 315 struct address_space *mapping) 316 { 317 if (vma_is_shared_maywrite(vma)) 318 mapping_unmap_writable(mapping); 319 320 flush_dcache_mmap_lock(mapping); 321 vma_interval_tree_remove(vma, &mapping->i_mmap); 322 flush_dcache_mmap_unlock(mapping); 323 } 324 325 /* 326 * vma has some anon_vma assigned, and is already inserted on that 327 * anon_vma's interval trees. 328 * 329 * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the 330 * vma must be removed from the anon_vma's interval trees using 331 * anon_vma_interval_tree_pre_update_vma(). 332 * 333 * After the update, the vma will be reinserted using 334 * anon_vma_interval_tree_post_update_vma(). 335 * 336 * The entire update must be protected by exclusive mmap_lock and by 337 * the root anon_vma's mutex. 338 */ 339 void 340 anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) 341 { 342 struct anon_vma_chain *avc; 343 344 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 345 anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); 346 } 347 348 void 349 anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) 350 { 351 struct anon_vma_chain *avc; 352 353 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 354 anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); 355 } 356 357 static void __vma_link_file(struct vm_area_struct *vma, 358 struct address_space *mapping) 359 { 360 if (vma_is_shared_maywrite(vma)) 361 mapping_allow_writable(mapping); 362 363 flush_dcache_mmap_lock(mapping); 364 vma_interval_tree_insert(vma, &mapping->i_mmap); 365 flush_dcache_mmap_unlock(mapping); 366 } 367 368 /* 369 * vma_prepare() - Helper function for handling locking VMAs prior to altering 370 * @vp: The initialized vma_prepare struct 371 */ 372 void vma_prepare(struct vma_prepare *vp) 373 { 374 if (vp->file) { 375 uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end); 376 377 if (vp->adj_next) 378 uprobe_munmap(vp->adj_next, vp->adj_next->vm_start, 379 vp->adj_next->vm_end); 380 381 i_mmap_lock_write(vp->mapping); 382 if (vp->insert && vp->insert->vm_file) { 383 /* 384 * Put into interval tree now, so instantiated pages 385 * are visible to arm/parisc __flush_dcache_page 386 * throughout; but we cannot insert into address 387 * space until vma start or end is updated. 388 */ 389 __vma_link_file(vp->insert, 390 vp->insert->vm_file->f_mapping); 391 } 392 } 393 394 if (vp->anon_vma) { 395 anon_vma_lock_write(vp->anon_vma); 396 anon_vma_interval_tree_pre_update_vma(vp->vma); 397 if (vp->adj_next) 398 anon_vma_interval_tree_pre_update_vma(vp->adj_next); 399 } 400 401 if (vp->file) { 402 flush_dcache_mmap_lock(vp->mapping); 403 vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap); 404 if (vp->adj_next) 405 vma_interval_tree_remove(vp->adj_next, 406 &vp->mapping->i_mmap); 407 } 408 409 } 410 411 /* 412 * dup_anon_vma() - Helper function to duplicate anon_vma 413 * @dst: The destination VMA 414 * @src: The source VMA 415 * @dup: Pointer to the destination VMA when successful. 416 * 417 * Returns: 0 on success. 418 */ 419 static int dup_anon_vma(struct vm_area_struct *dst, 420 struct vm_area_struct *src, struct vm_area_struct **dup) 421 { 422 /* 423 * Easily overlooked: when mprotect shifts the boundary, make sure the 424 * expanding vma has anon_vma set if the shrinking vma had, to cover any 425 * anon pages imported. 426 */ 427 if (src->anon_vma && !dst->anon_vma) { 428 int ret; 429 430 vma_assert_write_locked(dst); 431 dst->anon_vma = src->anon_vma; 432 ret = anon_vma_clone(dst, src); 433 if (ret) 434 return ret; 435 436 *dup = dst; 437 } 438 439 return 0; 440 } 441 442 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 443 void validate_mm(struct mm_struct *mm) 444 { 445 int bug = 0; 446 int i = 0; 447 struct vm_area_struct *vma; 448 VMA_ITERATOR(vmi, mm, 0); 449 450 mt_validate(&mm->mm_mt); 451 for_each_vma(vmi, vma) { 452 #ifdef CONFIG_DEBUG_VM_RB 453 struct anon_vma *anon_vma = vma->anon_vma; 454 struct anon_vma_chain *avc; 455 #endif 456 unsigned long vmi_start, vmi_end; 457 bool warn = 0; 458 459 vmi_start = vma_iter_addr(&vmi); 460 vmi_end = vma_iter_end(&vmi); 461 if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm)) 462 warn = 1; 463 464 if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm)) 465 warn = 1; 466 467 if (warn) { 468 pr_emerg("issue in %s\n", current->comm); 469 dump_stack(); 470 dump_vma(vma); 471 pr_emerg("tree range: %px start %lx end %lx\n", vma, 472 vmi_start, vmi_end - 1); 473 vma_iter_dump_tree(&vmi); 474 } 475 476 #ifdef CONFIG_DEBUG_VM_RB 477 if (anon_vma) { 478 anon_vma_lock_read(anon_vma); 479 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 480 anon_vma_interval_tree_verify(avc); 481 anon_vma_unlock_read(anon_vma); 482 } 483 #endif 484 i++; 485 } 486 if (i != mm->map_count) { 487 pr_emerg("map_count %d vma iterator %d\n", mm->map_count, i); 488 bug = 1; 489 } 490 VM_BUG_ON_MM(bug, mm); 491 } 492 #endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ 493 494 /* 495 * vma_expand - Expand an existing VMA 496 * 497 * @vmi: The vma iterator 498 * @vma: The vma to expand 499 * @start: The start of the vma 500 * @end: The exclusive end of the vma 501 * @pgoff: The page offset of vma 502 * @next: The current of next vma. 503 * 504 * Expand @vma to @start and @end. Can expand off the start and end. Will 505 * expand over @next if it's different from @vma and @end == @next->vm_end. 506 * Checking if the @vma can expand and merge with @next needs to be handled by 507 * the caller. 508 * 509 * Returns: 0 on success 510 */ 511 int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma, 512 unsigned long start, unsigned long end, pgoff_t pgoff, 513 struct vm_area_struct *next) 514 { 515 struct vm_area_struct *anon_dup = NULL; 516 bool remove_next = false; 517 struct vma_prepare vp; 518 519 vma_start_write(vma); 520 if (next && (vma != next) && (end == next->vm_end)) { 521 int ret; 522 523 remove_next = true; 524 vma_start_write(next); 525 ret = dup_anon_vma(vma, next, &anon_dup); 526 if (ret) 527 return ret; 528 } 529 530 init_multi_vma_prep(&vp, vma, NULL, remove_next ? next : NULL, NULL); 531 /* Not merging but overwriting any part of next is not handled. */ 532 VM_WARN_ON(next && !vp.remove && 533 next != vma && end > next->vm_start); 534 /* Only handles expanding */ 535 VM_WARN_ON(vma->vm_start < start || vma->vm_end > end); 536 537 /* Note: vma iterator must be pointing to 'start' */ 538 vma_iter_config(vmi, start, end); 539 if (vma_iter_prealloc(vmi, vma)) 540 goto nomem; 541 542 vma_prepare(&vp); 543 vma_adjust_trans_huge(vma, start, end, 0); 544 vma_set_range(vma, start, end, pgoff); 545 vma_iter_store(vmi, vma); 546 547 vma_complete(&vp, vmi, vma->vm_mm); 548 return 0; 549 550 nomem: 551 if (anon_dup) 552 unlink_anon_vmas(anon_dup); 553 return -ENOMEM; 554 } 555 556 /* 557 * vma_shrink() - Reduce an existing VMAs memory area 558 * @vmi: The vma iterator 559 * @vma: The VMA to modify 560 * @start: The new start 561 * @end: The new end 562 * 563 * Returns: 0 on success, -ENOMEM otherwise 564 */ 565 int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, 566 unsigned long start, unsigned long end, pgoff_t pgoff) 567 { 568 struct vma_prepare vp; 569 570 WARN_ON((vma->vm_start != start) && (vma->vm_end != end)); 571 572 if (vma->vm_start < start) 573 vma_iter_config(vmi, vma->vm_start, start); 574 else 575 vma_iter_config(vmi, end, vma->vm_end); 576 577 if (vma_iter_prealloc(vmi, NULL)) 578 return -ENOMEM; 579 580 vma_start_write(vma); 581 582 init_vma_prep(&vp, vma); 583 vma_prepare(&vp); 584 vma_adjust_trans_huge(vma, start, end, 0); 585 586 vma_iter_clear(vmi); 587 vma_set_range(vma, start, end, pgoff); 588 vma_complete(&vp, vmi, vma->vm_mm); 589 return 0; 590 } 591 592 /* 593 * vma_complete- Helper function for handling the unlocking after altering VMAs, 594 * or for inserting a VMA. 595 * 596 * @vp: The vma_prepare struct 597 * @vmi: The vma iterator 598 * @mm: The mm_struct 599 */ 600 void vma_complete(struct vma_prepare *vp, 601 struct vma_iterator *vmi, struct mm_struct *mm) 602 { 603 if (vp->file) { 604 if (vp->adj_next) 605 vma_interval_tree_insert(vp->adj_next, 606 &vp->mapping->i_mmap); 607 vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap); 608 flush_dcache_mmap_unlock(vp->mapping); 609 } 610 611 if (vp->remove && vp->file) { 612 __remove_shared_vm_struct(vp->remove, vp->mapping); 613 if (vp->remove2) 614 __remove_shared_vm_struct(vp->remove2, vp->mapping); 615 } else if (vp->insert) { 616 /* 617 * split_vma has split insert from vma, and needs 618 * us to insert it before dropping the locks 619 * (it may either follow vma or precede it). 620 */ 621 vma_iter_store(vmi, vp->insert); 622 mm->map_count++; 623 } 624 625 if (vp->anon_vma) { 626 anon_vma_interval_tree_post_update_vma(vp->vma); 627 if (vp->adj_next) 628 anon_vma_interval_tree_post_update_vma(vp->adj_next); 629 anon_vma_unlock_write(vp->anon_vma); 630 } 631 632 if (vp->file) { 633 i_mmap_unlock_write(vp->mapping); 634 uprobe_mmap(vp->vma); 635 636 if (vp->adj_next) 637 uprobe_mmap(vp->adj_next); 638 } 639 640 if (vp->remove) { 641 again: 642 vma_mark_detached(vp->remove, true); 643 if (vp->file) { 644 uprobe_munmap(vp->remove, vp->remove->vm_start, 645 vp->remove->vm_end); 646 fput(vp->file); 647 } 648 if (vp->remove->anon_vma) 649 anon_vma_merge(vp->vma, vp->remove); 650 mm->map_count--; 651 mpol_put(vma_policy(vp->remove)); 652 if (!vp->remove2) 653 WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end); 654 vm_area_free(vp->remove); 655 656 /* 657 * In mprotect's case 6 (see comments on vma_merge), 658 * we are removing both mid and next vmas 659 */ 660 if (vp->remove2) { 661 vp->remove = vp->remove2; 662 vp->remove2 = NULL; 663 goto again; 664 } 665 } 666 if (vp->insert && vp->file) 667 uprobe_mmap(vp->insert); 668 validate_mm(mm); 669 } 670 671 /* 672 * abort_munmap_vmas - Undo any munmap work and free resources 673 * 674 * Reattach any detached vmas and free up the maple tree used to track the vmas. 675 */ 676 static inline void abort_munmap_vmas(struct ma_state *mas_detach) 677 { 678 struct vm_area_struct *vma; 679 680 mas_set(mas_detach, 0); 681 mas_for_each(mas_detach, vma, ULONG_MAX) 682 vma_mark_detached(vma, false); 683 684 __mt_destroy(mas_detach->tree); 685 } 686 687 /* 688 * do_vmi_align_munmap() - munmap the aligned region from @start to @end. 689 * @vmi: The vma iterator 690 * @vma: The starting vm_area_struct 691 * @mm: The mm_struct 692 * @start: The aligned start address to munmap. 693 * @end: The aligned end address to munmap. 694 * @uf: The userfaultfd list_head 695 * @unlock: Set to true to drop the mmap_lock. unlocking only happens on 696 * success. 697 * 698 * Return: 0 on success and drops the lock if so directed, error and leaves the 699 * lock held otherwise. 700 */ 701 int 702 do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 703 struct mm_struct *mm, unsigned long start, 704 unsigned long end, struct list_head *uf, bool unlock) 705 { 706 struct vm_area_struct *prev, *next = NULL; 707 struct maple_tree mt_detach; 708 int count = 0; 709 int error = -ENOMEM; 710 unsigned long locked_vm = 0; 711 MA_STATE(mas_detach, &mt_detach, 0, 0); 712 mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK); 713 mt_on_stack(mt_detach); 714 715 /* 716 * If we need to split any vma, do it now to save pain later. 717 * 718 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially 719 * unmapped vm_area_struct will remain in use: so lower split_vma 720 * places tmp vma above, and higher split_vma places tmp vma below. 721 */ 722 723 /* Does it split the first one? */ 724 if (start > vma->vm_start) { 725 726 /* 727 * Make sure that map_count on return from munmap() will 728 * not exceed its limit; but let map_count go just above 729 * its limit temporarily, to help free resources as expected. 730 */ 731 if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count) 732 goto map_count_exceeded; 733 734 /* Don't bother splitting the VMA if we can't unmap it anyway */ 735 if (!can_modify_vma(vma)) { 736 error = -EPERM; 737 goto start_split_failed; 738 } 739 740 error = __split_vma(vmi, vma, start, 1); 741 if (error) 742 goto start_split_failed; 743 } 744 745 /* 746 * Detach a range of VMAs from the mm. Using next as a temp variable as 747 * it is always overwritten. 748 */ 749 next = vma; 750 do { 751 if (!can_modify_vma(next)) { 752 error = -EPERM; 753 goto modify_vma_failed; 754 } 755 756 /* Does it split the end? */ 757 if (next->vm_end > end) { 758 error = __split_vma(vmi, next, end, 0); 759 if (error) 760 goto end_split_failed; 761 } 762 vma_start_write(next); 763 mas_set(&mas_detach, count); 764 error = mas_store_gfp(&mas_detach, next, GFP_KERNEL); 765 if (error) 766 goto munmap_gather_failed; 767 vma_mark_detached(next, true); 768 if (next->vm_flags & VM_LOCKED) 769 locked_vm += vma_pages(next); 770 771 count++; 772 if (unlikely(uf)) { 773 /* 774 * If userfaultfd_unmap_prep returns an error the vmas 775 * will remain split, but userland will get a 776 * highly unexpected error anyway. This is no 777 * different than the case where the first of the two 778 * __split_vma fails, but we don't undo the first 779 * split, despite we could. This is unlikely enough 780 * failure that it's not worth optimizing it for. 781 */ 782 error = userfaultfd_unmap_prep(next, start, end, uf); 783 784 if (error) 785 goto userfaultfd_error; 786 } 787 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 788 BUG_ON(next->vm_start < start); 789 BUG_ON(next->vm_start > end); 790 #endif 791 } for_each_vma_range(*vmi, next, end); 792 793 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 794 /* Make sure no VMAs are about to be lost. */ 795 { 796 MA_STATE(test, &mt_detach, 0, 0); 797 struct vm_area_struct *vma_mas, *vma_test; 798 int test_count = 0; 799 800 vma_iter_set(vmi, start); 801 rcu_read_lock(); 802 vma_test = mas_find(&test, count - 1); 803 for_each_vma_range(*vmi, vma_mas, end) { 804 BUG_ON(vma_mas != vma_test); 805 test_count++; 806 vma_test = mas_next(&test, count - 1); 807 } 808 rcu_read_unlock(); 809 BUG_ON(count != test_count); 810 } 811 #endif 812 813 while (vma_iter_addr(vmi) > start) 814 vma_iter_prev_range(vmi); 815 816 error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL); 817 if (error) 818 goto clear_tree_failed; 819 820 /* Point of no return */ 821 mm->locked_vm -= locked_vm; 822 mm->map_count -= count; 823 if (unlock) 824 mmap_write_downgrade(mm); 825 826 prev = vma_iter_prev_range(vmi); 827 next = vma_next(vmi); 828 if (next) 829 vma_iter_prev_range(vmi); 830 831 /* 832 * We can free page tables without write-locking mmap_lock because VMAs 833 * were isolated before we downgraded mmap_lock. 834 */ 835 mas_set(&mas_detach, 1); 836 unmap_region(mm, &mas_detach, vma, prev, next, start, end, count, 837 !unlock); 838 /* Statistics and freeing VMAs */ 839 mas_set(&mas_detach, 0); 840 remove_mt(mm, &mas_detach); 841 validate_mm(mm); 842 if (unlock) 843 mmap_read_unlock(mm); 844 845 __mt_destroy(&mt_detach); 846 return 0; 847 848 modify_vma_failed: 849 clear_tree_failed: 850 userfaultfd_error: 851 munmap_gather_failed: 852 end_split_failed: 853 abort_munmap_vmas(&mas_detach); 854 start_split_failed: 855 map_count_exceeded: 856 validate_mm(mm); 857 return error; 858 } 859 860 /* 861 * do_vmi_munmap() - munmap a given range. 862 * @vmi: The vma iterator 863 * @mm: The mm_struct 864 * @start: The start address to munmap 865 * @len: The length of the range to munmap 866 * @uf: The userfaultfd list_head 867 * @unlock: set to true if the user wants to drop the mmap_lock on success 868 * 869 * This function takes a @mas that is either pointing to the previous VMA or set 870 * to MA_START and sets it up to remove the mapping(s). The @len will be 871 * aligned. 872 * 873 * Return: 0 on success and drops the lock if so directed, error and leaves the 874 * lock held otherwise. 875 */ 876 int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 877 unsigned long start, size_t len, struct list_head *uf, 878 bool unlock) 879 { 880 unsigned long end; 881 struct vm_area_struct *vma; 882 883 if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) 884 return -EINVAL; 885 886 end = start + PAGE_ALIGN(len); 887 if (end == start) 888 return -EINVAL; 889 890 /* Find the first overlapping VMA */ 891 vma = vma_find(vmi, end); 892 if (!vma) { 893 if (unlock) 894 mmap_write_unlock(mm); 895 return 0; 896 } 897 898 return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock); 899 } 900 901 /* 902 * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name), 903 * figure out whether that can be merged with its predecessor or its 904 * successor. Or both (it neatly fills a hole). 905 * 906 * In most cases - when called for mmap, brk or mremap - [addr,end) is 907 * certain not to be mapped by the time vma_merge is called; but when 908 * called for mprotect, it is certain to be already mapped (either at 909 * an offset within prev, or at the start of next), and the flags of 910 * this area are about to be changed to vm_flags - and the no-change 911 * case has already been eliminated. 912 * 913 * The following mprotect cases have to be considered, where **** is 914 * the area passed down from mprotect_fixup, never extending beyond one 915 * vma, PPPP is the previous vma, CCCC is a concurrent vma that starts 916 * at the same address as **** and is of the same or larger span, and 917 * NNNN the next vma after ****: 918 * 919 * **** **** **** 920 * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPCCCCCC 921 * cannot merge might become might become 922 * PPNNNNNNNNNN PPPPPPPPPPCC 923 * mmap, brk or case 4 below case 5 below 924 * mremap move: 925 * **** **** 926 * PPPP NNNN PPPPCCCCNNNN 927 * might become might become 928 * PPPPPPPPPPPP 1 or PPPPPPPPPPPP 6 or 929 * PPPPPPPPNNNN 2 or PPPPPPPPNNNN 7 or 930 * PPPPNNNNNNNN 3 PPPPNNNNNNNN 8 931 * 932 * It is important for case 8 that the vma CCCC overlapping the 933 * region **** is never going to extended over NNNN. Instead NNNN must 934 * be extended in region **** and CCCC must be removed. This way in 935 * all cases where vma_merge succeeds, the moment vma_merge drops the 936 * rmap_locks, the properties of the merged vma will be already 937 * correct for the whole merged range. Some of those properties like 938 * vm_page_prot/vm_flags may be accessed by rmap_walks and they must 939 * be correct for the whole merged range immediately after the 940 * rmap_locks are released. Otherwise if NNNN would be removed and 941 * CCCC would be extended over the NNNN range, remove_migration_ptes 942 * or other rmap walkers (if working on addresses beyond the "end" 943 * parameter) may establish ptes with the wrong permissions of CCCC 944 * instead of the right permissions of NNNN. 945 * 946 * In the code below: 947 * PPPP is represented by *prev 948 * CCCC is represented by *curr or not represented at all (NULL) 949 * NNNN is represented by *next or not represented at all (NULL) 950 * **** is not represented - it will be merged and the vma containing the 951 * area is returned, or the function will return NULL 952 */ 953 static struct vm_area_struct 954 *vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev, 955 struct vm_area_struct *src, unsigned long addr, unsigned long end, 956 unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy, 957 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 958 struct anon_vma_name *anon_name) 959 { 960 struct mm_struct *mm = src->vm_mm; 961 struct anon_vma *anon_vma = src->anon_vma; 962 struct file *file = src->vm_file; 963 struct vm_area_struct *curr, *next, *res; 964 struct vm_area_struct *vma, *adjust, *remove, *remove2; 965 struct vm_area_struct *anon_dup = NULL; 966 struct vma_prepare vp; 967 pgoff_t vma_pgoff; 968 int err = 0; 969 bool merge_prev = false; 970 bool merge_next = false; 971 bool vma_expanded = false; 972 unsigned long vma_start = addr; 973 unsigned long vma_end = end; 974 pgoff_t pglen = (end - addr) >> PAGE_SHIFT; 975 long adj_start = 0; 976 977 /* 978 * We later require that vma->vm_flags == vm_flags, 979 * so this tests vma->vm_flags & VM_SPECIAL, too. 980 */ 981 if (vm_flags & VM_SPECIAL) 982 return NULL; 983 984 /* Does the input range span an existing VMA? (cases 5 - 8) */ 985 curr = find_vma_intersection(mm, prev ? prev->vm_end : 0, end); 986 987 if (!curr || /* cases 1 - 4 */ 988 end == curr->vm_end) /* cases 6 - 8, adjacent VMA */ 989 next = vma_lookup(mm, end); 990 else 991 next = NULL; /* case 5 */ 992 993 if (prev) { 994 vma_start = prev->vm_start; 995 vma_pgoff = prev->vm_pgoff; 996 997 /* Can we merge the predecessor? */ 998 if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy) 999 && can_vma_merge_after(prev, vm_flags, anon_vma, file, 1000 pgoff, vm_userfaultfd_ctx, anon_name)) { 1001 merge_prev = true; 1002 vma_prev(vmi); 1003 } 1004 } 1005 1006 /* Can we merge the successor? */ 1007 if (next && mpol_equal(policy, vma_policy(next)) && 1008 can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen, 1009 vm_userfaultfd_ctx, anon_name)) { 1010 merge_next = true; 1011 } 1012 1013 /* Verify some invariant that must be enforced by the caller. */ 1014 VM_WARN_ON(prev && addr <= prev->vm_start); 1015 VM_WARN_ON(curr && (addr != curr->vm_start || end > curr->vm_end)); 1016 VM_WARN_ON(addr >= end); 1017 1018 if (!merge_prev && !merge_next) 1019 return NULL; /* Not mergeable. */ 1020 1021 if (merge_prev) 1022 vma_start_write(prev); 1023 1024 res = vma = prev; 1025 remove = remove2 = adjust = NULL; 1026 1027 /* Can we merge both the predecessor and the successor? */ 1028 if (merge_prev && merge_next && 1029 is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) { 1030 vma_start_write(next); 1031 remove = next; /* case 1 */ 1032 vma_end = next->vm_end; 1033 err = dup_anon_vma(prev, next, &anon_dup); 1034 if (curr) { /* case 6 */ 1035 vma_start_write(curr); 1036 remove = curr; 1037 remove2 = next; 1038 /* 1039 * Note that the dup_anon_vma below cannot overwrite err 1040 * since the first caller would do nothing unless next 1041 * has an anon_vma. 1042 */ 1043 if (!next->anon_vma) 1044 err = dup_anon_vma(prev, curr, &anon_dup); 1045 } 1046 } else if (merge_prev) { /* case 2 */ 1047 if (curr) { 1048 vma_start_write(curr); 1049 if (end == curr->vm_end) { /* case 7 */ 1050 /* 1051 * can_vma_merge_after() assumed we would not be 1052 * removing prev vma, so it skipped the check 1053 * for vm_ops->close, but we are removing curr 1054 */ 1055 if (curr->vm_ops && curr->vm_ops->close) 1056 err = -EINVAL; 1057 remove = curr; 1058 } else { /* case 5 */ 1059 adjust = curr; 1060 adj_start = (end - curr->vm_start); 1061 } 1062 if (!err) 1063 err = dup_anon_vma(prev, curr, &anon_dup); 1064 } 1065 } else { /* merge_next */ 1066 vma_start_write(next); 1067 res = next; 1068 if (prev && addr < prev->vm_end) { /* case 4 */ 1069 vma_start_write(prev); 1070 vma_end = addr; 1071 adjust = next; 1072 adj_start = -(prev->vm_end - addr); 1073 err = dup_anon_vma(next, prev, &anon_dup); 1074 } else { 1075 /* 1076 * Note that cases 3 and 8 are the ONLY ones where prev 1077 * is permitted to be (but is not necessarily) NULL. 1078 */ 1079 vma = next; /* case 3 */ 1080 vma_start = addr; 1081 vma_end = next->vm_end; 1082 vma_pgoff = next->vm_pgoff - pglen; 1083 if (curr) { /* case 8 */ 1084 vma_pgoff = curr->vm_pgoff; 1085 vma_start_write(curr); 1086 remove = curr; 1087 err = dup_anon_vma(next, curr, &anon_dup); 1088 } 1089 } 1090 } 1091 1092 /* Error in anon_vma clone. */ 1093 if (err) 1094 goto anon_vma_fail; 1095 1096 if (vma_start < vma->vm_start || vma_end > vma->vm_end) 1097 vma_expanded = true; 1098 1099 if (vma_expanded) { 1100 vma_iter_config(vmi, vma_start, vma_end); 1101 } else { 1102 vma_iter_config(vmi, adjust->vm_start + adj_start, 1103 adjust->vm_end); 1104 } 1105 1106 if (vma_iter_prealloc(vmi, vma)) 1107 goto prealloc_fail; 1108 1109 init_multi_vma_prep(&vp, vma, adjust, remove, remove2); 1110 VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma && 1111 vp.anon_vma != adjust->anon_vma); 1112 1113 vma_prepare(&vp); 1114 vma_adjust_trans_huge(vma, vma_start, vma_end, adj_start); 1115 vma_set_range(vma, vma_start, vma_end, vma_pgoff); 1116 1117 if (vma_expanded) 1118 vma_iter_store(vmi, vma); 1119 1120 if (adj_start) { 1121 adjust->vm_start += adj_start; 1122 adjust->vm_pgoff += adj_start >> PAGE_SHIFT; 1123 if (adj_start < 0) { 1124 WARN_ON(vma_expanded); 1125 vma_iter_store(vmi, next); 1126 } 1127 } 1128 1129 vma_complete(&vp, vmi, mm); 1130 khugepaged_enter_vma(res, vm_flags); 1131 return res; 1132 1133 prealloc_fail: 1134 if (anon_dup) 1135 unlink_anon_vmas(anon_dup); 1136 1137 anon_vma_fail: 1138 vma_iter_set(vmi, addr); 1139 vma_iter_load(vmi); 1140 return NULL; 1141 } 1142 1143 /* 1144 * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd 1145 * context and anonymous VMA name within the range [start, end). 1146 * 1147 * As a result, we might be able to merge the newly modified VMA range with an 1148 * adjacent VMA with identical properties. 1149 * 1150 * If no merge is possible and the range does not span the entirety of the VMA, 1151 * we then need to split the VMA to accommodate the change. 1152 * 1153 * The function returns either the merged VMA, the original VMA if a split was 1154 * required instead, or an error if the split failed. 1155 */ 1156 struct vm_area_struct *vma_modify(struct vma_iterator *vmi, 1157 struct vm_area_struct *prev, 1158 struct vm_area_struct *vma, 1159 unsigned long start, unsigned long end, 1160 unsigned long vm_flags, 1161 struct mempolicy *policy, 1162 struct vm_userfaultfd_ctx uffd_ctx, 1163 struct anon_vma_name *anon_name) 1164 { 1165 pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 1166 struct vm_area_struct *merged; 1167 1168 merged = vma_merge(vmi, prev, vma, start, end, vm_flags, 1169 pgoff, policy, uffd_ctx, anon_name); 1170 if (merged) 1171 return merged; 1172 1173 if (vma->vm_start < start) { 1174 int err = split_vma(vmi, vma, start, 1); 1175 1176 if (err) 1177 return ERR_PTR(err); 1178 } 1179 1180 if (vma->vm_end > end) { 1181 int err = split_vma(vmi, vma, end, 0); 1182 1183 if (err) 1184 return ERR_PTR(err); 1185 } 1186 1187 return vma; 1188 } 1189 1190 /* 1191 * Attempt to merge a newly mapped VMA with those adjacent to it. The caller 1192 * must ensure that [start, end) does not overlap any existing VMA. 1193 */ 1194 struct vm_area_struct 1195 *vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev, 1196 struct vm_area_struct *vma, unsigned long start, 1197 unsigned long end, pgoff_t pgoff) 1198 { 1199 return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff, 1200 vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 1201 } 1202 1203 /* 1204 * Expand vma by delta bytes, potentially merging with an immediately adjacent 1205 * VMA with identical properties. 1206 */ 1207 struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi, 1208 struct vm_area_struct *vma, 1209 unsigned long delta) 1210 { 1211 pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma); 1212 1213 /* vma is specified as prev, so case 1 or 2 will apply. */ 1214 return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta, 1215 vma->vm_flags, pgoff, vma_policy(vma), 1216 vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 1217 } 1218 1219 void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb) 1220 { 1221 vb->count = 0; 1222 } 1223 1224 static void unlink_file_vma_batch_process(struct unlink_vma_file_batch *vb) 1225 { 1226 struct address_space *mapping; 1227 int i; 1228 1229 mapping = vb->vmas[0]->vm_file->f_mapping; 1230 i_mmap_lock_write(mapping); 1231 for (i = 0; i < vb->count; i++) { 1232 VM_WARN_ON_ONCE(vb->vmas[i]->vm_file->f_mapping != mapping); 1233 __remove_shared_vm_struct(vb->vmas[i], mapping); 1234 } 1235 i_mmap_unlock_write(mapping); 1236 1237 unlink_file_vma_batch_init(vb); 1238 } 1239 1240 void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, 1241 struct vm_area_struct *vma) 1242 { 1243 if (vma->vm_file == NULL) 1244 return; 1245 1246 if ((vb->count > 0 && vb->vmas[0]->vm_file != vma->vm_file) || 1247 vb->count == ARRAY_SIZE(vb->vmas)) 1248 unlink_file_vma_batch_process(vb); 1249 1250 vb->vmas[vb->count] = vma; 1251 vb->count++; 1252 } 1253 1254 void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb) 1255 { 1256 if (vb->count > 0) 1257 unlink_file_vma_batch_process(vb); 1258 } 1259 1260 /* 1261 * Unlink a file-based vm structure from its interval tree, to hide 1262 * vma from rmap and vmtruncate before freeing its page tables. 1263 */ 1264 void unlink_file_vma(struct vm_area_struct *vma) 1265 { 1266 struct file *file = vma->vm_file; 1267 1268 if (file) { 1269 struct address_space *mapping = file->f_mapping; 1270 1271 i_mmap_lock_write(mapping); 1272 __remove_shared_vm_struct(vma, mapping); 1273 i_mmap_unlock_write(mapping); 1274 } 1275 } 1276 1277 void vma_link_file(struct vm_area_struct *vma) 1278 { 1279 struct file *file = vma->vm_file; 1280 struct address_space *mapping; 1281 1282 if (file) { 1283 mapping = file->f_mapping; 1284 i_mmap_lock_write(mapping); 1285 __vma_link_file(vma, mapping); 1286 i_mmap_unlock_write(mapping); 1287 } 1288 } 1289 1290 int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) 1291 { 1292 VMA_ITERATOR(vmi, mm, 0); 1293 1294 vma_iter_config(&vmi, vma->vm_start, vma->vm_end); 1295 if (vma_iter_prealloc(&vmi, vma)) 1296 return -ENOMEM; 1297 1298 vma_start_write(vma); 1299 vma_iter_store(&vmi, vma); 1300 vma_link_file(vma); 1301 mm->map_count++; 1302 validate_mm(mm); 1303 return 0; 1304 } 1305 1306 /* 1307 * Copy the vma structure to a new location in the same mm, 1308 * prior to moving page table entries, to effect an mremap move. 1309 */ 1310 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, 1311 unsigned long addr, unsigned long len, pgoff_t pgoff, 1312 bool *need_rmap_locks) 1313 { 1314 struct vm_area_struct *vma = *vmap; 1315 unsigned long vma_start = vma->vm_start; 1316 struct mm_struct *mm = vma->vm_mm; 1317 struct vm_area_struct *new_vma, *prev; 1318 bool faulted_in_anon_vma = true; 1319 VMA_ITERATOR(vmi, mm, addr); 1320 1321 /* 1322 * If anonymous vma has not yet been faulted, update new pgoff 1323 * to match new location, to increase its chance of merging. 1324 */ 1325 if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { 1326 pgoff = addr >> PAGE_SHIFT; 1327 faulted_in_anon_vma = false; 1328 } 1329 1330 new_vma = find_vma_prev(mm, addr, &prev); 1331 if (new_vma && new_vma->vm_start < addr + len) 1332 return NULL; /* should never get here */ 1333 1334 new_vma = vma_merge_new_vma(&vmi, prev, vma, addr, addr + len, pgoff); 1335 if (new_vma) { 1336 /* 1337 * Source vma may have been merged into new_vma 1338 */ 1339 if (unlikely(vma_start >= new_vma->vm_start && 1340 vma_start < new_vma->vm_end)) { 1341 /* 1342 * The only way we can get a vma_merge with 1343 * self during an mremap is if the vma hasn't 1344 * been faulted in yet and we were allowed to 1345 * reset the dst vma->vm_pgoff to the 1346 * destination address of the mremap to allow 1347 * the merge to happen. mremap must change the 1348 * vm_pgoff linearity between src and dst vmas 1349 * (in turn preventing a vma_merge) to be 1350 * safe. It is only safe to keep the vm_pgoff 1351 * linear if there are no pages mapped yet. 1352 */ 1353 VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); 1354 *vmap = vma = new_vma; 1355 } 1356 *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff); 1357 } else { 1358 new_vma = vm_area_dup(vma); 1359 if (!new_vma) 1360 goto out; 1361 vma_set_range(new_vma, addr, addr + len, pgoff); 1362 if (vma_dup_policy(vma, new_vma)) 1363 goto out_free_vma; 1364 if (anon_vma_clone(new_vma, vma)) 1365 goto out_free_mempol; 1366 if (new_vma->vm_file) 1367 get_file(new_vma->vm_file); 1368 if (new_vma->vm_ops && new_vma->vm_ops->open) 1369 new_vma->vm_ops->open(new_vma); 1370 if (vma_link(mm, new_vma)) 1371 goto out_vma_link; 1372 *need_rmap_locks = false; 1373 } 1374 return new_vma; 1375 1376 out_vma_link: 1377 if (new_vma->vm_ops && new_vma->vm_ops->close) 1378 new_vma->vm_ops->close(new_vma); 1379 1380 if (new_vma->vm_file) 1381 fput(new_vma->vm_file); 1382 1383 unlink_anon_vmas(new_vma); 1384 out_free_mempol: 1385 mpol_put(vma_policy(new_vma)); 1386 out_free_vma: 1387 vm_area_free(new_vma); 1388 out: 1389 return NULL; 1390 } 1391 1392 /* 1393 * Rough compatibility check to quickly see if it's even worth looking 1394 * at sharing an anon_vma. 1395 * 1396 * They need to have the same vm_file, and the flags can only differ 1397 * in things that mprotect may change. 1398 * 1399 * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that 1400 * we can merge the two vma's. For example, we refuse to merge a vma if 1401 * there is a vm_ops->close() function, because that indicates that the 1402 * driver is doing some kind of reference counting. But that doesn't 1403 * really matter for the anon_vma sharing case. 1404 */ 1405 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b) 1406 { 1407 return a->vm_end == b->vm_start && 1408 mpol_equal(vma_policy(a), vma_policy(b)) && 1409 a->vm_file == b->vm_file && 1410 !((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) && 1411 b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); 1412 } 1413 1414 /* 1415 * Do some basic sanity checking to see if we can re-use the anon_vma 1416 * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be 1417 * the same as 'old', the other will be the new one that is trying 1418 * to share the anon_vma. 1419 * 1420 * NOTE! This runs with mmap_lock held for reading, so it is possible that 1421 * the anon_vma of 'old' is concurrently in the process of being set up 1422 * by another page fault trying to merge _that_. But that's ok: if it 1423 * is being set up, that automatically means that it will be a singleton 1424 * acceptable for merging, so we can do all of this optimistically. But 1425 * we do that READ_ONCE() to make sure that we never re-load the pointer. 1426 * 1427 * IOW: that the "list_is_singular()" test on the anon_vma_chain only 1428 * matters for the 'stable anon_vma' case (ie the thing we want to avoid 1429 * is to return an anon_vma that is "complex" due to having gone through 1430 * a fork). 1431 * 1432 * We also make sure that the two vma's are compatible (adjacent, 1433 * and with the same memory policies). That's all stable, even with just 1434 * a read lock on the mmap_lock. 1435 */ 1436 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, 1437 struct vm_area_struct *a, 1438 struct vm_area_struct *b) 1439 { 1440 if (anon_vma_compatible(a, b)) { 1441 struct anon_vma *anon_vma = READ_ONCE(old->anon_vma); 1442 1443 if (anon_vma && list_is_singular(&old->anon_vma_chain)) 1444 return anon_vma; 1445 } 1446 return NULL; 1447 } 1448 1449 /* 1450 * find_mergeable_anon_vma is used by anon_vma_prepare, to check 1451 * neighbouring vmas for a suitable anon_vma, before it goes off 1452 * to allocate a new anon_vma. It checks because a repetitive 1453 * sequence of mprotects and faults may otherwise lead to distinct 1454 * anon_vmas being allocated, preventing vma merge in subsequent 1455 * mprotect. 1456 */ 1457 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) 1458 { 1459 struct anon_vma *anon_vma = NULL; 1460 struct vm_area_struct *prev, *next; 1461 VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_end); 1462 1463 /* Try next first. */ 1464 next = vma_iter_load(&vmi); 1465 if (next) { 1466 anon_vma = reusable_anon_vma(next, vma, next); 1467 if (anon_vma) 1468 return anon_vma; 1469 } 1470 1471 prev = vma_prev(&vmi); 1472 VM_BUG_ON_VMA(prev != vma, vma); 1473 prev = vma_prev(&vmi); 1474 /* Try prev next. */ 1475 if (prev) 1476 anon_vma = reusable_anon_vma(prev, prev, vma); 1477 1478 /* 1479 * We might reach here with anon_vma == NULL if we can't find 1480 * any reusable anon_vma. 1481 * There's no absolute need to look only at touching neighbours: 1482 * we could search further afield for "compatible" anon_vmas. 1483 * But it would probably just be a waste of time searching, 1484 * or lead to too many vmas hanging off the same anon_vma. 1485 * We're trying to allow mprotect remerging later on, 1486 * not trying to minimize memory used for anon_vmas. 1487 */ 1488 return anon_vma; 1489 } 1490 1491 static bool vm_ops_needs_writenotify(const struct vm_operations_struct *vm_ops) 1492 { 1493 return vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite); 1494 } 1495 1496 static bool vma_is_shared_writable(struct vm_area_struct *vma) 1497 { 1498 return (vma->vm_flags & (VM_WRITE | VM_SHARED)) == 1499 (VM_WRITE | VM_SHARED); 1500 } 1501 1502 static bool vma_fs_can_writeback(struct vm_area_struct *vma) 1503 { 1504 /* No managed pages to writeback. */ 1505 if (vma->vm_flags & VM_PFNMAP) 1506 return false; 1507 1508 return vma->vm_file && vma->vm_file->f_mapping && 1509 mapping_can_writeback(vma->vm_file->f_mapping); 1510 } 1511 1512 /* 1513 * Does this VMA require the underlying folios to have their dirty state 1514 * tracked? 1515 */ 1516 bool vma_needs_dirty_tracking(struct vm_area_struct *vma) 1517 { 1518 /* Only shared, writable VMAs require dirty tracking. */ 1519 if (!vma_is_shared_writable(vma)) 1520 return false; 1521 1522 /* Does the filesystem need to be notified? */ 1523 if (vm_ops_needs_writenotify(vma->vm_ops)) 1524 return true; 1525 1526 /* 1527 * Even if the filesystem doesn't indicate a need for writenotify, if it 1528 * can writeback, dirty tracking is still required. 1529 */ 1530 return vma_fs_can_writeback(vma); 1531 } 1532 1533 /* 1534 * Some shared mappings will want the pages marked read-only 1535 * to track write events. If so, we'll downgrade vm_page_prot 1536 * to the private version (using protection_map[] without the 1537 * VM_SHARED bit). 1538 */ 1539 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) 1540 { 1541 /* If it was private or non-writable, the write bit is already clear */ 1542 if (!vma_is_shared_writable(vma)) 1543 return false; 1544 1545 /* The backer wishes to know when pages are first written to? */ 1546 if (vm_ops_needs_writenotify(vma->vm_ops)) 1547 return true; 1548 1549 /* The open routine did something to the protections that pgprot_modify 1550 * won't preserve? */ 1551 if (pgprot_val(vm_page_prot) != 1552 pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags))) 1553 return false; 1554 1555 /* 1556 * Do we need to track softdirty? hugetlb does not support softdirty 1557 * tracking yet. 1558 */ 1559 if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma)) 1560 return true; 1561 1562 /* Do we need write faults for uffd-wp tracking? */ 1563 if (userfaultfd_wp(vma)) 1564 return true; 1565 1566 /* Can the mapping track the dirty pages? */ 1567 return vma_fs_can_writeback(vma); 1568 } 1569 1570 unsigned long count_vma_pages_range(struct mm_struct *mm, 1571 unsigned long addr, unsigned long end) 1572 { 1573 VMA_ITERATOR(vmi, mm, addr); 1574 struct vm_area_struct *vma; 1575 unsigned long nr_pages = 0; 1576 1577 for_each_vma_range(vmi, vma, end) { 1578 unsigned long vm_start = max(addr, vma->vm_start); 1579 unsigned long vm_end = min(end, vma->vm_end); 1580 1581 nr_pages += PHYS_PFN(vm_end - vm_start); 1582 } 1583 1584 return nr_pages; 1585 } 1586 1587 static DEFINE_MUTEX(mm_all_locks_mutex); 1588 1589 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma) 1590 { 1591 if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 1592 /* 1593 * The LSB of head.next can't change from under us 1594 * because we hold the mm_all_locks_mutex. 1595 */ 1596 down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock); 1597 /* 1598 * We can safely modify head.next after taking the 1599 * anon_vma->root->rwsem. If some other vma in this mm shares 1600 * the same anon_vma we won't take it again. 1601 * 1602 * No need of atomic instructions here, head.next 1603 * can't change from under us thanks to the 1604 * anon_vma->root->rwsem. 1605 */ 1606 if (__test_and_set_bit(0, (unsigned long *) 1607 &anon_vma->root->rb_root.rb_root.rb_node)) 1608 BUG(); 1609 } 1610 } 1611 1612 static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) 1613 { 1614 if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 1615 /* 1616 * AS_MM_ALL_LOCKS can't change from under us because 1617 * we hold the mm_all_locks_mutex. 1618 * 1619 * Operations on ->flags have to be atomic because 1620 * even if AS_MM_ALL_LOCKS is stable thanks to the 1621 * mm_all_locks_mutex, there may be other cpus 1622 * changing other bitflags in parallel to us. 1623 */ 1624 if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags)) 1625 BUG(); 1626 down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock); 1627 } 1628 } 1629 1630 /* 1631 * This operation locks against the VM for all pte/vma/mm related 1632 * operations that could ever happen on a certain mm. This includes 1633 * vmtruncate, try_to_unmap, and all page faults. 1634 * 1635 * The caller must take the mmap_lock in write mode before calling 1636 * mm_take_all_locks(). The caller isn't allowed to release the 1637 * mmap_lock until mm_drop_all_locks() returns. 1638 * 1639 * mmap_lock in write mode is required in order to block all operations 1640 * that could modify pagetables and free pages without need of 1641 * altering the vma layout. It's also needed in write mode to avoid new 1642 * anon_vmas to be associated with existing vmas. 1643 * 1644 * A single task can't take more than one mm_take_all_locks() in a row 1645 * or it would deadlock. 1646 * 1647 * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in 1648 * mapping->flags avoid to take the same lock twice, if more than one 1649 * vma in this mm is backed by the same anon_vma or address_space. 1650 * 1651 * We take locks in following order, accordingly to comment at beginning 1652 * of mm/rmap.c: 1653 * - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for 1654 * hugetlb mapping); 1655 * - all vmas marked locked 1656 * - all i_mmap_rwsem locks; 1657 * - all anon_vma->rwseml 1658 * 1659 * We can take all locks within these types randomly because the VM code 1660 * doesn't nest them and we protected from parallel mm_take_all_locks() by 1661 * mm_all_locks_mutex. 1662 * 1663 * mm_take_all_locks() and mm_drop_all_locks are expensive operations 1664 * that may have to take thousand of locks. 1665 * 1666 * mm_take_all_locks() can fail if it's interrupted by signals. 1667 */ 1668 int mm_take_all_locks(struct mm_struct *mm) 1669 { 1670 struct vm_area_struct *vma; 1671 struct anon_vma_chain *avc; 1672 VMA_ITERATOR(vmi, mm, 0); 1673 1674 mmap_assert_write_locked(mm); 1675 1676 mutex_lock(&mm_all_locks_mutex); 1677 1678 /* 1679 * vma_start_write() does not have a complement in mm_drop_all_locks() 1680 * because vma_start_write() is always asymmetrical; it marks a VMA as 1681 * being written to until mmap_write_unlock() or mmap_write_downgrade() 1682 * is reached. 1683 */ 1684 for_each_vma(vmi, vma) { 1685 if (signal_pending(current)) 1686 goto out_unlock; 1687 vma_start_write(vma); 1688 } 1689 1690 vma_iter_init(&vmi, mm, 0); 1691 for_each_vma(vmi, vma) { 1692 if (signal_pending(current)) 1693 goto out_unlock; 1694 if (vma->vm_file && vma->vm_file->f_mapping && 1695 is_vm_hugetlb_page(vma)) 1696 vm_lock_mapping(mm, vma->vm_file->f_mapping); 1697 } 1698 1699 vma_iter_init(&vmi, mm, 0); 1700 for_each_vma(vmi, vma) { 1701 if (signal_pending(current)) 1702 goto out_unlock; 1703 if (vma->vm_file && vma->vm_file->f_mapping && 1704 !is_vm_hugetlb_page(vma)) 1705 vm_lock_mapping(mm, vma->vm_file->f_mapping); 1706 } 1707 1708 vma_iter_init(&vmi, mm, 0); 1709 for_each_vma(vmi, vma) { 1710 if (signal_pending(current)) 1711 goto out_unlock; 1712 if (vma->anon_vma) 1713 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 1714 vm_lock_anon_vma(mm, avc->anon_vma); 1715 } 1716 1717 return 0; 1718 1719 out_unlock: 1720 mm_drop_all_locks(mm); 1721 return -EINTR; 1722 } 1723 1724 static void vm_unlock_anon_vma(struct anon_vma *anon_vma) 1725 { 1726 if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 1727 /* 1728 * The LSB of head.next can't change to 0 from under 1729 * us because we hold the mm_all_locks_mutex. 1730 * 1731 * We must however clear the bitflag before unlocking 1732 * the vma so the users using the anon_vma->rb_root will 1733 * never see our bitflag. 1734 * 1735 * No need of atomic instructions here, head.next 1736 * can't change from under us until we release the 1737 * anon_vma->root->rwsem. 1738 */ 1739 if (!__test_and_clear_bit(0, (unsigned long *) 1740 &anon_vma->root->rb_root.rb_root.rb_node)) 1741 BUG(); 1742 anon_vma_unlock_write(anon_vma); 1743 } 1744 } 1745 1746 static void vm_unlock_mapping(struct address_space *mapping) 1747 { 1748 if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 1749 /* 1750 * AS_MM_ALL_LOCKS can't change to 0 from under us 1751 * because we hold the mm_all_locks_mutex. 1752 */ 1753 i_mmap_unlock_write(mapping); 1754 if (!test_and_clear_bit(AS_MM_ALL_LOCKS, 1755 &mapping->flags)) 1756 BUG(); 1757 } 1758 } 1759 1760 /* 1761 * The mmap_lock cannot be released by the caller until 1762 * mm_drop_all_locks() returns. 1763 */ 1764 void mm_drop_all_locks(struct mm_struct *mm) 1765 { 1766 struct vm_area_struct *vma; 1767 struct anon_vma_chain *avc; 1768 VMA_ITERATOR(vmi, mm, 0); 1769 1770 mmap_assert_write_locked(mm); 1771 BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); 1772 1773 for_each_vma(vmi, vma) { 1774 if (vma->anon_vma) 1775 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 1776 vm_unlock_anon_vma(avc->anon_vma); 1777 if (vma->vm_file && vma->vm_file->f_mapping) 1778 vm_unlock_mapping(vma->vm_file->f_mapping); 1779 } 1780 1781 mutex_unlock(&mm_all_locks_mutex); 1782 } 1783