1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 /* 4 * VMA-specific functions. 5 */ 6 7 #include "vma_internal.h" 8 #include "vma.h" 9 10 /* 11 * If the vma has a ->close operation then the driver probably needs to release 12 * per-vma resources, so we don't attempt to merge those if the caller indicates 13 * the current vma may be removed as part of the merge. 14 */ 15 static inline bool is_mergeable_vma(struct vm_area_struct *vma, 16 struct file *file, unsigned long vm_flags, 17 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 18 struct anon_vma_name *anon_name, bool may_remove_vma) 19 { 20 /* 21 * VM_SOFTDIRTY should not prevent from VMA merging, if we 22 * match the flags but dirty bit -- the caller should mark 23 * merged VMA as dirty. If dirty bit won't be excluded from 24 * comparison, we increase pressure on the memory system forcing 25 * the kernel to generate new VMAs when old one could be 26 * extended instead. 27 */ 28 if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY) 29 return false; 30 if (vma->vm_file != file) 31 return false; 32 if (may_remove_vma && vma->vm_ops && vma->vm_ops->close) 33 return false; 34 if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx)) 35 return false; 36 if (!anon_vma_name_eq(anon_vma_name(vma), anon_name)) 37 return false; 38 return true; 39 } 40 41 static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1, 42 struct anon_vma *anon_vma2, struct vm_area_struct *vma) 43 { 44 /* 45 * The list_is_singular() test is to avoid merging VMA cloned from 46 * parents. This can improve scalability caused by anon_vma lock. 47 */ 48 if ((!anon_vma1 || !anon_vma2) && (!vma || 49 list_is_singular(&vma->anon_vma_chain))) 50 return true; 51 return anon_vma1 == anon_vma2; 52 } 53 54 /* 55 * init_multi_vma_prep() - Initializer for struct vma_prepare 56 * @vp: The vma_prepare struct 57 * @vma: The vma that will be altered once locked 58 * @next: The next vma if it is to be adjusted 59 * @remove: The first vma to be removed 60 * @remove2: The second vma to be removed 61 */ 62 static void init_multi_vma_prep(struct vma_prepare *vp, 63 struct vm_area_struct *vma, 64 struct vm_area_struct *next, 65 struct vm_area_struct *remove, 66 struct vm_area_struct *remove2) 67 { 68 memset(vp, 0, sizeof(struct vma_prepare)); 69 vp->vma = vma; 70 vp->anon_vma = vma->anon_vma; 71 vp->remove = remove; 72 vp->remove2 = remove2; 73 vp->adj_next = next; 74 if (!vp->anon_vma && next) 75 vp->anon_vma = next->anon_vma; 76 77 vp->file = vma->vm_file; 78 if (vp->file) 79 vp->mapping = vma->vm_file->f_mapping; 80 81 } 82 83 /* 84 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 85 * in front of (at a lower virtual address and file offset than) the vma. 86 * 87 * We cannot merge two vmas if they have differently assigned (non-NULL) 88 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 89 * 90 * We don't check here for the merged mmap wrapping around the end of pagecache 91 * indices (16TB on ia32) because do_mmap() does not permit mmap's which 92 * wrap, nor mmaps which cover the final page at index -1UL. 93 * 94 * We assume the vma may be removed as part of the merge. 95 */ 96 bool 97 can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, 98 struct anon_vma *anon_vma, struct file *file, 99 pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 100 struct anon_vma_name *anon_name) 101 { 102 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) && 103 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 104 if (vma->vm_pgoff == vm_pgoff) 105 return true; 106 } 107 return false; 108 } 109 110 /* 111 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 112 * beyond (at a higher virtual address and file offset than) the vma. 113 * 114 * We cannot merge two vmas if they have differently assigned (non-NULL) 115 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 116 * 117 * We assume that vma is not removed as part of the merge. 118 */ 119 bool 120 can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, 121 struct anon_vma *anon_vma, struct file *file, 122 pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 123 struct anon_vma_name *anon_name) 124 { 125 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) && 126 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 127 pgoff_t vm_pglen; 128 129 vm_pglen = vma_pages(vma); 130 if (vma->vm_pgoff + vm_pglen == vm_pgoff) 131 return true; 132 } 133 return false; 134 } 135 136 /* 137 * Close a vm structure and free it. 138 */ 139 void remove_vma(struct vm_area_struct *vma, bool unreachable) 140 { 141 might_sleep(); 142 if (vma->vm_ops && vma->vm_ops->close) 143 vma->vm_ops->close(vma); 144 if (vma->vm_file) 145 fput(vma->vm_file); 146 mpol_put(vma_policy(vma)); 147 if (unreachable) 148 __vm_area_free(vma); 149 else 150 vm_area_free(vma); 151 } 152 153 /* 154 * Get rid of page table information in the indicated region. 155 * 156 * Called with the mm semaphore held. 157 */ 158 void unmap_region(struct mm_struct *mm, struct ma_state *mas, 159 struct vm_area_struct *vma, struct vm_area_struct *prev, 160 struct vm_area_struct *next, unsigned long start, 161 unsigned long end, unsigned long tree_end, bool mm_wr_locked) 162 { 163 struct mmu_gather tlb; 164 unsigned long mt_start = mas->index; 165 166 lru_add_drain(); 167 tlb_gather_mmu(&tlb, mm); 168 update_hiwater_rss(mm); 169 unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked); 170 mas_set(mas, mt_start); 171 free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, 172 next ? next->vm_start : USER_PGTABLES_CEILING, 173 mm_wr_locked); 174 tlb_finish_mmu(&tlb); 175 } 176 177 /* 178 * __split_vma() bypasses sysctl_max_map_count checking. We use this where it 179 * has already been checked or doesn't make sense to fail. 180 * VMA Iterator will point to the original VMA. 181 */ 182 static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, 183 unsigned long addr, int new_below) 184 { 185 struct vma_prepare vp; 186 struct vm_area_struct *new; 187 int err; 188 189 WARN_ON(vma->vm_start >= addr); 190 WARN_ON(vma->vm_end <= addr); 191 192 if (vma->vm_ops && vma->vm_ops->may_split) { 193 err = vma->vm_ops->may_split(vma, addr); 194 if (err) 195 return err; 196 } 197 198 new = vm_area_dup(vma); 199 if (!new) 200 return -ENOMEM; 201 202 if (new_below) { 203 new->vm_end = addr; 204 } else { 205 new->vm_start = addr; 206 new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); 207 } 208 209 err = -ENOMEM; 210 vma_iter_config(vmi, new->vm_start, new->vm_end); 211 if (vma_iter_prealloc(vmi, new)) 212 goto out_free_vma; 213 214 err = vma_dup_policy(vma, new); 215 if (err) 216 goto out_free_vmi; 217 218 err = anon_vma_clone(new, vma); 219 if (err) 220 goto out_free_mpol; 221 222 if (new->vm_file) 223 get_file(new->vm_file); 224 225 if (new->vm_ops && new->vm_ops->open) 226 new->vm_ops->open(new); 227 228 vma_start_write(vma); 229 vma_start_write(new); 230 231 init_vma_prep(&vp, vma); 232 vp.insert = new; 233 vma_prepare(&vp); 234 vma_adjust_trans_huge(vma, vma->vm_start, addr, 0); 235 236 if (new_below) { 237 vma->vm_start = addr; 238 vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; 239 } else { 240 vma->vm_end = addr; 241 } 242 243 /* vma_complete stores the new vma */ 244 vma_complete(&vp, vmi, vma->vm_mm); 245 246 /* Success. */ 247 if (new_below) 248 vma_next(vmi); 249 else 250 vma_prev(vmi); 251 252 return 0; 253 254 out_free_mpol: 255 mpol_put(vma_policy(new)); 256 out_free_vmi: 257 vma_iter_free(vmi); 258 out_free_vma: 259 vm_area_free(new); 260 return err; 261 } 262 263 /* 264 * Split a vma into two pieces at address 'addr', a new vma is allocated 265 * either for the first part or the tail. 266 */ 267 static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, 268 unsigned long addr, int new_below) 269 { 270 if (vma->vm_mm->map_count >= sysctl_max_map_count) 271 return -ENOMEM; 272 273 return __split_vma(vmi, vma, addr, new_below); 274 } 275 276 /* 277 * Ok - we have the memory areas we should free on a maple tree so release them, 278 * and do the vma updates. 279 * 280 * Called with the mm semaphore held. 281 */ 282 static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas) 283 { 284 unsigned long nr_accounted = 0; 285 struct vm_area_struct *vma; 286 287 /* Update high watermark before we lower total_vm */ 288 update_hiwater_vm(mm); 289 mas_for_each(mas, vma, ULONG_MAX) { 290 long nrpages = vma_pages(vma); 291 292 if (vma->vm_flags & VM_ACCOUNT) 293 nr_accounted += nrpages; 294 vm_stat_account(mm, vma->vm_flags, -nrpages); 295 remove_vma(vma, false); 296 } 297 vm_unacct_memory(nr_accounted); 298 } 299 300 /* 301 * init_vma_prep() - Initializer wrapper for vma_prepare struct 302 * @vp: The vma_prepare struct 303 * @vma: The vma that will be altered once locked 304 */ 305 void init_vma_prep(struct vma_prepare *vp, 306 struct vm_area_struct *vma) 307 { 308 init_multi_vma_prep(vp, vma, NULL, NULL, NULL); 309 } 310 311 /* 312 * Requires inode->i_mapping->i_mmap_rwsem 313 */ 314 static void __remove_shared_vm_struct(struct vm_area_struct *vma, 315 struct address_space *mapping) 316 { 317 if (vma_is_shared_maywrite(vma)) 318 mapping_unmap_writable(mapping); 319 320 flush_dcache_mmap_lock(mapping); 321 vma_interval_tree_remove(vma, &mapping->i_mmap); 322 flush_dcache_mmap_unlock(mapping); 323 } 324 325 /* 326 * vma has some anon_vma assigned, and is already inserted on that 327 * anon_vma's interval trees. 328 * 329 * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the 330 * vma must be removed from the anon_vma's interval trees using 331 * anon_vma_interval_tree_pre_update_vma(). 332 * 333 * After the update, the vma will be reinserted using 334 * anon_vma_interval_tree_post_update_vma(). 335 * 336 * The entire update must be protected by exclusive mmap_lock and by 337 * the root anon_vma's mutex. 338 */ 339 void 340 anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) 341 { 342 struct anon_vma_chain *avc; 343 344 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 345 anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); 346 } 347 348 void 349 anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) 350 { 351 struct anon_vma_chain *avc; 352 353 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 354 anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); 355 } 356 357 static void __vma_link_file(struct vm_area_struct *vma, 358 struct address_space *mapping) 359 { 360 if (vma_is_shared_maywrite(vma)) 361 mapping_allow_writable(mapping); 362 363 flush_dcache_mmap_lock(mapping); 364 vma_interval_tree_insert(vma, &mapping->i_mmap); 365 flush_dcache_mmap_unlock(mapping); 366 } 367 368 /* 369 * vma_prepare() - Helper function for handling locking VMAs prior to altering 370 * @vp: The initialized vma_prepare struct 371 */ 372 void vma_prepare(struct vma_prepare *vp) 373 { 374 if (vp->file) { 375 uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end); 376 377 if (vp->adj_next) 378 uprobe_munmap(vp->adj_next, vp->adj_next->vm_start, 379 vp->adj_next->vm_end); 380 381 i_mmap_lock_write(vp->mapping); 382 if (vp->insert && vp->insert->vm_file) { 383 /* 384 * Put into interval tree now, so instantiated pages 385 * are visible to arm/parisc __flush_dcache_page 386 * throughout; but we cannot insert into address 387 * space until vma start or end is updated. 388 */ 389 __vma_link_file(vp->insert, 390 vp->insert->vm_file->f_mapping); 391 } 392 } 393 394 if (vp->anon_vma) { 395 anon_vma_lock_write(vp->anon_vma); 396 anon_vma_interval_tree_pre_update_vma(vp->vma); 397 if (vp->adj_next) 398 anon_vma_interval_tree_pre_update_vma(vp->adj_next); 399 } 400 401 if (vp->file) { 402 flush_dcache_mmap_lock(vp->mapping); 403 vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap); 404 if (vp->adj_next) 405 vma_interval_tree_remove(vp->adj_next, 406 &vp->mapping->i_mmap); 407 } 408 409 } 410 411 /* 412 * dup_anon_vma() - Helper function to duplicate anon_vma 413 * @dst: The destination VMA 414 * @src: The source VMA 415 * @dup: Pointer to the destination VMA when successful. 416 * 417 * Returns: 0 on success. 418 */ 419 static int dup_anon_vma(struct vm_area_struct *dst, 420 struct vm_area_struct *src, struct vm_area_struct **dup) 421 { 422 /* 423 * Easily overlooked: when mprotect shifts the boundary, make sure the 424 * expanding vma has anon_vma set if the shrinking vma had, to cover any 425 * anon pages imported. 426 */ 427 if (src->anon_vma && !dst->anon_vma) { 428 int ret; 429 430 vma_assert_write_locked(dst); 431 dst->anon_vma = src->anon_vma; 432 ret = anon_vma_clone(dst, src); 433 if (ret) 434 return ret; 435 436 *dup = dst; 437 } 438 439 return 0; 440 } 441 442 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 443 void validate_mm(struct mm_struct *mm) 444 { 445 int bug = 0; 446 int i = 0; 447 struct vm_area_struct *vma; 448 VMA_ITERATOR(vmi, mm, 0); 449 450 mt_validate(&mm->mm_mt); 451 for_each_vma(vmi, vma) { 452 #ifdef CONFIG_DEBUG_VM_RB 453 struct anon_vma *anon_vma = vma->anon_vma; 454 struct anon_vma_chain *avc; 455 #endif 456 unsigned long vmi_start, vmi_end; 457 bool warn = 0; 458 459 vmi_start = vma_iter_addr(&vmi); 460 vmi_end = vma_iter_end(&vmi); 461 if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm)) 462 warn = 1; 463 464 if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm)) 465 warn = 1; 466 467 if (warn) { 468 pr_emerg("issue in %s\n", current->comm); 469 dump_stack(); 470 dump_vma(vma); 471 pr_emerg("tree range: %px start %lx end %lx\n", vma, 472 vmi_start, vmi_end - 1); 473 vma_iter_dump_tree(&vmi); 474 } 475 476 #ifdef CONFIG_DEBUG_VM_RB 477 if (anon_vma) { 478 anon_vma_lock_read(anon_vma); 479 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 480 anon_vma_interval_tree_verify(avc); 481 anon_vma_unlock_read(anon_vma); 482 } 483 #endif 484 i++; 485 } 486 if (i != mm->map_count) { 487 pr_emerg("map_count %d vma iterator %d\n", mm->map_count, i); 488 bug = 1; 489 } 490 VM_BUG_ON_MM(bug, mm); 491 } 492 #endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ 493 494 /* 495 * vma_expand - Expand an existing VMA 496 * 497 * @vmi: The vma iterator 498 * @vma: The vma to expand 499 * @start: The start of the vma 500 * @end: The exclusive end of the vma 501 * @pgoff: The page offset of vma 502 * @next: The current of next vma. 503 * 504 * Expand @vma to @start and @end. Can expand off the start and end. Will 505 * expand over @next if it's different from @vma and @end == @next->vm_end. 506 * Checking if the @vma can expand and merge with @next needs to be handled by 507 * the caller. 508 * 509 * Returns: 0 on success 510 */ 511 int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma, 512 unsigned long start, unsigned long end, pgoff_t pgoff, 513 struct vm_area_struct *next) 514 { 515 struct vm_area_struct *anon_dup = NULL; 516 bool remove_next = false; 517 struct vma_prepare vp; 518 519 vma_start_write(vma); 520 if (next && (vma != next) && (end == next->vm_end)) { 521 int ret; 522 523 remove_next = true; 524 vma_start_write(next); 525 ret = dup_anon_vma(vma, next, &anon_dup); 526 if (ret) 527 return ret; 528 } 529 530 init_multi_vma_prep(&vp, vma, NULL, remove_next ? next : NULL, NULL); 531 /* Not merging but overwriting any part of next is not handled. */ 532 VM_WARN_ON(next && !vp.remove && 533 next != vma && end > next->vm_start); 534 /* Only handles expanding */ 535 VM_WARN_ON(vma->vm_start < start || vma->vm_end > end); 536 537 /* Note: vma iterator must be pointing to 'start' */ 538 vma_iter_config(vmi, start, end); 539 if (vma_iter_prealloc(vmi, vma)) 540 goto nomem; 541 542 vma_prepare(&vp); 543 vma_adjust_trans_huge(vma, start, end, 0); 544 vma_set_range(vma, start, end, pgoff); 545 vma_iter_store(vmi, vma); 546 547 vma_complete(&vp, vmi, vma->vm_mm); 548 return 0; 549 550 nomem: 551 if (anon_dup) 552 unlink_anon_vmas(anon_dup); 553 return -ENOMEM; 554 } 555 556 /* 557 * vma_shrink() - Reduce an existing VMAs memory area 558 * @vmi: The vma iterator 559 * @vma: The VMA to modify 560 * @start: The new start 561 * @end: The new end 562 * 563 * Returns: 0 on success, -ENOMEM otherwise 564 */ 565 int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, 566 unsigned long start, unsigned long end, pgoff_t pgoff) 567 { 568 struct vma_prepare vp; 569 570 WARN_ON((vma->vm_start != start) && (vma->vm_end != end)); 571 572 if (vma->vm_start < start) 573 vma_iter_config(vmi, vma->vm_start, start); 574 else 575 vma_iter_config(vmi, end, vma->vm_end); 576 577 if (vma_iter_prealloc(vmi, NULL)) 578 return -ENOMEM; 579 580 vma_start_write(vma); 581 582 init_vma_prep(&vp, vma); 583 vma_prepare(&vp); 584 vma_adjust_trans_huge(vma, start, end, 0); 585 586 vma_iter_clear(vmi); 587 vma_set_range(vma, start, end, pgoff); 588 vma_complete(&vp, vmi, vma->vm_mm); 589 return 0; 590 } 591 592 /* 593 * vma_complete- Helper function for handling the unlocking after altering VMAs, 594 * or for inserting a VMA. 595 * 596 * @vp: The vma_prepare struct 597 * @vmi: The vma iterator 598 * @mm: The mm_struct 599 */ 600 void vma_complete(struct vma_prepare *vp, 601 struct vma_iterator *vmi, struct mm_struct *mm) 602 { 603 if (vp->file) { 604 if (vp->adj_next) 605 vma_interval_tree_insert(vp->adj_next, 606 &vp->mapping->i_mmap); 607 vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap); 608 flush_dcache_mmap_unlock(vp->mapping); 609 } 610 611 if (vp->remove && vp->file) { 612 __remove_shared_vm_struct(vp->remove, vp->mapping); 613 if (vp->remove2) 614 __remove_shared_vm_struct(vp->remove2, vp->mapping); 615 } else if (vp->insert) { 616 /* 617 * split_vma has split insert from vma, and needs 618 * us to insert it before dropping the locks 619 * (it may either follow vma or precede it). 620 */ 621 vma_iter_store(vmi, vp->insert); 622 mm->map_count++; 623 } 624 625 if (vp->anon_vma) { 626 anon_vma_interval_tree_post_update_vma(vp->vma); 627 if (vp->adj_next) 628 anon_vma_interval_tree_post_update_vma(vp->adj_next); 629 anon_vma_unlock_write(vp->anon_vma); 630 } 631 632 if (vp->file) { 633 i_mmap_unlock_write(vp->mapping); 634 uprobe_mmap(vp->vma); 635 636 if (vp->adj_next) 637 uprobe_mmap(vp->adj_next); 638 } 639 640 if (vp->remove) { 641 again: 642 vma_mark_detached(vp->remove, true); 643 if (vp->file) { 644 uprobe_munmap(vp->remove, vp->remove->vm_start, 645 vp->remove->vm_end); 646 fput(vp->file); 647 } 648 if (vp->remove->anon_vma) 649 anon_vma_merge(vp->vma, vp->remove); 650 mm->map_count--; 651 mpol_put(vma_policy(vp->remove)); 652 if (!vp->remove2) 653 WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end); 654 vm_area_free(vp->remove); 655 656 /* 657 * In mprotect's case 6 (see comments on vma_merge), 658 * we are removing both mid and next vmas 659 */ 660 if (vp->remove2) { 661 vp->remove = vp->remove2; 662 vp->remove2 = NULL; 663 goto again; 664 } 665 } 666 if (vp->insert && vp->file) 667 uprobe_mmap(vp->insert); 668 validate_mm(mm); 669 } 670 671 /* 672 * abort_munmap_vmas - Undo any munmap work and free resources 673 * 674 * Reattach any detached vmas and free up the maple tree used to track the vmas. 675 */ 676 static inline void abort_munmap_vmas(struct ma_state *mas_detach) 677 { 678 struct vm_area_struct *vma; 679 680 mas_set(mas_detach, 0); 681 mas_for_each(mas_detach, vma, ULONG_MAX) 682 vma_mark_detached(vma, false); 683 684 __mt_destroy(mas_detach->tree); 685 } 686 687 /* 688 * vmi_complete_munmap_vmas() - Finish the munmap() operation 689 * @vmi: The vma iterator 690 * @vma: The first vma to be munmapped 691 * @mm: The mm struct 692 * @start: The start address 693 * @end: The end address 694 * @unlock: Unlock the mm or not 695 * @mas_detach: them maple state of the detached vma maple tree 696 * @locked_vm: The locked_vm count in the detached vmas 697 * 698 * This function updates the mm_struct, unmaps the region, frees the resources 699 * used for the munmap() and may downgrade the lock - if requested. Everything 700 * needed to be done once the vma maple tree is updated. 701 */ 702 static void 703 vmi_complete_munmap_vmas(struct vma_iterator *vmi, struct vm_area_struct *vma, 704 struct mm_struct *mm, unsigned long start, unsigned long end, 705 bool unlock, struct ma_state *mas_detach, 706 unsigned long locked_vm) 707 { 708 struct vm_area_struct *prev, *next; 709 int count; 710 711 count = mas_detach->index + 1; 712 mm->map_count -= count; 713 mm->locked_vm -= locked_vm; 714 if (unlock) 715 mmap_write_downgrade(mm); 716 717 prev = vma_iter_prev_range(vmi); 718 next = vma_next(vmi); 719 if (next) 720 vma_iter_prev_range(vmi); 721 722 /* 723 * We can free page tables without write-locking mmap_lock because VMAs 724 * were isolated before we downgraded mmap_lock. 725 */ 726 mas_set(mas_detach, 1); 727 unmap_region(mm, mas_detach, vma, prev, next, start, end, count, 728 !unlock); 729 /* Statistics and freeing VMAs */ 730 mas_set(mas_detach, 0); 731 remove_mt(mm, mas_detach); 732 validate_mm(mm); 733 if (unlock) 734 mmap_read_unlock(mm); 735 736 __mt_destroy(mas_detach->tree); 737 } 738 739 /* 740 * vmi_gather_munmap_vmas() - Put all VMAs within a range into a maple tree 741 * for removal at a later date. Handles splitting first and last if necessary 742 * and marking the vmas as isolated. 743 * 744 * @vmi: The vma iterator 745 * @vma: The starting vm_area_struct 746 * @mm: The mm_struct 747 * @start: The aligned start address to munmap. 748 * @end: The aligned end address to munmap. 749 * @uf: The userfaultfd list_head 750 * @mas_detach: The maple state tracking the detached tree 751 * @locked_vm: a pointer to store the VM_LOCKED pages count. 752 * 753 * Return: 0 on success, -EPERM on mseal vmas, -ENOMEM otherwise 754 */ 755 static int 756 vmi_gather_munmap_vmas(struct vma_iterator *vmi, struct vm_area_struct *vma, 757 struct mm_struct *mm, unsigned long start, 758 unsigned long end, struct list_head *uf, 759 struct ma_state *mas_detach, unsigned long *locked_vm) 760 { 761 struct vm_area_struct *next = NULL; 762 int count = 0; 763 int error = -ENOMEM; 764 765 /* 766 * If we need to split any vma, do it now to save pain later. 767 * 768 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially 769 * unmapped vm_area_struct will remain in use: so lower split_vma 770 * places tmp vma above, and higher split_vma places tmp vma below. 771 */ 772 773 /* Does it split the first one? */ 774 if (start > vma->vm_start) { 775 776 /* 777 * Make sure that map_count on return from munmap() will 778 * not exceed its limit; but let map_count go just above 779 * its limit temporarily, to help free resources as expected. 780 */ 781 if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count) 782 goto map_count_exceeded; 783 784 /* Don't bother splitting the VMA if we can't unmap it anyway */ 785 if (!can_modify_vma(vma)) { 786 error = -EPERM; 787 goto start_split_failed; 788 } 789 790 if (__split_vma(vmi, vma, start, 1)) 791 goto start_split_failed; 792 } 793 794 /* 795 * Detach a range of VMAs from the mm. Using next as a temp variable as 796 * it is always overwritten. 797 */ 798 next = vma; 799 do { 800 if (!can_modify_vma(next)) { 801 error = -EPERM; 802 goto modify_vma_failed; 803 } 804 805 /* Does it split the end? */ 806 if (next->vm_end > end) { 807 if (__split_vma(vmi, next, end, 0)) 808 goto end_split_failed; 809 } 810 vma_start_write(next); 811 mas_set(mas_detach, count++); 812 if (mas_store_gfp(mas_detach, next, GFP_KERNEL)) 813 goto munmap_gather_failed; 814 815 vma_mark_detached(next, true); 816 if (next->vm_flags & VM_LOCKED) 817 *locked_vm += vma_pages(next); 818 819 if (unlikely(uf)) { 820 /* 821 * If userfaultfd_unmap_prep returns an error the vmas 822 * will remain split, but userland will get a 823 * highly unexpected error anyway. This is no 824 * different than the case where the first of the two 825 * __split_vma fails, but we don't undo the first 826 * split, despite we could. This is unlikely enough 827 * failure that it's not worth optimizing it for. 828 */ 829 if (userfaultfd_unmap_prep(next, start, end, uf)) 830 goto userfaultfd_error; 831 } 832 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 833 BUG_ON(next->vm_start < start); 834 BUG_ON(next->vm_start > end); 835 #endif 836 } for_each_vma_range(*vmi, next, end); 837 838 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 839 /* Make sure no VMAs are about to be lost. */ 840 { 841 MA_STATE(test, mas_detach->tree, 0, 0); 842 struct vm_area_struct *vma_mas, *vma_test; 843 int test_count = 0; 844 845 vma_iter_set(vmi, start); 846 rcu_read_lock(); 847 vma_test = mas_find(&test, count - 1); 848 for_each_vma_range(*vmi, vma_mas, end) { 849 BUG_ON(vma_mas != vma_test); 850 test_count++; 851 vma_test = mas_next(&test, count - 1); 852 } 853 rcu_read_unlock(); 854 BUG_ON(count != test_count); 855 } 856 #endif 857 858 while (vma_iter_addr(vmi) > start) 859 vma_iter_prev_range(vmi); 860 861 return 0; 862 863 userfaultfd_error: 864 munmap_gather_failed: 865 end_split_failed: 866 modify_vma_failed: 867 abort_munmap_vmas(mas_detach); 868 start_split_failed: 869 map_count_exceeded: 870 return error; 871 } 872 873 /* 874 * do_vmi_align_munmap() - munmap the aligned region from @start to @end. 875 * @vmi: The vma iterator 876 * @vma: The starting vm_area_struct 877 * @mm: The mm_struct 878 * @start: The aligned start address to munmap. 879 * @end: The aligned end address to munmap. 880 * @uf: The userfaultfd list_head 881 * @unlock: Set to true to drop the mmap_lock. unlocking only happens on 882 * success. 883 * 884 * Return: 0 on success and drops the lock if so directed, error and leaves the 885 * lock held otherwise. 886 */ 887 int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 888 struct mm_struct *mm, unsigned long start, unsigned long end, 889 struct list_head *uf, bool unlock) 890 { 891 struct maple_tree mt_detach; 892 MA_STATE(mas_detach, &mt_detach, 0, 0); 893 mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK); 894 mt_on_stack(mt_detach); 895 int error; 896 unsigned long locked_vm = 0; 897 898 error = vmi_gather_munmap_vmas(vmi, vma, mm, start, end, uf, 899 &mas_detach, &locked_vm); 900 if (error) 901 goto gather_failed; 902 903 error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL); 904 if (error) 905 goto clear_tree_failed; 906 907 /* Point of no return */ 908 vmi_complete_munmap_vmas(vmi, vma, mm, start, end, unlock, &mas_detach, 909 locked_vm); 910 return 0; 911 912 clear_tree_failed: 913 abort_munmap_vmas(&mas_detach); 914 gather_failed: 915 validate_mm(mm); 916 return error; 917 } 918 919 /* 920 * do_vmi_munmap() - munmap a given range. 921 * @vmi: The vma iterator 922 * @mm: The mm_struct 923 * @start: The start address to munmap 924 * @len: The length of the range to munmap 925 * @uf: The userfaultfd list_head 926 * @unlock: set to true if the user wants to drop the mmap_lock on success 927 * 928 * This function takes a @mas that is either pointing to the previous VMA or set 929 * to MA_START and sets it up to remove the mapping(s). The @len will be 930 * aligned. 931 * 932 * Return: 0 on success and drops the lock if so directed, error and leaves the 933 * lock held otherwise. 934 */ 935 int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 936 unsigned long start, size_t len, struct list_head *uf, 937 bool unlock) 938 { 939 unsigned long end; 940 struct vm_area_struct *vma; 941 942 if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) 943 return -EINVAL; 944 945 end = start + PAGE_ALIGN(len); 946 if (end == start) 947 return -EINVAL; 948 949 /* Find the first overlapping VMA */ 950 vma = vma_find(vmi, end); 951 if (!vma) { 952 if (unlock) 953 mmap_write_unlock(mm); 954 return 0; 955 } 956 957 return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock); 958 } 959 960 /* 961 * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name), 962 * figure out whether that can be merged with its predecessor or its 963 * successor. Or both (it neatly fills a hole). 964 * 965 * In most cases - when called for mmap, brk or mremap - [addr,end) is 966 * certain not to be mapped by the time vma_merge is called; but when 967 * called for mprotect, it is certain to be already mapped (either at 968 * an offset within prev, or at the start of next), and the flags of 969 * this area are about to be changed to vm_flags - and the no-change 970 * case has already been eliminated. 971 * 972 * The following mprotect cases have to be considered, where **** is 973 * the area passed down from mprotect_fixup, never extending beyond one 974 * vma, PPPP is the previous vma, CCCC is a concurrent vma that starts 975 * at the same address as **** and is of the same or larger span, and 976 * NNNN the next vma after ****: 977 * 978 * **** **** **** 979 * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPCCCCCC 980 * cannot merge might become might become 981 * PPNNNNNNNNNN PPPPPPPPPPCC 982 * mmap, brk or case 4 below case 5 below 983 * mremap move: 984 * **** **** 985 * PPPP NNNN PPPPCCCCNNNN 986 * might become might become 987 * PPPPPPPPPPPP 1 or PPPPPPPPPPPP 6 or 988 * PPPPPPPPNNNN 2 or PPPPPPPPNNNN 7 or 989 * PPPPNNNNNNNN 3 PPPPNNNNNNNN 8 990 * 991 * It is important for case 8 that the vma CCCC overlapping the 992 * region **** is never going to extended over NNNN. Instead NNNN must 993 * be extended in region **** and CCCC must be removed. This way in 994 * all cases where vma_merge succeeds, the moment vma_merge drops the 995 * rmap_locks, the properties of the merged vma will be already 996 * correct for the whole merged range. Some of those properties like 997 * vm_page_prot/vm_flags may be accessed by rmap_walks and they must 998 * be correct for the whole merged range immediately after the 999 * rmap_locks are released. Otherwise if NNNN would be removed and 1000 * CCCC would be extended over the NNNN range, remove_migration_ptes 1001 * or other rmap walkers (if working on addresses beyond the "end" 1002 * parameter) may establish ptes with the wrong permissions of CCCC 1003 * instead of the right permissions of NNNN. 1004 * 1005 * In the code below: 1006 * PPPP is represented by *prev 1007 * CCCC is represented by *curr or not represented at all (NULL) 1008 * NNNN is represented by *next or not represented at all (NULL) 1009 * **** is not represented - it will be merged and the vma containing the 1010 * area is returned, or the function will return NULL 1011 */ 1012 static struct vm_area_struct 1013 *vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev, 1014 struct vm_area_struct *src, unsigned long addr, unsigned long end, 1015 unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy, 1016 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 1017 struct anon_vma_name *anon_name) 1018 { 1019 struct mm_struct *mm = src->vm_mm; 1020 struct anon_vma *anon_vma = src->anon_vma; 1021 struct file *file = src->vm_file; 1022 struct vm_area_struct *curr, *next, *res; 1023 struct vm_area_struct *vma, *adjust, *remove, *remove2; 1024 struct vm_area_struct *anon_dup = NULL; 1025 struct vma_prepare vp; 1026 pgoff_t vma_pgoff; 1027 int err = 0; 1028 bool merge_prev = false; 1029 bool merge_next = false; 1030 bool vma_expanded = false; 1031 unsigned long vma_start = addr; 1032 unsigned long vma_end = end; 1033 pgoff_t pglen = (end - addr) >> PAGE_SHIFT; 1034 long adj_start = 0; 1035 1036 /* 1037 * We later require that vma->vm_flags == vm_flags, 1038 * so this tests vma->vm_flags & VM_SPECIAL, too. 1039 */ 1040 if (vm_flags & VM_SPECIAL) 1041 return NULL; 1042 1043 /* Does the input range span an existing VMA? (cases 5 - 8) */ 1044 curr = find_vma_intersection(mm, prev ? prev->vm_end : 0, end); 1045 1046 if (!curr || /* cases 1 - 4 */ 1047 end == curr->vm_end) /* cases 6 - 8, adjacent VMA */ 1048 next = vma_lookup(mm, end); 1049 else 1050 next = NULL; /* case 5 */ 1051 1052 if (prev) { 1053 vma_start = prev->vm_start; 1054 vma_pgoff = prev->vm_pgoff; 1055 1056 /* Can we merge the predecessor? */ 1057 if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy) 1058 && can_vma_merge_after(prev, vm_flags, anon_vma, file, 1059 pgoff, vm_userfaultfd_ctx, anon_name)) { 1060 merge_prev = true; 1061 vma_prev(vmi); 1062 } 1063 } 1064 1065 /* Can we merge the successor? */ 1066 if (next && mpol_equal(policy, vma_policy(next)) && 1067 can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen, 1068 vm_userfaultfd_ctx, anon_name)) { 1069 merge_next = true; 1070 } 1071 1072 /* Verify some invariant that must be enforced by the caller. */ 1073 VM_WARN_ON(prev && addr <= prev->vm_start); 1074 VM_WARN_ON(curr && (addr != curr->vm_start || end > curr->vm_end)); 1075 VM_WARN_ON(addr >= end); 1076 1077 if (!merge_prev && !merge_next) 1078 return NULL; /* Not mergeable. */ 1079 1080 if (merge_prev) 1081 vma_start_write(prev); 1082 1083 res = vma = prev; 1084 remove = remove2 = adjust = NULL; 1085 1086 /* Can we merge both the predecessor and the successor? */ 1087 if (merge_prev && merge_next && 1088 is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) { 1089 vma_start_write(next); 1090 remove = next; /* case 1 */ 1091 vma_end = next->vm_end; 1092 err = dup_anon_vma(prev, next, &anon_dup); 1093 if (curr) { /* case 6 */ 1094 vma_start_write(curr); 1095 remove = curr; 1096 remove2 = next; 1097 /* 1098 * Note that the dup_anon_vma below cannot overwrite err 1099 * since the first caller would do nothing unless next 1100 * has an anon_vma. 1101 */ 1102 if (!next->anon_vma) 1103 err = dup_anon_vma(prev, curr, &anon_dup); 1104 } 1105 } else if (merge_prev) { /* case 2 */ 1106 if (curr) { 1107 vma_start_write(curr); 1108 if (end == curr->vm_end) { /* case 7 */ 1109 /* 1110 * can_vma_merge_after() assumed we would not be 1111 * removing prev vma, so it skipped the check 1112 * for vm_ops->close, but we are removing curr 1113 */ 1114 if (curr->vm_ops && curr->vm_ops->close) 1115 err = -EINVAL; 1116 remove = curr; 1117 } else { /* case 5 */ 1118 adjust = curr; 1119 adj_start = (end - curr->vm_start); 1120 } 1121 if (!err) 1122 err = dup_anon_vma(prev, curr, &anon_dup); 1123 } 1124 } else { /* merge_next */ 1125 vma_start_write(next); 1126 res = next; 1127 if (prev && addr < prev->vm_end) { /* case 4 */ 1128 vma_start_write(prev); 1129 vma_end = addr; 1130 adjust = next; 1131 adj_start = -(prev->vm_end - addr); 1132 err = dup_anon_vma(next, prev, &anon_dup); 1133 } else { 1134 /* 1135 * Note that cases 3 and 8 are the ONLY ones where prev 1136 * is permitted to be (but is not necessarily) NULL. 1137 */ 1138 vma = next; /* case 3 */ 1139 vma_start = addr; 1140 vma_end = next->vm_end; 1141 vma_pgoff = next->vm_pgoff - pglen; 1142 if (curr) { /* case 8 */ 1143 vma_pgoff = curr->vm_pgoff; 1144 vma_start_write(curr); 1145 remove = curr; 1146 err = dup_anon_vma(next, curr, &anon_dup); 1147 } 1148 } 1149 } 1150 1151 /* Error in anon_vma clone. */ 1152 if (err) 1153 goto anon_vma_fail; 1154 1155 if (vma_start < vma->vm_start || vma_end > vma->vm_end) 1156 vma_expanded = true; 1157 1158 if (vma_expanded) { 1159 vma_iter_config(vmi, vma_start, vma_end); 1160 } else { 1161 vma_iter_config(vmi, adjust->vm_start + adj_start, 1162 adjust->vm_end); 1163 } 1164 1165 if (vma_iter_prealloc(vmi, vma)) 1166 goto prealloc_fail; 1167 1168 init_multi_vma_prep(&vp, vma, adjust, remove, remove2); 1169 VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma && 1170 vp.anon_vma != adjust->anon_vma); 1171 1172 vma_prepare(&vp); 1173 vma_adjust_trans_huge(vma, vma_start, vma_end, adj_start); 1174 vma_set_range(vma, vma_start, vma_end, vma_pgoff); 1175 1176 if (vma_expanded) 1177 vma_iter_store(vmi, vma); 1178 1179 if (adj_start) { 1180 adjust->vm_start += adj_start; 1181 adjust->vm_pgoff += adj_start >> PAGE_SHIFT; 1182 if (adj_start < 0) { 1183 WARN_ON(vma_expanded); 1184 vma_iter_store(vmi, next); 1185 } 1186 } 1187 1188 vma_complete(&vp, vmi, mm); 1189 khugepaged_enter_vma(res, vm_flags); 1190 return res; 1191 1192 prealloc_fail: 1193 if (anon_dup) 1194 unlink_anon_vmas(anon_dup); 1195 1196 anon_vma_fail: 1197 vma_iter_set(vmi, addr); 1198 vma_iter_load(vmi); 1199 return NULL; 1200 } 1201 1202 /* 1203 * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd 1204 * context and anonymous VMA name within the range [start, end). 1205 * 1206 * As a result, we might be able to merge the newly modified VMA range with an 1207 * adjacent VMA with identical properties. 1208 * 1209 * If no merge is possible and the range does not span the entirety of the VMA, 1210 * we then need to split the VMA to accommodate the change. 1211 * 1212 * The function returns either the merged VMA, the original VMA if a split was 1213 * required instead, or an error if the split failed. 1214 */ 1215 struct vm_area_struct *vma_modify(struct vma_iterator *vmi, 1216 struct vm_area_struct *prev, 1217 struct vm_area_struct *vma, 1218 unsigned long start, unsigned long end, 1219 unsigned long vm_flags, 1220 struct mempolicy *policy, 1221 struct vm_userfaultfd_ctx uffd_ctx, 1222 struct anon_vma_name *anon_name) 1223 { 1224 pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 1225 struct vm_area_struct *merged; 1226 1227 merged = vma_merge(vmi, prev, vma, start, end, vm_flags, 1228 pgoff, policy, uffd_ctx, anon_name); 1229 if (merged) 1230 return merged; 1231 1232 if (vma->vm_start < start) { 1233 int err = split_vma(vmi, vma, start, 1); 1234 1235 if (err) 1236 return ERR_PTR(err); 1237 } 1238 1239 if (vma->vm_end > end) { 1240 int err = split_vma(vmi, vma, end, 0); 1241 1242 if (err) 1243 return ERR_PTR(err); 1244 } 1245 1246 return vma; 1247 } 1248 1249 /* 1250 * Attempt to merge a newly mapped VMA with those adjacent to it. The caller 1251 * must ensure that [start, end) does not overlap any existing VMA. 1252 */ 1253 struct vm_area_struct 1254 *vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev, 1255 struct vm_area_struct *vma, unsigned long start, 1256 unsigned long end, pgoff_t pgoff) 1257 { 1258 return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff, 1259 vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 1260 } 1261 1262 /* 1263 * Expand vma by delta bytes, potentially merging with an immediately adjacent 1264 * VMA with identical properties. 1265 */ 1266 struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi, 1267 struct vm_area_struct *vma, 1268 unsigned long delta) 1269 { 1270 pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma); 1271 1272 /* vma is specified as prev, so case 1 or 2 will apply. */ 1273 return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta, 1274 vma->vm_flags, pgoff, vma_policy(vma), 1275 vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 1276 } 1277 1278 void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb) 1279 { 1280 vb->count = 0; 1281 } 1282 1283 static void unlink_file_vma_batch_process(struct unlink_vma_file_batch *vb) 1284 { 1285 struct address_space *mapping; 1286 int i; 1287 1288 mapping = vb->vmas[0]->vm_file->f_mapping; 1289 i_mmap_lock_write(mapping); 1290 for (i = 0; i < vb->count; i++) { 1291 VM_WARN_ON_ONCE(vb->vmas[i]->vm_file->f_mapping != mapping); 1292 __remove_shared_vm_struct(vb->vmas[i], mapping); 1293 } 1294 i_mmap_unlock_write(mapping); 1295 1296 unlink_file_vma_batch_init(vb); 1297 } 1298 1299 void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, 1300 struct vm_area_struct *vma) 1301 { 1302 if (vma->vm_file == NULL) 1303 return; 1304 1305 if ((vb->count > 0 && vb->vmas[0]->vm_file != vma->vm_file) || 1306 vb->count == ARRAY_SIZE(vb->vmas)) 1307 unlink_file_vma_batch_process(vb); 1308 1309 vb->vmas[vb->count] = vma; 1310 vb->count++; 1311 } 1312 1313 void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb) 1314 { 1315 if (vb->count > 0) 1316 unlink_file_vma_batch_process(vb); 1317 } 1318 1319 /* 1320 * Unlink a file-based vm structure from its interval tree, to hide 1321 * vma from rmap and vmtruncate before freeing its page tables. 1322 */ 1323 void unlink_file_vma(struct vm_area_struct *vma) 1324 { 1325 struct file *file = vma->vm_file; 1326 1327 if (file) { 1328 struct address_space *mapping = file->f_mapping; 1329 1330 i_mmap_lock_write(mapping); 1331 __remove_shared_vm_struct(vma, mapping); 1332 i_mmap_unlock_write(mapping); 1333 } 1334 } 1335 1336 void vma_link_file(struct vm_area_struct *vma) 1337 { 1338 struct file *file = vma->vm_file; 1339 struct address_space *mapping; 1340 1341 if (file) { 1342 mapping = file->f_mapping; 1343 i_mmap_lock_write(mapping); 1344 __vma_link_file(vma, mapping); 1345 i_mmap_unlock_write(mapping); 1346 } 1347 } 1348 1349 int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) 1350 { 1351 VMA_ITERATOR(vmi, mm, 0); 1352 1353 vma_iter_config(&vmi, vma->vm_start, vma->vm_end); 1354 if (vma_iter_prealloc(&vmi, vma)) 1355 return -ENOMEM; 1356 1357 vma_start_write(vma); 1358 vma_iter_store(&vmi, vma); 1359 vma_link_file(vma); 1360 mm->map_count++; 1361 validate_mm(mm); 1362 return 0; 1363 } 1364 1365 /* 1366 * Copy the vma structure to a new location in the same mm, 1367 * prior to moving page table entries, to effect an mremap move. 1368 */ 1369 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, 1370 unsigned long addr, unsigned long len, pgoff_t pgoff, 1371 bool *need_rmap_locks) 1372 { 1373 struct vm_area_struct *vma = *vmap; 1374 unsigned long vma_start = vma->vm_start; 1375 struct mm_struct *mm = vma->vm_mm; 1376 struct vm_area_struct *new_vma, *prev; 1377 bool faulted_in_anon_vma = true; 1378 VMA_ITERATOR(vmi, mm, addr); 1379 1380 /* 1381 * If anonymous vma has not yet been faulted, update new pgoff 1382 * to match new location, to increase its chance of merging. 1383 */ 1384 if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { 1385 pgoff = addr >> PAGE_SHIFT; 1386 faulted_in_anon_vma = false; 1387 } 1388 1389 new_vma = find_vma_prev(mm, addr, &prev); 1390 if (new_vma && new_vma->vm_start < addr + len) 1391 return NULL; /* should never get here */ 1392 1393 new_vma = vma_merge_new_vma(&vmi, prev, vma, addr, addr + len, pgoff); 1394 if (new_vma) { 1395 /* 1396 * Source vma may have been merged into new_vma 1397 */ 1398 if (unlikely(vma_start >= new_vma->vm_start && 1399 vma_start < new_vma->vm_end)) { 1400 /* 1401 * The only way we can get a vma_merge with 1402 * self during an mremap is if the vma hasn't 1403 * been faulted in yet and we were allowed to 1404 * reset the dst vma->vm_pgoff to the 1405 * destination address of the mremap to allow 1406 * the merge to happen. mremap must change the 1407 * vm_pgoff linearity between src and dst vmas 1408 * (in turn preventing a vma_merge) to be 1409 * safe. It is only safe to keep the vm_pgoff 1410 * linear if there are no pages mapped yet. 1411 */ 1412 VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); 1413 *vmap = vma = new_vma; 1414 } 1415 *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff); 1416 } else { 1417 new_vma = vm_area_dup(vma); 1418 if (!new_vma) 1419 goto out; 1420 vma_set_range(new_vma, addr, addr + len, pgoff); 1421 if (vma_dup_policy(vma, new_vma)) 1422 goto out_free_vma; 1423 if (anon_vma_clone(new_vma, vma)) 1424 goto out_free_mempol; 1425 if (new_vma->vm_file) 1426 get_file(new_vma->vm_file); 1427 if (new_vma->vm_ops && new_vma->vm_ops->open) 1428 new_vma->vm_ops->open(new_vma); 1429 if (vma_link(mm, new_vma)) 1430 goto out_vma_link; 1431 *need_rmap_locks = false; 1432 } 1433 return new_vma; 1434 1435 out_vma_link: 1436 if (new_vma->vm_ops && new_vma->vm_ops->close) 1437 new_vma->vm_ops->close(new_vma); 1438 1439 if (new_vma->vm_file) 1440 fput(new_vma->vm_file); 1441 1442 unlink_anon_vmas(new_vma); 1443 out_free_mempol: 1444 mpol_put(vma_policy(new_vma)); 1445 out_free_vma: 1446 vm_area_free(new_vma); 1447 out: 1448 return NULL; 1449 } 1450 1451 /* 1452 * Rough compatibility check to quickly see if it's even worth looking 1453 * at sharing an anon_vma. 1454 * 1455 * They need to have the same vm_file, and the flags can only differ 1456 * in things that mprotect may change. 1457 * 1458 * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that 1459 * we can merge the two vma's. For example, we refuse to merge a vma if 1460 * there is a vm_ops->close() function, because that indicates that the 1461 * driver is doing some kind of reference counting. But that doesn't 1462 * really matter for the anon_vma sharing case. 1463 */ 1464 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b) 1465 { 1466 return a->vm_end == b->vm_start && 1467 mpol_equal(vma_policy(a), vma_policy(b)) && 1468 a->vm_file == b->vm_file && 1469 !((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) && 1470 b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); 1471 } 1472 1473 /* 1474 * Do some basic sanity checking to see if we can re-use the anon_vma 1475 * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be 1476 * the same as 'old', the other will be the new one that is trying 1477 * to share the anon_vma. 1478 * 1479 * NOTE! This runs with mmap_lock held for reading, so it is possible that 1480 * the anon_vma of 'old' is concurrently in the process of being set up 1481 * by another page fault trying to merge _that_. But that's ok: if it 1482 * is being set up, that automatically means that it will be a singleton 1483 * acceptable for merging, so we can do all of this optimistically. But 1484 * we do that READ_ONCE() to make sure that we never re-load the pointer. 1485 * 1486 * IOW: that the "list_is_singular()" test on the anon_vma_chain only 1487 * matters for the 'stable anon_vma' case (ie the thing we want to avoid 1488 * is to return an anon_vma that is "complex" due to having gone through 1489 * a fork). 1490 * 1491 * We also make sure that the two vma's are compatible (adjacent, 1492 * and with the same memory policies). That's all stable, even with just 1493 * a read lock on the mmap_lock. 1494 */ 1495 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, 1496 struct vm_area_struct *a, 1497 struct vm_area_struct *b) 1498 { 1499 if (anon_vma_compatible(a, b)) { 1500 struct anon_vma *anon_vma = READ_ONCE(old->anon_vma); 1501 1502 if (anon_vma && list_is_singular(&old->anon_vma_chain)) 1503 return anon_vma; 1504 } 1505 return NULL; 1506 } 1507 1508 /* 1509 * find_mergeable_anon_vma is used by anon_vma_prepare, to check 1510 * neighbouring vmas for a suitable anon_vma, before it goes off 1511 * to allocate a new anon_vma. It checks because a repetitive 1512 * sequence of mprotects and faults may otherwise lead to distinct 1513 * anon_vmas being allocated, preventing vma merge in subsequent 1514 * mprotect. 1515 */ 1516 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) 1517 { 1518 struct anon_vma *anon_vma = NULL; 1519 struct vm_area_struct *prev, *next; 1520 VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_end); 1521 1522 /* Try next first. */ 1523 next = vma_iter_load(&vmi); 1524 if (next) { 1525 anon_vma = reusable_anon_vma(next, vma, next); 1526 if (anon_vma) 1527 return anon_vma; 1528 } 1529 1530 prev = vma_prev(&vmi); 1531 VM_BUG_ON_VMA(prev != vma, vma); 1532 prev = vma_prev(&vmi); 1533 /* Try prev next. */ 1534 if (prev) 1535 anon_vma = reusable_anon_vma(prev, prev, vma); 1536 1537 /* 1538 * We might reach here with anon_vma == NULL if we can't find 1539 * any reusable anon_vma. 1540 * There's no absolute need to look only at touching neighbours: 1541 * we could search further afield for "compatible" anon_vmas. 1542 * But it would probably just be a waste of time searching, 1543 * or lead to too many vmas hanging off the same anon_vma. 1544 * We're trying to allow mprotect remerging later on, 1545 * not trying to minimize memory used for anon_vmas. 1546 */ 1547 return anon_vma; 1548 } 1549 1550 static bool vm_ops_needs_writenotify(const struct vm_operations_struct *vm_ops) 1551 { 1552 return vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite); 1553 } 1554 1555 static bool vma_is_shared_writable(struct vm_area_struct *vma) 1556 { 1557 return (vma->vm_flags & (VM_WRITE | VM_SHARED)) == 1558 (VM_WRITE | VM_SHARED); 1559 } 1560 1561 static bool vma_fs_can_writeback(struct vm_area_struct *vma) 1562 { 1563 /* No managed pages to writeback. */ 1564 if (vma->vm_flags & VM_PFNMAP) 1565 return false; 1566 1567 return vma->vm_file && vma->vm_file->f_mapping && 1568 mapping_can_writeback(vma->vm_file->f_mapping); 1569 } 1570 1571 /* 1572 * Does this VMA require the underlying folios to have their dirty state 1573 * tracked? 1574 */ 1575 bool vma_needs_dirty_tracking(struct vm_area_struct *vma) 1576 { 1577 /* Only shared, writable VMAs require dirty tracking. */ 1578 if (!vma_is_shared_writable(vma)) 1579 return false; 1580 1581 /* Does the filesystem need to be notified? */ 1582 if (vm_ops_needs_writenotify(vma->vm_ops)) 1583 return true; 1584 1585 /* 1586 * Even if the filesystem doesn't indicate a need for writenotify, if it 1587 * can writeback, dirty tracking is still required. 1588 */ 1589 return vma_fs_can_writeback(vma); 1590 } 1591 1592 /* 1593 * Some shared mappings will want the pages marked read-only 1594 * to track write events. If so, we'll downgrade vm_page_prot 1595 * to the private version (using protection_map[] without the 1596 * VM_SHARED bit). 1597 */ 1598 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) 1599 { 1600 /* If it was private or non-writable, the write bit is already clear */ 1601 if (!vma_is_shared_writable(vma)) 1602 return false; 1603 1604 /* The backer wishes to know when pages are first written to? */ 1605 if (vm_ops_needs_writenotify(vma->vm_ops)) 1606 return true; 1607 1608 /* The open routine did something to the protections that pgprot_modify 1609 * won't preserve? */ 1610 if (pgprot_val(vm_page_prot) != 1611 pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags))) 1612 return false; 1613 1614 /* 1615 * Do we need to track softdirty? hugetlb does not support softdirty 1616 * tracking yet. 1617 */ 1618 if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma)) 1619 return true; 1620 1621 /* Do we need write faults for uffd-wp tracking? */ 1622 if (userfaultfd_wp(vma)) 1623 return true; 1624 1625 /* Can the mapping track the dirty pages? */ 1626 return vma_fs_can_writeback(vma); 1627 } 1628 1629 unsigned long count_vma_pages_range(struct mm_struct *mm, 1630 unsigned long addr, unsigned long end) 1631 { 1632 VMA_ITERATOR(vmi, mm, addr); 1633 struct vm_area_struct *vma; 1634 unsigned long nr_pages = 0; 1635 1636 for_each_vma_range(vmi, vma, end) { 1637 unsigned long vm_start = max(addr, vma->vm_start); 1638 unsigned long vm_end = min(end, vma->vm_end); 1639 1640 nr_pages += PHYS_PFN(vm_end - vm_start); 1641 } 1642 1643 return nr_pages; 1644 } 1645 1646 static DEFINE_MUTEX(mm_all_locks_mutex); 1647 1648 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma) 1649 { 1650 if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 1651 /* 1652 * The LSB of head.next can't change from under us 1653 * because we hold the mm_all_locks_mutex. 1654 */ 1655 down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock); 1656 /* 1657 * We can safely modify head.next after taking the 1658 * anon_vma->root->rwsem. If some other vma in this mm shares 1659 * the same anon_vma we won't take it again. 1660 * 1661 * No need of atomic instructions here, head.next 1662 * can't change from under us thanks to the 1663 * anon_vma->root->rwsem. 1664 */ 1665 if (__test_and_set_bit(0, (unsigned long *) 1666 &anon_vma->root->rb_root.rb_root.rb_node)) 1667 BUG(); 1668 } 1669 } 1670 1671 static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) 1672 { 1673 if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 1674 /* 1675 * AS_MM_ALL_LOCKS can't change from under us because 1676 * we hold the mm_all_locks_mutex. 1677 * 1678 * Operations on ->flags have to be atomic because 1679 * even if AS_MM_ALL_LOCKS is stable thanks to the 1680 * mm_all_locks_mutex, there may be other cpus 1681 * changing other bitflags in parallel to us. 1682 */ 1683 if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags)) 1684 BUG(); 1685 down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock); 1686 } 1687 } 1688 1689 /* 1690 * This operation locks against the VM for all pte/vma/mm related 1691 * operations that could ever happen on a certain mm. This includes 1692 * vmtruncate, try_to_unmap, and all page faults. 1693 * 1694 * The caller must take the mmap_lock in write mode before calling 1695 * mm_take_all_locks(). The caller isn't allowed to release the 1696 * mmap_lock until mm_drop_all_locks() returns. 1697 * 1698 * mmap_lock in write mode is required in order to block all operations 1699 * that could modify pagetables and free pages without need of 1700 * altering the vma layout. It's also needed in write mode to avoid new 1701 * anon_vmas to be associated with existing vmas. 1702 * 1703 * A single task can't take more than one mm_take_all_locks() in a row 1704 * or it would deadlock. 1705 * 1706 * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in 1707 * mapping->flags avoid to take the same lock twice, if more than one 1708 * vma in this mm is backed by the same anon_vma or address_space. 1709 * 1710 * We take locks in following order, accordingly to comment at beginning 1711 * of mm/rmap.c: 1712 * - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for 1713 * hugetlb mapping); 1714 * - all vmas marked locked 1715 * - all i_mmap_rwsem locks; 1716 * - all anon_vma->rwseml 1717 * 1718 * We can take all locks within these types randomly because the VM code 1719 * doesn't nest them and we protected from parallel mm_take_all_locks() by 1720 * mm_all_locks_mutex. 1721 * 1722 * mm_take_all_locks() and mm_drop_all_locks are expensive operations 1723 * that may have to take thousand of locks. 1724 * 1725 * mm_take_all_locks() can fail if it's interrupted by signals. 1726 */ 1727 int mm_take_all_locks(struct mm_struct *mm) 1728 { 1729 struct vm_area_struct *vma; 1730 struct anon_vma_chain *avc; 1731 VMA_ITERATOR(vmi, mm, 0); 1732 1733 mmap_assert_write_locked(mm); 1734 1735 mutex_lock(&mm_all_locks_mutex); 1736 1737 /* 1738 * vma_start_write() does not have a complement in mm_drop_all_locks() 1739 * because vma_start_write() is always asymmetrical; it marks a VMA as 1740 * being written to until mmap_write_unlock() or mmap_write_downgrade() 1741 * is reached. 1742 */ 1743 for_each_vma(vmi, vma) { 1744 if (signal_pending(current)) 1745 goto out_unlock; 1746 vma_start_write(vma); 1747 } 1748 1749 vma_iter_init(&vmi, mm, 0); 1750 for_each_vma(vmi, vma) { 1751 if (signal_pending(current)) 1752 goto out_unlock; 1753 if (vma->vm_file && vma->vm_file->f_mapping && 1754 is_vm_hugetlb_page(vma)) 1755 vm_lock_mapping(mm, vma->vm_file->f_mapping); 1756 } 1757 1758 vma_iter_init(&vmi, mm, 0); 1759 for_each_vma(vmi, vma) { 1760 if (signal_pending(current)) 1761 goto out_unlock; 1762 if (vma->vm_file && vma->vm_file->f_mapping && 1763 !is_vm_hugetlb_page(vma)) 1764 vm_lock_mapping(mm, vma->vm_file->f_mapping); 1765 } 1766 1767 vma_iter_init(&vmi, mm, 0); 1768 for_each_vma(vmi, vma) { 1769 if (signal_pending(current)) 1770 goto out_unlock; 1771 if (vma->anon_vma) 1772 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 1773 vm_lock_anon_vma(mm, avc->anon_vma); 1774 } 1775 1776 return 0; 1777 1778 out_unlock: 1779 mm_drop_all_locks(mm); 1780 return -EINTR; 1781 } 1782 1783 static void vm_unlock_anon_vma(struct anon_vma *anon_vma) 1784 { 1785 if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 1786 /* 1787 * The LSB of head.next can't change to 0 from under 1788 * us because we hold the mm_all_locks_mutex. 1789 * 1790 * We must however clear the bitflag before unlocking 1791 * the vma so the users using the anon_vma->rb_root will 1792 * never see our bitflag. 1793 * 1794 * No need of atomic instructions here, head.next 1795 * can't change from under us until we release the 1796 * anon_vma->root->rwsem. 1797 */ 1798 if (!__test_and_clear_bit(0, (unsigned long *) 1799 &anon_vma->root->rb_root.rb_root.rb_node)) 1800 BUG(); 1801 anon_vma_unlock_write(anon_vma); 1802 } 1803 } 1804 1805 static void vm_unlock_mapping(struct address_space *mapping) 1806 { 1807 if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 1808 /* 1809 * AS_MM_ALL_LOCKS can't change to 0 from under us 1810 * because we hold the mm_all_locks_mutex. 1811 */ 1812 i_mmap_unlock_write(mapping); 1813 if (!test_and_clear_bit(AS_MM_ALL_LOCKS, 1814 &mapping->flags)) 1815 BUG(); 1816 } 1817 } 1818 1819 /* 1820 * The mmap_lock cannot be released by the caller until 1821 * mm_drop_all_locks() returns. 1822 */ 1823 void mm_drop_all_locks(struct mm_struct *mm) 1824 { 1825 struct vm_area_struct *vma; 1826 struct anon_vma_chain *avc; 1827 VMA_ITERATOR(vmi, mm, 0); 1828 1829 mmap_assert_write_locked(mm); 1830 BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); 1831 1832 for_each_vma(vmi, vma) { 1833 if (vma->anon_vma) 1834 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 1835 vm_unlock_anon_vma(avc->anon_vma); 1836 if (vma->vm_file && vma->vm_file->f_mapping) 1837 vm_unlock_mapping(vma->vm_file->f_mapping); 1838 } 1839 1840 mutex_unlock(&mm_all_locks_mutex); 1841 } 1842