1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 /* 4 * VMA-specific functions. 5 */ 6 7 #include "vma_internal.h" 8 #include "vma.h" 9 10 /* 11 * If the vma has a ->close operation then the driver probably needs to release 12 * per-vma resources, so we don't attempt to merge those if the caller indicates 13 * the current vma may be removed as part of the merge. 14 */ 15 static inline bool is_mergeable_vma(struct vm_area_struct *vma, 16 struct file *file, unsigned long vm_flags, 17 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 18 struct anon_vma_name *anon_name, bool may_remove_vma) 19 { 20 /* 21 * VM_SOFTDIRTY should not prevent from VMA merging, if we 22 * match the flags but dirty bit -- the caller should mark 23 * merged VMA as dirty. If dirty bit won't be excluded from 24 * comparison, we increase pressure on the memory system forcing 25 * the kernel to generate new VMAs when old one could be 26 * extended instead. 27 */ 28 if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY) 29 return false; 30 if (vma->vm_file != file) 31 return false; 32 if (may_remove_vma && vma->vm_ops && vma->vm_ops->close) 33 return false; 34 if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx)) 35 return false; 36 if (!anon_vma_name_eq(anon_vma_name(vma), anon_name)) 37 return false; 38 return true; 39 } 40 41 static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1, 42 struct anon_vma *anon_vma2, struct vm_area_struct *vma) 43 { 44 /* 45 * The list_is_singular() test is to avoid merging VMA cloned from 46 * parents. This can improve scalability caused by anon_vma lock. 47 */ 48 if ((!anon_vma1 || !anon_vma2) && (!vma || 49 list_is_singular(&vma->anon_vma_chain))) 50 return true; 51 return anon_vma1 == anon_vma2; 52 } 53 54 /* 55 * init_multi_vma_prep() - Initializer for struct vma_prepare 56 * @vp: The vma_prepare struct 57 * @vma: The vma that will be altered once locked 58 * @next: The next vma if it is to be adjusted 59 * @remove: The first vma to be removed 60 * @remove2: The second vma to be removed 61 */ 62 static void init_multi_vma_prep(struct vma_prepare *vp, 63 struct vm_area_struct *vma, 64 struct vm_area_struct *next, 65 struct vm_area_struct *remove, 66 struct vm_area_struct *remove2) 67 { 68 memset(vp, 0, sizeof(struct vma_prepare)); 69 vp->vma = vma; 70 vp->anon_vma = vma->anon_vma; 71 vp->remove = remove; 72 vp->remove2 = remove2; 73 vp->adj_next = next; 74 if (!vp->anon_vma && next) 75 vp->anon_vma = next->anon_vma; 76 77 vp->file = vma->vm_file; 78 if (vp->file) 79 vp->mapping = vma->vm_file->f_mapping; 80 81 } 82 83 /* 84 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 85 * in front of (at a lower virtual address and file offset than) the vma. 86 * 87 * We cannot merge two vmas if they have differently assigned (non-NULL) 88 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 89 * 90 * We don't check here for the merged mmap wrapping around the end of pagecache 91 * indices (16TB on ia32) because do_mmap() does not permit mmap's which 92 * wrap, nor mmaps which cover the final page at index -1UL. 93 * 94 * We assume the vma may be removed as part of the merge. 95 */ 96 bool 97 can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, 98 struct anon_vma *anon_vma, struct file *file, 99 pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 100 struct anon_vma_name *anon_name) 101 { 102 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) && 103 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 104 if (vma->vm_pgoff == vm_pgoff) 105 return true; 106 } 107 return false; 108 } 109 110 /* 111 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) 112 * beyond (at a higher virtual address and file offset than) the vma. 113 * 114 * We cannot merge two vmas if they have differently assigned (non-NULL) 115 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. 116 * 117 * We assume that vma is not removed as part of the merge. 118 */ 119 bool 120 can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, 121 struct anon_vma *anon_vma, struct file *file, 122 pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 123 struct anon_vma_name *anon_name) 124 { 125 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) && 126 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { 127 pgoff_t vm_pglen; 128 129 vm_pglen = vma_pages(vma); 130 if (vma->vm_pgoff + vm_pglen == vm_pgoff) 131 return true; 132 } 133 return false; 134 } 135 136 /* 137 * Close a vm structure and free it. 138 */ 139 void remove_vma(struct vm_area_struct *vma, bool unreachable) 140 { 141 might_sleep(); 142 if (vma->vm_ops && vma->vm_ops->close) 143 vma->vm_ops->close(vma); 144 if (vma->vm_file) 145 fput(vma->vm_file); 146 mpol_put(vma_policy(vma)); 147 if (unreachable) 148 __vm_area_free(vma); 149 else 150 vm_area_free(vma); 151 } 152 153 /* 154 * Get rid of page table information in the indicated region. 155 * 156 * Called with the mm semaphore held. 157 */ 158 void unmap_region(struct ma_state *mas, struct vm_area_struct *vma, 159 struct vm_area_struct *prev, struct vm_area_struct *next) 160 { 161 struct mm_struct *mm = vma->vm_mm; 162 struct mmu_gather tlb; 163 164 lru_add_drain(); 165 tlb_gather_mmu(&tlb, mm); 166 update_hiwater_rss(mm); 167 unmap_vmas(&tlb, mas, vma, vma->vm_start, vma->vm_end, vma->vm_end, 168 /* mm_wr_locked = */ true); 169 mas_set(mas, vma->vm_end); 170 free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, 171 next ? next->vm_start : USER_PGTABLES_CEILING, 172 /* mm_wr_locked = */ true); 173 tlb_finish_mmu(&tlb); 174 } 175 176 /* 177 * __split_vma() bypasses sysctl_max_map_count checking. We use this where it 178 * has already been checked or doesn't make sense to fail. 179 * VMA Iterator will point to the original VMA. 180 */ 181 static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, 182 unsigned long addr, int new_below) 183 { 184 struct vma_prepare vp; 185 struct vm_area_struct *new; 186 int err; 187 188 WARN_ON(vma->vm_start >= addr); 189 WARN_ON(vma->vm_end <= addr); 190 191 if (vma->vm_ops && vma->vm_ops->may_split) { 192 err = vma->vm_ops->may_split(vma, addr); 193 if (err) 194 return err; 195 } 196 197 new = vm_area_dup(vma); 198 if (!new) 199 return -ENOMEM; 200 201 if (new_below) { 202 new->vm_end = addr; 203 } else { 204 new->vm_start = addr; 205 new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); 206 } 207 208 err = -ENOMEM; 209 vma_iter_config(vmi, new->vm_start, new->vm_end); 210 if (vma_iter_prealloc(vmi, new)) 211 goto out_free_vma; 212 213 err = vma_dup_policy(vma, new); 214 if (err) 215 goto out_free_vmi; 216 217 err = anon_vma_clone(new, vma); 218 if (err) 219 goto out_free_mpol; 220 221 if (new->vm_file) 222 get_file(new->vm_file); 223 224 if (new->vm_ops && new->vm_ops->open) 225 new->vm_ops->open(new); 226 227 vma_start_write(vma); 228 vma_start_write(new); 229 230 init_vma_prep(&vp, vma); 231 vp.insert = new; 232 vma_prepare(&vp); 233 vma_adjust_trans_huge(vma, vma->vm_start, addr, 0); 234 235 if (new_below) { 236 vma->vm_start = addr; 237 vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; 238 } else { 239 vma->vm_end = addr; 240 } 241 242 /* vma_complete stores the new vma */ 243 vma_complete(&vp, vmi, vma->vm_mm); 244 validate_mm(vma->vm_mm); 245 246 /* Success. */ 247 if (new_below) 248 vma_next(vmi); 249 else 250 vma_prev(vmi); 251 252 return 0; 253 254 out_free_mpol: 255 mpol_put(vma_policy(new)); 256 out_free_vmi: 257 vma_iter_free(vmi); 258 out_free_vma: 259 vm_area_free(new); 260 return err; 261 } 262 263 /* 264 * Split a vma into two pieces at address 'addr', a new vma is allocated 265 * either for the first part or the tail. 266 */ 267 static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, 268 unsigned long addr, int new_below) 269 { 270 if (vma->vm_mm->map_count >= sysctl_max_map_count) 271 return -ENOMEM; 272 273 return __split_vma(vmi, vma, addr, new_below); 274 } 275 276 /* 277 * init_vma_prep() - Initializer wrapper for vma_prepare struct 278 * @vp: The vma_prepare struct 279 * @vma: The vma that will be altered once locked 280 */ 281 void init_vma_prep(struct vma_prepare *vp, 282 struct vm_area_struct *vma) 283 { 284 init_multi_vma_prep(vp, vma, NULL, NULL, NULL); 285 } 286 287 /* 288 * Requires inode->i_mapping->i_mmap_rwsem 289 */ 290 static void __remove_shared_vm_struct(struct vm_area_struct *vma, 291 struct address_space *mapping) 292 { 293 if (vma_is_shared_maywrite(vma)) 294 mapping_unmap_writable(mapping); 295 296 flush_dcache_mmap_lock(mapping); 297 vma_interval_tree_remove(vma, &mapping->i_mmap); 298 flush_dcache_mmap_unlock(mapping); 299 } 300 301 /* 302 * vma has some anon_vma assigned, and is already inserted on that 303 * anon_vma's interval trees. 304 * 305 * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the 306 * vma must be removed from the anon_vma's interval trees using 307 * anon_vma_interval_tree_pre_update_vma(). 308 * 309 * After the update, the vma will be reinserted using 310 * anon_vma_interval_tree_post_update_vma(). 311 * 312 * The entire update must be protected by exclusive mmap_lock and by 313 * the root anon_vma's mutex. 314 */ 315 void 316 anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) 317 { 318 struct anon_vma_chain *avc; 319 320 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 321 anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); 322 } 323 324 void 325 anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) 326 { 327 struct anon_vma_chain *avc; 328 329 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 330 anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); 331 } 332 333 static void __vma_link_file(struct vm_area_struct *vma, 334 struct address_space *mapping) 335 { 336 if (vma_is_shared_maywrite(vma)) 337 mapping_allow_writable(mapping); 338 339 flush_dcache_mmap_lock(mapping); 340 vma_interval_tree_insert(vma, &mapping->i_mmap); 341 flush_dcache_mmap_unlock(mapping); 342 } 343 344 /* 345 * vma_prepare() - Helper function for handling locking VMAs prior to altering 346 * @vp: The initialized vma_prepare struct 347 */ 348 void vma_prepare(struct vma_prepare *vp) 349 { 350 if (vp->file) { 351 uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end); 352 353 if (vp->adj_next) 354 uprobe_munmap(vp->adj_next, vp->adj_next->vm_start, 355 vp->adj_next->vm_end); 356 357 i_mmap_lock_write(vp->mapping); 358 if (vp->insert && vp->insert->vm_file) { 359 /* 360 * Put into interval tree now, so instantiated pages 361 * are visible to arm/parisc __flush_dcache_page 362 * throughout; but we cannot insert into address 363 * space until vma start or end is updated. 364 */ 365 __vma_link_file(vp->insert, 366 vp->insert->vm_file->f_mapping); 367 } 368 } 369 370 if (vp->anon_vma) { 371 anon_vma_lock_write(vp->anon_vma); 372 anon_vma_interval_tree_pre_update_vma(vp->vma); 373 if (vp->adj_next) 374 anon_vma_interval_tree_pre_update_vma(vp->adj_next); 375 } 376 377 if (vp->file) { 378 flush_dcache_mmap_lock(vp->mapping); 379 vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap); 380 if (vp->adj_next) 381 vma_interval_tree_remove(vp->adj_next, 382 &vp->mapping->i_mmap); 383 } 384 385 } 386 387 /* 388 * dup_anon_vma() - Helper function to duplicate anon_vma 389 * @dst: The destination VMA 390 * @src: The source VMA 391 * @dup: Pointer to the destination VMA when successful. 392 * 393 * Returns: 0 on success. 394 */ 395 static int dup_anon_vma(struct vm_area_struct *dst, 396 struct vm_area_struct *src, struct vm_area_struct **dup) 397 { 398 /* 399 * Easily overlooked: when mprotect shifts the boundary, make sure the 400 * expanding vma has anon_vma set if the shrinking vma had, to cover any 401 * anon pages imported. 402 */ 403 if (src->anon_vma && !dst->anon_vma) { 404 int ret; 405 406 vma_assert_write_locked(dst); 407 dst->anon_vma = src->anon_vma; 408 ret = anon_vma_clone(dst, src); 409 if (ret) 410 return ret; 411 412 *dup = dst; 413 } 414 415 return 0; 416 } 417 418 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 419 void validate_mm(struct mm_struct *mm) 420 { 421 int bug = 0; 422 int i = 0; 423 struct vm_area_struct *vma; 424 VMA_ITERATOR(vmi, mm, 0); 425 426 mt_validate(&mm->mm_mt); 427 for_each_vma(vmi, vma) { 428 #ifdef CONFIG_DEBUG_VM_RB 429 struct anon_vma *anon_vma = vma->anon_vma; 430 struct anon_vma_chain *avc; 431 #endif 432 unsigned long vmi_start, vmi_end; 433 bool warn = 0; 434 435 vmi_start = vma_iter_addr(&vmi); 436 vmi_end = vma_iter_end(&vmi); 437 if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm)) 438 warn = 1; 439 440 if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm)) 441 warn = 1; 442 443 if (warn) { 444 pr_emerg("issue in %s\n", current->comm); 445 dump_stack(); 446 dump_vma(vma); 447 pr_emerg("tree range: %px start %lx end %lx\n", vma, 448 vmi_start, vmi_end - 1); 449 vma_iter_dump_tree(&vmi); 450 } 451 452 #ifdef CONFIG_DEBUG_VM_RB 453 if (anon_vma) { 454 anon_vma_lock_read(anon_vma); 455 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 456 anon_vma_interval_tree_verify(avc); 457 anon_vma_unlock_read(anon_vma); 458 } 459 #endif 460 i++; 461 } 462 if (i != mm->map_count) { 463 pr_emerg("map_count %d vma iterator %d\n", mm->map_count, i); 464 bug = 1; 465 } 466 VM_BUG_ON_MM(bug, mm); 467 } 468 #endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ 469 470 /* 471 * vma_expand - Expand an existing VMA 472 * 473 * @vmi: The vma iterator 474 * @vma: The vma to expand 475 * @start: The start of the vma 476 * @end: The exclusive end of the vma 477 * @pgoff: The page offset of vma 478 * @next: The current of next vma. 479 * 480 * Expand @vma to @start and @end. Can expand off the start and end. Will 481 * expand over @next if it's different from @vma and @end == @next->vm_end. 482 * Checking if the @vma can expand and merge with @next needs to be handled by 483 * the caller. 484 * 485 * Returns: 0 on success 486 */ 487 int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma, 488 unsigned long start, unsigned long end, pgoff_t pgoff, 489 struct vm_area_struct *next) 490 { 491 struct vm_area_struct *anon_dup = NULL; 492 bool remove_next = false; 493 struct vma_prepare vp; 494 495 vma_start_write(vma); 496 if (next && (vma != next) && (end == next->vm_end)) { 497 int ret; 498 499 remove_next = true; 500 vma_start_write(next); 501 ret = dup_anon_vma(vma, next, &anon_dup); 502 if (ret) 503 return ret; 504 } 505 506 init_multi_vma_prep(&vp, vma, NULL, remove_next ? next : NULL, NULL); 507 /* Not merging but overwriting any part of next is not handled. */ 508 VM_WARN_ON(next && !vp.remove && 509 next != vma && end > next->vm_start); 510 /* Only handles expanding */ 511 VM_WARN_ON(vma->vm_start < start || vma->vm_end > end); 512 513 /* Note: vma iterator must be pointing to 'start' */ 514 vma_iter_config(vmi, start, end); 515 if (vma_iter_prealloc(vmi, vma)) 516 goto nomem; 517 518 vma_prepare(&vp); 519 vma_adjust_trans_huge(vma, start, end, 0); 520 vma_set_range(vma, start, end, pgoff); 521 vma_iter_store(vmi, vma); 522 523 vma_complete(&vp, vmi, vma->vm_mm); 524 validate_mm(vma->vm_mm); 525 return 0; 526 527 nomem: 528 if (anon_dup) 529 unlink_anon_vmas(anon_dup); 530 return -ENOMEM; 531 } 532 533 /* 534 * vma_shrink() - Reduce an existing VMAs memory area 535 * @vmi: The vma iterator 536 * @vma: The VMA to modify 537 * @start: The new start 538 * @end: The new end 539 * 540 * Returns: 0 on success, -ENOMEM otherwise 541 */ 542 int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, 543 unsigned long start, unsigned long end, pgoff_t pgoff) 544 { 545 struct vma_prepare vp; 546 547 WARN_ON((vma->vm_start != start) && (vma->vm_end != end)); 548 549 if (vma->vm_start < start) 550 vma_iter_config(vmi, vma->vm_start, start); 551 else 552 vma_iter_config(vmi, end, vma->vm_end); 553 554 if (vma_iter_prealloc(vmi, NULL)) 555 return -ENOMEM; 556 557 vma_start_write(vma); 558 559 init_vma_prep(&vp, vma); 560 vma_prepare(&vp); 561 vma_adjust_trans_huge(vma, start, end, 0); 562 563 vma_iter_clear(vmi); 564 vma_set_range(vma, start, end, pgoff); 565 vma_complete(&vp, vmi, vma->vm_mm); 566 validate_mm(vma->vm_mm); 567 return 0; 568 } 569 570 /* 571 * vma_complete- Helper function for handling the unlocking after altering VMAs, 572 * or for inserting a VMA. 573 * 574 * @vp: The vma_prepare struct 575 * @vmi: The vma iterator 576 * @mm: The mm_struct 577 */ 578 void vma_complete(struct vma_prepare *vp, 579 struct vma_iterator *vmi, struct mm_struct *mm) 580 { 581 if (vp->file) { 582 if (vp->adj_next) 583 vma_interval_tree_insert(vp->adj_next, 584 &vp->mapping->i_mmap); 585 vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap); 586 flush_dcache_mmap_unlock(vp->mapping); 587 } 588 589 if (vp->remove && vp->file) { 590 __remove_shared_vm_struct(vp->remove, vp->mapping); 591 if (vp->remove2) 592 __remove_shared_vm_struct(vp->remove2, vp->mapping); 593 } else if (vp->insert) { 594 /* 595 * split_vma has split insert from vma, and needs 596 * us to insert it before dropping the locks 597 * (it may either follow vma or precede it). 598 */ 599 vma_iter_store(vmi, vp->insert); 600 mm->map_count++; 601 } 602 603 if (vp->anon_vma) { 604 anon_vma_interval_tree_post_update_vma(vp->vma); 605 if (vp->adj_next) 606 anon_vma_interval_tree_post_update_vma(vp->adj_next); 607 anon_vma_unlock_write(vp->anon_vma); 608 } 609 610 if (vp->file) { 611 i_mmap_unlock_write(vp->mapping); 612 uprobe_mmap(vp->vma); 613 614 if (vp->adj_next) 615 uprobe_mmap(vp->adj_next); 616 } 617 618 if (vp->remove) { 619 again: 620 vma_mark_detached(vp->remove, true); 621 if (vp->file) { 622 uprobe_munmap(vp->remove, vp->remove->vm_start, 623 vp->remove->vm_end); 624 fput(vp->file); 625 } 626 if (vp->remove->anon_vma) 627 anon_vma_merge(vp->vma, vp->remove); 628 mm->map_count--; 629 mpol_put(vma_policy(vp->remove)); 630 if (!vp->remove2) 631 WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end); 632 vm_area_free(vp->remove); 633 634 /* 635 * In mprotect's case 6 (see comments on vma_merge), 636 * we are removing both mid and next vmas 637 */ 638 if (vp->remove2) { 639 vp->remove = vp->remove2; 640 vp->remove2 = NULL; 641 goto again; 642 } 643 } 644 if (vp->insert && vp->file) 645 uprobe_mmap(vp->insert); 646 } 647 648 static void vms_complete_pte_clear(struct vma_munmap_struct *vms, 649 struct ma_state *mas_detach, bool mm_wr_locked) 650 { 651 struct mmu_gather tlb; 652 653 /* 654 * We can free page tables without write-locking mmap_lock because VMAs 655 * were isolated before we downgraded mmap_lock. 656 */ 657 mas_set(mas_detach, 1); 658 lru_add_drain(); 659 tlb_gather_mmu(&tlb, vms->mm); 660 update_hiwater_rss(vms->mm); 661 unmap_vmas(&tlb, mas_detach, vms->vma, vms->start, vms->end, vms->vma_count, mm_wr_locked); 662 mas_set(mas_detach, 1); 663 /* start and end may be different if there is no prev or next vma. */ 664 free_pgtables(&tlb, mas_detach, vms->vma, vms->unmap_start, vms->unmap_end, mm_wr_locked); 665 tlb_finish_mmu(&tlb); 666 } 667 668 /* 669 * vms_complete_munmap_vmas() - Finish the munmap() operation 670 * @vms: The vma munmap struct 671 * @mas_detach: The maple state of the detached vmas 672 * 673 * This updates the mm_struct, unmaps the region, frees the resources 674 * used for the munmap() and may downgrade the lock - if requested. Everything 675 * needed to be done once the vma maple tree is updated. 676 */ 677 void vms_complete_munmap_vmas(struct vma_munmap_struct *vms, 678 struct ma_state *mas_detach) 679 { 680 struct vm_area_struct *vma; 681 struct mm_struct *mm; 682 683 mm = vms->mm; 684 mm->map_count -= vms->vma_count; 685 mm->locked_vm -= vms->locked_vm; 686 if (vms->unlock) 687 mmap_write_downgrade(mm); 688 689 vms_complete_pte_clear(vms, mas_detach, !vms->unlock); 690 /* Update high watermark before we lower total_vm */ 691 update_hiwater_vm(mm); 692 /* Stat accounting */ 693 WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm) - vms->nr_pages); 694 /* Paranoid bookkeeping */ 695 VM_WARN_ON(vms->exec_vm > mm->exec_vm); 696 VM_WARN_ON(vms->stack_vm > mm->stack_vm); 697 VM_WARN_ON(vms->data_vm > mm->data_vm); 698 mm->exec_vm -= vms->exec_vm; 699 mm->stack_vm -= vms->stack_vm; 700 mm->data_vm -= vms->data_vm; 701 702 /* Remove and clean up vmas */ 703 mas_set(mas_detach, 0); 704 mas_for_each(mas_detach, vma, ULONG_MAX) 705 remove_vma(vma, false); 706 707 vm_unacct_memory(vms->nr_accounted); 708 validate_mm(mm); 709 if (vms->unlock) 710 mmap_read_unlock(mm); 711 712 __mt_destroy(mas_detach->tree); 713 } 714 715 /* 716 * vms_gather_munmap_vmas() - Put all VMAs within a range into a maple tree 717 * for removal at a later date. Handles splitting first and last if necessary 718 * and marking the vmas as isolated. 719 * 720 * @vms: The vma munmap struct 721 * @mas_detach: The maple state tracking the detached tree 722 * 723 * Return: 0 on success, -EPERM on mseal vmas, -ENOMEM otherwise 724 */ 725 int vms_gather_munmap_vmas(struct vma_munmap_struct *vms, 726 struct ma_state *mas_detach) 727 { 728 struct vm_area_struct *next = NULL; 729 int error = -ENOMEM; 730 731 /* 732 * If we need to split any vma, do it now to save pain later. 733 * 734 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially 735 * unmapped vm_area_struct will remain in use: so lower split_vma 736 * places tmp vma above, and higher split_vma places tmp vma below. 737 */ 738 739 /* Does it split the first one? */ 740 if (vms->start > vms->vma->vm_start) { 741 742 /* 743 * Make sure that map_count on return from munmap() will 744 * not exceed its limit; but let map_count go just above 745 * its limit temporarily, to help free resources as expected. 746 */ 747 if (vms->end < vms->vma->vm_end && 748 vms->mm->map_count >= sysctl_max_map_count) 749 goto map_count_exceeded; 750 751 /* Don't bother splitting the VMA if we can't unmap it anyway */ 752 if (!can_modify_vma(vms->vma)) { 753 error = -EPERM; 754 goto start_split_failed; 755 } 756 757 if (__split_vma(vms->vmi, vms->vma, vms->start, 1)) 758 goto start_split_failed; 759 } 760 vms->prev = vma_prev(vms->vmi); 761 if (vms->prev) 762 vms->unmap_start = vms->prev->vm_end; 763 764 /* 765 * Detach a range of VMAs from the mm. Using next as a temp variable as 766 * it is always overwritten. 767 */ 768 for_each_vma_range(*(vms->vmi), next, vms->end) { 769 long nrpages; 770 771 if (!can_modify_vma(next)) { 772 error = -EPERM; 773 goto modify_vma_failed; 774 } 775 /* Does it split the end? */ 776 if (next->vm_end > vms->end) { 777 if (__split_vma(vms->vmi, next, vms->end, 0)) 778 goto end_split_failed; 779 } 780 vma_start_write(next); 781 mas_set(mas_detach, vms->vma_count++); 782 if (mas_store_gfp(mas_detach, next, GFP_KERNEL)) 783 goto munmap_gather_failed; 784 785 vma_mark_detached(next, true); 786 nrpages = vma_pages(next); 787 788 vms->nr_pages += nrpages; 789 if (next->vm_flags & VM_LOCKED) 790 vms->locked_vm += nrpages; 791 792 if (next->vm_flags & VM_ACCOUNT) 793 vms->nr_accounted += nrpages; 794 795 if (is_exec_mapping(next->vm_flags)) 796 vms->exec_vm += nrpages; 797 else if (is_stack_mapping(next->vm_flags)) 798 vms->stack_vm += nrpages; 799 else if (is_data_mapping(next->vm_flags)) 800 vms->data_vm += nrpages; 801 802 if (unlikely(vms->uf)) { 803 /* 804 * If userfaultfd_unmap_prep returns an error the vmas 805 * will remain split, but userland will get a 806 * highly unexpected error anyway. This is no 807 * different than the case where the first of the two 808 * __split_vma fails, but we don't undo the first 809 * split, despite we could. This is unlikely enough 810 * failure that it's not worth optimizing it for. 811 */ 812 if (userfaultfd_unmap_prep(next, vms->start, vms->end, 813 vms->uf)) 814 goto userfaultfd_error; 815 } 816 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 817 BUG_ON(next->vm_start < vms->start); 818 BUG_ON(next->vm_start > vms->end); 819 #endif 820 } 821 822 vms->next = vma_next(vms->vmi); 823 if (vms->next) 824 vms->unmap_end = vms->next->vm_start; 825 826 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 827 /* Make sure no VMAs are about to be lost. */ 828 { 829 MA_STATE(test, mas_detach->tree, 0, 0); 830 struct vm_area_struct *vma_mas, *vma_test; 831 int test_count = 0; 832 833 vma_iter_set(vms->vmi, vms->start); 834 rcu_read_lock(); 835 vma_test = mas_find(&test, vms->vma_count - 1); 836 for_each_vma_range(*(vms->vmi), vma_mas, vms->end) { 837 BUG_ON(vma_mas != vma_test); 838 test_count++; 839 vma_test = mas_next(&test, vms->vma_count - 1); 840 } 841 rcu_read_unlock(); 842 BUG_ON(vms->vma_count != test_count); 843 } 844 #endif 845 846 while (vma_iter_addr(vms->vmi) > vms->start) 847 vma_iter_prev_range(vms->vmi); 848 849 return 0; 850 851 userfaultfd_error: 852 munmap_gather_failed: 853 end_split_failed: 854 modify_vma_failed: 855 abort_munmap_vmas(mas_detach); 856 start_split_failed: 857 map_count_exceeded: 858 return error; 859 } 860 861 /* 862 * do_vmi_align_munmap() - munmap the aligned region from @start to @end. 863 * @vmi: The vma iterator 864 * @vma: The starting vm_area_struct 865 * @mm: The mm_struct 866 * @start: The aligned start address to munmap. 867 * @end: The aligned end address to munmap. 868 * @uf: The userfaultfd list_head 869 * @unlock: Set to true to drop the mmap_lock. unlocking only happens on 870 * success. 871 * 872 * Return: 0 on success and drops the lock if so directed, error and leaves the 873 * lock held otherwise. 874 */ 875 int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 876 struct mm_struct *mm, unsigned long start, unsigned long end, 877 struct list_head *uf, bool unlock) 878 { 879 struct maple_tree mt_detach; 880 MA_STATE(mas_detach, &mt_detach, 0, 0); 881 mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK); 882 mt_on_stack(mt_detach); 883 struct vma_munmap_struct vms; 884 int error; 885 886 init_vma_munmap(&vms, vmi, vma, start, end, uf, unlock); 887 error = vms_gather_munmap_vmas(&vms, &mas_detach); 888 if (error) 889 goto gather_failed; 890 891 error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL); 892 if (error) 893 goto clear_tree_failed; 894 895 /* Point of no return */ 896 vms_complete_munmap_vmas(&vms, &mas_detach); 897 return 0; 898 899 clear_tree_failed: 900 abort_munmap_vmas(&mas_detach); 901 gather_failed: 902 validate_mm(mm); 903 return error; 904 } 905 906 /* 907 * do_vmi_munmap() - munmap a given range. 908 * @vmi: The vma iterator 909 * @mm: The mm_struct 910 * @start: The start address to munmap 911 * @len: The length of the range to munmap 912 * @uf: The userfaultfd list_head 913 * @unlock: set to true if the user wants to drop the mmap_lock on success 914 * 915 * This function takes a @mas that is either pointing to the previous VMA or set 916 * to MA_START and sets it up to remove the mapping(s). The @len will be 917 * aligned. 918 * 919 * Return: 0 on success and drops the lock if so directed, error and leaves the 920 * lock held otherwise. 921 */ 922 int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 923 unsigned long start, size_t len, struct list_head *uf, 924 bool unlock) 925 { 926 unsigned long end; 927 struct vm_area_struct *vma; 928 929 if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) 930 return -EINVAL; 931 932 end = start + PAGE_ALIGN(len); 933 if (end == start) 934 return -EINVAL; 935 936 /* Find the first overlapping VMA */ 937 vma = vma_find(vmi, end); 938 if (!vma) { 939 if (unlock) 940 mmap_write_unlock(mm); 941 return 0; 942 } 943 944 return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock); 945 } 946 947 /* 948 * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name), 949 * figure out whether that can be merged with its predecessor or its 950 * successor. Or both (it neatly fills a hole). 951 * 952 * In most cases - when called for mmap, brk or mremap - [addr,end) is 953 * certain not to be mapped by the time vma_merge is called; but when 954 * called for mprotect, it is certain to be already mapped (either at 955 * an offset within prev, or at the start of next), and the flags of 956 * this area are about to be changed to vm_flags - and the no-change 957 * case has already been eliminated. 958 * 959 * The following mprotect cases have to be considered, where **** is 960 * the area passed down from mprotect_fixup, never extending beyond one 961 * vma, PPPP is the previous vma, CCCC is a concurrent vma that starts 962 * at the same address as **** and is of the same or larger span, and 963 * NNNN the next vma after ****: 964 * 965 * **** **** **** 966 * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPCCCCCC 967 * cannot merge might become might become 968 * PPNNNNNNNNNN PPPPPPPPPPCC 969 * mmap, brk or case 4 below case 5 below 970 * mremap move: 971 * **** **** 972 * PPPP NNNN PPPPCCCCNNNN 973 * might become might become 974 * PPPPPPPPPPPP 1 or PPPPPPPPPPPP 6 or 975 * PPPPPPPPNNNN 2 or PPPPPPPPNNNN 7 or 976 * PPPPNNNNNNNN 3 PPPPNNNNNNNN 8 977 * 978 * It is important for case 8 that the vma CCCC overlapping the 979 * region **** is never going to extended over NNNN. Instead NNNN must 980 * be extended in region **** and CCCC must be removed. This way in 981 * all cases where vma_merge succeeds, the moment vma_merge drops the 982 * rmap_locks, the properties of the merged vma will be already 983 * correct for the whole merged range. Some of those properties like 984 * vm_page_prot/vm_flags may be accessed by rmap_walks and they must 985 * be correct for the whole merged range immediately after the 986 * rmap_locks are released. Otherwise if NNNN would be removed and 987 * CCCC would be extended over the NNNN range, remove_migration_ptes 988 * or other rmap walkers (if working on addresses beyond the "end" 989 * parameter) may establish ptes with the wrong permissions of CCCC 990 * instead of the right permissions of NNNN. 991 * 992 * In the code below: 993 * PPPP is represented by *prev 994 * CCCC is represented by *curr or not represented at all (NULL) 995 * NNNN is represented by *next or not represented at all (NULL) 996 * **** is not represented - it will be merged and the vma containing the 997 * area is returned, or the function will return NULL 998 */ 999 static struct vm_area_struct 1000 *vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev, 1001 struct vm_area_struct *src, unsigned long addr, unsigned long end, 1002 unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy, 1003 struct vm_userfaultfd_ctx vm_userfaultfd_ctx, 1004 struct anon_vma_name *anon_name) 1005 { 1006 struct mm_struct *mm = src->vm_mm; 1007 struct anon_vma *anon_vma = src->anon_vma; 1008 struct file *file = src->vm_file; 1009 struct vm_area_struct *curr, *next, *res; 1010 struct vm_area_struct *vma, *adjust, *remove, *remove2; 1011 struct vm_area_struct *anon_dup = NULL; 1012 struct vma_prepare vp; 1013 pgoff_t vma_pgoff; 1014 int err = 0; 1015 bool merge_prev = false; 1016 bool merge_next = false; 1017 bool vma_expanded = false; 1018 unsigned long vma_start = addr; 1019 unsigned long vma_end = end; 1020 pgoff_t pglen = (end - addr) >> PAGE_SHIFT; 1021 long adj_start = 0; 1022 1023 /* 1024 * We later require that vma->vm_flags == vm_flags, 1025 * so this tests vma->vm_flags & VM_SPECIAL, too. 1026 */ 1027 if (vm_flags & VM_SPECIAL) 1028 return NULL; 1029 1030 /* Does the input range span an existing VMA? (cases 5 - 8) */ 1031 curr = find_vma_intersection(mm, prev ? prev->vm_end : 0, end); 1032 1033 if (!curr || /* cases 1 - 4 */ 1034 end == curr->vm_end) /* cases 6 - 8, adjacent VMA */ 1035 next = vma_lookup(mm, end); 1036 else 1037 next = NULL; /* case 5 */ 1038 1039 if (prev) { 1040 vma_start = prev->vm_start; 1041 vma_pgoff = prev->vm_pgoff; 1042 1043 /* Can we merge the predecessor? */ 1044 if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy) 1045 && can_vma_merge_after(prev, vm_flags, anon_vma, file, 1046 pgoff, vm_userfaultfd_ctx, anon_name)) { 1047 merge_prev = true; 1048 vma_prev(vmi); 1049 } 1050 } 1051 1052 /* Can we merge the successor? */ 1053 if (next && mpol_equal(policy, vma_policy(next)) && 1054 can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen, 1055 vm_userfaultfd_ctx, anon_name)) { 1056 merge_next = true; 1057 } 1058 1059 /* Verify some invariant that must be enforced by the caller. */ 1060 VM_WARN_ON(prev && addr <= prev->vm_start); 1061 VM_WARN_ON(curr && (addr != curr->vm_start || end > curr->vm_end)); 1062 VM_WARN_ON(addr >= end); 1063 1064 if (!merge_prev && !merge_next) 1065 return NULL; /* Not mergeable. */ 1066 1067 if (merge_prev) 1068 vma_start_write(prev); 1069 1070 res = vma = prev; 1071 remove = remove2 = adjust = NULL; 1072 1073 /* Can we merge both the predecessor and the successor? */ 1074 if (merge_prev && merge_next && 1075 is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) { 1076 vma_start_write(next); 1077 remove = next; /* case 1 */ 1078 vma_end = next->vm_end; 1079 err = dup_anon_vma(prev, next, &anon_dup); 1080 if (curr) { /* case 6 */ 1081 vma_start_write(curr); 1082 remove = curr; 1083 remove2 = next; 1084 /* 1085 * Note that the dup_anon_vma below cannot overwrite err 1086 * since the first caller would do nothing unless next 1087 * has an anon_vma. 1088 */ 1089 if (!next->anon_vma) 1090 err = dup_anon_vma(prev, curr, &anon_dup); 1091 } 1092 } else if (merge_prev) { /* case 2 */ 1093 if (curr) { 1094 vma_start_write(curr); 1095 if (end == curr->vm_end) { /* case 7 */ 1096 /* 1097 * can_vma_merge_after() assumed we would not be 1098 * removing prev vma, so it skipped the check 1099 * for vm_ops->close, but we are removing curr 1100 */ 1101 if (curr->vm_ops && curr->vm_ops->close) 1102 err = -EINVAL; 1103 remove = curr; 1104 } else { /* case 5 */ 1105 adjust = curr; 1106 adj_start = (end - curr->vm_start); 1107 } 1108 if (!err) 1109 err = dup_anon_vma(prev, curr, &anon_dup); 1110 } 1111 } else { /* merge_next */ 1112 vma_start_write(next); 1113 res = next; 1114 if (prev && addr < prev->vm_end) { /* case 4 */ 1115 vma_start_write(prev); 1116 vma_end = addr; 1117 adjust = next; 1118 adj_start = -(prev->vm_end - addr); 1119 err = dup_anon_vma(next, prev, &anon_dup); 1120 } else { 1121 /* 1122 * Note that cases 3 and 8 are the ONLY ones where prev 1123 * is permitted to be (but is not necessarily) NULL. 1124 */ 1125 vma = next; /* case 3 */ 1126 vma_start = addr; 1127 vma_end = next->vm_end; 1128 vma_pgoff = next->vm_pgoff - pglen; 1129 if (curr) { /* case 8 */ 1130 vma_pgoff = curr->vm_pgoff; 1131 vma_start_write(curr); 1132 remove = curr; 1133 err = dup_anon_vma(next, curr, &anon_dup); 1134 } 1135 } 1136 } 1137 1138 /* Error in anon_vma clone. */ 1139 if (err) 1140 goto anon_vma_fail; 1141 1142 if (vma_start < vma->vm_start || vma_end > vma->vm_end) 1143 vma_expanded = true; 1144 1145 if (vma_expanded) { 1146 vma_iter_config(vmi, vma_start, vma_end); 1147 } else { 1148 vma_iter_config(vmi, adjust->vm_start + adj_start, 1149 adjust->vm_end); 1150 } 1151 1152 if (vma_iter_prealloc(vmi, vma)) 1153 goto prealloc_fail; 1154 1155 init_multi_vma_prep(&vp, vma, adjust, remove, remove2); 1156 VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma && 1157 vp.anon_vma != adjust->anon_vma); 1158 1159 vma_prepare(&vp); 1160 vma_adjust_trans_huge(vma, vma_start, vma_end, adj_start); 1161 vma_set_range(vma, vma_start, vma_end, vma_pgoff); 1162 1163 if (vma_expanded) 1164 vma_iter_store(vmi, vma); 1165 1166 if (adj_start) { 1167 adjust->vm_start += adj_start; 1168 adjust->vm_pgoff += adj_start >> PAGE_SHIFT; 1169 if (adj_start < 0) { 1170 WARN_ON(vma_expanded); 1171 vma_iter_store(vmi, next); 1172 } 1173 } 1174 1175 vma_complete(&vp, vmi, mm); 1176 validate_mm(mm); 1177 khugepaged_enter_vma(res, vm_flags); 1178 return res; 1179 1180 prealloc_fail: 1181 if (anon_dup) 1182 unlink_anon_vmas(anon_dup); 1183 1184 anon_vma_fail: 1185 vma_iter_set(vmi, addr); 1186 vma_iter_load(vmi); 1187 return NULL; 1188 } 1189 1190 /* 1191 * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd 1192 * context and anonymous VMA name within the range [start, end). 1193 * 1194 * As a result, we might be able to merge the newly modified VMA range with an 1195 * adjacent VMA with identical properties. 1196 * 1197 * If no merge is possible and the range does not span the entirety of the VMA, 1198 * we then need to split the VMA to accommodate the change. 1199 * 1200 * The function returns either the merged VMA, the original VMA if a split was 1201 * required instead, or an error if the split failed. 1202 */ 1203 struct vm_area_struct *vma_modify(struct vma_iterator *vmi, 1204 struct vm_area_struct *prev, 1205 struct vm_area_struct *vma, 1206 unsigned long start, unsigned long end, 1207 unsigned long vm_flags, 1208 struct mempolicy *policy, 1209 struct vm_userfaultfd_ctx uffd_ctx, 1210 struct anon_vma_name *anon_name) 1211 { 1212 pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 1213 struct vm_area_struct *merged; 1214 1215 merged = vma_merge(vmi, prev, vma, start, end, vm_flags, 1216 pgoff, policy, uffd_ctx, anon_name); 1217 if (merged) 1218 return merged; 1219 1220 if (vma->vm_start < start) { 1221 int err = split_vma(vmi, vma, start, 1); 1222 1223 if (err) 1224 return ERR_PTR(err); 1225 } 1226 1227 if (vma->vm_end > end) { 1228 int err = split_vma(vmi, vma, end, 0); 1229 1230 if (err) 1231 return ERR_PTR(err); 1232 } 1233 1234 return vma; 1235 } 1236 1237 /* 1238 * Attempt to merge a newly mapped VMA with those adjacent to it. The caller 1239 * must ensure that [start, end) does not overlap any existing VMA. 1240 */ 1241 struct vm_area_struct 1242 *vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev, 1243 struct vm_area_struct *vma, unsigned long start, 1244 unsigned long end, pgoff_t pgoff) 1245 { 1246 return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff, 1247 vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 1248 } 1249 1250 /* 1251 * Expand vma by delta bytes, potentially merging with an immediately adjacent 1252 * VMA with identical properties. 1253 */ 1254 struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi, 1255 struct vm_area_struct *vma, 1256 unsigned long delta) 1257 { 1258 pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma); 1259 1260 /* vma is specified as prev, so case 1 or 2 will apply. */ 1261 return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta, 1262 vma->vm_flags, pgoff, vma_policy(vma), 1263 vma->vm_userfaultfd_ctx, anon_vma_name(vma)); 1264 } 1265 1266 void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb) 1267 { 1268 vb->count = 0; 1269 } 1270 1271 static void unlink_file_vma_batch_process(struct unlink_vma_file_batch *vb) 1272 { 1273 struct address_space *mapping; 1274 int i; 1275 1276 mapping = vb->vmas[0]->vm_file->f_mapping; 1277 i_mmap_lock_write(mapping); 1278 for (i = 0; i < vb->count; i++) { 1279 VM_WARN_ON_ONCE(vb->vmas[i]->vm_file->f_mapping != mapping); 1280 __remove_shared_vm_struct(vb->vmas[i], mapping); 1281 } 1282 i_mmap_unlock_write(mapping); 1283 1284 unlink_file_vma_batch_init(vb); 1285 } 1286 1287 void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, 1288 struct vm_area_struct *vma) 1289 { 1290 if (vma->vm_file == NULL) 1291 return; 1292 1293 if ((vb->count > 0 && vb->vmas[0]->vm_file != vma->vm_file) || 1294 vb->count == ARRAY_SIZE(vb->vmas)) 1295 unlink_file_vma_batch_process(vb); 1296 1297 vb->vmas[vb->count] = vma; 1298 vb->count++; 1299 } 1300 1301 void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb) 1302 { 1303 if (vb->count > 0) 1304 unlink_file_vma_batch_process(vb); 1305 } 1306 1307 /* 1308 * Unlink a file-based vm structure from its interval tree, to hide 1309 * vma from rmap and vmtruncate before freeing its page tables. 1310 */ 1311 void unlink_file_vma(struct vm_area_struct *vma) 1312 { 1313 struct file *file = vma->vm_file; 1314 1315 if (file) { 1316 struct address_space *mapping = file->f_mapping; 1317 1318 i_mmap_lock_write(mapping); 1319 __remove_shared_vm_struct(vma, mapping); 1320 i_mmap_unlock_write(mapping); 1321 } 1322 } 1323 1324 void vma_link_file(struct vm_area_struct *vma) 1325 { 1326 struct file *file = vma->vm_file; 1327 struct address_space *mapping; 1328 1329 if (file) { 1330 mapping = file->f_mapping; 1331 i_mmap_lock_write(mapping); 1332 __vma_link_file(vma, mapping); 1333 i_mmap_unlock_write(mapping); 1334 } 1335 } 1336 1337 int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) 1338 { 1339 VMA_ITERATOR(vmi, mm, 0); 1340 1341 vma_iter_config(&vmi, vma->vm_start, vma->vm_end); 1342 if (vma_iter_prealloc(&vmi, vma)) 1343 return -ENOMEM; 1344 1345 vma_start_write(vma); 1346 vma_iter_store(&vmi, vma); 1347 vma_link_file(vma); 1348 mm->map_count++; 1349 validate_mm(mm); 1350 return 0; 1351 } 1352 1353 /* 1354 * Copy the vma structure to a new location in the same mm, 1355 * prior to moving page table entries, to effect an mremap move. 1356 */ 1357 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, 1358 unsigned long addr, unsigned long len, pgoff_t pgoff, 1359 bool *need_rmap_locks) 1360 { 1361 struct vm_area_struct *vma = *vmap; 1362 unsigned long vma_start = vma->vm_start; 1363 struct mm_struct *mm = vma->vm_mm; 1364 struct vm_area_struct *new_vma, *prev; 1365 bool faulted_in_anon_vma = true; 1366 VMA_ITERATOR(vmi, mm, addr); 1367 1368 /* 1369 * If anonymous vma has not yet been faulted, update new pgoff 1370 * to match new location, to increase its chance of merging. 1371 */ 1372 if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { 1373 pgoff = addr >> PAGE_SHIFT; 1374 faulted_in_anon_vma = false; 1375 } 1376 1377 new_vma = find_vma_prev(mm, addr, &prev); 1378 if (new_vma && new_vma->vm_start < addr + len) 1379 return NULL; /* should never get here */ 1380 1381 new_vma = vma_merge_new_vma(&vmi, prev, vma, addr, addr + len, pgoff); 1382 if (new_vma) { 1383 /* 1384 * Source vma may have been merged into new_vma 1385 */ 1386 if (unlikely(vma_start >= new_vma->vm_start && 1387 vma_start < new_vma->vm_end)) { 1388 /* 1389 * The only way we can get a vma_merge with 1390 * self during an mremap is if the vma hasn't 1391 * been faulted in yet and we were allowed to 1392 * reset the dst vma->vm_pgoff to the 1393 * destination address of the mremap to allow 1394 * the merge to happen. mremap must change the 1395 * vm_pgoff linearity between src and dst vmas 1396 * (in turn preventing a vma_merge) to be 1397 * safe. It is only safe to keep the vm_pgoff 1398 * linear if there are no pages mapped yet. 1399 */ 1400 VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); 1401 *vmap = vma = new_vma; 1402 } 1403 *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff); 1404 } else { 1405 new_vma = vm_area_dup(vma); 1406 if (!new_vma) 1407 goto out; 1408 vma_set_range(new_vma, addr, addr + len, pgoff); 1409 if (vma_dup_policy(vma, new_vma)) 1410 goto out_free_vma; 1411 if (anon_vma_clone(new_vma, vma)) 1412 goto out_free_mempol; 1413 if (new_vma->vm_file) 1414 get_file(new_vma->vm_file); 1415 if (new_vma->vm_ops && new_vma->vm_ops->open) 1416 new_vma->vm_ops->open(new_vma); 1417 if (vma_link(mm, new_vma)) 1418 goto out_vma_link; 1419 *need_rmap_locks = false; 1420 } 1421 return new_vma; 1422 1423 out_vma_link: 1424 if (new_vma->vm_ops && new_vma->vm_ops->close) 1425 new_vma->vm_ops->close(new_vma); 1426 1427 if (new_vma->vm_file) 1428 fput(new_vma->vm_file); 1429 1430 unlink_anon_vmas(new_vma); 1431 out_free_mempol: 1432 mpol_put(vma_policy(new_vma)); 1433 out_free_vma: 1434 vm_area_free(new_vma); 1435 out: 1436 return NULL; 1437 } 1438 1439 /* 1440 * Rough compatibility check to quickly see if it's even worth looking 1441 * at sharing an anon_vma. 1442 * 1443 * They need to have the same vm_file, and the flags can only differ 1444 * in things that mprotect may change. 1445 * 1446 * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that 1447 * we can merge the two vma's. For example, we refuse to merge a vma if 1448 * there is a vm_ops->close() function, because that indicates that the 1449 * driver is doing some kind of reference counting. But that doesn't 1450 * really matter for the anon_vma sharing case. 1451 */ 1452 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b) 1453 { 1454 return a->vm_end == b->vm_start && 1455 mpol_equal(vma_policy(a), vma_policy(b)) && 1456 a->vm_file == b->vm_file && 1457 !((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) && 1458 b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); 1459 } 1460 1461 /* 1462 * Do some basic sanity checking to see if we can re-use the anon_vma 1463 * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be 1464 * the same as 'old', the other will be the new one that is trying 1465 * to share the anon_vma. 1466 * 1467 * NOTE! This runs with mmap_lock held for reading, so it is possible that 1468 * the anon_vma of 'old' is concurrently in the process of being set up 1469 * by another page fault trying to merge _that_. But that's ok: if it 1470 * is being set up, that automatically means that it will be a singleton 1471 * acceptable for merging, so we can do all of this optimistically. But 1472 * we do that READ_ONCE() to make sure that we never re-load the pointer. 1473 * 1474 * IOW: that the "list_is_singular()" test on the anon_vma_chain only 1475 * matters for the 'stable anon_vma' case (ie the thing we want to avoid 1476 * is to return an anon_vma that is "complex" due to having gone through 1477 * a fork). 1478 * 1479 * We also make sure that the two vma's are compatible (adjacent, 1480 * and with the same memory policies). That's all stable, even with just 1481 * a read lock on the mmap_lock. 1482 */ 1483 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, 1484 struct vm_area_struct *a, 1485 struct vm_area_struct *b) 1486 { 1487 if (anon_vma_compatible(a, b)) { 1488 struct anon_vma *anon_vma = READ_ONCE(old->anon_vma); 1489 1490 if (anon_vma && list_is_singular(&old->anon_vma_chain)) 1491 return anon_vma; 1492 } 1493 return NULL; 1494 } 1495 1496 /* 1497 * find_mergeable_anon_vma is used by anon_vma_prepare, to check 1498 * neighbouring vmas for a suitable anon_vma, before it goes off 1499 * to allocate a new anon_vma. It checks because a repetitive 1500 * sequence of mprotects and faults may otherwise lead to distinct 1501 * anon_vmas being allocated, preventing vma merge in subsequent 1502 * mprotect. 1503 */ 1504 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) 1505 { 1506 struct anon_vma *anon_vma = NULL; 1507 struct vm_area_struct *prev, *next; 1508 VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_end); 1509 1510 /* Try next first. */ 1511 next = vma_iter_load(&vmi); 1512 if (next) { 1513 anon_vma = reusable_anon_vma(next, vma, next); 1514 if (anon_vma) 1515 return anon_vma; 1516 } 1517 1518 prev = vma_prev(&vmi); 1519 VM_BUG_ON_VMA(prev != vma, vma); 1520 prev = vma_prev(&vmi); 1521 /* Try prev next. */ 1522 if (prev) 1523 anon_vma = reusable_anon_vma(prev, prev, vma); 1524 1525 /* 1526 * We might reach here with anon_vma == NULL if we can't find 1527 * any reusable anon_vma. 1528 * There's no absolute need to look only at touching neighbours: 1529 * we could search further afield for "compatible" anon_vmas. 1530 * But it would probably just be a waste of time searching, 1531 * or lead to too many vmas hanging off the same anon_vma. 1532 * We're trying to allow mprotect remerging later on, 1533 * not trying to minimize memory used for anon_vmas. 1534 */ 1535 return anon_vma; 1536 } 1537 1538 static bool vm_ops_needs_writenotify(const struct vm_operations_struct *vm_ops) 1539 { 1540 return vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite); 1541 } 1542 1543 static bool vma_is_shared_writable(struct vm_area_struct *vma) 1544 { 1545 return (vma->vm_flags & (VM_WRITE | VM_SHARED)) == 1546 (VM_WRITE | VM_SHARED); 1547 } 1548 1549 static bool vma_fs_can_writeback(struct vm_area_struct *vma) 1550 { 1551 /* No managed pages to writeback. */ 1552 if (vma->vm_flags & VM_PFNMAP) 1553 return false; 1554 1555 return vma->vm_file && vma->vm_file->f_mapping && 1556 mapping_can_writeback(vma->vm_file->f_mapping); 1557 } 1558 1559 /* 1560 * Does this VMA require the underlying folios to have their dirty state 1561 * tracked? 1562 */ 1563 bool vma_needs_dirty_tracking(struct vm_area_struct *vma) 1564 { 1565 /* Only shared, writable VMAs require dirty tracking. */ 1566 if (!vma_is_shared_writable(vma)) 1567 return false; 1568 1569 /* Does the filesystem need to be notified? */ 1570 if (vm_ops_needs_writenotify(vma->vm_ops)) 1571 return true; 1572 1573 /* 1574 * Even if the filesystem doesn't indicate a need for writenotify, if it 1575 * can writeback, dirty tracking is still required. 1576 */ 1577 return vma_fs_can_writeback(vma); 1578 } 1579 1580 /* 1581 * Some shared mappings will want the pages marked read-only 1582 * to track write events. If so, we'll downgrade vm_page_prot 1583 * to the private version (using protection_map[] without the 1584 * VM_SHARED bit). 1585 */ 1586 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) 1587 { 1588 /* If it was private or non-writable, the write bit is already clear */ 1589 if (!vma_is_shared_writable(vma)) 1590 return false; 1591 1592 /* The backer wishes to know when pages are first written to? */ 1593 if (vm_ops_needs_writenotify(vma->vm_ops)) 1594 return true; 1595 1596 /* The open routine did something to the protections that pgprot_modify 1597 * won't preserve? */ 1598 if (pgprot_val(vm_page_prot) != 1599 pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags))) 1600 return false; 1601 1602 /* 1603 * Do we need to track softdirty? hugetlb does not support softdirty 1604 * tracking yet. 1605 */ 1606 if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma)) 1607 return true; 1608 1609 /* Do we need write faults for uffd-wp tracking? */ 1610 if (userfaultfd_wp(vma)) 1611 return true; 1612 1613 /* Can the mapping track the dirty pages? */ 1614 return vma_fs_can_writeback(vma); 1615 } 1616 1617 unsigned long count_vma_pages_range(struct mm_struct *mm, 1618 unsigned long addr, unsigned long end) 1619 { 1620 VMA_ITERATOR(vmi, mm, addr); 1621 struct vm_area_struct *vma; 1622 unsigned long nr_pages = 0; 1623 1624 for_each_vma_range(vmi, vma, end) { 1625 unsigned long vm_start = max(addr, vma->vm_start); 1626 unsigned long vm_end = min(end, vma->vm_end); 1627 1628 nr_pages += PHYS_PFN(vm_end - vm_start); 1629 } 1630 1631 return nr_pages; 1632 } 1633 1634 static DEFINE_MUTEX(mm_all_locks_mutex); 1635 1636 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma) 1637 { 1638 if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 1639 /* 1640 * The LSB of head.next can't change from under us 1641 * because we hold the mm_all_locks_mutex. 1642 */ 1643 down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock); 1644 /* 1645 * We can safely modify head.next after taking the 1646 * anon_vma->root->rwsem. If some other vma in this mm shares 1647 * the same anon_vma we won't take it again. 1648 * 1649 * No need of atomic instructions here, head.next 1650 * can't change from under us thanks to the 1651 * anon_vma->root->rwsem. 1652 */ 1653 if (__test_and_set_bit(0, (unsigned long *) 1654 &anon_vma->root->rb_root.rb_root.rb_node)) 1655 BUG(); 1656 } 1657 } 1658 1659 static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) 1660 { 1661 if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 1662 /* 1663 * AS_MM_ALL_LOCKS can't change from under us because 1664 * we hold the mm_all_locks_mutex. 1665 * 1666 * Operations on ->flags have to be atomic because 1667 * even if AS_MM_ALL_LOCKS is stable thanks to the 1668 * mm_all_locks_mutex, there may be other cpus 1669 * changing other bitflags in parallel to us. 1670 */ 1671 if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags)) 1672 BUG(); 1673 down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock); 1674 } 1675 } 1676 1677 /* 1678 * This operation locks against the VM for all pte/vma/mm related 1679 * operations that could ever happen on a certain mm. This includes 1680 * vmtruncate, try_to_unmap, and all page faults. 1681 * 1682 * The caller must take the mmap_lock in write mode before calling 1683 * mm_take_all_locks(). The caller isn't allowed to release the 1684 * mmap_lock until mm_drop_all_locks() returns. 1685 * 1686 * mmap_lock in write mode is required in order to block all operations 1687 * that could modify pagetables and free pages without need of 1688 * altering the vma layout. It's also needed in write mode to avoid new 1689 * anon_vmas to be associated with existing vmas. 1690 * 1691 * A single task can't take more than one mm_take_all_locks() in a row 1692 * or it would deadlock. 1693 * 1694 * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in 1695 * mapping->flags avoid to take the same lock twice, if more than one 1696 * vma in this mm is backed by the same anon_vma or address_space. 1697 * 1698 * We take locks in following order, accordingly to comment at beginning 1699 * of mm/rmap.c: 1700 * - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for 1701 * hugetlb mapping); 1702 * - all vmas marked locked 1703 * - all i_mmap_rwsem locks; 1704 * - all anon_vma->rwseml 1705 * 1706 * We can take all locks within these types randomly because the VM code 1707 * doesn't nest them and we protected from parallel mm_take_all_locks() by 1708 * mm_all_locks_mutex. 1709 * 1710 * mm_take_all_locks() and mm_drop_all_locks are expensive operations 1711 * that may have to take thousand of locks. 1712 * 1713 * mm_take_all_locks() can fail if it's interrupted by signals. 1714 */ 1715 int mm_take_all_locks(struct mm_struct *mm) 1716 { 1717 struct vm_area_struct *vma; 1718 struct anon_vma_chain *avc; 1719 VMA_ITERATOR(vmi, mm, 0); 1720 1721 mmap_assert_write_locked(mm); 1722 1723 mutex_lock(&mm_all_locks_mutex); 1724 1725 /* 1726 * vma_start_write() does not have a complement in mm_drop_all_locks() 1727 * because vma_start_write() is always asymmetrical; it marks a VMA as 1728 * being written to until mmap_write_unlock() or mmap_write_downgrade() 1729 * is reached. 1730 */ 1731 for_each_vma(vmi, vma) { 1732 if (signal_pending(current)) 1733 goto out_unlock; 1734 vma_start_write(vma); 1735 } 1736 1737 vma_iter_init(&vmi, mm, 0); 1738 for_each_vma(vmi, vma) { 1739 if (signal_pending(current)) 1740 goto out_unlock; 1741 if (vma->vm_file && vma->vm_file->f_mapping && 1742 is_vm_hugetlb_page(vma)) 1743 vm_lock_mapping(mm, vma->vm_file->f_mapping); 1744 } 1745 1746 vma_iter_init(&vmi, mm, 0); 1747 for_each_vma(vmi, vma) { 1748 if (signal_pending(current)) 1749 goto out_unlock; 1750 if (vma->vm_file && vma->vm_file->f_mapping && 1751 !is_vm_hugetlb_page(vma)) 1752 vm_lock_mapping(mm, vma->vm_file->f_mapping); 1753 } 1754 1755 vma_iter_init(&vmi, mm, 0); 1756 for_each_vma(vmi, vma) { 1757 if (signal_pending(current)) 1758 goto out_unlock; 1759 if (vma->anon_vma) 1760 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 1761 vm_lock_anon_vma(mm, avc->anon_vma); 1762 } 1763 1764 return 0; 1765 1766 out_unlock: 1767 mm_drop_all_locks(mm); 1768 return -EINTR; 1769 } 1770 1771 static void vm_unlock_anon_vma(struct anon_vma *anon_vma) 1772 { 1773 if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) { 1774 /* 1775 * The LSB of head.next can't change to 0 from under 1776 * us because we hold the mm_all_locks_mutex. 1777 * 1778 * We must however clear the bitflag before unlocking 1779 * the vma so the users using the anon_vma->rb_root will 1780 * never see our bitflag. 1781 * 1782 * No need of atomic instructions here, head.next 1783 * can't change from under us until we release the 1784 * anon_vma->root->rwsem. 1785 */ 1786 if (!__test_and_clear_bit(0, (unsigned long *) 1787 &anon_vma->root->rb_root.rb_root.rb_node)) 1788 BUG(); 1789 anon_vma_unlock_write(anon_vma); 1790 } 1791 } 1792 1793 static void vm_unlock_mapping(struct address_space *mapping) 1794 { 1795 if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) { 1796 /* 1797 * AS_MM_ALL_LOCKS can't change to 0 from under us 1798 * because we hold the mm_all_locks_mutex. 1799 */ 1800 i_mmap_unlock_write(mapping); 1801 if (!test_and_clear_bit(AS_MM_ALL_LOCKS, 1802 &mapping->flags)) 1803 BUG(); 1804 } 1805 } 1806 1807 /* 1808 * The mmap_lock cannot be released by the caller until 1809 * mm_drop_all_locks() returns. 1810 */ 1811 void mm_drop_all_locks(struct mm_struct *mm) 1812 { 1813 struct vm_area_struct *vma; 1814 struct anon_vma_chain *avc; 1815 VMA_ITERATOR(vmi, mm, 0); 1816 1817 mmap_assert_write_locked(mm); 1818 BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); 1819 1820 for_each_vma(vmi, vma) { 1821 if (vma->anon_vma) 1822 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) 1823 vm_unlock_anon_vma(avc->anon_vma); 1824 if (vma->vm_file && vma->vm_file->f_mapping) 1825 vm_unlock_mapping(vma->vm_file->f_mapping); 1826 } 1827 1828 mutex_unlock(&mm_all_locks_mutex); 1829 } 1830