1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * vma.h 4 * 5 * Core VMA manipulation API implemented in vma.c, vma_init.c and vma_exec.c. 6 * 7 * Note that, in order for VMA logic to be userland testable, this header 8 * intentionally includes no dependencies. 9 * 10 * This is specifically scoped to mm-only. Users of this functionality (other 11 * than the core VMA implementation itself) should not include this header 12 * directly, but rather include internal.h. 13 */ 14 #ifndef __MM_VMA_H 15 #define __MM_VMA_H 16 17 /* 18 * VMA lock generalization 19 */ 20 struct vma_prepare { 21 struct vm_area_struct *vma; 22 struct vm_area_struct *adj_next; 23 struct file *file; 24 struct address_space *mapping; 25 struct anon_vma *anon_vma; 26 struct vm_area_struct *insert; 27 struct vm_area_struct *remove; 28 struct vm_area_struct *remove2; 29 30 bool skip_vma_uprobe :1; 31 }; 32 33 struct unlink_vma_file_batch { 34 int count; 35 struct vm_area_struct *vmas[8]; 36 }; 37 38 /* 39 * vma munmap operation 40 */ 41 struct vma_munmap_struct { 42 struct vma_iterator *vmi; 43 struct vm_area_struct *vma; /* The first vma to munmap */ 44 struct vm_area_struct *prev; /* vma before the munmap area */ 45 struct vm_area_struct *next; /* vma after the munmap area */ 46 struct list_head *uf; /* Userfaultfd list_head */ 47 unsigned long start; /* Aligned start addr (inclusive) */ 48 unsigned long end; /* Aligned end addr (exclusive) */ 49 unsigned long unmap_start; /* Unmap PTE start */ 50 unsigned long unmap_end; /* Unmap PTE end */ 51 int vma_count; /* Number of vmas that will be removed */ 52 bool unlock; /* Unlock after the munmap */ 53 bool clear_ptes; /* If there are outstanding PTE to be cleared */ 54 /* 2 byte hole */ 55 unsigned long nr_pages; /* Number of pages being removed */ 56 unsigned long locked_vm; /* Number of locked pages */ 57 unsigned long nr_accounted; /* Number of VM_ACCOUNT pages */ 58 unsigned long exec_vm; 59 unsigned long stack_vm; 60 unsigned long data_vm; 61 }; 62 63 enum vma_merge_state { 64 VMA_MERGE_START, 65 VMA_MERGE_ERROR_NOMEM, 66 VMA_MERGE_NOMERGE, 67 VMA_MERGE_SUCCESS, 68 }; 69 70 /* 71 * Describes a VMA merge operation and is threaded throughout it. 72 * 73 * Any of the fields may be mutated by the merge operation, so no guarantees are 74 * made to the contents of this structure after a merge operation has completed. 75 */ 76 struct vma_merge_struct { 77 struct mm_struct *mm; 78 struct vma_iterator *vmi; 79 /* 80 * Adjacent VMAs, any of which may be NULL if not present: 81 * 82 * |------|--------|------| 83 * | prev | middle | next | 84 * |------|--------|------| 85 * 86 * middle may not yet exist in the case of a proposed new VMA being 87 * merged, or it may be an existing VMA. 88 * 89 * next may be assigned by the caller. 90 */ 91 struct vm_area_struct *prev; 92 struct vm_area_struct *middle; 93 struct vm_area_struct *next; 94 /* This is the VMA we ultimately target to become the merged VMA. */ 95 struct vm_area_struct *target; 96 /* 97 * Initially, the start, end, pgoff fields are provided by the caller 98 * and describe the proposed new VMA range, whether modifying an 99 * existing VMA (which will be 'middle'), or adding a new one. 100 * 101 * During the merge process these fields are updated to describe the new 102 * range _including those VMAs which will be merged_. 103 */ 104 unsigned long start; 105 unsigned long end; 106 pgoff_t pgoff; 107 108 union { 109 /* Temporary while VMA flags are being converted. */ 110 vm_flags_t vm_flags; 111 vma_flags_t vma_flags; 112 }; 113 struct file *file; 114 struct anon_vma *anon_vma; 115 struct mempolicy *policy; 116 struct vm_userfaultfd_ctx uffd_ctx; 117 struct anon_vma_name *anon_name; 118 enum vma_merge_state state; 119 120 /* If copied from (i.e. mremap()'d) the VMA from which we are copying. */ 121 struct vm_area_struct *copied_from; 122 123 /* Flags which callers can use to modify merge behaviour: */ 124 125 /* 126 * If we can expand, simply do so. We know there is nothing to merge to 127 * the right. Does not reset state upon failure to merge. The VMA 128 * iterator is assumed to be positioned at the previous VMA, rather than 129 * at the gap. 130 */ 131 bool just_expand :1; 132 133 /* 134 * If a merge is possible, but an OOM error occurs, give up and don't 135 * execute the merge, returning NULL. 136 */ 137 bool give_up_on_oom :1; 138 139 /* 140 * If set, skip uprobe_mmap upon merged vma. 141 */ 142 bool skip_vma_uprobe :1; 143 144 /* Internal flags set during merge process: */ 145 146 /* 147 * Internal flag indicating the merge increases vmg->middle->vm_start 148 * (and thereby, vmg->prev->vm_end). 149 */ 150 bool __adjust_middle_start :1; 151 /* 152 * Internal flag indicating the merge decreases vmg->next->vm_start 153 * (and thereby, vmg->middle->vm_end). 154 */ 155 bool __adjust_next_start :1; 156 /* 157 * Internal flag used during the merge operation to indicate we will 158 * remove vmg->middle. 159 */ 160 bool __remove_middle :1; 161 /* 162 * Internal flag used during the merge operation to indicate we will 163 * remove vmg->next. 164 */ 165 bool __remove_next :1; 166 167 }; 168 169 struct unmap_desc { 170 struct ma_state *mas; /* the maple state point to the first vma */ 171 struct vm_area_struct *first; /* The first vma */ 172 unsigned long pg_start; /* The first pagetable address to free (floor) */ 173 unsigned long pg_end; /* The last pagetable address to free (ceiling) */ 174 unsigned long vma_start; /* The min vma address */ 175 unsigned long vma_end; /* The max vma address */ 176 unsigned long tree_end; /* Maximum for the vma tree search */ 177 unsigned long tree_reset; /* Where to reset the vma tree walk */ 178 bool mm_wr_locked; /* If the mmap write lock is held */ 179 }; 180 181 /* 182 * unmap_all_init() - Initialize unmap_desc to remove all vmas, point the 183 * pg_start and pg_end to a safe location. 184 */ 185 static inline void unmap_all_init(struct unmap_desc *unmap, 186 struct vma_iterator *vmi, struct vm_area_struct *vma) 187 { 188 unmap->mas = &vmi->mas; 189 unmap->first = vma; 190 unmap->pg_start = FIRST_USER_ADDRESS; 191 unmap->pg_end = USER_PGTABLES_CEILING; 192 unmap->vma_start = 0; 193 unmap->vma_end = ULONG_MAX; 194 unmap->tree_end = ULONG_MAX; 195 unmap->tree_reset = vma->vm_end; 196 unmap->mm_wr_locked = false; 197 } 198 199 /* 200 * unmap_pgtable_init() - Initialize unmap_desc to remove all page tables within 201 * the user range. 202 * 203 * ARM can have mappings outside of vmas. 204 * See: e2cdef8c847b4 ("[PATCH] freepgt: free_pgtables from FIRST_USER_ADDRESS") 205 * 206 * ARM LPAE uses page table mappings beyond the USER_PGTABLES_CEILING 207 * See: CONFIG_ARM_LPAE in arch/arm/include/asm/pgtable.h 208 */ 209 static inline void unmap_pgtable_init(struct unmap_desc *unmap, 210 struct vma_iterator *vmi) 211 { 212 vma_iter_set(vmi, unmap->tree_reset); 213 unmap->vma_start = FIRST_USER_ADDRESS; 214 unmap->vma_end = USER_PGTABLES_CEILING; 215 unmap->tree_end = USER_PGTABLES_CEILING; 216 } 217 218 #define UNMAP_STATE(name, _vmi, _vma, _vma_start, _vma_end, _prev, _next) \ 219 struct unmap_desc name = { \ 220 .mas = &(_vmi)->mas, \ 221 .first = _vma, \ 222 .pg_start = _prev ? ((struct vm_area_struct *)_prev)->vm_end : \ 223 FIRST_USER_ADDRESS, \ 224 .pg_end = _next ? ((struct vm_area_struct *)_next)->vm_start : \ 225 USER_PGTABLES_CEILING, \ 226 .vma_start = _vma_start, \ 227 .vma_end = _vma_end, \ 228 .tree_end = _next ? \ 229 ((struct vm_area_struct *)_next)->vm_start : \ 230 USER_PGTABLES_CEILING, \ 231 .tree_reset = _vma->vm_end, \ 232 .mm_wr_locked = true, \ 233 } 234 235 static inline bool vmg_nomem(struct vma_merge_struct *vmg) 236 { 237 return vmg->state == VMA_MERGE_ERROR_NOMEM; 238 } 239 240 static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg) 241 { 242 return vmg->pgoff; 243 } 244 245 static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg) 246 { 247 const unsigned long size = vmg->end - vmg->start; 248 249 return size >> PAGE_SHIFT; 250 } 251 252 static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) 253 { 254 return vmg_start_pgoff(vmg) + vmg_pages(vmg); 255 } 256 257 static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) 258 { 259 /* nommu doesn't set a virtual pgoff for anon VMAs. */ 260 if (!IS_ENABLED(CONFIG_MMU)) 261 return; 262 /* 263 * File-backed VMAs have arbitrary page offset (either page offset into 264 * file or for pfnmap the PFN of the start of the range or drivers may 265 * set arbitrary page offset). 266 */ 267 if (!vma_is_anonymous(vma)) 268 return; 269 /* MAP_PRIVATE-/dev/zero is anon, non-NULL vm_file, but has file pgoff. */ 270 if (vma->vm_file) 271 return; 272 /* If faulted in, could have been remapped. */ 273 if (vma->anon_vma) 274 return; 275 /* OK this is really an anon VMA - expect virtual page offset. */ 276 VM_WARN_ON_ONCE(pgoff != vma->vm_start >> PAGE_SHIFT); 277 } 278 279 static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) 280 { 281 vma_assert_can_modify(vma); 282 assert_sane_pgoff(vma, pgoff); 283 vma->vm_pgoff = pgoff; 284 } 285 286 static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta) 287 { 288 vma_assert_can_modify(vma); 289 vma_set_pgoff(vma, vma_start_pgoff(vma) + delta); 290 } 291 292 static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta) 293 { 294 vma_assert_can_modify(vma); 295 vma_set_pgoff(vma, vma_start_pgoff(vma) - delta); 296 } 297 298 #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ 299 struct vma_merge_struct name = { \ 300 .mm = mm_, \ 301 .vmi = vmi_, \ 302 .start = start_, \ 303 .end = end_, \ 304 .vma_flags = vma_flags_, \ 305 .pgoff = pgoff_, \ 306 .state = VMA_MERGE_START, \ 307 } 308 309 #define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \ 310 struct vma_merge_struct name = { \ 311 .mm = vma_->vm_mm, \ 312 .vmi = vmi_, \ 313 .prev = prev_, \ 314 .middle = vma_, \ 315 .next = NULL, \ 316 .start = start_, \ 317 .end = end_, \ 318 .vm_flags = vma_->vm_flags, \ 319 .pgoff = linear_page_index(vma_, start_), \ 320 .file = vma_->vm_file, \ 321 .anon_vma = vma_->anon_vma, \ 322 .policy = vma_policy(vma_), \ 323 .uffd_ctx = vma_->vm_userfaultfd_ctx, \ 324 .anon_name = anon_vma_name(vma_), \ 325 .state = VMA_MERGE_START, \ 326 } 327 328 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 329 void validate_mm(struct mm_struct *mm); 330 #else 331 #define validate_mm(mm) do { } while (0) 332 #endif 333 334 __must_check int vma_expand(struct vma_merge_struct *vmg); 335 __must_check int vma_shrink(struct vma_iterator *vmi, 336 struct vm_area_struct *vma, unsigned long end); 337 338 static inline int vma_iter_store_gfp(struct vma_iterator *vmi, 339 struct vm_area_struct *vma, gfp_t gfp) 340 341 { 342 if (vmi->mas.status != ma_start && 343 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start))) 344 vma_iter_invalidate(vmi); 345 346 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1); 347 mas_store_gfp(&vmi->mas, vma, gfp); 348 if (unlikely(mas_is_err(&vmi->mas))) 349 return -ENOMEM; 350 351 vma_mark_attached(vma); 352 return 0; 353 } 354 355 /* 356 * Temporary helper function for stacked mmap handlers which specify 357 * f_op->mmap() but which might have an underlying file system which implements 358 * f_op->mmap_prepare(). 359 */ 360 static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, 361 struct vm_area_desc *desc) 362 { 363 /* 364 * Since we're invoking .mmap_prepare() despite having a partially 365 * established VMA, we must take care to handle setting fields 366 * correctly. 367 */ 368 369 /* Mutable fields. Populated with initial state. */ 370 vma_set_pgoff(vma, desc->pgoff); 371 if (desc->vm_file != vma->vm_file) 372 vma_set_file(vma, desc->vm_file); 373 vma->flags = desc->vma_flags; 374 vma->vm_page_prot = desc->page_prot; 375 376 /* User-defined fields. */ 377 vma->vm_ops = desc->vm_ops; 378 vma->vm_private_data = desc->private_data; 379 } 380 381 int 382 do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 383 struct mm_struct *mm, unsigned long start, 384 unsigned long end, struct list_head *uf, bool unlock); 385 386 int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 387 unsigned long start, size_t len, struct list_head *uf, 388 bool unlock); 389 390 void remove_vma(struct vm_area_struct *vma); 391 void unmap_region(struct unmap_desc *unmap); 392 393 /** 394 * vma_modify_flags() - Perform any necessary split/merge in preparation for 395 * setting VMA flags to *@vm_flags in the range @start to @end contained within 396 * @vma. 397 * @vmi: Valid VMA iterator positioned at @vma. 398 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 399 * @vma: The VMA containing the range @start to @end to be updated. 400 * @start: The start of the range to update. May be offset within @vma. 401 * @end: The exclusive end of the range to update, may be offset within @vma. 402 * @vma_flags_ptr: A pointer to the VMA flags that the @start to @end range is 403 * about to be set to. On merge, this will be updated to include sticky flags. 404 * 405 * IMPORTANT: The actual modification being requested here is NOT applied, 406 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 407 * and the caller is expected to perform the actual modification. 408 * 409 * In order to account for sticky VMA flags, the @vma_flags_ptr parameter points 410 * to the requested flags which are then updated so the caller, should they 411 * overwrite any existing flags, correctly retains these. 412 * 413 * Returns: A VMA which contains the range @start to @end ready to have its 414 * flags altered to *@vma_flags. 415 */ 416 __must_check struct vm_area_struct *vma_modify_flags(struct vma_iterator *vmi, 417 struct vm_area_struct *prev, struct vm_area_struct *vma, 418 unsigned long start, unsigned long end, vma_flags_t *vma_flags_ptr); 419 420 /** 421 * vma_modify_name() - Perform any necessary split/merge in preparation for 422 * setting anonymous VMA name to @new_name in the range @start to @end contained 423 * within @vma. 424 * @vmi: Valid VMA iterator positioned at @vma. 425 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 426 * @vma: The VMA containing the range @start to @end to be updated. 427 * @start: The start of the range to update. May be offset within @vma. 428 * @end: The exclusive end of the range to update, may be offset within @vma. 429 * @new_name: The anonymous VMA name that the @start to @end range is about to 430 * be set to. 431 * 432 * IMPORTANT: The actual modification being requested here is NOT applied, 433 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 434 * and the caller is expected to perform the actual modification. 435 * 436 * Returns: A VMA which contains the range @start to @end ready to have its 437 * anonymous VMA name changed to @new_name. 438 */ 439 __must_check struct vm_area_struct *vma_modify_name(struct vma_iterator *vmi, 440 struct vm_area_struct *prev, struct vm_area_struct *vma, 441 unsigned long start, unsigned long end, 442 struct anon_vma_name *new_name); 443 444 /** 445 * vma_modify_policy() - Perform any necessary split/merge in preparation for 446 * setting NUMA policy to @new_pol in the range @start to @end contained 447 * within @vma. 448 * @vmi: Valid VMA iterator positioned at @vma. 449 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 450 * @vma: The VMA containing the range @start to @end to be updated. 451 * @start: The start of the range to update. May be offset within @vma. 452 * @end: The exclusive end of the range to update, may be offset within @vma. 453 * @new_pol: The NUMA policy that the @start to @end range is about to be set 454 * to. 455 * 456 * IMPORTANT: The actual modification being requested here is NOT applied, 457 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 458 * and the caller is expected to perform the actual modification. 459 * 460 * Returns: A VMA which contains the range @start to @end ready to have its 461 * NUMA policy changed to @new_pol. 462 */ 463 __must_check struct vm_area_struct *vma_modify_policy(struct vma_iterator *vmi, 464 struct vm_area_struct *prev, struct vm_area_struct *vma, 465 unsigned long start, unsigned long end, 466 struct mempolicy *new_pol); 467 468 /** 469 * vma_modify_flags_uffd() - Perform any necessary split/merge in preparation for 470 * setting VMA flags to @vm_flags and UFFD context to @new_ctx in the range 471 * @start to @end contained within @vma. 472 * @vmi: Valid VMA iterator positioned at @vma. 473 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 474 * @vma: The VMA containing the range @start to @end to be updated. 475 * @start: The start of the range to update. May be offset within @vma. 476 * @end: The exclusive end of the range to update, may be offset within @vma. 477 * @vma_flags: The VMA flags that the @start to @end range is about to be set to. 478 * @new_ctx: The userfaultfd context that the @start to @end range is about to 479 * be set to. 480 * @give_up_on_oom: If an out of memory condition occurs on merge, simply give 481 * up on it and treat the merge as best-effort. 482 * 483 * IMPORTANT: The actual modification being requested here is NOT applied, 484 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 485 * and the caller is expected to perform the actual modification. 486 * 487 * Returns: A VMA which contains the range @start to @end ready to have its VMA 488 * flags changed to @vma_flags and its userfaultfd context changed to @new_ctx. 489 */ 490 __must_check struct vm_area_struct *vma_modify_flags_uffd(struct vma_iterator *vmi, 491 struct vm_area_struct *prev, struct vm_area_struct *vma, 492 unsigned long start, unsigned long end, const vma_flags_t *vma_flags, 493 struct vm_userfaultfd_ctx new_ctx, bool give_up_on_oom); 494 495 __must_check struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg); 496 497 __must_check struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi, 498 struct vm_area_struct *vma, unsigned long delta); 499 500 void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb); 501 502 void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb); 503 504 void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, 505 struct vm_area_struct *vma); 506 507 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, 508 unsigned long addr, unsigned long len, pgoff_t pgoff, 509 bool *need_rmap_locks); 510 511 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); 512 513 bool vma_needs_dirty_tracking(struct vm_area_struct *vma); 514 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot); 515 516 int mm_take_all_locks(struct mm_struct *mm); 517 void mm_drop_all_locks(struct mm_struct *mm); 518 519 unsigned long mmap_region(struct file *file, unsigned long addr, 520 unsigned long len, vma_flags_t vma_flags, unsigned long pgoff, 521 struct list_head *uf); 522 523 int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, 524 unsigned long addr, unsigned long request, 525 vma_flags_t vma_flags); 526 527 unsigned long unmapped_area(struct vm_unmapped_area_info *info); 528 unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info); 529 530 static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma) 531 { 532 /* 533 * We want to check manually if we can change individual PTEs writable 534 * if we can't do that automatically for all PTEs in a mapping. For 535 * private mappings, that's always the case when we have write 536 * permissions as we properly have to handle COW. 537 */ 538 if (vma->vm_flags & VM_SHARED) 539 return vma_wants_writenotify(vma, vma->vm_page_prot); 540 return !!(vma->vm_flags & VM_WRITE); 541 } 542 543 #ifdef CONFIG_MMU 544 static inline pgprot_t vma_pgprot_modify(pgprot_t oldprot, vma_flags_t vma_flags) 545 { 546 const pgprot_t prot = vma_flags_to_page_prot(vma_flags); 547 548 return pgprot_modify(oldprot, prot); 549 } 550 #endif 551 552 static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi, 553 unsigned long min) 554 { 555 return mas_prev(&vmi->mas, min); 556 } 557 558 /* 559 * These three helpers classifies VMAs for virtual memory accounting. 560 */ 561 562 /* 563 * Executable code area - executable, not writable, not stack 564 */ 565 static inline bool is_exec_mapping(vm_flags_t flags) 566 { 567 return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC; 568 } 569 570 /* 571 * Stack area (including shadow stacks) 572 * 573 * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous: 574 * do_mmap() forbids all other combinations. 575 */ 576 static inline bool is_stack_mapping(vm_flags_t flags) 577 { 578 return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK); 579 } 580 581 /* 582 * Data area - private, writable, not stack 583 */ 584 static inline bool is_data_mapping(vm_flags_t flags) 585 { 586 return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE; 587 } 588 589 static inline bool is_data_mapping_vma_flags(const vma_flags_t *vma_flags) 590 { 591 return vma_flags_test(vma_flags, VMA_WRITE_BIT) && 592 !vma_flags_test_any(vma_flags, VMA_SHARED_BIT, VMA_STACK_BIT); 593 } 594 595 static inline void vma_iter_config(struct vma_iterator *vmi, 596 unsigned long index, unsigned long last) 597 { 598 __mas_set_range(&vmi->mas, index, last - 1); 599 } 600 601 static inline void vma_iter_reset(struct vma_iterator *vmi) 602 { 603 mas_reset(&vmi->mas); 604 } 605 606 static inline 607 struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min) 608 { 609 return mas_prev_range(&vmi->mas, min); 610 } 611 612 static inline 613 struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max) 614 { 615 return mas_next_range(&vmi->mas, max); 616 } 617 618 static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min, 619 unsigned long max, unsigned long size) 620 { 621 return mas_empty_area(&vmi->mas, min, max - 1, size); 622 } 623 624 static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min, 625 unsigned long max, unsigned long size) 626 { 627 return mas_empty_area_rev(&vmi->mas, min, max - 1, size); 628 } 629 630 /* 631 * VMA Iterator functions shared between nommu and mmap 632 */ 633 static inline int vma_iter_prealloc(struct vma_iterator *vmi, 634 struct vm_area_struct *vma) 635 { 636 return mas_preallocate(&vmi->mas, vma, GFP_KERNEL); 637 } 638 639 static inline void vma_iter_clear(struct vma_iterator *vmi) 640 { 641 mas_store_prealloc(&vmi->mas, NULL); 642 } 643 644 static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi) 645 { 646 return mas_walk(&vmi->mas); 647 } 648 649 /* Store a VMA with preallocated memory */ 650 static inline void vma_iter_store_overwrite(struct vma_iterator *vmi, 651 struct vm_area_struct *vma) 652 { 653 vma_assert_attached(vma); 654 655 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 656 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start && 657 vmi->mas.index > vma->vm_start)) { 658 pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n", 659 vmi->mas.index, vma->vm_start, vma->vm_start, 660 vma->vm_end, vmi->mas.index, vmi->mas.last); 661 } 662 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start && 663 vmi->mas.last < vma->vm_start)) { 664 pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n", 665 vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end, 666 vmi->mas.index, vmi->mas.last); 667 } 668 #endif 669 670 if (vmi->mas.status != ma_start && 671 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start))) 672 vma_iter_invalidate(vmi); 673 674 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1); 675 mas_store_prealloc(&vmi->mas, vma); 676 } 677 678 static inline void vma_iter_store_new(struct vma_iterator *vmi, 679 struct vm_area_struct *vma) 680 { 681 vma_mark_attached(vma); 682 vma_iter_store_overwrite(vmi, vma); 683 } 684 685 static inline unsigned long vma_iter_addr(struct vma_iterator *vmi) 686 { 687 return vmi->mas.index; 688 } 689 690 static inline unsigned long vma_iter_end(struct vma_iterator *vmi) 691 { 692 return vmi->mas.last + 1; 693 } 694 695 static inline 696 struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi) 697 { 698 return mas_prev_range(&vmi->mas, 0); 699 } 700 701 /* 702 * Retrieve the next VMA and rewind the iterator to end of the previous VMA, or 703 * if no previous VMA, to index 0. 704 */ 705 static inline 706 struct vm_area_struct *vma_iter_next_rewind(struct vma_iterator *vmi, 707 struct vm_area_struct **pprev) 708 { 709 struct vm_area_struct *next = vma_next(vmi); 710 struct vm_area_struct *prev = vma_prev(vmi); 711 712 /* 713 * Consider the case where no previous VMA exists. We advance to the 714 * next VMA, skipping any gap, then rewind to the start of the range. 715 * 716 * If we were to unconditionally advance to the next range we'd wind up 717 * at the next VMA again, so we check to ensure there is a previous VMA 718 * to skip over. 719 */ 720 if (prev) 721 vma_iter_next_range(vmi); 722 723 if (pprev) 724 *pprev = prev; 725 726 return next; 727 } 728 729 #ifdef CONFIG_64BIT 730 static inline bool vma_is_sealed(struct vm_area_struct *vma) 731 { 732 return (vma->vm_flags & VM_SEALED); 733 } 734 #else 735 static inline bool vma_is_sealed(struct vm_area_struct *vma) 736 { 737 return false; 738 } 739 #endif 740 741 #if defined(CONFIG_STACK_GROWSUP) 742 int expand_upwards(struct vm_area_struct *vma, unsigned long address); 743 #endif 744 745 int expand_downwards(struct vm_area_struct *vma, unsigned long address); 746 747 int __vm_munmap(unsigned long start, size_t len, bool unlock); 748 749 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma); 750 751 /* vma_init.h, shared between CONFIG_MMU and nommu. */ 752 void __init vma_state_init(void); 753 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm); 754 struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig); 755 void vm_area_free(struct vm_area_struct *vma); 756 757 /* vma_exec.c */ 758 #ifdef CONFIG_MMU 759 int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, 760 unsigned long *top_mem_p); 761 int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift); 762 #endif 763 764 #ifdef CONFIG_MMU 765 /* 766 * Denies creating a writable executable mapping or gaining executable permissions. 767 * 768 * This denies the following: 769 * 770 * a) mmap(PROT_WRITE | PROT_EXEC) 771 * 772 * b) mmap(PROT_WRITE) 773 * mprotect(PROT_EXEC) 774 * 775 * c) mmap(PROT_WRITE) 776 * mprotect(PROT_READ) 777 * mprotect(PROT_EXEC) 778 * 779 * But allows the following: 780 * 781 * d) mmap(PROT_READ | PROT_EXEC) 782 * mmap(PROT_READ | PROT_EXEC | PROT_BTI) 783 * 784 * This is only applicable if the user has set the Memory-Deny-Write-Execute 785 * (MDWE) protection mask for the current process. 786 * 787 * @old specifies the VMA flags the VMA originally possessed, and @new the ones 788 * we propose to set. 789 * 790 * Return: false if proposed change is OK, true if not ok and should be denied. 791 */ 792 static inline bool map_deny_write_exec(const vma_flags_t *old, 793 const vma_flags_t *new) 794 { 795 /* If MDWE is disabled, we have nothing to deny. */ 796 if (!mm_flags_test(MMF_HAS_MDWE, current->mm)) 797 return false; 798 799 /* If the new VMA is not executable, we have nothing to deny. */ 800 if (!vma_flags_test(new, VMA_EXEC_BIT)) 801 return false; 802 803 /* Under MDWE we do not accept newly writably executable VMAs... */ 804 if (vma_flags_test(new, VMA_WRITE_BIT)) 805 return true; 806 807 /* ...nor previously non-executable VMAs becoming executable. */ 808 if (!vma_flags_test(old, VMA_EXEC_BIT)) 809 return true; 810 811 return false; 812 } 813 #endif 814 815 struct vm_area_struct *__install_special_mapping(struct mm_struct *mm, 816 unsigned long addr, unsigned long len, 817 vm_flags_t vm_flags, void *priv, 818 const struct vm_operations_struct *ops); 819 820 #endif /* __MM_VMA_H */ 821