1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * vma.h 4 * 5 * Core VMA manipulation API implemented in vma.c, vma_init.c and vma_exec.c. 6 * 7 * Note that, in order for VMA logic to be userland testable, this header 8 * intentionally includes no dependencies. 9 * 10 * This is specifically scoped to mm-only. Users of this functionality (other 11 * than the core VMA implementation itself) should not include this header 12 * directly, but rather include internal.h. 13 */ 14 #ifndef __MM_VMA_H 15 #define __MM_VMA_H 16 17 /* 18 * VMA lock generalization 19 */ 20 struct vma_prepare { 21 struct vm_area_struct *vma; 22 struct vm_area_struct *adj_next; 23 struct file *file; 24 struct address_space *mapping; 25 struct anon_vma *anon_vma; 26 struct vm_area_struct *insert; 27 struct vm_area_struct *remove; 28 struct vm_area_struct *remove2; 29 30 bool skip_vma_uprobe :1; 31 }; 32 33 struct unlink_vma_file_batch { 34 int count; 35 struct vm_area_struct *vmas[8]; 36 }; 37 38 /* 39 * vma munmap operation 40 */ 41 struct vma_munmap_struct { 42 struct vma_iterator *vmi; 43 struct vm_area_struct *vma; /* The first vma to munmap */ 44 struct vm_area_struct *prev; /* vma before the munmap area */ 45 struct vm_area_struct *next; /* vma after the munmap area */ 46 struct list_head *uf; /* Userfaultfd list_head */ 47 unsigned long start; /* Aligned start addr (inclusive) */ 48 unsigned long end; /* Aligned end addr (exclusive) */ 49 unsigned long unmap_start; /* Unmap PTE start */ 50 unsigned long unmap_end; /* Unmap PTE end */ 51 int vma_count; /* Number of vmas that will be removed */ 52 bool unlock; /* Unlock after the munmap */ 53 bool clear_ptes; /* If there are outstanding PTE to be cleared */ 54 /* 2 byte hole */ 55 unsigned long nr_pages; /* Number of pages being removed */ 56 unsigned long locked_vm; /* Number of locked pages */ 57 unsigned long nr_accounted; /* Number of VM_ACCOUNT pages */ 58 unsigned long exec_vm; 59 unsigned long stack_vm; 60 unsigned long data_vm; 61 }; 62 63 enum vma_merge_state { 64 VMA_MERGE_START, 65 VMA_MERGE_ERROR_NOMEM, 66 VMA_MERGE_NOMERGE, 67 VMA_MERGE_SUCCESS, 68 }; 69 70 /* 71 * Describes a VMA merge operation and is threaded throughout it. 72 * 73 * Any of the fields may be mutated by the merge operation, so no guarantees are 74 * made to the contents of this structure after a merge operation has completed. 75 */ 76 struct vma_merge_struct { 77 struct mm_struct *mm; 78 struct vma_iterator *vmi; 79 /* 80 * Adjacent VMAs, any of which may be NULL if not present: 81 * 82 * |------|--------|------| 83 * | prev | middle | next | 84 * |------|--------|------| 85 * 86 * middle may not yet exist in the case of a proposed new VMA being 87 * merged, or it may be an existing VMA. 88 * 89 * next may be assigned by the caller. 90 */ 91 struct vm_area_struct *prev; 92 struct vm_area_struct *middle; 93 struct vm_area_struct *next; 94 /* This is the VMA we ultimately target to become the merged VMA. */ 95 struct vm_area_struct *target; 96 /* 97 * Initially, the start, end, pgoff fields are provided by the caller 98 * and describe the proposed new VMA range, whether modifying an 99 * existing VMA (which will be 'middle'), or adding a new one. 100 * 101 * During the merge process these fields are updated to describe the new 102 * range _including those VMAs which will be merged_. 103 */ 104 unsigned long start; 105 unsigned long end; 106 pgoff_t pgoff; 107 pgoff_t anon_pgoff; 108 109 union { 110 /* Temporary while VMA flags are being converted. */ 111 vm_flags_t vm_flags; 112 vma_flags_t vma_flags; 113 }; 114 struct file *file; 115 struct anon_vma *anon_vma; 116 struct mempolicy *policy; 117 struct vm_userfaultfd_ctx uffd_ctx; 118 struct anon_vma_name *anon_name; 119 enum vma_merge_state state; 120 121 /* If copied from (i.e. mremap()'d) the VMA from which we are copying. */ 122 struct vm_area_struct *copied_from; 123 124 /* Flags which callers can use to modify merge behaviour: */ 125 126 /* 127 * If we can expand, simply do so. We know there is nothing to merge to 128 * the right. Does not reset state upon failure to merge. The VMA 129 * iterator is assumed to be positioned at the previous VMA, rather than 130 * at the gap. 131 */ 132 bool just_expand :1; 133 134 /* 135 * If a merge is possible, but an OOM error occurs, give up and don't 136 * execute the merge, returning NULL. 137 */ 138 bool give_up_on_oom :1; 139 140 /* 141 * If set, skip uprobe_mmap upon merged vma. 142 */ 143 bool skip_vma_uprobe :1; 144 145 /* Internal flags set during merge process: */ 146 147 /* 148 * Internal flag indicating the merge increases vmg->middle->vm_start 149 * (and thereby, vmg->prev->vm_end). 150 */ 151 bool __adjust_middle_start :1; 152 /* 153 * Internal flag indicating the merge decreases vmg->next->vm_start 154 * (and thereby, vmg->middle->vm_end). 155 */ 156 bool __adjust_next_start :1; 157 /* 158 * Internal flag used during the merge operation to indicate we will 159 * remove vmg->middle. 160 */ 161 bool __remove_middle :1; 162 /* 163 * Internal flag used during the merge operation to indicate we will 164 * remove vmg->next. 165 */ 166 bool __remove_next :1; 167 168 }; 169 170 struct unmap_desc { 171 struct ma_state *mas; /* the maple state point to the first vma */ 172 struct vm_area_struct *first; /* The first vma */ 173 unsigned long pg_start; /* The first pagetable address to free (floor) */ 174 unsigned long pg_end; /* The last pagetable address to free (ceiling) */ 175 unsigned long vma_start; /* The min vma address */ 176 unsigned long vma_end; /* The max vma address */ 177 unsigned long tree_end; /* Maximum for the vma tree search */ 178 unsigned long tree_reset; /* Where to reset the vma tree walk */ 179 bool mm_wr_locked; /* If the mmap write lock is held */ 180 }; 181 182 /* 183 * unmap_all_init() - Initialize unmap_desc to remove all vmas, point the 184 * pg_start and pg_end to a safe location. 185 */ 186 static inline void unmap_all_init(struct unmap_desc *unmap, 187 struct vma_iterator *vmi, struct vm_area_struct *vma) 188 { 189 unmap->mas = &vmi->mas; 190 unmap->first = vma; 191 unmap->pg_start = FIRST_USER_ADDRESS; 192 unmap->pg_end = USER_PGTABLES_CEILING; 193 unmap->vma_start = 0; 194 unmap->vma_end = ULONG_MAX; 195 unmap->tree_end = ULONG_MAX; 196 unmap->tree_reset = vma->vm_end; 197 unmap->mm_wr_locked = false; 198 } 199 200 /* 201 * unmap_pgtable_init() - Initialize unmap_desc to remove all page tables within 202 * the user range. 203 * 204 * ARM can have mappings outside of vmas. 205 * See: e2cdef8c847b4 ("[PATCH] freepgt: free_pgtables from FIRST_USER_ADDRESS") 206 * 207 * ARM LPAE uses page table mappings beyond the USER_PGTABLES_CEILING 208 * See: CONFIG_ARM_LPAE in arch/arm/include/asm/pgtable.h 209 */ 210 static inline void unmap_pgtable_init(struct unmap_desc *unmap, 211 struct vma_iterator *vmi) 212 { 213 vma_iter_set(vmi, unmap->tree_reset); 214 unmap->vma_start = FIRST_USER_ADDRESS; 215 unmap->vma_end = USER_PGTABLES_CEILING; 216 unmap->tree_end = USER_PGTABLES_CEILING; 217 } 218 219 #define UNMAP_STATE(name, _vmi, _vma, _vma_start, _vma_end, _prev, _next) \ 220 struct unmap_desc name = { \ 221 .mas = &(_vmi)->mas, \ 222 .first = _vma, \ 223 .pg_start = _prev ? ((struct vm_area_struct *)_prev)->vm_end : \ 224 FIRST_USER_ADDRESS, \ 225 .pg_end = _next ? ((struct vm_area_struct *)_next)->vm_start : \ 226 USER_PGTABLES_CEILING, \ 227 .vma_start = _vma_start, \ 228 .vma_end = _vma_end, \ 229 .tree_end = _next ? \ 230 ((struct vm_area_struct *)_next)->vm_start : \ 231 USER_PGTABLES_CEILING, \ 232 .tree_reset = _vma->vm_end, \ 233 .mm_wr_locked = true, \ 234 } 235 236 static inline bool vmg_nomem(struct vma_merge_struct *vmg) 237 { 238 return vmg->state == VMA_MERGE_ERROR_NOMEM; 239 } 240 241 static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg) 242 { 243 const unsigned long size = vmg->end - vmg->start; 244 245 return size >> PAGE_SHIFT; 246 } 247 248 static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg) 249 { 250 return vmg->pgoff; 251 } 252 253 static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) 254 { 255 return vmg_start_pgoff(vmg) + vmg_pages(vmg); 256 } 257 258 static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) 259 { 260 /* nommu doesn't set a virtual pgoff for anon VMAs. */ 261 if (!IS_ENABLED(CONFIG_MMU)) 262 return; 263 /* 264 * File-backed VMAs have arbitrary page offset (either page offset into 265 * file or for pfnmap the PFN of the start of the range or drivers may 266 * set arbitrary page offset). 267 */ 268 if (!vma_is_anonymous(vma)) 269 return; 270 /* If faulted in, could have been remapped. */ 271 if (vma->anon_vma) 272 return; 273 /* OK this is really an anon VMA - expect virtual page offset. */ 274 VM_WARN_ON_ONCE(pgoff != vma->vm_start >> PAGE_SHIFT); 275 } 276 277 static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) 278 { 279 vma_assert_can_modify(vma); 280 assert_sane_pgoff(vma, pgoff); 281 vma->vm_pgoff = pgoff; 282 } 283 284 static inline pgoff_t vmg_start_anon_pgoff(const struct vma_merge_struct *vmg) 285 { 286 return vmg->anon_pgoff; 287 } 288 289 static inline pgoff_t vmg_end_anon_pgoff(const struct vma_merge_struct *vmg) 290 { 291 return vmg_start_anon_pgoff(vmg) + vmg_pages(vmg); 292 } 293 294 static inline void __vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) 295 { 296 #ifdef CONFIG_64BIT 297 vma->__vm_anon_pgoff_hi = pgoff >> 32; 298 #endif 299 vma->__vm_anon_pgoff_lo = pgoff & GENMASK(31, 0); 300 } 301 302 static inline void vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) 303 { 304 vma_assert_can_modify(vma); 305 __vma_set_anon_pgoff(vma, pgoff); 306 } 307 308 static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta) 309 { 310 vma_assert_can_modify(vma); 311 vma_set_pgoff(vma, vma_start_pgoff(vma) + delta); 312 vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) + delta); 313 } 314 315 static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta) 316 { 317 vma_assert_can_modify(vma); 318 vma_set_pgoff(vma, vma_start_pgoff(vma) - delta); 319 vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) - delta); 320 } 321 322 #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_, anon_pgoff_) \ 323 struct vma_merge_struct name = { \ 324 .mm = mm_, \ 325 .vmi = vmi_, \ 326 .start = start_, \ 327 .end = end_, \ 328 .vma_flags = vma_flags_, \ 329 .pgoff = pgoff_, \ 330 .anon_pgoff = anon_pgoff_, \ 331 .state = VMA_MERGE_START, \ 332 } 333 334 #define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \ 335 struct vma_merge_struct name = { \ 336 .mm = vma_->vm_mm, \ 337 .vmi = vmi_, \ 338 .prev = prev_, \ 339 .middle = vma_, \ 340 .next = NULL, \ 341 .start = start_, \ 342 .end = end_, \ 343 .vm_flags = vma_->vm_flags, \ 344 .pgoff = linear_page_index(vma_, start_), \ 345 .anon_pgoff = __linear_anon_page_index(vma_, start_), \ 346 .file = vma_->vm_file, \ 347 .anon_vma = vma_->anon_vma, \ 348 .policy = vma_policy(vma_), \ 349 .uffd_ctx = vma_->vm_userfaultfd_ctx, \ 350 .anon_name = anon_vma_name(vma_), \ 351 .state = VMA_MERGE_START, \ 352 } 353 354 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 355 void validate_mm(struct mm_struct *mm); 356 #else 357 #define validate_mm(mm) do { } while (0) 358 #endif 359 360 __must_check int vma_expand(struct vma_merge_struct *vmg); 361 __must_check int vma_shrink(struct vma_iterator *vmi, 362 struct vm_area_struct *vma, unsigned long end); 363 364 static inline int vma_iter_store_gfp(struct vma_iterator *vmi, 365 struct vm_area_struct *vma, gfp_t gfp) 366 367 { 368 if (vmi->mas.status != ma_start && 369 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start))) 370 vma_iter_invalidate(vmi); 371 372 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1); 373 mas_store_gfp(&vmi->mas, vma, gfp); 374 if (unlikely(mas_is_err(&vmi->mas))) 375 return -ENOMEM; 376 377 vma_mark_attached(vma); 378 return 0; 379 } 380 381 /* 382 * Temporary helper function for stacked mmap handlers which specify 383 * f_op->mmap() but which might have an underlying file system which implements 384 * f_op->mmap_prepare(). 385 */ 386 static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, 387 struct vm_area_desc *desc) 388 { 389 /* 390 * Since we're invoking .mmap_prepare() despite having a partially 391 * established VMA, we must take care to handle setting fields 392 * correctly. 393 */ 394 395 /* Mutable fields. Populated with initial state. */ 396 vma_set_pgoff(vma, desc->pgoff); 397 if (desc->vm_file != vma->vm_file) 398 vma_set_file(vma, desc->vm_file); 399 vma->flags = desc->vma_flags; 400 vma->vm_page_prot = desc->page_prot; 401 402 /* User-defined fields. */ 403 vma->vm_ops = desc->vm_ops; 404 vma->vm_private_data = desc->private_data; 405 } 406 407 int 408 do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 409 struct mm_struct *mm, unsigned long start, 410 unsigned long end, struct list_head *uf, bool unlock); 411 412 int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 413 unsigned long start, size_t len, struct list_head *uf, 414 bool unlock); 415 416 void remove_vma(struct vm_area_struct *vma); 417 void unmap_region(struct unmap_desc *unmap); 418 419 /** 420 * vma_modify_flags() - Perform any necessary split/merge in preparation for 421 * setting VMA flags to *@vm_flags in the range @start to @end contained within 422 * @vma. 423 * @vmi: Valid VMA iterator positioned at @vma. 424 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 425 * @vma: The VMA containing the range @start to @end to be updated. 426 * @start: The start of the range to update. May be offset within @vma. 427 * @end: The exclusive end of the range to update, may be offset within @vma. 428 * @vma_flags_ptr: A pointer to the VMA flags that the @start to @end range is 429 * about to be set to. On merge, this will be updated to include sticky flags. 430 * 431 * IMPORTANT: The actual modification being requested here is NOT applied, 432 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 433 * and the caller is expected to perform the actual modification. 434 * 435 * In order to account for sticky VMA flags, the @vma_flags_ptr parameter points 436 * to the requested flags which are then updated so the caller, should they 437 * overwrite any existing flags, correctly retains these. 438 * 439 * Returns: A VMA which contains the range @start to @end ready to have its 440 * flags altered to *@vma_flags. 441 */ 442 __must_check struct vm_area_struct *vma_modify_flags(struct vma_iterator *vmi, 443 struct vm_area_struct *prev, struct vm_area_struct *vma, 444 unsigned long start, unsigned long end, vma_flags_t *vma_flags_ptr); 445 446 /** 447 * vma_modify_name() - Perform any necessary split/merge in preparation for 448 * setting anonymous VMA name to @new_name in the range @start to @end contained 449 * within @vma. 450 * @vmi: Valid VMA iterator positioned at @vma. 451 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 452 * @vma: The VMA containing the range @start to @end to be updated. 453 * @start: The start of the range to update. May be offset within @vma. 454 * @end: The exclusive end of the range to update, may be offset within @vma. 455 * @new_name: The anonymous VMA name that the @start to @end range is about to 456 * be set to. 457 * 458 * IMPORTANT: The actual modification being requested here is NOT applied, 459 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 460 * and the caller is expected to perform the actual modification. 461 * 462 * Returns: A VMA which contains the range @start to @end ready to have its 463 * anonymous VMA name changed to @new_name. 464 */ 465 __must_check struct vm_area_struct *vma_modify_name(struct vma_iterator *vmi, 466 struct vm_area_struct *prev, struct vm_area_struct *vma, 467 unsigned long start, unsigned long end, 468 struct anon_vma_name *new_name); 469 470 /** 471 * vma_modify_policy() - Perform any necessary split/merge in preparation for 472 * setting NUMA policy to @new_pol in the range @start to @end contained 473 * within @vma. 474 * @vmi: Valid VMA iterator positioned at @vma. 475 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 476 * @vma: The VMA containing the range @start to @end to be updated. 477 * @start: The start of the range to update. May be offset within @vma. 478 * @end: The exclusive end of the range to update, may be offset within @vma. 479 * @new_pol: The NUMA policy that the @start to @end range is about to be set 480 * to. 481 * 482 * IMPORTANT: The actual modification being requested here is NOT applied, 483 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 484 * and the caller is expected to perform the actual modification. 485 * 486 * Returns: A VMA which contains the range @start to @end ready to have its 487 * NUMA policy changed to @new_pol. 488 */ 489 __must_check struct vm_area_struct *vma_modify_policy(struct vma_iterator *vmi, 490 struct vm_area_struct *prev, struct vm_area_struct *vma, 491 unsigned long start, unsigned long end, 492 struct mempolicy *new_pol); 493 494 /** 495 * vma_modify_flags_uffd() - Perform any necessary split/merge in preparation for 496 * setting VMA flags to @vm_flags and UFFD context to @new_ctx in the range 497 * @start to @end contained within @vma. 498 * @vmi: Valid VMA iterator positioned at @vma. 499 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 500 * @vma: The VMA containing the range @start to @end to be updated. 501 * @start: The start of the range to update. May be offset within @vma. 502 * @end: The exclusive end of the range to update, may be offset within @vma. 503 * @vma_flags: The VMA flags that the @start to @end range is about to be set to. 504 * @new_ctx: The userfaultfd context that the @start to @end range is about to 505 * be set to. 506 * @give_up_on_oom: If an out of memory condition occurs on merge, simply give 507 * up on it and treat the merge as best-effort. 508 * 509 * IMPORTANT: The actual modification being requested here is NOT applied, 510 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 511 * and the caller is expected to perform the actual modification. 512 * 513 * Returns: A VMA which contains the range @start to @end ready to have its VMA 514 * flags changed to @vma_flags and its userfaultfd context changed to @new_ctx. 515 */ 516 __must_check struct vm_area_struct *vma_modify_flags_uffd(struct vma_iterator *vmi, 517 struct vm_area_struct *prev, struct vm_area_struct *vma, 518 unsigned long start, unsigned long end, const vma_flags_t *vma_flags, 519 struct vm_userfaultfd_ctx new_ctx, bool give_up_on_oom); 520 521 __must_check struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg); 522 523 __must_check struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi, 524 struct vm_area_struct *vma, unsigned long delta); 525 526 void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb); 527 528 void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb); 529 530 void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, 531 struct vm_area_struct *vma); 532 533 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, 534 unsigned long addr, unsigned long len, pgoff_t pgoff, 535 pgoff_t anon_pgoff, bool *need_rmap_locks, bool keep_source); 536 537 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); 538 539 bool vma_needs_dirty_tracking(struct vm_area_struct *vma); 540 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot); 541 542 int mm_take_all_locks(struct mm_struct *mm); 543 void mm_drop_all_locks(struct mm_struct *mm); 544 545 unsigned long mmap_region(struct file *file, unsigned long addr, 546 unsigned long len, vma_flags_t vma_flags, unsigned long pgoff, 547 struct list_head *uf); 548 549 int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, 550 unsigned long addr, unsigned long request, 551 vma_flags_t vma_flags); 552 553 unsigned long unmapped_area(struct vm_unmapped_area_info *info); 554 unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info); 555 556 #ifdef CONFIG_MMU 557 int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, 558 unsigned long addr, int new_below); 559 #endif 560 561 static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma) 562 { 563 /* 564 * We want to check manually if we can change individual PTEs writable 565 * if we can't do that automatically for all PTEs in a mapping. For 566 * private mappings, that's always the case when we have write 567 * permissions as we properly have to handle COW. 568 */ 569 if (vma->vm_flags & VM_SHARED) 570 return vma_wants_writenotify(vma, vma->vm_page_prot); 571 return !!(vma->vm_flags & VM_WRITE); 572 } 573 574 #ifdef CONFIG_MMU 575 static inline pgprot_t vma_pgprot_modify(pgprot_t oldprot, vma_flags_t vma_flags) 576 { 577 const pgprot_t prot = vma_flags_to_page_prot(vma_flags); 578 579 return pgprot_modify(oldprot, prot); 580 } 581 #endif 582 583 static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi, 584 unsigned long min) 585 { 586 return mas_prev(&vmi->mas, min); 587 } 588 589 /* 590 * These three helpers classifies VMAs for virtual memory accounting. 591 */ 592 593 /* 594 * Executable code area - executable, not writable, not stack 595 */ 596 static inline bool is_exec_mapping(vm_flags_t flags) 597 { 598 return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC; 599 } 600 601 /* 602 * Stack area (including shadow stacks) 603 * 604 * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous: 605 * do_mmap() forbids all other combinations. 606 */ 607 static inline bool is_stack_mapping(vm_flags_t flags) 608 { 609 return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK); 610 } 611 612 /* 613 * Data area - private, writable, not stack 614 */ 615 static inline bool is_data_mapping(vm_flags_t flags) 616 { 617 return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE; 618 } 619 620 static inline bool is_data_mapping_vma_flags(const vma_flags_t *vma_flags) 621 { 622 return vma_flags_test(vma_flags, VMA_WRITE_BIT) && 623 !vma_flags_test_any(vma_flags, VMA_SHARED_BIT, VMA_STACK_BIT); 624 } 625 626 static inline void vma_iter_config(struct vma_iterator *vmi, 627 unsigned long index, unsigned long last) 628 { 629 __mas_set_range(&vmi->mas, index, last - 1); 630 } 631 632 static inline void vma_iter_reset(struct vma_iterator *vmi) 633 { 634 mas_reset(&vmi->mas); 635 } 636 637 static inline 638 struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min) 639 { 640 return mas_prev_range(&vmi->mas, min); 641 } 642 643 static inline 644 struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max) 645 { 646 return mas_next_range(&vmi->mas, max); 647 } 648 649 static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min, 650 unsigned long max, unsigned long size) 651 { 652 return mas_empty_area(&vmi->mas, min, max - 1, size); 653 } 654 655 static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min, 656 unsigned long max, unsigned long size) 657 { 658 return mas_empty_area_rev(&vmi->mas, min, max - 1, size); 659 } 660 661 /* 662 * VMA Iterator functions shared between nommu and mmap 663 */ 664 static inline int vma_iter_prealloc(struct vma_iterator *vmi, 665 struct vm_area_struct *vma) 666 { 667 return mas_preallocate(&vmi->mas, vma, GFP_KERNEL); 668 } 669 670 static inline void vma_iter_clear(struct vma_iterator *vmi) 671 { 672 mas_store_prealloc(&vmi->mas, NULL); 673 } 674 675 static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi) 676 { 677 return mas_walk(&vmi->mas); 678 } 679 680 /* Store a VMA with preallocated memory */ 681 static inline void vma_iter_store_overwrite(struct vma_iterator *vmi, 682 struct vm_area_struct *vma) 683 { 684 vma_assert_attached(vma); 685 686 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 687 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start && 688 vmi->mas.index > vma->vm_start)) { 689 pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n", 690 vmi->mas.index, vma->vm_start, vma->vm_start, 691 vma->vm_end, vmi->mas.index, vmi->mas.last); 692 } 693 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start && 694 vmi->mas.last < vma->vm_start)) { 695 pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n", 696 vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end, 697 vmi->mas.index, vmi->mas.last); 698 } 699 #endif 700 701 if (vmi->mas.status != ma_start && 702 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start))) 703 vma_iter_invalidate(vmi); 704 705 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1); 706 mas_store_prealloc(&vmi->mas, vma); 707 } 708 709 static inline void vma_iter_store_new(struct vma_iterator *vmi, 710 struct vm_area_struct *vma) 711 { 712 vma_mark_attached(vma); 713 vma_iter_store_overwrite(vmi, vma); 714 } 715 716 static inline unsigned long vma_iter_addr(struct vma_iterator *vmi) 717 { 718 return vmi->mas.index; 719 } 720 721 static inline unsigned long vma_iter_end(struct vma_iterator *vmi) 722 { 723 return vmi->mas.last + 1; 724 } 725 726 static inline 727 struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi) 728 { 729 return mas_prev_range(&vmi->mas, 0); 730 } 731 732 /* 733 * Retrieve the next VMA and rewind the iterator to end of the previous VMA, or 734 * if no previous VMA, to index 0. 735 */ 736 static inline 737 struct vm_area_struct *vma_iter_next_rewind(struct vma_iterator *vmi, 738 struct vm_area_struct **pprev) 739 { 740 struct vm_area_struct *next = vma_next(vmi); 741 struct vm_area_struct *prev = vma_prev(vmi); 742 743 /* 744 * Consider the case where no previous VMA exists. We advance to the 745 * next VMA, skipping any gap, then rewind to the start of the range. 746 * 747 * If we were to unconditionally advance to the next range we'd wind up 748 * at the next VMA again, so we check to ensure there is a previous VMA 749 * to skip over. 750 */ 751 if (prev) 752 vma_iter_next_range(vmi); 753 754 if (pprev) 755 *pprev = prev; 756 757 return next; 758 } 759 760 #ifdef CONFIG_64BIT 761 static inline bool vma_is_sealed(struct vm_area_struct *vma) 762 { 763 return (vma->vm_flags & VM_SEALED); 764 } 765 #else 766 static inline bool vma_is_sealed(struct vm_area_struct *vma) 767 { 768 return false; 769 } 770 #endif 771 772 #if defined(CONFIG_STACK_GROWSUP) 773 int expand_upwards(struct vm_area_struct *vma, unsigned long address); 774 #endif 775 776 int expand_downwards(struct vm_area_struct *vma, unsigned long address); 777 778 int __vm_munmap(unsigned long start, size_t len, bool unlock); 779 780 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma); 781 782 /* vma_init.h, shared between CONFIG_MMU and nommu. */ 783 void __init vma_state_init(void); 784 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm); 785 struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig); 786 void vm_area_free(struct vm_area_struct *vma); 787 788 /* vma_exec.c */ 789 #ifdef CONFIG_MMU 790 int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, 791 unsigned long *top_mem_p); 792 int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift); 793 #endif 794 795 #ifdef CONFIG_MMU 796 /* 797 * Denies creating a writable executable mapping or gaining executable permissions. 798 * 799 * This denies the following: 800 * 801 * a) mmap(PROT_WRITE | PROT_EXEC) 802 * 803 * b) mmap(PROT_WRITE) 804 * mprotect(PROT_EXEC) 805 * 806 * c) mmap(PROT_WRITE) 807 * mprotect(PROT_READ) 808 * mprotect(PROT_EXEC) 809 * 810 * But allows the following: 811 * 812 * d) mmap(PROT_READ | PROT_EXEC) 813 * mmap(PROT_READ | PROT_EXEC | PROT_BTI) 814 * 815 * This is only applicable if the user has set the Memory-Deny-Write-Execute 816 * (MDWE) protection mask for the current process. 817 * 818 * @old specifies the VMA flags the VMA originally possessed, and @new the ones 819 * we propose to set. 820 * 821 * Return: false if proposed change is OK, true if not ok and should be denied. 822 */ 823 static inline bool map_deny_write_exec(const vma_flags_t *old, 824 const vma_flags_t *new) 825 { 826 /* If MDWE is disabled, we have nothing to deny. */ 827 if (!mm_flags_test(MMF_HAS_MDWE, current->mm)) 828 return false; 829 830 /* If the new VMA is not executable, we have nothing to deny. */ 831 if (!vma_flags_test(new, VMA_EXEC_BIT)) 832 return false; 833 834 /* Under MDWE we do not accept newly writably executable VMAs... */ 835 if (vma_flags_test(new, VMA_WRITE_BIT)) 836 return true; 837 838 /* ...nor previously non-executable VMAs becoming executable. */ 839 if (!vma_flags_test(old, VMA_EXEC_BIT)) 840 return true; 841 842 return false; 843 } 844 #endif 845 846 struct vm_area_struct *__install_special_mapping(struct mm_struct *mm, 847 unsigned long addr, unsigned long len, 848 vm_flags_t vm_flags, void *priv, 849 const struct vm_operations_struct *ops); 850 851 #endif /* __MM_VMA_H */ 852