1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * vma.h 4 * 5 * Core VMA manipulation API implemented in vma.c, vma_init.c and vma_exec.c. 6 * 7 * Note that, in order for VMA logic to be userland testable, this header 8 * intentionally includes no dependencies. 9 * 10 * This is specifically scoped to mm-only. Users of this functionality (other 11 * than the core VMA implementation itself) should not include this header 12 * directly, but rather include internal.h. 13 */ 14 #ifndef __MM_VMA_H 15 #define __MM_VMA_H 16 17 /* 18 * VMA lock generalization 19 */ 20 struct vma_prepare { 21 struct vm_area_struct *vma; 22 struct vm_area_struct *adj_next; 23 struct file *file; 24 struct address_space *mapping; 25 struct anon_vma *anon_vma; 26 struct vm_area_struct *insert; 27 struct vm_area_struct *remove; 28 struct vm_area_struct *remove2; 29 30 bool skip_vma_uprobe :1; 31 }; 32 33 struct unlink_vma_file_batch { 34 int count; 35 struct vm_area_struct *vmas[8]; 36 }; 37 38 /* 39 * vma munmap operation 40 */ 41 struct vma_munmap_struct { 42 struct vma_iterator *vmi; 43 struct vm_area_struct *vma; /* The first vma to munmap */ 44 struct vm_area_struct *prev; /* vma before the munmap area */ 45 struct vm_area_struct *next; /* vma after the munmap area */ 46 struct list_head *uf; /* Userfaultfd list_head */ 47 unsigned long start; /* Aligned start addr (inclusive) */ 48 unsigned long end; /* Aligned end addr (exclusive) */ 49 unsigned long unmap_start; /* Unmap PTE start */ 50 unsigned long unmap_end; /* Unmap PTE end */ 51 int vma_count; /* Number of vmas that will be removed */ 52 bool unlock; /* Unlock after the munmap */ 53 bool clear_ptes; /* If there are outstanding PTE to be cleared */ 54 /* 2 byte hole */ 55 unsigned long nr_pages; /* Number of pages being removed */ 56 unsigned long locked_vm; /* Number of locked pages */ 57 unsigned long nr_accounted; /* Number of VM_ACCOUNT pages */ 58 unsigned long exec_vm; 59 unsigned long stack_vm; 60 unsigned long data_vm; 61 }; 62 63 enum vma_merge_state { 64 VMA_MERGE_START, 65 VMA_MERGE_ERROR_NOMEM, 66 VMA_MERGE_NOMERGE, 67 VMA_MERGE_SUCCESS, 68 }; 69 70 /* 71 * Describes a VMA merge operation and is threaded throughout it. 72 * 73 * Any of the fields may be mutated by the merge operation, so no guarantees are 74 * made to the contents of this structure after a merge operation has completed. 75 */ 76 struct vma_merge_struct { 77 struct mm_struct *mm; 78 struct vma_iterator *vmi; 79 /* 80 * Adjacent VMAs, any of which may be NULL if not present: 81 * 82 * |------|--------|------| 83 * | prev | middle | next | 84 * |------|--------|------| 85 * 86 * middle may not yet exist in the case of a proposed new VMA being 87 * merged, or it may be an existing VMA. 88 * 89 * next may be assigned by the caller. 90 */ 91 struct vm_area_struct *prev; 92 struct vm_area_struct *middle; 93 struct vm_area_struct *next; 94 /* This is the VMA we ultimately target to become the merged VMA. */ 95 struct vm_area_struct *target; 96 /* 97 * Initially, the start, end, pgoff fields are provided by the caller 98 * and describe the proposed new VMA range, whether modifying an 99 * existing VMA (which will be 'middle'), or adding a new one. 100 * 101 * During the merge process these fields are updated to describe the new 102 * range _including those VMAs which will be merged_. 103 */ 104 unsigned long start; 105 unsigned long end; 106 pgoff_t pgoff; 107 pgoff_t anon_pgoff; 108 109 union { 110 /* Temporary while VMA flags are being converted. */ 111 vm_flags_t vm_flags; 112 vma_flags_t vma_flags; 113 }; 114 struct file *file; 115 struct anon_vma *anon_vma; 116 struct mempolicy *policy; 117 struct vm_userfaultfd_ctx uffd_ctx; 118 struct anon_vma_name *anon_name; 119 enum vma_merge_state state; 120 121 /* If copied from (i.e. mremap()'d) the VMA from which we are copying. */ 122 struct vm_area_struct *copied_from; 123 124 /* Flags which callers can use to modify merge behaviour: */ 125 126 /* 127 * If we can expand, simply do so. We know there is nothing to merge to 128 * the right. Does not reset state upon failure to merge. The VMA 129 * iterator is assumed to be positioned at the previous VMA, rather than 130 * at the gap. 131 */ 132 bool just_expand :1; 133 134 /* 135 * If a merge is possible, but an OOM error occurs, give up and don't 136 * execute the merge, returning NULL. 137 */ 138 bool give_up_on_oom :1; 139 140 /* 141 * If set, skip uprobe_mmap upon merged vma. 142 */ 143 bool skip_vma_uprobe :1; 144 145 /* Internal flags set during merge process: */ 146 147 /* 148 * Internal flag indicating the merge increases vmg->middle->vm_start 149 * (and thereby, vmg->prev->vm_end). 150 */ 151 bool __adjust_middle_start :1; 152 /* 153 * Internal flag indicating the merge decreases vmg->next->vm_start 154 * (and thereby, vmg->middle->vm_end). 155 */ 156 bool __adjust_next_start :1; 157 /* 158 * Internal flag used during the merge operation to indicate we will 159 * remove vmg->middle. 160 */ 161 bool __remove_middle :1; 162 /* 163 * Internal flag used during the merge operation to indicate we will 164 * remove vmg->next. 165 */ 166 bool __remove_next :1; 167 168 }; 169 170 struct unmap_desc { 171 struct ma_state *mas; /* the maple state point to the first vma */ 172 struct vm_area_struct *first; /* The first vma */ 173 unsigned long pg_start; /* The first pagetable address to free (floor) */ 174 unsigned long pg_end; /* The last pagetable address to free (ceiling) */ 175 unsigned long vma_start; /* The min vma address */ 176 unsigned long vma_end; /* The max vma address */ 177 unsigned long tree_end; /* Maximum for the vma tree search */ 178 unsigned long tree_reset; /* Where to reset the vma tree walk */ 179 bool mm_wr_locked; /* If the mmap write lock is held */ 180 }; 181 182 /* 183 * unmap_all_init() - Initialize unmap_desc to remove all vmas, point the 184 * pg_start and pg_end to a safe location. 185 */ 186 static inline void unmap_all_init(struct unmap_desc *unmap, 187 struct vma_iterator *vmi, struct vm_area_struct *vma) 188 { 189 unmap->mas = &vmi->mas; 190 unmap->first = vma; 191 unmap->pg_start = FIRST_USER_ADDRESS; 192 unmap->pg_end = USER_PGTABLES_CEILING; 193 unmap->vma_start = 0; 194 unmap->vma_end = ULONG_MAX; 195 unmap->tree_end = ULONG_MAX; 196 unmap->tree_reset = vma->vm_end; 197 unmap->mm_wr_locked = false; 198 } 199 200 /* 201 * unmap_pgtable_init() - Initialize unmap_desc to remove all page tables within 202 * the user range. 203 * 204 * ARM can have mappings outside of vmas. 205 * See: e2cdef8c847b4 ("[PATCH] freepgt: free_pgtables from FIRST_USER_ADDRESS") 206 * 207 * ARM LPAE uses page table mappings beyond the USER_PGTABLES_CEILING 208 * See: CONFIG_ARM_LPAE in arch/arm/include/asm/pgtable.h 209 */ 210 static inline void unmap_pgtable_init(struct unmap_desc *unmap, 211 struct vma_iterator *vmi) 212 { 213 vma_iter_set(vmi, unmap->tree_reset); 214 unmap->vma_start = FIRST_USER_ADDRESS; 215 unmap->vma_end = USER_PGTABLES_CEILING; 216 unmap->tree_end = USER_PGTABLES_CEILING; 217 } 218 219 #define UNMAP_STATE(name, _vmi, _vma, _vma_start, _vma_end, _prev, _next) \ 220 struct unmap_desc name = { \ 221 .mas = &(_vmi)->mas, \ 222 .first = _vma, \ 223 .pg_start = _prev ? ((struct vm_area_struct *)_prev)->vm_end : \ 224 FIRST_USER_ADDRESS, \ 225 .pg_end = _next ? ((struct vm_area_struct *)_next)->vm_start : \ 226 USER_PGTABLES_CEILING, \ 227 .vma_start = _vma_start, \ 228 .vma_end = _vma_end, \ 229 .tree_end = _next ? \ 230 ((struct vm_area_struct *)_next)->vm_start : \ 231 USER_PGTABLES_CEILING, \ 232 .tree_reset = _vma->vm_end, \ 233 .mm_wr_locked = true, \ 234 } 235 236 static inline bool vmg_nomem(struct vma_merge_struct *vmg) 237 { 238 return vmg->state == VMA_MERGE_ERROR_NOMEM; 239 } 240 241 static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg) 242 { 243 const unsigned long size = vmg->end - vmg->start; 244 245 return size >> PAGE_SHIFT; 246 } 247 248 static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg) 249 { 250 return vmg->pgoff; 251 } 252 253 static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) 254 { 255 return vmg_start_pgoff(vmg) + vmg_pages(vmg); 256 } 257 258 static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) 259 { 260 /* nommu doesn't set a virtual pgoff for anon VMAs. */ 261 if (!IS_ENABLED(CONFIG_MMU)) 262 return; 263 /* 264 * File-backed VMAs have arbitrary page offset (either page offset into 265 * file or for pfnmap the PFN of the start of the range or drivers may 266 * set arbitrary page offset). 267 */ 268 if (!vma_is_anonymous(vma)) 269 return; 270 /* MAP_PRIVATE-/dev/zero is anon, non-NULL vm_file, but has file pgoff. */ 271 if (vma->vm_file) 272 return; 273 /* If faulted in, could have been remapped. */ 274 if (vma->anon_vma) 275 return; 276 /* OK this is really an anon VMA - expect virtual page offset. */ 277 VM_WARN_ON_ONCE(pgoff != vma->vm_start >> PAGE_SHIFT); 278 } 279 280 static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) 281 { 282 vma_assert_can_modify(vma); 283 assert_sane_pgoff(vma, pgoff); 284 vma->vm_pgoff = pgoff; 285 } 286 287 static inline pgoff_t vmg_start_anon_pgoff(const struct vma_merge_struct *vmg) 288 { 289 return vmg->anon_pgoff; 290 } 291 292 static inline pgoff_t vmg_end_anon_pgoff(const struct vma_merge_struct *vmg) 293 { 294 return vmg_start_anon_pgoff(vmg) + vmg_pages(vmg); 295 } 296 297 static inline void __vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) 298 { 299 #ifdef CONFIG_64BIT 300 vma->__vm_anon_pgoff_hi = pgoff >> 32; 301 #endif 302 vma->__vm_anon_pgoff_lo = pgoff & GENMASK(31, 0); 303 } 304 305 static inline void vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) 306 { 307 vma_assert_can_modify(vma); 308 __vma_set_anon_pgoff(vma, pgoff); 309 } 310 311 static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta) 312 { 313 vma_assert_can_modify(vma); 314 vma_set_pgoff(vma, vma_start_pgoff(vma) + delta); 315 vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) + delta); 316 } 317 318 static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta) 319 { 320 vma_assert_can_modify(vma); 321 vma_set_pgoff(vma, vma_start_pgoff(vma) - delta); 322 vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) - delta); 323 } 324 325 #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_, anon_pgoff_) \ 326 struct vma_merge_struct name = { \ 327 .mm = mm_, \ 328 .vmi = vmi_, \ 329 .start = start_, \ 330 .end = end_, \ 331 .vma_flags = vma_flags_, \ 332 .pgoff = pgoff_, \ 333 .anon_pgoff = anon_pgoff_, \ 334 .state = VMA_MERGE_START, \ 335 } 336 337 #define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \ 338 struct vma_merge_struct name = { \ 339 .mm = vma_->vm_mm, \ 340 .vmi = vmi_, \ 341 .prev = prev_, \ 342 .middle = vma_, \ 343 .next = NULL, \ 344 .start = start_, \ 345 .end = end_, \ 346 .vm_flags = vma_->vm_flags, \ 347 .pgoff = linear_page_index(vma_, start_), \ 348 .anon_pgoff = __linear_anon_page_index(vma_, start_), \ 349 .file = vma_->vm_file, \ 350 .anon_vma = vma_->anon_vma, \ 351 .policy = vma_policy(vma_), \ 352 .uffd_ctx = vma_->vm_userfaultfd_ctx, \ 353 .anon_name = anon_vma_name(vma_), \ 354 .state = VMA_MERGE_START, \ 355 } 356 357 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 358 void validate_mm(struct mm_struct *mm); 359 #else 360 #define validate_mm(mm) do { } while (0) 361 #endif 362 363 __must_check int vma_expand(struct vma_merge_struct *vmg); 364 __must_check int vma_shrink(struct vma_iterator *vmi, 365 struct vm_area_struct *vma, unsigned long end); 366 367 static inline int vma_iter_store_gfp(struct vma_iterator *vmi, 368 struct vm_area_struct *vma, gfp_t gfp) 369 370 { 371 if (vmi->mas.status != ma_start && 372 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start))) 373 vma_iter_invalidate(vmi); 374 375 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1); 376 mas_store_gfp(&vmi->mas, vma, gfp); 377 if (unlikely(mas_is_err(&vmi->mas))) 378 return -ENOMEM; 379 380 vma_mark_attached(vma); 381 return 0; 382 } 383 384 /* 385 * Temporary helper function for stacked mmap handlers which specify 386 * f_op->mmap() but which might have an underlying file system which implements 387 * f_op->mmap_prepare(). 388 */ 389 static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, 390 struct vm_area_desc *desc) 391 { 392 /* 393 * Since we're invoking .mmap_prepare() despite having a partially 394 * established VMA, we must take care to handle setting fields 395 * correctly. 396 */ 397 398 /* Mutable fields. Populated with initial state. */ 399 vma_set_pgoff(vma, desc->pgoff); 400 if (desc->vm_file != vma->vm_file) 401 vma_set_file(vma, desc->vm_file); 402 vma->flags = desc->vma_flags; 403 vma->vm_page_prot = desc->page_prot; 404 405 /* User-defined fields. */ 406 vma->vm_ops = desc->vm_ops; 407 vma->vm_private_data = desc->private_data; 408 } 409 410 int 411 do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 412 struct mm_struct *mm, unsigned long start, 413 unsigned long end, struct list_head *uf, bool unlock); 414 415 int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 416 unsigned long start, size_t len, struct list_head *uf, 417 bool unlock); 418 419 void remove_vma(struct vm_area_struct *vma); 420 void unmap_region(struct unmap_desc *unmap); 421 422 /** 423 * vma_modify_flags() - Perform any necessary split/merge in preparation for 424 * setting VMA flags to *@vm_flags in the range @start to @end contained within 425 * @vma. 426 * @vmi: Valid VMA iterator positioned at @vma. 427 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 428 * @vma: The VMA containing the range @start to @end to be updated. 429 * @start: The start of the range to update. May be offset within @vma. 430 * @end: The exclusive end of the range to update, may be offset within @vma. 431 * @vma_flags_ptr: A pointer to the VMA flags that the @start to @end range is 432 * about to be set to. On merge, this will be updated to include sticky flags. 433 * 434 * IMPORTANT: The actual modification being requested here is NOT applied, 435 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 436 * and the caller is expected to perform the actual modification. 437 * 438 * In order to account for sticky VMA flags, the @vma_flags_ptr parameter points 439 * to the requested flags which are then updated so the caller, should they 440 * overwrite any existing flags, correctly retains these. 441 * 442 * Returns: A VMA which contains the range @start to @end ready to have its 443 * flags altered to *@vma_flags. 444 */ 445 __must_check struct vm_area_struct *vma_modify_flags(struct vma_iterator *vmi, 446 struct vm_area_struct *prev, struct vm_area_struct *vma, 447 unsigned long start, unsigned long end, vma_flags_t *vma_flags_ptr); 448 449 /** 450 * vma_modify_name() - Perform any necessary split/merge in preparation for 451 * setting anonymous VMA name to @new_name in the range @start to @end contained 452 * within @vma. 453 * @vmi: Valid VMA iterator positioned at @vma. 454 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 455 * @vma: The VMA containing the range @start to @end to be updated. 456 * @start: The start of the range to update. May be offset within @vma. 457 * @end: The exclusive end of the range to update, may be offset within @vma. 458 * @new_name: The anonymous VMA name that the @start to @end range is about to 459 * be set to. 460 * 461 * IMPORTANT: The actual modification being requested here is NOT applied, 462 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 463 * and the caller is expected to perform the actual modification. 464 * 465 * Returns: A VMA which contains the range @start to @end ready to have its 466 * anonymous VMA name changed to @new_name. 467 */ 468 __must_check struct vm_area_struct *vma_modify_name(struct vma_iterator *vmi, 469 struct vm_area_struct *prev, struct vm_area_struct *vma, 470 unsigned long start, unsigned long end, 471 struct anon_vma_name *new_name); 472 473 /** 474 * vma_modify_policy() - Perform any necessary split/merge in preparation for 475 * setting NUMA policy to @new_pol in the range @start to @end contained 476 * within @vma. 477 * @vmi: Valid VMA iterator positioned at @vma. 478 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 479 * @vma: The VMA containing the range @start to @end to be updated. 480 * @start: The start of the range to update. May be offset within @vma. 481 * @end: The exclusive end of the range to update, may be offset within @vma. 482 * @new_pol: The NUMA policy that the @start to @end range is about to be set 483 * to. 484 * 485 * IMPORTANT: The actual modification being requested here is NOT applied, 486 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 487 * and the caller is expected to perform the actual modification. 488 * 489 * Returns: A VMA which contains the range @start to @end ready to have its 490 * NUMA policy changed to @new_pol. 491 */ 492 __must_check struct vm_area_struct *vma_modify_policy(struct vma_iterator *vmi, 493 struct vm_area_struct *prev, struct vm_area_struct *vma, 494 unsigned long start, unsigned long end, 495 struct mempolicy *new_pol); 496 497 /** 498 * vma_modify_flags_uffd() - Perform any necessary split/merge in preparation for 499 * setting VMA flags to @vm_flags and UFFD context to @new_ctx in the range 500 * @start to @end contained within @vma. 501 * @vmi: Valid VMA iterator positioned at @vma. 502 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first. 503 * @vma: The VMA containing the range @start to @end to be updated. 504 * @start: The start of the range to update. May be offset within @vma. 505 * @end: The exclusive end of the range to update, may be offset within @vma. 506 * @vma_flags: The VMA flags that the @start to @end range is about to be set to. 507 * @new_ctx: The userfaultfd context that the @start to @end range is about to 508 * be set to. 509 * @give_up_on_oom: If an out of memory condition occurs on merge, simply give 510 * up on it and treat the merge as best-effort. 511 * 512 * IMPORTANT: The actual modification being requested here is NOT applied, 513 * rather the VMA is perhaps split, perhaps merged to accommodate the change, 514 * and the caller is expected to perform the actual modification. 515 * 516 * Returns: A VMA which contains the range @start to @end ready to have its VMA 517 * flags changed to @vma_flags and its userfaultfd context changed to @new_ctx. 518 */ 519 __must_check struct vm_area_struct *vma_modify_flags_uffd(struct vma_iterator *vmi, 520 struct vm_area_struct *prev, struct vm_area_struct *vma, 521 unsigned long start, unsigned long end, const vma_flags_t *vma_flags, 522 struct vm_userfaultfd_ctx new_ctx, bool give_up_on_oom); 523 524 __must_check struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg); 525 526 __must_check struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi, 527 struct vm_area_struct *vma, unsigned long delta); 528 529 void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb); 530 531 void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb); 532 533 void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, 534 struct vm_area_struct *vma); 535 536 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, 537 unsigned long addr, unsigned long len, pgoff_t pgoff, 538 pgoff_t anon_pgoff, bool *need_rmap_locks, bool keep_source); 539 540 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); 541 542 bool vma_needs_dirty_tracking(struct vm_area_struct *vma); 543 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot); 544 545 int mm_take_all_locks(struct mm_struct *mm); 546 void mm_drop_all_locks(struct mm_struct *mm); 547 548 unsigned long mmap_region(struct file *file, unsigned long addr, 549 unsigned long len, vma_flags_t vma_flags, unsigned long pgoff, 550 struct list_head *uf); 551 552 int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, 553 unsigned long addr, unsigned long request, 554 vma_flags_t vma_flags); 555 556 unsigned long unmapped_area(struct vm_unmapped_area_info *info); 557 unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info); 558 559 #ifdef CONFIG_MMU 560 int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, 561 unsigned long addr, int new_below); 562 #endif 563 564 static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma) 565 { 566 /* 567 * We want to check manually if we can change individual PTEs writable 568 * if we can't do that automatically for all PTEs in a mapping. For 569 * private mappings, that's always the case when we have write 570 * permissions as we properly have to handle COW. 571 */ 572 if (vma->vm_flags & VM_SHARED) 573 return vma_wants_writenotify(vma, vma->vm_page_prot); 574 return !!(vma->vm_flags & VM_WRITE); 575 } 576 577 #ifdef CONFIG_MMU 578 static inline pgprot_t vma_pgprot_modify(pgprot_t oldprot, vma_flags_t vma_flags) 579 { 580 const pgprot_t prot = vma_flags_to_page_prot(vma_flags); 581 582 return pgprot_modify(oldprot, prot); 583 } 584 #endif 585 586 static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi, 587 unsigned long min) 588 { 589 return mas_prev(&vmi->mas, min); 590 } 591 592 /* 593 * These three helpers classifies VMAs for virtual memory accounting. 594 */ 595 596 /* 597 * Executable code area - executable, not writable, not stack 598 */ 599 static inline bool is_exec_mapping(vm_flags_t flags) 600 { 601 return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC; 602 } 603 604 /* 605 * Stack area (including shadow stacks) 606 * 607 * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous: 608 * do_mmap() forbids all other combinations. 609 */ 610 static inline bool is_stack_mapping(vm_flags_t flags) 611 { 612 return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK); 613 } 614 615 /* 616 * Data area - private, writable, not stack 617 */ 618 static inline bool is_data_mapping(vm_flags_t flags) 619 { 620 return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE; 621 } 622 623 static inline bool is_data_mapping_vma_flags(const vma_flags_t *vma_flags) 624 { 625 return vma_flags_test(vma_flags, VMA_WRITE_BIT) && 626 !vma_flags_test_any(vma_flags, VMA_SHARED_BIT, VMA_STACK_BIT); 627 } 628 629 static inline void vma_iter_config(struct vma_iterator *vmi, 630 unsigned long index, unsigned long last) 631 { 632 __mas_set_range(&vmi->mas, index, last - 1); 633 } 634 635 static inline void vma_iter_reset(struct vma_iterator *vmi) 636 { 637 mas_reset(&vmi->mas); 638 } 639 640 static inline 641 struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min) 642 { 643 return mas_prev_range(&vmi->mas, min); 644 } 645 646 static inline 647 struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max) 648 { 649 return mas_next_range(&vmi->mas, max); 650 } 651 652 static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min, 653 unsigned long max, unsigned long size) 654 { 655 return mas_empty_area(&vmi->mas, min, max - 1, size); 656 } 657 658 static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min, 659 unsigned long max, unsigned long size) 660 { 661 return mas_empty_area_rev(&vmi->mas, min, max - 1, size); 662 } 663 664 /* 665 * VMA Iterator functions shared between nommu and mmap 666 */ 667 static inline int vma_iter_prealloc(struct vma_iterator *vmi, 668 struct vm_area_struct *vma) 669 { 670 return mas_preallocate(&vmi->mas, vma, GFP_KERNEL); 671 } 672 673 static inline void vma_iter_clear(struct vma_iterator *vmi) 674 { 675 mas_store_prealloc(&vmi->mas, NULL); 676 } 677 678 static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi) 679 { 680 return mas_walk(&vmi->mas); 681 } 682 683 /* Store a VMA with preallocated memory */ 684 static inline void vma_iter_store_overwrite(struct vma_iterator *vmi, 685 struct vm_area_struct *vma) 686 { 687 vma_assert_attached(vma); 688 689 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 690 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start && 691 vmi->mas.index > vma->vm_start)) { 692 pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n", 693 vmi->mas.index, vma->vm_start, vma->vm_start, 694 vma->vm_end, vmi->mas.index, vmi->mas.last); 695 } 696 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start && 697 vmi->mas.last < vma->vm_start)) { 698 pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n", 699 vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end, 700 vmi->mas.index, vmi->mas.last); 701 } 702 #endif 703 704 if (vmi->mas.status != ma_start && 705 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start))) 706 vma_iter_invalidate(vmi); 707 708 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1); 709 mas_store_prealloc(&vmi->mas, vma); 710 } 711 712 static inline void vma_iter_store_new(struct vma_iterator *vmi, 713 struct vm_area_struct *vma) 714 { 715 vma_mark_attached(vma); 716 vma_iter_store_overwrite(vmi, vma); 717 } 718 719 static inline unsigned long vma_iter_addr(struct vma_iterator *vmi) 720 { 721 return vmi->mas.index; 722 } 723 724 static inline unsigned long vma_iter_end(struct vma_iterator *vmi) 725 { 726 return vmi->mas.last + 1; 727 } 728 729 static inline 730 struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi) 731 { 732 return mas_prev_range(&vmi->mas, 0); 733 } 734 735 /* 736 * Retrieve the next VMA and rewind the iterator to end of the previous VMA, or 737 * if no previous VMA, to index 0. 738 */ 739 static inline 740 struct vm_area_struct *vma_iter_next_rewind(struct vma_iterator *vmi, 741 struct vm_area_struct **pprev) 742 { 743 struct vm_area_struct *next = vma_next(vmi); 744 struct vm_area_struct *prev = vma_prev(vmi); 745 746 /* 747 * Consider the case where no previous VMA exists. We advance to the 748 * next VMA, skipping any gap, then rewind to the start of the range. 749 * 750 * If we were to unconditionally advance to the next range we'd wind up 751 * at the next VMA again, so we check to ensure there is a previous VMA 752 * to skip over. 753 */ 754 if (prev) 755 vma_iter_next_range(vmi); 756 757 if (pprev) 758 *pprev = prev; 759 760 return next; 761 } 762 763 #ifdef CONFIG_64BIT 764 static inline bool vma_is_sealed(struct vm_area_struct *vma) 765 { 766 return (vma->vm_flags & VM_SEALED); 767 } 768 #else 769 static inline bool vma_is_sealed(struct vm_area_struct *vma) 770 { 771 return false; 772 } 773 #endif 774 775 #if defined(CONFIG_STACK_GROWSUP) 776 int expand_upwards(struct vm_area_struct *vma, unsigned long address); 777 #endif 778 779 int expand_downwards(struct vm_area_struct *vma, unsigned long address); 780 781 int __vm_munmap(unsigned long start, size_t len, bool unlock); 782 783 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma); 784 785 /* vma_init.h, shared between CONFIG_MMU and nommu. */ 786 void __init vma_state_init(void); 787 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm); 788 struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig); 789 void vm_area_free(struct vm_area_struct *vma); 790 791 /* vma_exec.c */ 792 #ifdef CONFIG_MMU 793 int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, 794 unsigned long *top_mem_p); 795 int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift); 796 #endif 797 798 #ifdef CONFIG_MMU 799 /* 800 * Denies creating a writable executable mapping or gaining executable permissions. 801 * 802 * This denies the following: 803 * 804 * a) mmap(PROT_WRITE | PROT_EXEC) 805 * 806 * b) mmap(PROT_WRITE) 807 * mprotect(PROT_EXEC) 808 * 809 * c) mmap(PROT_WRITE) 810 * mprotect(PROT_READ) 811 * mprotect(PROT_EXEC) 812 * 813 * But allows the following: 814 * 815 * d) mmap(PROT_READ | PROT_EXEC) 816 * mmap(PROT_READ | PROT_EXEC | PROT_BTI) 817 * 818 * This is only applicable if the user has set the Memory-Deny-Write-Execute 819 * (MDWE) protection mask for the current process. 820 * 821 * @old specifies the VMA flags the VMA originally possessed, and @new the ones 822 * we propose to set. 823 * 824 * Return: false if proposed change is OK, true if not ok and should be denied. 825 */ 826 static inline bool map_deny_write_exec(const vma_flags_t *old, 827 const vma_flags_t *new) 828 { 829 /* If MDWE is disabled, we have nothing to deny. */ 830 if (!mm_flags_test(MMF_HAS_MDWE, current->mm)) 831 return false; 832 833 /* If the new VMA is not executable, we have nothing to deny. */ 834 if (!vma_flags_test(new, VMA_EXEC_BIT)) 835 return false; 836 837 /* Under MDWE we do not accept newly writably executable VMAs... */ 838 if (vma_flags_test(new, VMA_WRITE_BIT)) 839 return true; 840 841 /* ...nor previously non-executable VMAs becoming executable. */ 842 if (!vma_flags_test(old, VMA_EXEC_BIT)) 843 return true; 844 845 return false; 846 } 847 #endif 848 849 struct vm_area_struct *__install_special_mapping(struct mm_struct *mm, 850 unsigned long addr, unsigned long len, 851 vm_flags_t vm_flags, void *priv, 852 const struct vm_operations_struct *ops); 853 854 #endif /* __MM_VMA_H */ 855