1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * vma.h
4 *
5 * Core VMA manipulation API implemented in vma.c, vma_init.c and vma_exec.c.
6 *
7 * Note that, in order for VMA logic to be userland testable, this header
8 * intentionally includes no dependencies.
9 *
10 * This is specifically scoped to mm-only. Users of this functionality (other
11 * than the core VMA implementation itself) should not include this header
12 * directly, but rather include internal.h.
13 */
14 #ifndef __MM_VMA_H
15 #define __MM_VMA_H
16
17 /*
18 * VMA lock generalization
19 */
20 struct vma_prepare {
21 struct vm_area_struct *vma;
22 struct vm_area_struct *adj_next;
23 struct file *file;
24 struct address_space *mapping;
25 struct anon_vma *anon_vma;
26 struct vm_area_struct *insert;
27 struct vm_area_struct *remove;
28 struct vm_area_struct *remove2;
29
30 bool skip_vma_uprobe :1;
31 };
32
33 struct unlink_vma_file_batch {
34 int count;
35 struct vm_area_struct *vmas[8];
36 };
37
38 /*
39 * vma munmap operation
40 */
41 struct vma_munmap_struct {
42 struct vma_iterator *vmi;
43 struct vm_area_struct *vma; /* The first vma to munmap */
44 struct vm_area_struct *prev; /* vma before the munmap area */
45 struct vm_area_struct *next; /* vma after the munmap area */
46 struct list_head *uf; /* Userfaultfd list_head */
47 unsigned long start; /* Aligned start addr (inclusive) */
48 unsigned long end; /* Aligned end addr (exclusive) */
49 unsigned long unmap_start; /* Unmap PTE start */
50 unsigned long unmap_end; /* Unmap PTE end */
51 int vma_count; /* Number of vmas that will be removed */
52 bool unlock; /* Unlock after the munmap */
53 bool clear_ptes; /* If there are outstanding PTE to be cleared */
54 /* 2 byte hole */
55 unsigned long nr_pages; /* Number of pages being removed */
56 unsigned long locked_vm; /* Number of locked pages */
57 unsigned long nr_accounted; /* Number of VM_ACCOUNT pages */
58 unsigned long exec_vm;
59 unsigned long stack_vm;
60 unsigned long data_vm;
61 };
62
63 enum vma_merge_state {
64 VMA_MERGE_START,
65 VMA_MERGE_ERROR_NOMEM,
66 VMA_MERGE_NOMERGE,
67 VMA_MERGE_SUCCESS,
68 };
69
70 /*
71 * Describes a VMA merge operation and is threaded throughout it.
72 *
73 * Any of the fields may be mutated by the merge operation, so no guarantees are
74 * made to the contents of this structure after a merge operation has completed.
75 */
76 struct vma_merge_struct {
77 struct mm_struct *mm;
78 struct vma_iterator *vmi;
79 /*
80 * Adjacent VMAs, any of which may be NULL if not present:
81 *
82 * |------|--------|------|
83 * | prev | middle | next |
84 * |------|--------|------|
85 *
86 * middle may not yet exist in the case of a proposed new VMA being
87 * merged, or it may be an existing VMA.
88 *
89 * next may be assigned by the caller.
90 */
91 struct vm_area_struct *prev;
92 struct vm_area_struct *middle;
93 struct vm_area_struct *next;
94 /* This is the VMA we ultimately target to become the merged VMA. */
95 struct vm_area_struct *target;
96 /*
97 * Initially, the start, end, pgoff fields are provided by the caller
98 * and describe the proposed new VMA range, whether modifying an
99 * existing VMA (which will be 'middle'), or adding a new one.
100 *
101 * During the merge process these fields are updated to describe the new
102 * range _including those VMAs which will be merged_.
103 */
104 unsigned long start;
105 unsigned long end;
106 pgoff_t pgoff;
107 pgoff_t anon_pgoff;
108
109 union {
110 /* Temporary while VMA flags are being converted. */
111 vm_flags_t vm_flags;
112 vma_flags_t vma_flags;
113 };
114 struct file *file;
115 struct anon_vma *anon_vma;
116 struct mempolicy *policy;
117 struct vm_userfaultfd_ctx uffd_ctx;
118 struct anon_vma_name *anon_name;
119 enum vma_merge_state state;
120
121 /* If copied from (i.e. mremap()'d) the VMA from which we are copying. */
122 struct vm_area_struct *copied_from;
123
124 /* Flags which callers can use to modify merge behaviour: */
125
126 /*
127 * If we can expand, simply do so. We know there is nothing to merge to
128 * the right. Does not reset state upon failure to merge. The VMA
129 * iterator is assumed to be positioned at the previous VMA, rather than
130 * at the gap.
131 */
132 bool just_expand :1;
133
134 /*
135 * If a merge is possible, but an OOM error occurs, give up and don't
136 * execute the merge, returning NULL.
137 */
138 bool give_up_on_oom :1;
139
140 /*
141 * If set, skip uprobe_mmap upon merged vma.
142 */
143 bool skip_vma_uprobe :1;
144
145 /* Internal flags set during merge process: */
146
147 /*
148 * Internal flag indicating the merge increases vmg->middle->vm_start
149 * (and thereby, vmg->prev->vm_end).
150 */
151 bool __adjust_middle_start :1;
152 /*
153 * Internal flag indicating the merge decreases vmg->next->vm_start
154 * (and thereby, vmg->middle->vm_end).
155 */
156 bool __adjust_next_start :1;
157 /*
158 * Internal flag used during the merge operation to indicate we will
159 * remove vmg->middle.
160 */
161 bool __remove_middle :1;
162 /*
163 * Internal flag used during the merge operation to indicate we will
164 * remove vmg->next.
165 */
166 bool __remove_next :1;
167
168 };
169
170 struct unmap_desc {
171 struct ma_state *mas; /* the maple state point to the first vma */
172 struct vm_area_struct *first; /* The first vma */
173 unsigned long pg_start; /* The first pagetable address to free (floor) */
174 unsigned long pg_end; /* The last pagetable address to free (ceiling) */
175 unsigned long vma_start; /* The min vma address */
176 unsigned long vma_end; /* The max vma address */
177 unsigned long tree_end; /* Maximum for the vma tree search */
178 unsigned long tree_reset; /* Where to reset the vma tree walk */
179 bool mm_wr_locked; /* If the mmap write lock is held */
180 };
181
182 /*
183 * unmap_all_init() - Initialize unmap_desc to remove all vmas, point the
184 * pg_start and pg_end to a safe location.
185 */
unmap_all_init(struct unmap_desc * unmap,struct vma_iterator * vmi,struct vm_area_struct * vma)186 static inline void unmap_all_init(struct unmap_desc *unmap,
187 struct vma_iterator *vmi, struct vm_area_struct *vma)
188 {
189 unmap->mas = &vmi->mas;
190 unmap->first = vma;
191 unmap->pg_start = FIRST_USER_ADDRESS;
192 unmap->pg_end = USER_PGTABLES_CEILING;
193 unmap->vma_start = 0;
194 unmap->vma_end = ULONG_MAX;
195 unmap->tree_end = ULONG_MAX;
196 unmap->tree_reset = vma->vm_end;
197 unmap->mm_wr_locked = false;
198 }
199
200 /*
201 * unmap_pgtable_init() - Initialize unmap_desc to remove all page tables within
202 * the user range.
203 *
204 * ARM can have mappings outside of vmas.
205 * See: e2cdef8c847b4 ("[PATCH] freepgt: free_pgtables from FIRST_USER_ADDRESS")
206 *
207 * ARM LPAE uses page table mappings beyond the USER_PGTABLES_CEILING
208 * See: CONFIG_ARM_LPAE in arch/arm/include/asm/pgtable.h
209 */
unmap_pgtable_init(struct unmap_desc * unmap,struct vma_iterator * vmi)210 static inline void unmap_pgtable_init(struct unmap_desc *unmap,
211 struct vma_iterator *vmi)
212 {
213 vma_iter_set(vmi, unmap->tree_reset);
214 unmap->vma_start = FIRST_USER_ADDRESS;
215 unmap->vma_end = USER_PGTABLES_CEILING;
216 unmap->tree_end = USER_PGTABLES_CEILING;
217 }
218
219 #define UNMAP_STATE(name, _vmi, _vma, _vma_start, _vma_end, _prev, _next) \
220 struct unmap_desc name = { \
221 .mas = &(_vmi)->mas, \
222 .first = _vma, \
223 .pg_start = _prev ? ((struct vm_area_struct *)_prev)->vm_end : \
224 FIRST_USER_ADDRESS, \
225 .pg_end = _next ? ((struct vm_area_struct *)_next)->vm_start : \
226 USER_PGTABLES_CEILING, \
227 .vma_start = _vma_start, \
228 .vma_end = _vma_end, \
229 .tree_end = _next ? \
230 ((struct vm_area_struct *)_next)->vm_start : \
231 USER_PGTABLES_CEILING, \
232 .tree_reset = _vma->vm_end, \
233 .mm_wr_locked = true, \
234 }
235
vmg_nomem(struct vma_merge_struct * vmg)236 static inline bool vmg_nomem(struct vma_merge_struct *vmg)
237 {
238 return vmg->state == VMA_MERGE_ERROR_NOMEM;
239 }
240
vmg_pages(const struct vma_merge_struct * vmg)241 static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg)
242 {
243 const unsigned long size = vmg->end - vmg->start;
244
245 return size >> PAGE_SHIFT;
246 }
247
vmg_start_pgoff(const struct vma_merge_struct * vmg)248 static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg)
249 {
250 return vmg->pgoff;
251 }
252
vmg_end_pgoff(const struct vma_merge_struct * vmg)253 static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg)
254 {
255 return vmg_start_pgoff(vmg) + vmg_pages(vmg);
256 }
257
assert_sane_pgoff(struct vm_area_struct * vma,pgoff_t pgoff)258 static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
259 {
260 /* nommu doesn't set a virtual pgoff for anon VMAs. */
261 if (!IS_ENABLED(CONFIG_MMU))
262 return;
263 /*
264 * File-backed VMAs have arbitrary page offset (either page offset into
265 * file or for pfnmap the PFN of the start of the range or drivers may
266 * set arbitrary page offset).
267 */
268 if (!vma_is_anonymous(vma))
269 return;
270 /* MAP_PRIVATE-/dev/zero is anon, non-NULL vm_file, but has file pgoff. */
271 if (vma->vm_file)
272 return;
273 /* If faulted in, could have been remapped. */
274 if (vma->anon_vma)
275 return;
276 /* OK this is really an anon VMA - expect virtual page offset. */
277 VM_WARN_ON_ONCE(pgoff != vma->vm_start >> PAGE_SHIFT);
278 }
279
vma_set_pgoff(struct vm_area_struct * vma,pgoff_t pgoff)280 static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
281 {
282 vma_assert_can_modify(vma);
283 assert_sane_pgoff(vma, pgoff);
284 vma->vm_pgoff = pgoff;
285 }
286
vmg_start_anon_pgoff(const struct vma_merge_struct * vmg)287 static inline pgoff_t vmg_start_anon_pgoff(const struct vma_merge_struct *vmg)
288 {
289 return vmg->anon_pgoff;
290 }
291
vmg_end_anon_pgoff(const struct vma_merge_struct * vmg)292 static inline pgoff_t vmg_end_anon_pgoff(const struct vma_merge_struct *vmg)
293 {
294 return vmg_start_anon_pgoff(vmg) + vmg_pages(vmg);
295 }
296
__vma_set_anon_pgoff(struct vm_area_struct * vma,pgoff_t pgoff)297 static inline void __vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
298 {
299 #ifdef CONFIG_64BIT
300 vma->__vm_anon_pgoff_hi = pgoff >> 32;
301 #endif
302 vma->__vm_anon_pgoff_lo = pgoff & GENMASK(31, 0);
303 }
304
vma_set_anon_pgoff(struct vm_area_struct * vma,pgoff_t pgoff)305 static inline void vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
306 {
307 vma_assert_can_modify(vma);
308 __vma_set_anon_pgoff(vma, pgoff);
309 }
310
vma_add_pgoff(struct vm_area_struct * vma,pgoff_t delta)311 static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta)
312 {
313 vma_assert_can_modify(vma);
314 vma_set_pgoff(vma, vma_start_pgoff(vma) + delta);
315 vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) + delta);
316 }
317
vma_sub_pgoff(struct vm_area_struct * vma,pgoff_t delta)318 static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta)
319 {
320 vma_assert_can_modify(vma);
321 vma_set_pgoff(vma, vma_start_pgoff(vma) - delta);
322 vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) - delta);
323 }
324
325 #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_, anon_pgoff_) \
326 struct vma_merge_struct name = { \
327 .mm = mm_, \
328 .vmi = vmi_, \
329 .start = start_, \
330 .end = end_, \
331 .vma_flags = vma_flags_, \
332 .pgoff = pgoff_, \
333 .anon_pgoff = anon_pgoff_, \
334 .state = VMA_MERGE_START, \
335 }
336
337 #define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \
338 struct vma_merge_struct name = { \
339 .mm = vma_->vm_mm, \
340 .vmi = vmi_, \
341 .prev = prev_, \
342 .middle = vma_, \
343 .next = NULL, \
344 .start = start_, \
345 .end = end_, \
346 .vm_flags = vma_->vm_flags, \
347 .pgoff = linear_page_index(vma_, start_), \
348 .anon_pgoff = __linear_anon_page_index(vma_, start_), \
349 .file = vma_->vm_file, \
350 .anon_vma = vma_->anon_vma, \
351 .policy = vma_policy(vma_), \
352 .uffd_ctx = vma_->vm_userfaultfd_ctx, \
353 .anon_name = anon_vma_name(vma_), \
354 .state = VMA_MERGE_START, \
355 }
356
357 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE
358 void validate_mm(struct mm_struct *mm);
359 #else
360 #define validate_mm(mm) do { } while (0)
361 #endif
362
363 __must_check int vma_expand(struct vma_merge_struct *vmg);
364 __must_check int vma_shrink(struct vma_iterator *vmi,
365 struct vm_area_struct *vma, unsigned long end);
366
vma_iter_store_gfp(struct vma_iterator * vmi,struct vm_area_struct * vma,gfp_t gfp)367 static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
368 struct vm_area_struct *vma, gfp_t gfp)
369
370 {
371 if (vmi->mas.status != ma_start &&
372 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
373 vma_iter_invalidate(vmi);
374
375 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
376 mas_store_gfp(&vmi->mas, vma, gfp);
377 if (unlikely(mas_is_err(&vmi->mas)))
378 return -ENOMEM;
379
380 vma_mark_attached(vma);
381 return 0;
382 }
383
384 /*
385 * Temporary helper function for stacked mmap handlers which specify
386 * f_op->mmap() but which might have an underlying file system which implements
387 * f_op->mmap_prepare().
388 */
compat_set_vma_from_desc(struct vm_area_struct * vma,struct vm_area_desc * desc)389 static inline void compat_set_vma_from_desc(struct vm_area_struct *vma,
390 struct vm_area_desc *desc)
391 {
392 /*
393 * Since we're invoking .mmap_prepare() despite having a partially
394 * established VMA, we must take care to handle setting fields
395 * correctly.
396 */
397
398 /* Mutable fields. Populated with initial state. */
399 vma_set_pgoff(vma, desc->pgoff);
400 if (desc->vm_file != vma->vm_file)
401 vma_set_file(vma, desc->vm_file);
402 vma->flags = desc->vma_flags;
403 vma->vm_page_prot = desc->page_prot;
404
405 /* User-defined fields. */
406 vma->vm_ops = desc->vm_ops;
407 vma->vm_private_data = desc->private_data;
408 }
409
410 int
411 do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
412 struct mm_struct *mm, unsigned long start,
413 unsigned long end, struct list_head *uf, bool unlock);
414
415 int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
416 unsigned long start, size_t len, struct list_head *uf,
417 bool unlock);
418
419 void remove_vma(struct vm_area_struct *vma);
420 void unmap_region(struct unmap_desc *unmap);
421
422 /**
423 * vma_modify_flags() - Perform any necessary split/merge in preparation for
424 * setting VMA flags to *@vm_flags in the range @start to @end contained within
425 * @vma.
426 * @vmi: Valid VMA iterator positioned at @vma.
427 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
428 * @vma: The VMA containing the range @start to @end to be updated.
429 * @start: The start of the range to update. May be offset within @vma.
430 * @end: The exclusive end of the range to update, may be offset within @vma.
431 * @vma_flags_ptr: A pointer to the VMA flags that the @start to @end range is
432 * about to be set to. On merge, this will be updated to include sticky flags.
433 *
434 * IMPORTANT: The actual modification being requested here is NOT applied,
435 * rather the VMA is perhaps split, perhaps merged to accommodate the change,
436 * and the caller is expected to perform the actual modification.
437 *
438 * In order to account for sticky VMA flags, the @vma_flags_ptr parameter points
439 * to the requested flags which are then updated so the caller, should they
440 * overwrite any existing flags, correctly retains these.
441 *
442 * Returns: A VMA which contains the range @start to @end ready to have its
443 * flags altered to *@vma_flags.
444 */
445 __must_check struct vm_area_struct *vma_modify_flags(struct vma_iterator *vmi,
446 struct vm_area_struct *prev, struct vm_area_struct *vma,
447 unsigned long start, unsigned long end, vma_flags_t *vma_flags_ptr);
448
449 /**
450 * vma_modify_name() - Perform any necessary split/merge in preparation for
451 * setting anonymous VMA name to @new_name in the range @start to @end contained
452 * within @vma.
453 * @vmi: Valid VMA iterator positioned at @vma.
454 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
455 * @vma: The VMA containing the range @start to @end to be updated.
456 * @start: The start of the range to update. May be offset within @vma.
457 * @end: The exclusive end of the range to update, may be offset within @vma.
458 * @new_name: The anonymous VMA name that the @start to @end range is about to
459 * be set to.
460 *
461 * IMPORTANT: The actual modification being requested here is NOT applied,
462 * rather the VMA is perhaps split, perhaps merged to accommodate the change,
463 * and the caller is expected to perform the actual modification.
464 *
465 * Returns: A VMA which contains the range @start to @end ready to have its
466 * anonymous VMA name changed to @new_name.
467 */
468 __must_check struct vm_area_struct *vma_modify_name(struct vma_iterator *vmi,
469 struct vm_area_struct *prev, struct vm_area_struct *vma,
470 unsigned long start, unsigned long end,
471 struct anon_vma_name *new_name);
472
473 /**
474 * vma_modify_policy() - Perform any necessary split/merge in preparation for
475 * setting NUMA policy to @new_pol in the range @start to @end contained
476 * within @vma.
477 * @vmi: Valid VMA iterator positioned at @vma.
478 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
479 * @vma: The VMA containing the range @start to @end to be updated.
480 * @start: The start of the range to update. May be offset within @vma.
481 * @end: The exclusive end of the range to update, may be offset within @vma.
482 * @new_pol: The NUMA policy that the @start to @end range is about to be set
483 * to.
484 *
485 * IMPORTANT: The actual modification being requested here is NOT applied,
486 * rather the VMA is perhaps split, perhaps merged to accommodate the change,
487 * and the caller is expected to perform the actual modification.
488 *
489 * Returns: A VMA which contains the range @start to @end ready to have its
490 * NUMA policy changed to @new_pol.
491 */
492 __must_check struct vm_area_struct *vma_modify_policy(struct vma_iterator *vmi,
493 struct vm_area_struct *prev, struct vm_area_struct *vma,
494 unsigned long start, unsigned long end,
495 struct mempolicy *new_pol);
496
497 /**
498 * vma_modify_flags_uffd() - Perform any necessary split/merge in preparation for
499 * setting VMA flags to @vm_flags and UFFD context to @new_ctx in the range
500 * @start to @end contained within @vma.
501 * @vmi: Valid VMA iterator positioned at @vma.
502 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
503 * @vma: The VMA containing the range @start to @end to be updated.
504 * @start: The start of the range to update. May be offset within @vma.
505 * @end: The exclusive end of the range to update, may be offset within @vma.
506 * @vma_flags: The VMA flags that the @start to @end range is about to be set to.
507 * @new_ctx: The userfaultfd context that the @start to @end range is about to
508 * be set to.
509 * @give_up_on_oom: If an out of memory condition occurs on merge, simply give
510 * up on it and treat the merge as best-effort.
511 *
512 * IMPORTANT: The actual modification being requested here is NOT applied,
513 * rather the VMA is perhaps split, perhaps merged to accommodate the change,
514 * and the caller is expected to perform the actual modification.
515 *
516 * Returns: A VMA which contains the range @start to @end ready to have its VMA
517 * flags changed to @vma_flags and its userfaultfd context changed to @new_ctx.
518 */
519 __must_check struct vm_area_struct *vma_modify_flags_uffd(struct vma_iterator *vmi,
520 struct vm_area_struct *prev, struct vm_area_struct *vma,
521 unsigned long start, unsigned long end, const vma_flags_t *vma_flags,
522 struct vm_userfaultfd_ctx new_ctx, bool give_up_on_oom);
523
524 __must_check struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg);
525
526 __must_check struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
527 struct vm_area_struct *vma, unsigned long delta);
528
529 void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb);
530
531 void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb);
532
533 void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
534 struct vm_area_struct *vma);
535
536 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
537 unsigned long addr, unsigned long len, pgoff_t pgoff,
538 pgoff_t anon_pgoff, bool *need_rmap_locks);
539
540 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma);
541
542 bool vma_needs_dirty_tracking(struct vm_area_struct *vma);
543 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
544
545 int mm_take_all_locks(struct mm_struct *mm);
546 void mm_drop_all_locks(struct mm_struct *mm);
547
548 unsigned long mmap_region(struct file *file, unsigned long addr,
549 unsigned long len, vma_flags_t vma_flags, unsigned long pgoff,
550 struct list_head *uf);
551
552 int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma,
553 unsigned long addr, unsigned long request,
554 vma_flags_t vma_flags);
555
556 unsigned long unmapped_area(struct vm_unmapped_area_info *info);
557 unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
558
vma_wants_manual_pte_write_upgrade(struct vm_area_struct * vma)559 static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
560 {
561 /*
562 * We want to check manually if we can change individual PTEs writable
563 * if we can't do that automatically for all PTEs in a mapping. For
564 * private mappings, that's always the case when we have write
565 * permissions as we properly have to handle COW.
566 */
567 if (vma->vm_flags & VM_SHARED)
568 return vma_wants_writenotify(vma, vma->vm_page_prot);
569 return !!(vma->vm_flags & VM_WRITE);
570 }
571
572 #ifdef CONFIG_MMU
vma_pgprot_modify(pgprot_t oldprot,vma_flags_t vma_flags)573 static inline pgprot_t vma_pgprot_modify(pgprot_t oldprot, vma_flags_t vma_flags)
574 {
575 const pgprot_t prot = vma_flags_to_page_prot(vma_flags);
576
577 return pgprot_modify(oldprot, prot);
578 }
579 #endif
580
vma_prev_limit(struct vma_iterator * vmi,unsigned long min)581 static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi,
582 unsigned long min)
583 {
584 return mas_prev(&vmi->mas, min);
585 }
586
587 /*
588 * These three helpers classifies VMAs for virtual memory accounting.
589 */
590
591 /*
592 * Executable code area - executable, not writable, not stack
593 */
is_exec_mapping(vm_flags_t flags)594 static inline bool is_exec_mapping(vm_flags_t flags)
595 {
596 return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
597 }
598
599 /*
600 * Stack area (including shadow stacks)
601 *
602 * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
603 * do_mmap() forbids all other combinations.
604 */
is_stack_mapping(vm_flags_t flags)605 static inline bool is_stack_mapping(vm_flags_t flags)
606 {
607 return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
608 }
609
610 /*
611 * Data area - private, writable, not stack
612 */
is_data_mapping(vm_flags_t flags)613 static inline bool is_data_mapping(vm_flags_t flags)
614 {
615 return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
616 }
617
is_data_mapping_vma_flags(const vma_flags_t * vma_flags)618 static inline bool is_data_mapping_vma_flags(const vma_flags_t *vma_flags)
619 {
620 return vma_flags_test(vma_flags, VMA_WRITE_BIT) &&
621 !vma_flags_test_any(vma_flags, VMA_SHARED_BIT, VMA_STACK_BIT);
622 }
623
vma_iter_config(struct vma_iterator * vmi,unsigned long index,unsigned long last)624 static inline void vma_iter_config(struct vma_iterator *vmi,
625 unsigned long index, unsigned long last)
626 {
627 __mas_set_range(&vmi->mas, index, last - 1);
628 }
629
vma_iter_reset(struct vma_iterator * vmi)630 static inline void vma_iter_reset(struct vma_iterator *vmi)
631 {
632 mas_reset(&vmi->mas);
633 }
634
635 static inline
vma_iter_prev_range_limit(struct vma_iterator * vmi,unsigned long min)636 struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min)
637 {
638 return mas_prev_range(&vmi->mas, min);
639 }
640
641 static inline
vma_iter_next_range_limit(struct vma_iterator * vmi,unsigned long max)642 struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max)
643 {
644 return mas_next_range(&vmi->mas, max);
645 }
646
vma_iter_area_lowest(struct vma_iterator * vmi,unsigned long min,unsigned long max,unsigned long size)647 static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min,
648 unsigned long max, unsigned long size)
649 {
650 return mas_empty_area(&vmi->mas, min, max - 1, size);
651 }
652
vma_iter_area_highest(struct vma_iterator * vmi,unsigned long min,unsigned long max,unsigned long size)653 static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min,
654 unsigned long max, unsigned long size)
655 {
656 return mas_empty_area_rev(&vmi->mas, min, max - 1, size);
657 }
658
659 /*
660 * VMA Iterator functions shared between nommu and mmap
661 */
vma_iter_prealloc(struct vma_iterator * vmi,struct vm_area_struct * vma)662 static inline int vma_iter_prealloc(struct vma_iterator *vmi,
663 struct vm_area_struct *vma)
664 {
665 return mas_preallocate(&vmi->mas, vma, GFP_KERNEL);
666 }
667
vma_iter_clear(struct vma_iterator * vmi)668 static inline void vma_iter_clear(struct vma_iterator *vmi)
669 {
670 mas_store_prealloc(&vmi->mas, NULL);
671 }
672
vma_iter_load(struct vma_iterator * vmi)673 static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
674 {
675 return mas_walk(&vmi->mas);
676 }
677
678 /* Store a VMA with preallocated memory */
vma_iter_store_overwrite(struct vma_iterator * vmi,struct vm_area_struct * vma)679 static inline void vma_iter_store_overwrite(struct vma_iterator *vmi,
680 struct vm_area_struct *vma)
681 {
682 vma_assert_attached(vma);
683
684 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
685 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
686 vmi->mas.index > vma->vm_start)) {
687 pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n",
688 vmi->mas.index, vma->vm_start, vma->vm_start,
689 vma->vm_end, vmi->mas.index, vmi->mas.last);
690 }
691 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
692 vmi->mas.last < vma->vm_start)) {
693 pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n",
694 vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end,
695 vmi->mas.index, vmi->mas.last);
696 }
697 #endif
698
699 if (vmi->mas.status != ma_start &&
700 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
701 vma_iter_invalidate(vmi);
702
703 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
704 mas_store_prealloc(&vmi->mas, vma);
705 }
706
vma_iter_store_new(struct vma_iterator * vmi,struct vm_area_struct * vma)707 static inline void vma_iter_store_new(struct vma_iterator *vmi,
708 struct vm_area_struct *vma)
709 {
710 vma_mark_attached(vma);
711 vma_iter_store_overwrite(vmi, vma);
712 }
713
vma_iter_addr(struct vma_iterator * vmi)714 static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
715 {
716 return vmi->mas.index;
717 }
718
vma_iter_end(struct vma_iterator * vmi)719 static inline unsigned long vma_iter_end(struct vma_iterator *vmi)
720 {
721 return vmi->mas.last + 1;
722 }
723
724 static inline
vma_iter_prev_range(struct vma_iterator * vmi)725 struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi)
726 {
727 return mas_prev_range(&vmi->mas, 0);
728 }
729
730 /*
731 * Retrieve the next VMA and rewind the iterator to end of the previous VMA, or
732 * if no previous VMA, to index 0.
733 */
734 static inline
vma_iter_next_rewind(struct vma_iterator * vmi,struct vm_area_struct ** pprev)735 struct vm_area_struct *vma_iter_next_rewind(struct vma_iterator *vmi,
736 struct vm_area_struct **pprev)
737 {
738 struct vm_area_struct *next = vma_next(vmi);
739 struct vm_area_struct *prev = vma_prev(vmi);
740
741 /*
742 * Consider the case where no previous VMA exists. We advance to the
743 * next VMA, skipping any gap, then rewind to the start of the range.
744 *
745 * If we were to unconditionally advance to the next range we'd wind up
746 * at the next VMA again, so we check to ensure there is a previous VMA
747 * to skip over.
748 */
749 if (prev)
750 vma_iter_next_range(vmi);
751
752 if (pprev)
753 *pprev = prev;
754
755 return next;
756 }
757
758 #ifdef CONFIG_64BIT
vma_is_sealed(struct vm_area_struct * vma)759 static inline bool vma_is_sealed(struct vm_area_struct *vma)
760 {
761 return (vma->vm_flags & VM_SEALED);
762 }
763 #else
vma_is_sealed(struct vm_area_struct * vma)764 static inline bool vma_is_sealed(struct vm_area_struct *vma)
765 {
766 return false;
767 }
768 #endif
769
770 #if defined(CONFIG_STACK_GROWSUP)
771 int expand_upwards(struct vm_area_struct *vma, unsigned long address);
772 #endif
773
774 int expand_downwards(struct vm_area_struct *vma, unsigned long address);
775
776 int __vm_munmap(unsigned long start, size_t len, bool unlock);
777
778 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma);
779
780 /* vma_init.h, shared between CONFIG_MMU and nommu. */
781 void __init vma_state_init(void);
782 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm);
783 struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig);
784 void vm_area_free(struct vm_area_struct *vma);
785
786 /* vma_exec.c */
787 #ifdef CONFIG_MMU
788 int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap,
789 unsigned long *top_mem_p);
790 int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift);
791 #endif
792
793 #ifdef CONFIG_MMU
794 /*
795 * Denies creating a writable executable mapping or gaining executable permissions.
796 *
797 * This denies the following:
798 *
799 * a) mmap(PROT_WRITE | PROT_EXEC)
800 *
801 * b) mmap(PROT_WRITE)
802 * mprotect(PROT_EXEC)
803 *
804 * c) mmap(PROT_WRITE)
805 * mprotect(PROT_READ)
806 * mprotect(PROT_EXEC)
807 *
808 * But allows the following:
809 *
810 * d) mmap(PROT_READ | PROT_EXEC)
811 * mmap(PROT_READ | PROT_EXEC | PROT_BTI)
812 *
813 * This is only applicable if the user has set the Memory-Deny-Write-Execute
814 * (MDWE) protection mask for the current process.
815 *
816 * @old specifies the VMA flags the VMA originally possessed, and @new the ones
817 * we propose to set.
818 *
819 * Return: false if proposed change is OK, true if not ok and should be denied.
820 */
map_deny_write_exec(const vma_flags_t * old,const vma_flags_t * new)821 static inline bool map_deny_write_exec(const vma_flags_t *old,
822 const vma_flags_t *new)
823 {
824 /* If MDWE is disabled, we have nothing to deny. */
825 if (!mm_flags_test(MMF_HAS_MDWE, current->mm))
826 return false;
827
828 /* If the new VMA is not executable, we have nothing to deny. */
829 if (!vma_flags_test(new, VMA_EXEC_BIT))
830 return false;
831
832 /* Under MDWE we do not accept newly writably executable VMAs... */
833 if (vma_flags_test(new, VMA_WRITE_BIT))
834 return true;
835
836 /* ...nor previously non-executable VMAs becoming executable. */
837 if (!vma_flags_test(old, VMA_EXEC_BIT))
838 return true;
839
840 return false;
841 }
842 #endif
843
844 struct vm_area_struct *__install_special_mapping(struct mm_struct *mm,
845 unsigned long addr, unsigned long len,
846 vm_flags_t vm_flags, void *priv,
847 const struct vm_operations_struct *ops);
848
849 #endif /* __MM_VMA_H */
850