xref: /linux/mm/mmap.c (revision 0b32d436c015d5a88b3368405e3d8fe82f195a54)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * mm/mmap.c
4  *
5  * Written by obz.
6  *
7  * Address space accounting code	<alan@lxorguk.ukuu.org.uk>
8  */
9 
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11 
12 #include <linux/kernel.h>
13 #include <linux/slab.h>
14 #include <linux/backing-dev.h>
15 #include <linux/mm.h>
16 #include <linux/mm_inline.h>
17 #include <linux/shm.h>
18 #include <linux/mman.h>
19 #include <linux/pagemap.h>
20 #include <linux/swap.h>
21 #include <linux/syscalls.h>
22 #include <linux/capability.h>
23 #include <linux/init.h>
24 #include <linux/file.h>
25 #include <linux/fs.h>
26 #include <linux/personality.h>
27 #include <linux/security.h>
28 #include <linux/hugetlb.h>
29 #include <linux/shmem_fs.h>
30 #include <linux/profile.h>
31 #include <linux/export.h>
32 #include <linux/mount.h>
33 #include <linux/mempolicy.h>
34 #include <linux/rmap.h>
35 #include <linux/mmu_notifier.h>
36 #include <linux/mmdebug.h>
37 #include <linux/perf_event.h>
38 #include <linux/audit.h>
39 #include <linux/khugepaged.h>
40 #include <linux/uprobes.h>
41 #include <linux/notifier.h>
42 #include <linux/memory.h>
43 #include <linux/printk.h>
44 #include <linux/userfaultfd_k.h>
45 #include <linux/moduleparam.h>
46 #include <linux/pkeys.h>
47 #include <linux/oom.h>
48 #include <linux/sched/mm.h>
49 #include <linux/ksm.h>
50 
51 #include <linux/uaccess.h>
52 #include <asm/cacheflush.h>
53 #include <asm/tlb.h>
54 #include <asm/mmu_context.h>
55 
56 #define CREATE_TRACE_POINTS
57 #include <trace/events/mmap.h>
58 
59 #include "internal.h"
60 
61 #ifndef arch_mmap_check
62 #define arch_mmap_check(addr, len, flags)	(0)
63 #endif
64 
65 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
66 const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN;
67 int mmap_rnd_bits_max __ro_after_init = CONFIG_ARCH_MMAP_RND_BITS_MAX;
68 int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS;
69 #endif
70 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
71 const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN;
72 const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX;
73 int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS;
74 #endif
75 
76 static bool ignore_rlimit_data;
77 core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
78 
79 static void unmap_region(struct mm_struct *mm, struct ma_state *mas,
80 		struct vm_area_struct *vma, struct vm_area_struct *prev,
81 		struct vm_area_struct *next, unsigned long start,
82 		unsigned long end, unsigned long tree_end, bool mm_wr_locked);
83 
vm_pgprot_modify(pgprot_t oldprot,unsigned long vm_flags)84 static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
85 {
86 	return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
87 }
88 
89 /* Update vma->vm_page_prot to reflect vma->vm_flags. */
vma_set_page_prot(struct vm_area_struct * vma)90 void vma_set_page_prot(struct vm_area_struct *vma)
91 {
92 	unsigned long vm_flags = vma->vm_flags;
93 	pgprot_t vm_page_prot;
94 
95 	vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
96 	if (vma_wants_writenotify(vma, vm_page_prot)) {
97 		vm_flags &= ~VM_SHARED;
98 		vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags);
99 	}
100 	/* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */
101 	WRITE_ONCE(vma->vm_page_prot, vm_page_prot);
102 }
103 
104 /*
105  * Requires inode->i_mapping->i_mmap_rwsem
106  */
__remove_shared_vm_struct(struct vm_area_struct * vma,struct address_space * mapping)107 static void __remove_shared_vm_struct(struct vm_area_struct *vma,
108 				      struct address_space *mapping)
109 {
110 	if (vma_is_shared_maywrite(vma))
111 		mapping_unmap_writable(mapping);
112 
113 	flush_dcache_mmap_lock(mapping);
114 	vma_interval_tree_remove(vma, &mapping->i_mmap);
115 	flush_dcache_mmap_unlock(mapping);
116 }
117 
118 /*
119  * Unlink a file-based vm structure from its interval tree, to hide
120  * vma from rmap and vmtruncate before freeing its page tables.
121  */
unlink_file_vma(struct vm_area_struct * vma)122 void unlink_file_vma(struct vm_area_struct *vma)
123 {
124 	struct file *file = vma->vm_file;
125 
126 	if (file) {
127 		struct address_space *mapping = file->f_mapping;
128 		i_mmap_lock_write(mapping);
129 		__remove_shared_vm_struct(vma, mapping);
130 		i_mmap_unlock_write(mapping);
131 	}
132 }
133 
134 /*
135  * Close a vm structure and free it.
136  */
remove_vma(struct vm_area_struct * vma,bool unreachable)137 static void remove_vma(struct vm_area_struct *vma, bool unreachable)
138 {
139 	might_sleep();
140 	if (vma->vm_ops && vma->vm_ops->close)
141 		vma->vm_ops->close(vma);
142 	if (vma->vm_file)
143 		fput(vma->vm_file);
144 	mpol_put(vma_policy(vma));
145 	if (unreachable)
146 		__vm_area_free(vma);
147 	else
148 		vm_area_free(vma);
149 }
150 
vma_prev_limit(struct vma_iterator * vmi,unsigned long min)151 static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi,
152 						    unsigned long min)
153 {
154 	return mas_prev(&vmi->mas, min);
155 }
156 
157 /*
158  * check_brk_limits() - Use platform specific check of range & verify mlock
159  * limits.
160  * @addr: The address to check
161  * @len: The size of increase.
162  *
163  * Return: 0 on success.
164  */
check_brk_limits(unsigned long addr,unsigned long len)165 static int check_brk_limits(unsigned long addr, unsigned long len)
166 {
167 	unsigned long mapped_addr;
168 
169 	mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
170 	if (IS_ERR_VALUE(mapped_addr))
171 		return mapped_addr;
172 
173 	return mlock_future_ok(current->mm, current->mm->def_flags, len)
174 		? 0 : -EAGAIN;
175 }
176 static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma,
177 		unsigned long addr, unsigned long request, unsigned long flags);
SYSCALL_DEFINE1(brk,unsigned long,brk)178 SYSCALL_DEFINE1(brk, unsigned long, brk)
179 {
180 	unsigned long newbrk, oldbrk, origbrk;
181 	struct mm_struct *mm = current->mm;
182 	struct vm_area_struct *brkvma, *next = NULL;
183 	unsigned long min_brk;
184 	bool populate = false;
185 	LIST_HEAD(uf);
186 	struct vma_iterator vmi;
187 
188 	if (mmap_write_lock_killable(mm))
189 		return -EINTR;
190 
191 	origbrk = mm->brk;
192 
193 #ifdef CONFIG_COMPAT_BRK
194 	/*
195 	 * CONFIG_COMPAT_BRK can still be overridden by setting
196 	 * randomize_va_space to 2, which will still cause mm->start_brk
197 	 * to be arbitrarily shifted
198 	 */
199 	if (current->brk_randomized)
200 		min_brk = mm->start_brk;
201 	else
202 		min_brk = mm->end_data;
203 #else
204 	min_brk = mm->start_brk;
205 #endif
206 	if (brk < min_brk)
207 		goto out;
208 
209 	/*
210 	 * Check against rlimit here. If this check is done later after the test
211 	 * of oldbrk with newbrk then it can escape the test and let the data
212 	 * segment grow beyond its set limit the in case where the limit is
213 	 * not page aligned -Ram Gupta
214 	 */
215 	if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
216 			      mm->end_data, mm->start_data))
217 		goto out;
218 
219 	newbrk = PAGE_ALIGN(brk);
220 	oldbrk = PAGE_ALIGN(mm->brk);
221 	if (oldbrk == newbrk) {
222 		mm->brk = brk;
223 		goto success;
224 	}
225 
226 	/* Always allow shrinking brk. */
227 	if (brk <= mm->brk) {
228 		/* Search one past newbrk */
229 		vma_iter_init(&vmi, mm, newbrk);
230 		brkvma = vma_find(&vmi, oldbrk);
231 		if (!brkvma || brkvma->vm_start >= oldbrk)
232 			goto out; /* mapping intersects with an existing non-brk vma. */
233 		/*
234 		 * mm->brk must be protected by write mmap_lock.
235 		 * do_vma_munmap() will drop the lock on success,  so update it
236 		 * before calling do_vma_munmap().
237 		 */
238 		mm->brk = brk;
239 		if (do_vma_munmap(&vmi, brkvma, newbrk, oldbrk, &uf, true))
240 			goto out;
241 
242 		goto success_unlocked;
243 	}
244 
245 	if (check_brk_limits(oldbrk, newbrk - oldbrk))
246 		goto out;
247 
248 	/*
249 	 * Only check if the next VMA is within the stack_guard_gap of the
250 	 * expansion area
251 	 */
252 	vma_iter_init(&vmi, mm, oldbrk);
253 	next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap);
254 	if (next && newbrk + PAGE_SIZE > vm_start_gap(next))
255 		goto out;
256 
257 	brkvma = vma_prev_limit(&vmi, mm->start_brk);
258 	/* Ok, looks good - let it rip. */
259 	if (do_brk_flags(&vmi, brkvma, oldbrk, newbrk - oldbrk, 0) < 0)
260 		goto out;
261 
262 	mm->brk = brk;
263 	if (mm->def_flags & VM_LOCKED)
264 		populate = true;
265 
266 success:
267 	mmap_write_unlock(mm);
268 success_unlocked:
269 	userfaultfd_unmap_complete(mm, &uf);
270 	if (populate)
271 		mm_populate(oldbrk, newbrk - oldbrk);
272 	return brk;
273 
274 out:
275 	mm->brk = origbrk;
276 	mmap_write_unlock(mm);
277 	return origbrk;
278 }
279 
280 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
validate_mm(struct mm_struct * mm)281 static void validate_mm(struct mm_struct *mm)
282 {
283 	int bug = 0;
284 	int i = 0;
285 	struct vm_area_struct *vma;
286 	VMA_ITERATOR(vmi, mm, 0);
287 
288 	mt_validate(&mm->mm_mt);
289 	for_each_vma(vmi, vma) {
290 #ifdef CONFIG_DEBUG_VM_RB
291 		struct anon_vma *anon_vma = vma->anon_vma;
292 		struct anon_vma_chain *avc;
293 #endif
294 		unsigned long vmi_start, vmi_end;
295 		bool warn = 0;
296 
297 		vmi_start = vma_iter_addr(&vmi);
298 		vmi_end = vma_iter_end(&vmi);
299 		if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm))
300 			warn = 1;
301 
302 		if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm))
303 			warn = 1;
304 
305 		if (warn) {
306 			pr_emerg("issue in %s\n", current->comm);
307 			dump_stack();
308 			dump_vma(vma);
309 			pr_emerg("tree range: %px start %lx end %lx\n", vma,
310 				 vmi_start, vmi_end - 1);
311 			vma_iter_dump_tree(&vmi);
312 		}
313 
314 #ifdef CONFIG_DEBUG_VM_RB
315 		if (anon_vma) {
316 			anon_vma_lock_read(anon_vma);
317 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
318 				anon_vma_interval_tree_verify(avc);
319 			anon_vma_unlock_read(anon_vma);
320 		}
321 #endif
322 		i++;
323 	}
324 	if (i != mm->map_count) {
325 		pr_emerg("map_count %d vma iterator %d\n", mm->map_count, i);
326 		bug = 1;
327 	}
328 	VM_BUG_ON_MM(bug, mm);
329 }
330 
331 #else /* !CONFIG_DEBUG_VM_MAPLE_TREE */
332 #define validate_mm(mm) do { } while (0)
333 #endif /* CONFIG_DEBUG_VM_MAPLE_TREE */
334 
335 /*
336  * vma has some anon_vma assigned, and is already inserted on that
337  * anon_vma's interval trees.
338  *
339  * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
340  * vma must be removed from the anon_vma's interval trees using
341  * anon_vma_interval_tree_pre_update_vma().
342  *
343  * After the update, the vma will be reinserted using
344  * anon_vma_interval_tree_post_update_vma().
345  *
346  * The entire update must be protected by exclusive mmap_lock and by
347  * the root anon_vma's mutex.
348  */
349 static inline void
anon_vma_interval_tree_pre_update_vma(struct vm_area_struct * vma)350 anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
351 {
352 	struct anon_vma_chain *avc;
353 
354 	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
355 		anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
356 }
357 
358 static inline void
anon_vma_interval_tree_post_update_vma(struct vm_area_struct * vma)359 anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
360 {
361 	struct anon_vma_chain *avc;
362 
363 	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
364 		anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
365 }
366 
count_vma_pages_range(struct mm_struct * mm,unsigned long addr,unsigned long end)367 static unsigned long count_vma_pages_range(struct mm_struct *mm,
368 		unsigned long addr, unsigned long end)
369 {
370 	VMA_ITERATOR(vmi, mm, addr);
371 	struct vm_area_struct *vma;
372 	unsigned long nr_pages = 0;
373 
374 	for_each_vma_range(vmi, vma, end) {
375 		unsigned long vm_start = max(addr, vma->vm_start);
376 		unsigned long vm_end = min(end, vma->vm_end);
377 
378 		nr_pages += PHYS_PFN(vm_end - vm_start);
379 	}
380 
381 	return nr_pages;
382 }
383 
__vma_link_file(struct vm_area_struct * vma,struct address_space * mapping)384 static void __vma_link_file(struct vm_area_struct *vma,
385 			    struct address_space *mapping)
386 {
387 	if (vma_is_shared_maywrite(vma))
388 		mapping_allow_writable(mapping);
389 
390 	flush_dcache_mmap_lock(mapping);
391 	vma_interval_tree_insert(vma, &mapping->i_mmap);
392 	flush_dcache_mmap_unlock(mapping);
393 }
394 
vma_link_file(struct vm_area_struct * vma)395 static void vma_link_file(struct vm_area_struct *vma)
396 {
397 	struct file *file = vma->vm_file;
398 	struct address_space *mapping;
399 
400 	if (file) {
401 		mapping = file->f_mapping;
402 		i_mmap_lock_write(mapping);
403 		__vma_link_file(vma, mapping);
404 		i_mmap_unlock_write(mapping);
405 	}
406 }
407 
vma_link(struct mm_struct * mm,struct vm_area_struct * vma)408 static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma)
409 {
410 	VMA_ITERATOR(vmi, mm, 0);
411 
412 	vma_iter_config(&vmi, vma->vm_start, vma->vm_end);
413 	if (vma_iter_prealloc(&vmi, vma))
414 		return -ENOMEM;
415 
416 	vma_start_write(vma);
417 	vma_iter_store(&vmi, vma);
418 	vma_link_file(vma);
419 	mm->map_count++;
420 	validate_mm(mm);
421 	return 0;
422 }
423 
424 /*
425  * init_multi_vma_prep() - Initializer for struct vma_prepare
426  * @vp: The vma_prepare struct
427  * @vma: The vma that will be altered once locked
428  * @next: The next vma if it is to be adjusted
429  * @remove: The first vma to be removed
430  * @remove2: The second vma to be removed
431  */
init_multi_vma_prep(struct vma_prepare * vp,struct vm_area_struct * vma,struct vm_area_struct * next,struct vm_area_struct * remove,struct vm_area_struct * remove2)432 static inline void init_multi_vma_prep(struct vma_prepare *vp,
433 		struct vm_area_struct *vma, struct vm_area_struct *next,
434 		struct vm_area_struct *remove, struct vm_area_struct *remove2)
435 {
436 	memset(vp, 0, sizeof(struct vma_prepare));
437 	vp->vma = vma;
438 	vp->anon_vma = vma->anon_vma;
439 	vp->remove = remove;
440 	vp->remove2 = remove2;
441 	vp->adj_next = next;
442 	if (!vp->anon_vma && next)
443 		vp->anon_vma = next->anon_vma;
444 
445 	vp->file = vma->vm_file;
446 	if (vp->file)
447 		vp->mapping = vma->vm_file->f_mapping;
448 
449 }
450 
451 /*
452  * init_vma_prep() - Initializer wrapper for vma_prepare struct
453  * @vp: The vma_prepare struct
454  * @vma: The vma that will be altered once locked
455  */
init_vma_prep(struct vma_prepare * vp,struct vm_area_struct * vma)456 static inline void init_vma_prep(struct vma_prepare *vp,
457 				 struct vm_area_struct *vma)
458 {
459 	init_multi_vma_prep(vp, vma, NULL, NULL, NULL);
460 }
461 
462 
463 /*
464  * vma_prepare() - Helper function for handling locking VMAs prior to altering
465  * @vp: The initialized vma_prepare struct
466  */
vma_prepare(struct vma_prepare * vp)467 static inline void vma_prepare(struct vma_prepare *vp)
468 {
469 	if (vp->file) {
470 		uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end);
471 
472 		if (vp->adj_next)
473 			uprobe_munmap(vp->adj_next, vp->adj_next->vm_start,
474 				      vp->adj_next->vm_end);
475 
476 		i_mmap_lock_write(vp->mapping);
477 		if (vp->insert && vp->insert->vm_file) {
478 			/*
479 			 * Put into interval tree now, so instantiated pages
480 			 * are visible to arm/parisc __flush_dcache_page
481 			 * throughout; but we cannot insert into address
482 			 * space until vma start or end is updated.
483 			 */
484 			__vma_link_file(vp->insert,
485 					vp->insert->vm_file->f_mapping);
486 		}
487 	}
488 
489 	if (vp->anon_vma) {
490 		anon_vma_lock_write(vp->anon_vma);
491 		anon_vma_interval_tree_pre_update_vma(vp->vma);
492 		if (vp->adj_next)
493 			anon_vma_interval_tree_pre_update_vma(vp->adj_next);
494 	}
495 
496 	if (vp->file) {
497 		flush_dcache_mmap_lock(vp->mapping);
498 		vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap);
499 		if (vp->adj_next)
500 			vma_interval_tree_remove(vp->adj_next,
501 						 &vp->mapping->i_mmap);
502 	}
503 
504 }
505 
506 /*
507  * vma_complete- Helper function for handling the unlocking after altering VMAs,
508  * or for inserting a VMA.
509  *
510  * @vp: The vma_prepare struct
511  * @vmi: The vma iterator
512  * @mm: The mm_struct
513  */
vma_complete(struct vma_prepare * vp,struct vma_iterator * vmi,struct mm_struct * mm)514 static inline void vma_complete(struct vma_prepare *vp,
515 				struct vma_iterator *vmi, struct mm_struct *mm)
516 {
517 	if (vp->file) {
518 		if (vp->adj_next)
519 			vma_interval_tree_insert(vp->adj_next,
520 						 &vp->mapping->i_mmap);
521 		vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap);
522 		flush_dcache_mmap_unlock(vp->mapping);
523 	}
524 
525 	if (vp->remove && vp->file) {
526 		__remove_shared_vm_struct(vp->remove, vp->mapping);
527 		if (vp->remove2)
528 			__remove_shared_vm_struct(vp->remove2, vp->mapping);
529 	} else if (vp->insert) {
530 		/*
531 		 * split_vma has split insert from vma, and needs
532 		 * us to insert it before dropping the locks
533 		 * (it may either follow vma or precede it).
534 		 */
535 		vma_iter_store(vmi, vp->insert);
536 		mm->map_count++;
537 	}
538 
539 	if (vp->anon_vma) {
540 		anon_vma_interval_tree_post_update_vma(vp->vma);
541 		if (vp->adj_next)
542 			anon_vma_interval_tree_post_update_vma(vp->adj_next);
543 		anon_vma_unlock_write(vp->anon_vma);
544 	}
545 
546 	if (vp->file) {
547 		i_mmap_unlock_write(vp->mapping);
548 		uprobe_mmap(vp->vma);
549 
550 		if (vp->adj_next)
551 			uprobe_mmap(vp->adj_next);
552 	}
553 
554 	if (vp->remove) {
555 again:
556 		vma_mark_detached(vp->remove, true);
557 		if (vp->file) {
558 			uprobe_munmap(vp->remove, vp->remove->vm_start,
559 				      vp->remove->vm_end);
560 			fput(vp->file);
561 		}
562 		if (vp->remove->anon_vma)
563 			anon_vma_merge(vp->vma, vp->remove);
564 		mm->map_count--;
565 		mpol_put(vma_policy(vp->remove));
566 		if (!vp->remove2)
567 			WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end);
568 		vm_area_free(vp->remove);
569 
570 		/*
571 		 * In mprotect's case 6 (see comments on vma_merge),
572 		 * we are removing both mid and next vmas
573 		 */
574 		if (vp->remove2) {
575 			vp->remove = vp->remove2;
576 			vp->remove2 = NULL;
577 			goto again;
578 		}
579 	}
580 	if (vp->insert && vp->file)
581 		uprobe_mmap(vp->insert);
582 	validate_mm(mm);
583 }
584 
585 /*
586  * dup_anon_vma() - Helper function to duplicate anon_vma
587  * @dst: The destination VMA
588  * @src: The source VMA
589  * @dup: Pointer to the destination VMA when successful.
590  *
591  * Returns: 0 on success.
592  */
dup_anon_vma(struct vm_area_struct * dst,struct vm_area_struct * src,struct vm_area_struct ** dup)593 static inline int dup_anon_vma(struct vm_area_struct *dst,
594 		struct vm_area_struct *src, struct vm_area_struct **dup)
595 {
596 	/*
597 	 * Easily overlooked: when mprotect shifts the boundary, make sure the
598 	 * expanding vma has anon_vma set if the shrinking vma had, to cover any
599 	 * anon pages imported.
600 	 */
601 	if (src->anon_vma && !dst->anon_vma) {
602 		int ret;
603 
604 		vma_assert_write_locked(dst);
605 		dst->anon_vma = src->anon_vma;
606 		ret = anon_vma_clone(dst, src);
607 		if (ret)
608 			return ret;
609 
610 		*dup = dst;
611 	}
612 
613 	return 0;
614 }
615 
616 /*
617  * vma_expand - Expand an existing VMA
618  *
619  * @vmi: The vma iterator
620  * @vma: The vma to expand
621  * @start: The start of the vma
622  * @end: The exclusive end of the vma
623  * @pgoff: The page offset of vma
624  * @next: The current of next vma.
625  *
626  * Expand @vma to @start and @end.  Can expand off the start and end.  Will
627  * expand over @next if it's different from @vma and @end == @next->vm_end.
628  * Checking if the @vma can expand and merge with @next needs to be handled by
629  * the caller.
630  *
631  * Returns: 0 on success
632  */
vma_expand(struct vma_iterator * vmi,struct vm_area_struct * vma,unsigned long start,unsigned long end,pgoff_t pgoff,struct vm_area_struct * next)633 int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
634 	       unsigned long start, unsigned long end, pgoff_t pgoff,
635 	       struct vm_area_struct *next)
636 {
637 	struct vm_area_struct *anon_dup = NULL;
638 	bool remove_next = false;
639 	struct vma_prepare vp;
640 
641 	vma_start_write(vma);
642 	if (next && (vma != next) && (end == next->vm_end)) {
643 		int ret;
644 
645 		remove_next = true;
646 		vma_start_write(next);
647 		ret = dup_anon_vma(vma, next, &anon_dup);
648 		if (ret)
649 			return ret;
650 	}
651 
652 	init_multi_vma_prep(&vp, vma, NULL, remove_next ? next : NULL, NULL);
653 	/* Not merging but overwriting any part of next is not handled. */
654 	VM_WARN_ON(next && !vp.remove &&
655 		  next != vma && end > next->vm_start);
656 	/* Only handles expanding */
657 	VM_WARN_ON(vma->vm_start < start || vma->vm_end > end);
658 
659 	/* Note: vma iterator must be pointing to 'start' */
660 	vma_iter_config(vmi, start, end);
661 	if (vma_iter_prealloc(vmi, vma))
662 		goto nomem;
663 
664 	vma_prepare(&vp);
665 	vma_adjust_trans_huge(vma, start, end, 0);
666 	vma_set_range(vma, start, end, pgoff);
667 	vma_iter_store(vmi, vma);
668 
669 	vma_complete(&vp, vmi, vma->vm_mm);
670 	return 0;
671 
672 nomem:
673 	if (anon_dup)
674 		unlink_anon_vmas(anon_dup);
675 	return -ENOMEM;
676 }
677 
678 /*
679  * vma_shrink() - Reduce an existing VMAs memory area
680  * @vmi: The vma iterator
681  * @vma: The VMA to modify
682  * @start: The new start
683  * @end: The new end
684  *
685  * Returns: 0 on success, -ENOMEM otherwise
686  */
vma_shrink(struct vma_iterator * vmi,struct vm_area_struct * vma,unsigned long start,unsigned long end,pgoff_t pgoff)687 int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
688 	       unsigned long start, unsigned long end, pgoff_t pgoff)
689 {
690 	struct vma_prepare vp;
691 
692 	WARN_ON((vma->vm_start != start) && (vma->vm_end != end));
693 
694 	if (vma->vm_start < start)
695 		vma_iter_config(vmi, vma->vm_start, start);
696 	else
697 		vma_iter_config(vmi, end, vma->vm_end);
698 
699 	if (vma_iter_prealloc(vmi, NULL))
700 		return -ENOMEM;
701 
702 	vma_start_write(vma);
703 
704 	init_vma_prep(&vp, vma);
705 	vma_prepare(&vp);
706 	vma_adjust_trans_huge(vma, start, end, 0);
707 
708 	vma_iter_clear(vmi);
709 	vma_set_range(vma, start, end, pgoff);
710 	vma_complete(&vp, vmi, vma->vm_mm);
711 	return 0;
712 }
713 
714 /*
715  * If the vma has a ->close operation then the driver probably needs to release
716  * per-vma resources, so we don't attempt to merge those if the caller indicates
717  * the current vma may be removed as part of the merge.
718  */
is_mergeable_vma(struct vm_area_struct * vma,struct file * file,unsigned long vm_flags,struct vm_userfaultfd_ctx vm_userfaultfd_ctx,struct anon_vma_name * anon_name,bool may_remove_vma)719 static inline bool is_mergeable_vma(struct vm_area_struct *vma,
720 		struct file *file, unsigned long vm_flags,
721 		struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
722 		struct anon_vma_name *anon_name, bool may_remove_vma)
723 {
724 	/*
725 	 * VM_SOFTDIRTY should not prevent from VMA merging, if we
726 	 * match the flags but dirty bit -- the caller should mark
727 	 * merged VMA as dirty. If dirty bit won't be excluded from
728 	 * comparison, we increase pressure on the memory system forcing
729 	 * the kernel to generate new VMAs when old one could be
730 	 * extended instead.
731 	 */
732 	if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
733 		return false;
734 	if (vma->vm_file != file)
735 		return false;
736 	if (may_remove_vma && vma->vm_ops && vma->vm_ops->close)
737 		return false;
738 	if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
739 		return false;
740 	if (!anon_vma_name_eq(anon_vma_name(vma), anon_name))
741 		return false;
742 	return true;
743 }
744 
is_mergeable_anon_vma(struct anon_vma * anon_vma1,struct anon_vma * anon_vma2,struct vm_area_struct * vma)745 static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1,
746 		 struct anon_vma *anon_vma2, struct vm_area_struct *vma)
747 {
748 	/*
749 	 * The list_is_singular() test is to avoid merging VMA cloned from
750 	 * parents. This can improve scalability caused by anon_vma lock.
751 	 */
752 	if ((!anon_vma1 || !anon_vma2) && (!vma ||
753 		list_is_singular(&vma->anon_vma_chain)))
754 		return true;
755 	return anon_vma1 == anon_vma2;
756 }
757 
758 /*
759  * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
760  * in front of (at a lower virtual address and file offset than) the vma.
761  *
762  * We cannot merge two vmas if they have differently assigned (non-NULL)
763  * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
764  *
765  * We don't check here for the merged mmap wrapping around the end of pagecache
766  * indices (16TB on ia32) because do_mmap() does not permit mmap's which
767  * wrap, nor mmaps which cover the final page at index -1UL.
768  *
769  * We assume the vma may be removed as part of the merge.
770  */
771 static bool
can_vma_merge_before(struct vm_area_struct * vma,unsigned long vm_flags,struct anon_vma * anon_vma,struct file * file,pgoff_t vm_pgoff,struct vm_userfaultfd_ctx vm_userfaultfd_ctx,struct anon_vma_name * anon_name)772 can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
773 		struct anon_vma *anon_vma, struct file *file,
774 		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
775 		struct anon_vma_name *anon_name)
776 {
777 	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) &&
778 	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
779 		if (vma->vm_pgoff == vm_pgoff)
780 			return true;
781 	}
782 	return false;
783 }
784 
785 /*
786  * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
787  * beyond (at a higher virtual address and file offset than) the vma.
788  *
789  * We cannot merge two vmas if they have differently assigned (non-NULL)
790  * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
791  *
792  * We assume that vma is not removed as part of the merge.
793  */
794 static bool
can_vma_merge_after(struct vm_area_struct * vma,unsigned long vm_flags,struct anon_vma * anon_vma,struct file * file,pgoff_t vm_pgoff,struct vm_userfaultfd_ctx vm_userfaultfd_ctx,struct anon_vma_name * anon_name)795 can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
796 		struct anon_vma *anon_vma, struct file *file,
797 		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
798 		struct anon_vma_name *anon_name)
799 {
800 	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) &&
801 	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
802 		pgoff_t vm_pglen;
803 		vm_pglen = vma_pages(vma);
804 		if (vma->vm_pgoff + vm_pglen == vm_pgoff)
805 			return true;
806 	}
807 	return false;
808 }
809 
810 /*
811  * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name),
812  * figure out whether that can be merged with its predecessor or its
813  * successor.  Or both (it neatly fills a hole).
814  *
815  * In most cases - when called for mmap, brk or mremap - [addr,end) is
816  * certain not to be mapped by the time vma_merge is called; but when
817  * called for mprotect, it is certain to be already mapped (either at
818  * an offset within prev, or at the start of next), and the flags of
819  * this area are about to be changed to vm_flags - and the no-change
820  * case has already been eliminated.
821  *
822  * The following mprotect cases have to be considered, where **** is
823  * the area passed down from mprotect_fixup, never extending beyond one
824  * vma, PPPP is the previous vma, CCCC is a concurrent vma that starts
825  * at the same address as **** and is of the same or larger span, and
826  * NNNN the next vma after ****:
827  *
828  *     ****             ****                   ****
829  *    PPPPPPNNNNNN    PPPPPPNNNNNN       PPPPPPCCCCCC
830  *    cannot merge    might become       might become
831  *                    PPNNNNNNNNNN       PPPPPPPPPPCC
832  *    mmap, brk or    case 4 below       case 5 below
833  *    mremap move:
834  *                        ****               ****
835  *                    PPPP    NNNN       PPPPCCCCNNNN
836  *                    might become       might become
837  *                    PPPPPPPPPPPP 1 or  PPPPPPPPPPPP 6 or
838  *                    PPPPPPPPNNNN 2 or  PPPPPPPPNNNN 7 or
839  *                    PPPPNNNNNNNN 3     PPPPNNNNNNNN 8
840  *
841  * It is important for case 8 that the vma CCCC overlapping the
842  * region **** is never going to extended over NNNN. Instead NNNN must
843  * be extended in region **** and CCCC must be removed. This way in
844  * all cases where vma_merge succeeds, the moment vma_merge drops the
845  * rmap_locks, the properties of the merged vma will be already
846  * correct for the whole merged range. Some of those properties like
847  * vm_page_prot/vm_flags may be accessed by rmap_walks and they must
848  * be correct for the whole merged range immediately after the
849  * rmap_locks are released. Otherwise if NNNN would be removed and
850  * CCCC would be extended over the NNNN range, remove_migration_ptes
851  * or other rmap walkers (if working on addresses beyond the "end"
852  * parameter) may establish ptes with the wrong permissions of CCCC
853  * instead of the right permissions of NNNN.
854  *
855  * In the code below:
856  * PPPP is represented by *prev
857  * CCCC is represented by *curr or not represented at all (NULL)
858  * NNNN is represented by *next or not represented at all (NULL)
859  * **** is not represented - it will be merged and the vma containing the
860  *      area is returned, or the function will return NULL
861  */
862 static struct vm_area_struct
vma_merge(struct vma_iterator * vmi,struct vm_area_struct * prev,struct vm_area_struct * src,unsigned long addr,unsigned long end,unsigned long vm_flags,pgoff_t pgoff,struct mempolicy * policy,struct vm_userfaultfd_ctx vm_userfaultfd_ctx,struct anon_vma_name * anon_name)863 *vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev,
864 	   struct vm_area_struct *src, unsigned long addr, unsigned long end,
865 	   unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy,
866 	   struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
867 	   struct anon_vma_name *anon_name)
868 {
869 	struct mm_struct *mm = src->vm_mm;
870 	struct anon_vma *anon_vma = src->anon_vma;
871 	struct file *file = src->vm_file;
872 	struct vm_area_struct *curr, *next, *res;
873 	struct vm_area_struct *vma, *adjust, *remove, *remove2;
874 	struct vm_area_struct *anon_dup = NULL;
875 	struct vma_prepare vp;
876 	pgoff_t vma_pgoff;
877 	int err = 0;
878 	bool merge_prev = false;
879 	bool merge_next = false;
880 	bool vma_expanded = false;
881 	unsigned long vma_start = addr;
882 	unsigned long vma_end = end;
883 	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
884 	long adj_start = 0;
885 
886 	/*
887 	 * We later require that vma->vm_flags == vm_flags,
888 	 * so this tests vma->vm_flags & VM_SPECIAL, too.
889 	 */
890 	if (vm_flags & VM_SPECIAL)
891 		return NULL;
892 
893 	/* Does the input range span an existing VMA? (cases 5 - 8) */
894 	curr = find_vma_intersection(mm, prev ? prev->vm_end : 0, end);
895 
896 	if (!curr ||			/* cases 1 - 4 */
897 	    end == curr->vm_end)	/* cases 6 - 8, adjacent VMA */
898 		next = vma_lookup(mm, end);
899 	else
900 		next = NULL;		/* case 5 */
901 
902 	if (prev) {
903 		vma_start = prev->vm_start;
904 		vma_pgoff = prev->vm_pgoff;
905 
906 		/* Can we merge the predecessor? */
907 		if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy)
908 		    && can_vma_merge_after(prev, vm_flags, anon_vma, file,
909 					   pgoff, vm_userfaultfd_ctx, anon_name)) {
910 			merge_prev = true;
911 			vma_prev(vmi);
912 		}
913 	}
914 
915 	/* Can we merge the successor? */
916 	if (next && mpol_equal(policy, vma_policy(next)) &&
917 	    can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen,
918 				 vm_userfaultfd_ctx, anon_name)) {
919 		merge_next = true;
920 	}
921 
922 	/* Verify some invariant that must be enforced by the caller. */
923 	VM_WARN_ON(prev && addr <= prev->vm_start);
924 	VM_WARN_ON(curr && (addr != curr->vm_start || end > curr->vm_end));
925 	VM_WARN_ON(addr >= end);
926 
927 	if (!merge_prev && !merge_next)
928 		return NULL; /* Not mergeable. */
929 
930 	if (merge_prev)
931 		vma_start_write(prev);
932 
933 	res = vma = prev;
934 	remove = remove2 = adjust = NULL;
935 
936 	/* Can we merge both the predecessor and the successor? */
937 	if (merge_prev && merge_next &&
938 	    is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) {
939 		vma_start_write(next);
940 		remove = next;				/* case 1 */
941 		vma_end = next->vm_end;
942 		err = dup_anon_vma(prev, next, &anon_dup);
943 		if (curr) {				/* case 6 */
944 			vma_start_write(curr);
945 			remove = curr;
946 			remove2 = next;
947 			/*
948 			 * Note that the dup_anon_vma below cannot overwrite err
949 			 * since the first caller would do nothing unless next
950 			 * has an anon_vma.
951 			 */
952 			if (!next->anon_vma)
953 				err = dup_anon_vma(prev, curr, &anon_dup);
954 		}
955 	} else if (merge_prev) {			/* case 2 */
956 		if (curr) {
957 			vma_start_write(curr);
958 			if (end == curr->vm_end) {	/* case 7 */
959 				/*
960 				 * can_vma_merge_after() assumed we would not be
961 				 * removing prev vma, so it skipped the check
962 				 * for vm_ops->close, but we are removing curr
963 				 */
964 				if (curr->vm_ops && curr->vm_ops->close)
965 					err = -EINVAL;
966 				remove = curr;
967 			} else {			/* case 5 */
968 				adjust = curr;
969 				adj_start = (end - curr->vm_start);
970 			}
971 			if (!err)
972 				err = dup_anon_vma(prev, curr, &anon_dup);
973 		}
974 	} else { /* merge_next */
975 		vma_start_write(next);
976 		res = next;
977 		if (prev && addr < prev->vm_end) {	/* case 4 */
978 			vma_start_write(prev);
979 			vma_end = addr;
980 			adjust = next;
981 			adj_start = -(prev->vm_end - addr);
982 			err = dup_anon_vma(next, prev, &anon_dup);
983 		} else {
984 			/*
985 			 * Note that cases 3 and 8 are the ONLY ones where prev
986 			 * is permitted to be (but is not necessarily) NULL.
987 			 */
988 			vma = next;			/* case 3 */
989 			vma_start = addr;
990 			vma_end = next->vm_end;
991 			vma_pgoff = next->vm_pgoff - pglen;
992 			if (curr) {			/* case 8 */
993 				vma_pgoff = curr->vm_pgoff;
994 				vma_start_write(curr);
995 				remove = curr;
996 				err = dup_anon_vma(next, curr, &anon_dup);
997 			}
998 		}
999 	}
1000 
1001 	/* Error in anon_vma clone. */
1002 	if (err)
1003 		goto anon_vma_fail;
1004 
1005 	if (vma_start < vma->vm_start || vma_end > vma->vm_end)
1006 		vma_expanded = true;
1007 
1008 	if (vma_expanded) {
1009 		vma_iter_config(vmi, vma_start, vma_end);
1010 	} else {
1011 		vma_iter_config(vmi, adjust->vm_start + adj_start,
1012 				adjust->vm_end);
1013 	}
1014 
1015 	if (vma_iter_prealloc(vmi, vma))
1016 		goto prealloc_fail;
1017 
1018 	init_multi_vma_prep(&vp, vma, adjust, remove, remove2);
1019 	VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma &&
1020 		   vp.anon_vma != adjust->anon_vma);
1021 
1022 	vma_prepare(&vp);
1023 	vma_adjust_trans_huge(vma, vma_start, vma_end, adj_start);
1024 	vma_set_range(vma, vma_start, vma_end, vma_pgoff);
1025 
1026 	if (vma_expanded)
1027 		vma_iter_store(vmi, vma);
1028 
1029 	if (adj_start) {
1030 		adjust->vm_start += adj_start;
1031 		adjust->vm_pgoff += adj_start >> PAGE_SHIFT;
1032 		if (adj_start < 0) {
1033 			WARN_ON(vma_expanded);
1034 			vma_iter_store(vmi, next);
1035 		}
1036 	}
1037 
1038 	vma_complete(&vp, vmi, mm);
1039 	khugepaged_enter_vma(res, vm_flags);
1040 	return res;
1041 
1042 prealloc_fail:
1043 	if (anon_dup)
1044 		unlink_anon_vmas(anon_dup);
1045 
1046 anon_vma_fail:
1047 	vma_iter_set(vmi, addr);
1048 	vma_iter_load(vmi);
1049 	return NULL;
1050 }
1051 
1052 /*
1053  * Rough compatibility check to quickly see if it's even worth looking
1054  * at sharing an anon_vma.
1055  *
1056  * They need to have the same vm_file, and the flags can only differ
1057  * in things that mprotect may change.
1058  *
1059  * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
1060  * we can merge the two vma's. For example, we refuse to merge a vma if
1061  * there is a vm_ops->close() function, because that indicates that the
1062  * driver is doing some kind of reference counting. But that doesn't
1063  * really matter for the anon_vma sharing case.
1064  */
anon_vma_compatible(struct vm_area_struct * a,struct vm_area_struct * b)1065 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
1066 {
1067 	return a->vm_end == b->vm_start &&
1068 		mpol_equal(vma_policy(a), vma_policy(b)) &&
1069 		a->vm_file == b->vm_file &&
1070 		!((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) &&
1071 		b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
1072 }
1073 
1074 /*
1075  * Do some basic sanity checking to see if we can re-use the anon_vma
1076  * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
1077  * the same as 'old', the other will be the new one that is trying
1078  * to share the anon_vma.
1079  *
1080  * NOTE! This runs with mmap_lock held for reading, so it is possible that
1081  * the anon_vma of 'old' is concurrently in the process of being set up
1082  * by another page fault trying to merge _that_. But that's ok: if it
1083  * is being set up, that automatically means that it will be a singleton
1084  * acceptable for merging, so we can do all of this optimistically. But
1085  * we do that READ_ONCE() to make sure that we never re-load the pointer.
1086  *
1087  * IOW: that the "list_is_singular()" test on the anon_vma_chain only
1088  * matters for the 'stable anon_vma' case (ie the thing we want to avoid
1089  * is to return an anon_vma that is "complex" due to having gone through
1090  * a fork).
1091  *
1092  * We also make sure that the two vma's are compatible (adjacent,
1093  * and with the same memory policies). That's all stable, even with just
1094  * a read lock on the mmap_lock.
1095  */
reusable_anon_vma(struct vm_area_struct * old,struct vm_area_struct * a,struct vm_area_struct * b)1096 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
1097 {
1098 	if (anon_vma_compatible(a, b)) {
1099 		struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
1100 
1101 		if (anon_vma && list_is_singular(&old->anon_vma_chain))
1102 			return anon_vma;
1103 	}
1104 	return NULL;
1105 }
1106 
1107 /*
1108  * find_mergeable_anon_vma is used by anon_vma_prepare, to check
1109  * neighbouring vmas for a suitable anon_vma, before it goes off
1110  * to allocate a new anon_vma.  It checks because a repetitive
1111  * sequence of mprotects and faults may otherwise lead to distinct
1112  * anon_vmas being allocated, preventing vma merge in subsequent
1113  * mprotect.
1114  */
find_mergeable_anon_vma(struct vm_area_struct * vma)1115 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
1116 {
1117 	struct anon_vma *anon_vma = NULL;
1118 	struct vm_area_struct *prev, *next;
1119 	VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_end);
1120 
1121 	/* Try next first. */
1122 	next = vma_iter_load(&vmi);
1123 	if (next) {
1124 		anon_vma = reusable_anon_vma(next, vma, next);
1125 		if (anon_vma)
1126 			return anon_vma;
1127 	}
1128 
1129 	prev = vma_prev(&vmi);
1130 	VM_BUG_ON_VMA(prev != vma, vma);
1131 	prev = vma_prev(&vmi);
1132 	/* Try prev next. */
1133 	if (prev)
1134 		anon_vma = reusable_anon_vma(prev, prev, vma);
1135 
1136 	/*
1137 	 * We might reach here with anon_vma == NULL if we can't find
1138 	 * any reusable anon_vma.
1139 	 * There's no absolute need to look only at touching neighbours:
1140 	 * we could search further afield for "compatible" anon_vmas.
1141 	 * But it would probably just be a waste of time searching,
1142 	 * or lead to too many vmas hanging off the same anon_vma.
1143 	 * We're trying to allow mprotect remerging later on,
1144 	 * not trying to minimize memory used for anon_vmas.
1145 	 */
1146 	return anon_vma;
1147 }
1148 
1149 /*
1150  * If a hint addr is less than mmap_min_addr change hint to be as
1151  * low as possible but still greater than mmap_min_addr
1152  */
round_hint_to_min(unsigned long hint)1153 static inline unsigned long round_hint_to_min(unsigned long hint)
1154 {
1155 	hint &= PAGE_MASK;
1156 	if (((void *)hint != NULL) &&
1157 	    (hint < mmap_min_addr))
1158 		return PAGE_ALIGN(mmap_min_addr);
1159 	return hint;
1160 }
1161 
mlock_future_ok(struct mm_struct * mm,unsigned long flags,unsigned long bytes)1162 bool mlock_future_ok(struct mm_struct *mm, unsigned long flags,
1163 			unsigned long bytes)
1164 {
1165 	unsigned long locked_pages, limit_pages;
1166 
1167 	if (!(flags & VM_LOCKED) || capable(CAP_IPC_LOCK))
1168 		return true;
1169 
1170 	locked_pages = bytes >> PAGE_SHIFT;
1171 	locked_pages += mm->locked_vm;
1172 
1173 	limit_pages = rlimit(RLIMIT_MEMLOCK);
1174 	limit_pages >>= PAGE_SHIFT;
1175 
1176 	return locked_pages <= limit_pages;
1177 }
1178 
file_mmap_size_max(struct file * file,struct inode * inode)1179 static inline u64 file_mmap_size_max(struct file *file, struct inode *inode)
1180 {
1181 	if (S_ISREG(inode->i_mode))
1182 		return MAX_LFS_FILESIZE;
1183 
1184 	if (S_ISBLK(inode->i_mode))
1185 		return MAX_LFS_FILESIZE;
1186 
1187 	if (S_ISSOCK(inode->i_mode))
1188 		return MAX_LFS_FILESIZE;
1189 
1190 	/* Special "we do even unsigned file positions" case */
1191 	if (file->f_mode & FMODE_UNSIGNED_OFFSET)
1192 		return 0;
1193 
1194 	/* Yes, random drivers might want more. But I'm tired of buggy drivers */
1195 	return ULONG_MAX;
1196 }
1197 
file_mmap_ok(struct file * file,struct inode * inode,unsigned long pgoff,unsigned long len)1198 static inline bool file_mmap_ok(struct file *file, struct inode *inode,
1199 				unsigned long pgoff, unsigned long len)
1200 {
1201 	u64 maxsize = file_mmap_size_max(file, inode);
1202 
1203 	if (maxsize && len > maxsize)
1204 		return false;
1205 	maxsize -= len;
1206 	if (pgoff > maxsize >> PAGE_SHIFT)
1207 		return false;
1208 	return true;
1209 }
1210 
1211 /*
1212  * The caller must write-lock current->mm->mmap_lock.
1213  */
do_mmap(struct file * file,unsigned long addr,unsigned long len,unsigned long prot,unsigned long flags,vm_flags_t vm_flags,unsigned long pgoff,unsigned long * populate,struct list_head * uf)1214 unsigned long do_mmap(struct file *file, unsigned long addr,
1215 			unsigned long len, unsigned long prot,
1216 			unsigned long flags, vm_flags_t vm_flags,
1217 			unsigned long pgoff, unsigned long *populate,
1218 			struct list_head *uf)
1219 {
1220 	struct mm_struct *mm = current->mm;
1221 	int pkey = 0;
1222 
1223 	*populate = 0;
1224 
1225 	if (!len)
1226 		return -EINVAL;
1227 
1228 	/*
1229 	 * Does the application expect PROT_READ to imply PROT_EXEC?
1230 	 *
1231 	 * (the exception is when the underlying filesystem is noexec
1232 	 *  mounted, in which case we don't add PROT_EXEC.)
1233 	 */
1234 	if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
1235 		if (!(file && path_noexec(&file->f_path)))
1236 			prot |= PROT_EXEC;
1237 
1238 	/* force arch specific MAP_FIXED handling in get_unmapped_area */
1239 	if (flags & MAP_FIXED_NOREPLACE)
1240 		flags |= MAP_FIXED;
1241 
1242 	if (!(flags & MAP_FIXED))
1243 		addr = round_hint_to_min(addr);
1244 
1245 	/* Careful about overflows.. */
1246 	len = PAGE_ALIGN(len);
1247 	if (!len)
1248 		return -ENOMEM;
1249 
1250 	/* offset overflow? */
1251 	if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
1252 		return -EOVERFLOW;
1253 
1254 	/* Too many mappings? */
1255 	if (mm->map_count > sysctl_max_map_count)
1256 		return -ENOMEM;
1257 
1258 	/*
1259 	 * addr is returned from get_unmapped_area,
1260 	 * There are two cases:
1261 	 * 1> MAP_FIXED == false
1262 	 *	unallocated memory, no need to check sealing.
1263 	 * 1> MAP_FIXED == true
1264 	 *	sealing is checked inside mmap_region when
1265 	 *	do_vmi_munmap is called.
1266 	 */
1267 
1268 	if (prot == PROT_EXEC) {
1269 		pkey = execute_only_pkey(mm);
1270 		if (pkey < 0)
1271 			pkey = 0;
1272 	}
1273 
1274 	/* Do simple checking here so the lower-level routines won't have
1275 	 * to. we assume access permissions have been handled by the open
1276 	 * of the memory object, so we don't do any here.
1277 	 */
1278 	vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
1279 			mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
1280 
1281 	/* Obtain the address to map to. we verify (or select) it and ensure
1282 	 * that it represents a valid section of the address space.
1283 	 */
1284 	addr = __get_unmapped_area(file, addr, len, pgoff, flags, vm_flags);
1285 	if (IS_ERR_VALUE(addr))
1286 		return addr;
1287 
1288 	if (flags & MAP_FIXED_NOREPLACE) {
1289 		if (find_vma_intersection(mm, addr, addr + len))
1290 			return -EEXIST;
1291 	}
1292 
1293 	if (flags & MAP_LOCKED)
1294 		if (!can_do_mlock())
1295 			return -EPERM;
1296 
1297 	if (!mlock_future_ok(mm, vm_flags, len))
1298 		return -EAGAIN;
1299 
1300 	if (file) {
1301 		struct inode *inode = file_inode(file);
1302 		unsigned long flags_mask;
1303 
1304 		if (!file_mmap_ok(file, inode, pgoff, len))
1305 			return -EOVERFLOW;
1306 
1307 		flags_mask = LEGACY_MAP_MASK;
1308 		if (file->f_op->fop_flags & FOP_MMAP_SYNC)
1309 			flags_mask |= MAP_SYNC;
1310 
1311 		switch (flags & MAP_TYPE) {
1312 		case MAP_SHARED:
1313 			/*
1314 			 * Force use of MAP_SHARED_VALIDATE with non-legacy
1315 			 * flags. E.g. MAP_SYNC is dangerous to use with
1316 			 * MAP_SHARED as you don't know which consistency model
1317 			 * you will get. We silently ignore unsupported flags
1318 			 * with MAP_SHARED to preserve backward compatibility.
1319 			 */
1320 			flags &= LEGACY_MAP_MASK;
1321 			fallthrough;
1322 		case MAP_SHARED_VALIDATE:
1323 			if (flags & ~flags_mask)
1324 				return -EOPNOTSUPP;
1325 			if (prot & PROT_WRITE) {
1326 				if (!(file->f_mode & FMODE_WRITE))
1327 					return -EACCES;
1328 				if (IS_SWAPFILE(file->f_mapping->host))
1329 					return -ETXTBSY;
1330 			}
1331 
1332 			/*
1333 			 * Make sure we don't allow writing to an append-only
1334 			 * file..
1335 			 */
1336 			if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
1337 				return -EACCES;
1338 
1339 			vm_flags |= VM_SHARED | VM_MAYSHARE;
1340 			if (!(file->f_mode & FMODE_WRITE))
1341 				vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
1342 			fallthrough;
1343 		case MAP_PRIVATE:
1344 			if (!(file->f_mode & FMODE_READ))
1345 				return -EACCES;
1346 			if (path_noexec(&file->f_path)) {
1347 				if (vm_flags & VM_EXEC)
1348 					return -EPERM;
1349 				vm_flags &= ~VM_MAYEXEC;
1350 			}
1351 
1352 			if (!file->f_op->mmap)
1353 				return -ENODEV;
1354 			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1355 				return -EINVAL;
1356 			break;
1357 
1358 		default:
1359 			return -EINVAL;
1360 		}
1361 	} else {
1362 		switch (flags & MAP_TYPE) {
1363 		case MAP_SHARED:
1364 			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1365 				return -EINVAL;
1366 			/*
1367 			 * Ignore pgoff.
1368 			 */
1369 			pgoff = 0;
1370 			vm_flags |= VM_SHARED | VM_MAYSHARE;
1371 			break;
1372 		case MAP_PRIVATE:
1373 			/*
1374 			 * Set pgoff according to addr for anon_vma.
1375 			 */
1376 			pgoff = addr >> PAGE_SHIFT;
1377 			break;
1378 		default:
1379 			return -EINVAL;
1380 		}
1381 	}
1382 
1383 	/*
1384 	 * Set 'VM_NORESERVE' if we should not account for the
1385 	 * memory use of this mapping.
1386 	 */
1387 	if (flags & MAP_NORESERVE) {
1388 		/* We honor MAP_NORESERVE if allowed to overcommit */
1389 		if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
1390 			vm_flags |= VM_NORESERVE;
1391 
1392 		/* hugetlb applies strict overcommit unless MAP_NORESERVE */
1393 		if (file && is_file_hugepages(file))
1394 			vm_flags |= VM_NORESERVE;
1395 	}
1396 
1397 	addr = mmap_region(file, addr, len, vm_flags, pgoff, uf);
1398 	if (!IS_ERR_VALUE(addr) &&
1399 	    ((vm_flags & VM_LOCKED) ||
1400 	     (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
1401 		*populate = len;
1402 	return addr;
1403 }
1404 
ksys_mmap_pgoff(unsigned long addr,unsigned long len,unsigned long prot,unsigned long flags,unsigned long fd,unsigned long pgoff)1405 unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
1406 			      unsigned long prot, unsigned long flags,
1407 			      unsigned long fd, unsigned long pgoff)
1408 {
1409 	struct file *file = NULL;
1410 	unsigned long retval;
1411 
1412 	if (!(flags & MAP_ANONYMOUS)) {
1413 		audit_mmap_fd(fd, flags);
1414 		file = fget(fd);
1415 		if (!file)
1416 			return -EBADF;
1417 		if (is_file_hugepages(file)) {
1418 			len = ALIGN(len, huge_page_size(hstate_file(file)));
1419 		} else if (unlikely(flags & MAP_HUGETLB)) {
1420 			retval = -EINVAL;
1421 			goto out_fput;
1422 		}
1423 	} else if (flags & MAP_HUGETLB) {
1424 		struct hstate *hs;
1425 
1426 		hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
1427 		if (!hs)
1428 			return -EINVAL;
1429 
1430 		len = ALIGN(len, huge_page_size(hs));
1431 		/*
1432 		 * VM_NORESERVE is used because the reservations will be
1433 		 * taken when vm_ops->mmap() is called
1434 		 */
1435 		file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
1436 				VM_NORESERVE,
1437 				HUGETLB_ANONHUGE_INODE,
1438 				(flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
1439 		if (IS_ERR(file))
1440 			return PTR_ERR(file);
1441 	}
1442 
1443 	retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1444 out_fput:
1445 	if (file)
1446 		fput(file);
1447 	return retval;
1448 }
1449 
SYSCALL_DEFINE6(mmap_pgoff,unsigned long,addr,unsigned long,len,unsigned long,prot,unsigned long,flags,unsigned long,fd,unsigned long,pgoff)1450 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1451 		unsigned long, prot, unsigned long, flags,
1452 		unsigned long, fd, unsigned long, pgoff)
1453 {
1454 	return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
1455 }
1456 
1457 #ifdef __ARCH_WANT_SYS_OLD_MMAP
1458 struct mmap_arg_struct {
1459 	unsigned long addr;
1460 	unsigned long len;
1461 	unsigned long prot;
1462 	unsigned long flags;
1463 	unsigned long fd;
1464 	unsigned long offset;
1465 };
1466 
SYSCALL_DEFINE1(old_mmap,struct mmap_arg_struct __user *,arg)1467 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1468 {
1469 	struct mmap_arg_struct a;
1470 
1471 	if (copy_from_user(&a, arg, sizeof(a)))
1472 		return -EFAULT;
1473 	if (offset_in_page(a.offset))
1474 		return -EINVAL;
1475 
1476 	return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1477 			       a.offset >> PAGE_SHIFT);
1478 }
1479 #endif /* __ARCH_WANT_SYS_OLD_MMAP */
1480 
vm_ops_needs_writenotify(const struct vm_operations_struct * vm_ops)1481 static bool vm_ops_needs_writenotify(const struct vm_operations_struct *vm_ops)
1482 {
1483 	return vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite);
1484 }
1485 
vma_is_shared_writable(struct vm_area_struct * vma)1486 static bool vma_is_shared_writable(struct vm_area_struct *vma)
1487 {
1488 	return (vma->vm_flags & (VM_WRITE | VM_SHARED)) ==
1489 		(VM_WRITE | VM_SHARED);
1490 }
1491 
vma_fs_can_writeback(struct vm_area_struct * vma)1492 static bool vma_fs_can_writeback(struct vm_area_struct *vma)
1493 {
1494 	/* No managed pages to writeback. */
1495 	if (vma->vm_flags & VM_PFNMAP)
1496 		return false;
1497 
1498 	return vma->vm_file && vma->vm_file->f_mapping &&
1499 		mapping_can_writeback(vma->vm_file->f_mapping);
1500 }
1501 
1502 /*
1503  * Does this VMA require the underlying folios to have their dirty state
1504  * tracked?
1505  */
vma_needs_dirty_tracking(struct vm_area_struct * vma)1506 bool vma_needs_dirty_tracking(struct vm_area_struct *vma)
1507 {
1508 	/* Only shared, writable VMAs require dirty tracking. */
1509 	if (!vma_is_shared_writable(vma))
1510 		return false;
1511 
1512 	/* Does the filesystem need to be notified? */
1513 	if (vm_ops_needs_writenotify(vma->vm_ops))
1514 		return true;
1515 
1516 	/*
1517 	 * Even if the filesystem doesn't indicate a need for writenotify, if it
1518 	 * can writeback, dirty tracking is still required.
1519 	 */
1520 	return vma_fs_can_writeback(vma);
1521 }
1522 
1523 /*
1524  * Some shared mappings will want the pages marked read-only
1525  * to track write events. If so, we'll downgrade vm_page_prot
1526  * to the private version (using protection_map[] without the
1527  * VM_SHARED bit).
1528  */
vma_wants_writenotify(struct vm_area_struct * vma,pgprot_t vm_page_prot)1529 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
1530 {
1531 	/* If it was private or non-writable, the write bit is already clear */
1532 	if (!vma_is_shared_writable(vma))
1533 		return false;
1534 
1535 	/* The backer wishes to know when pages are first written to? */
1536 	if (vm_ops_needs_writenotify(vma->vm_ops))
1537 		return true;
1538 
1539 	/* The open routine did something to the protections that pgprot_modify
1540 	 * won't preserve? */
1541 	if (pgprot_val(vm_page_prot) !=
1542 	    pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags)))
1543 		return false;
1544 
1545 	/*
1546 	 * Do we need to track softdirty? hugetlb does not support softdirty
1547 	 * tracking yet.
1548 	 */
1549 	if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma))
1550 		return true;
1551 
1552 	/* Do we need write faults for uffd-wp tracking? */
1553 	if (userfaultfd_wp(vma))
1554 		return true;
1555 
1556 	/* Can the mapping track the dirty pages? */
1557 	return vma_fs_can_writeback(vma);
1558 }
1559 
1560 /*
1561  * We account for memory if it's a private writeable mapping,
1562  * not hugepages and VM_NORESERVE wasn't set.
1563  */
accountable_mapping(struct file * file,vm_flags_t vm_flags)1564 static inline bool accountable_mapping(struct file *file, vm_flags_t vm_flags)
1565 {
1566 	/*
1567 	 * hugetlb has its own accounting separate from the core VM
1568 	 * VM_HUGETLB may not be set yet so we cannot check for that flag.
1569 	 */
1570 	if (file && is_file_hugepages(file))
1571 		return false;
1572 
1573 	return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
1574 }
1575 
1576 /**
1577  * unmapped_area() - Find an area between the low_limit and the high_limit with
1578  * the correct alignment and offset, all from @info. Note: current->mm is used
1579  * for the search.
1580  *
1581  * @info: The unmapped area information including the range [low_limit -
1582  * high_limit), the alignment offset and mask.
1583  *
1584  * Return: A memory address or -ENOMEM.
1585  */
unmapped_area(struct vm_unmapped_area_info * info)1586 static unsigned long unmapped_area(struct vm_unmapped_area_info *info)
1587 {
1588 	unsigned long length, gap;
1589 	unsigned long low_limit, high_limit;
1590 	struct vm_area_struct *tmp;
1591 	VMA_ITERATOR(vmi, current->mm, 0);
1592 
1593 	/* Adjust search length to account for worst case alignment overhead */
1594 	length = info->length + info->align_mask + info->start_gap;
1595 	if (length < info->length)
1596 		return -ENOMEM;
1597 
1598 	low_limit = info->low_limit;
1599 	if (low_limit < mmap_min_addr)
1600 		low_limit = mmap_min_addr;
1601 	high_limit = info->high_limit;
1602 retry:
1603 	if (vma_iter_area_lowest(&vmi, low_limit, high_limit, length))
1604 		return -ENOMEM;
1605 
1606 	/*
1607 	 * Adjust for the gap first so it doesn't interfere with the
1608 	 * later alignment. The first step is the minimum needed to
1609 	 * fulill the start gap, the next steps is the minimum to align
1610 	 * that. It is the minimum needed to fulill both.
1611 	 */
1612 	gap = vma_iter_addr(&vmi) + info->start_gap;
1613 	gap += (info->align_offset - gap) & info->align_mask;
1614 	tmp = vma_next(&vmi);
1615 	if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
1616 		if (vm_start_gap(tmp) < gap + length - 1) {
1617 			low_limit = tmp->vm_end;
1618 			vma_iter_reset(&vmi);
1619 			goto retry;
1620 		}
1621 	} else {
1622 		tmp = vma_prev(&vmi);
1623 		if (tmp && vm_end_gap(tmp) > gap) {
1624 			low_limit = vm_end_gap(tmp);
1625 			vma_iter_reset(&vmi);
1626 			goto retry;
1627 		}
1628 	}
1629 
1630 	return gap;
1631 }
1632 
1633 /**
1634  * unmapped_area_topdown() - Find an area between the low_limit and the
1635  * high_limit with the correct alignment and offset at the highest available
1636  * address, all from @info. Note: current->mm is used for the search.
1637  *
1638  * @info: The unmapped area information including the range [low_limit -
1639  * high_limit), the alignment offset and mask.
1640  *
1641  * Return: A memory address or -ENOMEM.
1642  */
unmapped_area_topdown(struct vm_unmapped_area_info * info)1643 static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
1644 {
1645 	unsigned long length, gap, gap_end;
1646 	unsigned long low_limit, high_limit;
1647 	struct vm_area_struct *tmp;
1648 	VMA_ITERATOR(vmi, current->mm, 0);
1649 
1650 	/* Adjust search length to account for worst case alignment overhead */
1651 	length = info->length + info->align_mask + info->start_gap;
1652 	if (length < info->length)
1653 		return -ENOMEM;
1654 
1655 	low_limit = info->low_limit;
1656 	if (low_limit < mmap_min_addr)
1657 		low_limit = mmap_min_addr;
1658 	high_limit = info->high_limit;
1659 retry:
1660 	if (vma_iter_area_highest(&vmi, low_limit, high_limit, length))
1661 		return -ENOMEM;
1662 
1663 	gap = vma_iter_end(&vmi) - info->length;
1664 	gap -= (gap - info->align_offset) & info->align_mask;
1665 	gap_end = vma_iter_end(&vmi);
1666 	tmp = vma_next(&vmi);
1667 	if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
1668 		if (vm_start_gap(tmp) < gap_end) {
1669 			high_limit = vm_start_gap(tmp);
1670 			vma_iter_reset(&vmi);
1671 			goto retry;
1672 		}
1673 	} else {
1674 		tmp = vma_prev(&vmi);
1675 		if (tmp && vm_end_gap(tmp) > gap) {
1676 			high_limit = tmp->vm_start;
1677 			vma_iter_reset(&vmi);
1678 			goto retry;
1679 		}
1680 	}
1681 
1682 	return gap;
1683 }
1684 
1685 /*
1686  * Search for an unmapped address range.
1687  *
1688  * We are looking for a range that:
1689  * - does not intersect with any VMA;
1690  * - is contained within the [low_limit, high_limit) interval;
1691  * - is at least the desired size.
1692  * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
1693  */
vm_unmapped_area(struct vm_unmapped_area_info * info)1694 unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info)
1695 {
1696 	unsigned long addr;
1697 
1698 	if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
1699 		addr = unmapped_area_topdown(info);
1700 	else
1701 		addr = unmapped_area(info);
1702 
1703 	trace_vm_unmapped_area(addr, info);
1704 	return addr;
1705 }
1706 
1707 /* Get an address range which is currently unmapped.
1708  * For shmat() with addr=0.
1709  *
1710  * Ugly calling convention alert:
1711  * Return value with the low bits set means error value,
1712  * ie
1713  *	if (ret & ~PAGE_MASK)
1714  *		error = ret;
1715  *
1716  * This function "knows" that -ENOMEM has the bits set.
1717  */
1718 unsigned long
generic_get_unmapped_area(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)1719 generic_get_unmapped_area(struct file *filp, unsigned long addr,
1720 			  unsigned long len, unsigned long pgoff,
1721 			  unsigned long flags)
1722 {
1723 	struct mm_struct *mm = current->mm;
1724 	struct vm_area_struct *vma, *prev;
1725 	struct vm_unmapped_area_info info = {};
1726 	const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags);
1727 
1728 	if (len > mmap_end - mmap_min_addr)
1729 		return -ENOMEM;
1730 
1731 	if (flags & MAP_FIXED)
1732 		return addr;
1733 
1734 	if (addr) {
1735 		addr = PAGE_ALIGN(addr);
1736 		vma = find_vma_prev(mm, addr, &prev);
1737 		if (mmap_end - len >= addr && addr >= mmap_min_addr &&
1738 		    (!vma || addr + len <= vm_start_gap(vma)) &&
1739 		    (!prev || addr >= vm_end_gap(prev)))
1740 			return addr;
1741 	}
1742 
1743 	info.length = len;
1744 	info.low_limit = mm->mmap_base;
1745 	info.high_limit = mmap_end;
1746 	return vm_unmapped_area(&info);
1747 }
1748 
1749 #ifndef HAVE_ARCH_UNMAPPED_AREA
1750 unsigned long
arch_get_unmapped_area(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)1751 arch_get_unmapped_area(struct file *filp, unsigned long addr,
1752 		       unsigned long len, unsigned long pgoff,
1753 		       unsigned long flags)
1754 {
1755 	return generic_get_unmapped_area(filp, addr, len, pgoff, flags);
1756 }
1757 #endif
1758 
1759 /*
1760  * This mmap-allocator allocates new areas top-down from below the
1761  * stack's low limit (the base):
1762  */
1763 unsigned long
generic_get_unmapped_area_topdown(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)1764 generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
1765 				  unsigned long len, unsigned long pgoff,
1766 				  unsigned long flags)
1767 {
1768 	struct vm_area_struct *vma, *prev;
1769 	struct mm_struct *mm = current->mm;
1770 	struct vm_unmapped_area_info info = {};
1771 	const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags);
1772 
1773 	/* requested length too big for entire address space */
1774 	if (len > mmap_end - mmap_min_addr)
1775 		return -ENOMEM;
1776 
1777 	if (flags & MAP_FIXED)
1778 		return addr;
1779 
1780 	/* requesting a specific address */
1781 	if (addr) {
1782 		addr = PAGE_ALIGN(addr);
1783 		vma = find_vma_prev(mm, addr, &prev);
1784 		if (mmap_end - len >= addr && addr >= mmap_min_addr &&
1785 				(!vma || addr + len <= vm_start_gap(vma)) &&
1786 				(!prev || addr >= vm_end_gap(prev)))
1787 			return addr;
1788 	}
1789 
1790 	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
1791 	info.length = len;
1792 	info.low_limit = PAGE_SIZE;
1793 	info.high_limit = arch_get_mmap_base(addr, mm->mmap_base);
1794 	addr = vm_unmapped_area(&info);
1795 
1796 	/*
1797 	 * A failed mmap() very likely causes application failure,
1798 	 * so fall back to the bottom-up function here. This scenario
1799 	 * can happen with large stack limits and large mmap()
1800 	 * allocations.
1801 	 */
1802 	if (offset_in_page(addr)) {
1803 		VM_BUG_ON(addr != -ENOMEM);
1804 		info.flags = 0;
1805 		info.low_limit = TASK_UNMAPPED_BASE;
1806 		info.high_limit = mmap_end;
1807 		addr = vm_unmapped_area(&info);
1808 	}
1809 
1810 	return addr;
1811 }
1812 
1813 #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1814 unsigned long
arch_get_unmapped_area_topdown(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)1815 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
1816 			       unsigned long len, unsigned long pgoff,
1817 			       unsigned long flags)
1818 {
1819 	return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags);
1820 }
1821 #endif
1822 
1823 #ifndef HAVE_ARCH_UNMAPPED_AREA_VMFLAGS
1824 unsigned long
arch_get_unmapped_area_vmflags(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags,vm_flags_t vm_flags)1825 arch_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, unsigned long len,
1826 			       unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags)
1827 {
1828 	return arch_get_unmapped_area(filp, addr, len, pgoff, flags);
1829 }
1830 
1831 unsigned long
arch_get_unmapped_area_topdown_vmflags(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags,vm_flags_t vm_flags)1832 arch_get_unmapped_area_topdown_vmflags(struct file *filp, unsigned long addr,
1833 				       unsigned long len, unsigned long pgoff,
1834 				       unsigned long flags, vm_flags_t vm_flags)
1835 {
1836 	return arch_get_unmapped_area_topdown(filp, addr, len, pgoff, flags);
1837 }
1838 #endif
1839 
mm_get_unmapped_area_vmflags(struct mm_struct * mm,struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags,vm_flags_t vm_flags)1840 unsigned long mm_get_unmapped_area_vmflags(struct mm_struct *mm, struct file *filp,
1841 					   unsigned long addr, unsigned long len,
1842 					   unsigned long pgoff, unsigned long flags,
1843 					   vm_flags_t vm_flags)
1844 {
1845 	if (test_bit(MMF_TOPDOWN, &mm->flags))
1846 		return arch_get_unmapped_area_topdown_vmflags(filp, addr, len, pgoff,
1847 							      flags, vm_flags);
1848 	return arch_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, vm_flags);
1849 }
1850 
1851 unsigned long
__get_unmapped_area(struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags,vm_flags_t vm_flags)1852 __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
1853 		unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags)
1854 {
1855 	unsigned long (*get_area)(struct file *, unsigned long,
1856 				  unsigned long, unsigned long, unsigned long)
1857 				  = NULL;
1858 
1859 	unsigned long error = arch_mmap_check(addr, len, flags);
1860 	if (error)
1861 		return error;
1862 
1863 	/* Careful about overflows.. */
1864 	if (len > TASK_SIZE)
1865 		return -ENOMEM;
1866 
1867 	if (file) {
1868 		if (file->f_op->get_unmapped_area)
1869 			get_area = file->f_op->get_unmapped_area;
1870 	} else if (flags & MAP_SHARED) {
1871 		/*
1872 		 * mmap_region() will call shmem_zero_setup() to create a file,
1873 		 * so use shmem's get_unmapped_area in case it can be huge.
1874 		 */
1875 		get_area = shmem_get_unmapped_area;
1876 	}
1877 
1878 	/* Always treat pgoff as zero for anonymous memory. */
1879 	if (!file)
1880 		pgoff = 0;
1881 
1882 	if (get_area) {
1883 		addr = get_area(file, addr, len, pgoff, flags);
1884 	} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
1885 		/* Ensures that larger anonymous mappings are THP aligned. */
1886 		addr = thp_get_unmapped_area_vmflags(file, addr, len,
1887 						     pgoff, flags, vm_flags);
1888 	} else {
1889 		addr = mm_get_unmapped_area_vmflags(current->mm, file, addr, len,
1890 						    pgoff, flags, vm_flags);
1891 	}
1892 	if (IS_ERR_VALUE(addr))
1893 		return addr;
1894 
1895 	if (addr > TASK_SIZE - len)
1896 		return -ENOMEM;
1897 	if (offset_in_page(addr))
1898 		return -EINVAL;
1899 
1900 	error = security_mmap_addr(addr);
1901 	return error ? error : addr;
1902 }
1903 
1904 unsigned long
mm_get_unmapped_area(struct mm_struct * mm,struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)1905 mm_get_unmapped_area(struct mm_struct *mm, struct file *file,
1906 		     unsigned long addr, unsigned long len,
1907 		     unsigned long pgoff, unsigned long flags)
1908 {
1909 	if (test_bit(MMF_TOPDOWN, &mm->flags))
1910 		return arch_get_unmapped_area_topdown(file, addr, len, pgoff, flags);
1911 	return arch_get_unmapped_area(file, addr, len, pgoff, flags);
1912 }
1913 EXPORT_SYMBOL(mm_get_unmapped_area);
1914 
1915 /**
1916  * find_vma_intersection() - Look up the first VMA which intersects the interval
1917  * @mm: The process address space.
1918  * @start_addr: The inclusive start user address.
1919  * @end_addr: The exclusive end user address.
1920  *
1921  * Returns: The first VMA within the provided range, %NULL otherwise.  Assumes
1922  * start_addr < end_addr.
1923  */
find_vma_intersection(struct mm_struct * mm,unsigned long start_addr,unsigned long end_addr)1924 struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
1925 					     unsigned long start_addr,
1926 					     unsigned long end_addr)
1927 {
1928 	unsigned long index = start_addr;
1929 
1930 	mmap_assert_locked(mm);
1931 	return mt_find(&mm->mm_mt, &index, end_addr - 1);
1932 }
1933 EXPORT_SYMBOL(find_vma_intersection);
1934 
1935 /**
1936  * find_vma() - Find the VMA for a given address, or the next VMA.
1937  * @mm: The mm_struct to check
1938  * @addr: The address
1939  *
1940  * Returns: The VMA associated with addr, or the next VMA.
1941  * May return %NULL in the case of no VMA at addr or above.
1942  */
find_vma(struct mm_struct * mm,unsigned long addr)1943 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
1944 {
1945 	unsigned long index = addr;
1946 
1947 	mmap_assert_locked(mm);
1948 	return mt_find(&mm->mm_mt, &index, ULONG_MAX);
1949 }
1950 EXPORT_SYMBOL(find_vma);
1951 
1952 /**
1953  * find_vma_prev() - Find the VMA for a given address, or the next vma and
1954  * set %pprev to the previous VMA, if any.
1955  * @mm: The mm_struct to check
1956  * @addr: The address
1957  * @pprev: The pointer to set to the previous VMA
1958  *
1959  * Note that RCU lock is missing here since the external mmap_lock() is used
1960  * instead.
1961  *
1962  * Returns: The VMA associated with @addr, or the next vma.
1963  * May return %NULL in the case of no vma at addr or above.
1964  */
1965 struct vm_area_struct *
find_vma_prev(struct mm_struct * mm,unsigned long addr,struct vm_area_struct ** pprev)1966 find_vma_prev(struct mm_struct *mm, unsigned long addr,
1967 			struct vm_area_struct **pprev)
1968 {
1969 	struct vm_area_struct *vma;
1970 	VMA_ITERATOR(vmi, mm, addr);
1971 
1972 	vma = vma_iter_load(&vmi);
1973 	*pprev = vma_prev(&vmi);
1974 	if (!vma)
1975 		vma = vma_next(&vmi);
1976 	return vma;
1977 }
1978 
1979 /*
1980  * Verify that the stack growth is acceptable and
1981  * update accounting. This is shared with both the
1982  * grow-up and grow-down cases.
1983  */
acct_stack_growth(struct vm_area_struct * vma,unsigned long size,unsigned long grow)1984 static int acct_stack_growth(struct vm_area_struct *vma,
1985 			     unsigned long size, unsigned long grow)
1986 {
1987 	struct mm_struct *mm = vma->vm_mm;
1988 	unsigned long new_start;
1989 
1990 	/* address space limit tests */
1991 	if (!may_expand_vm(mm, vma->vm_flags, grow))
1992 		return -ENOMEM;
1993 
1994 	/* Stack limit test */
1995 	if (size > rlimit(RLIMIT_STACK))
1996 		return -ENOMEM;
1997 
1998 	/* mlock limit tests */
1999 	if (!mlock_future_ok(mm, vma->vm_flags, grow << PAGE_SHIFT))
2000 		return -ENOMEM;
2001 
2002 	/* Check to ensure the stack will not grow into a hugetlb-only region */
2003 	new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
2004 			vma->vm_end - size;
2005 	if (is_hugepage_only_range(vma->vm_mm, new_start, size))
2006 		return -EFAULT;
2007 
2008 	/*
2009 	 * Overcommit..  This must be the final test, as it will
2010 	 * update security statistics.
2011 	 */
2012 	if (security_vm_enough_memory_mm(mm, grow))
2013 		return -ENOMEM;
2014 
2015 	return 0;
2016 }
2017 
2018 #if defined(CONFIG_STACK_GROWSUP)
2019 /*
2020  * PA-RISC uses this for its stack.
2021  * vma is the last one with address > vma->vm_end.  Have to extend vma.
2022  */
expand_upwards(struct vm_area_struct * vma,unsigned long address)2023 static int expand_upwards(struct vm_area_struct *vma, unsigned long address)
2024 {
2025 	struct mm_struct *mm = vma->vm_mm;
2026 	struct vm_area_struct *next;
2027 	unsigned long gap_addr;
2028 	int error = 0;
2029 	VMA_ITERATOR(vmi, mm, vma->vm_start);
2030 
2031 	if (!(vma->vm_flags & VM_GROWSUP))
2032 		return -EFAULT;
2033 
2034 	/* Guard against exceeding limits of the address space. */
2035 	address &= PAGE_MASK;
2036 	if (address >= (TASK_SIZE & PAGE_MASK))
2037 		return -ENOMEM;
2038 	address += PAGE_SIZE;
2039 
2040 	/* Enforce stack_guard_gap */
2041 	gap_addr = address + stack_guard_gap;
2042 
2043 	/* Guard against overflow */
2044 	if (gap_addr < address || gap_addr > TASK_SIZE)
2045 		gap_addr = TASK_SIZE;
2046 
2047 	next = find_vma_intersection(mm, vma->vm_end, gap_addr);
2048 	if (next && vma_is_accessible(next)) {
2049 		if (!(next->vm_flags & VM_GROWSUP))
2050 			return -ENOMEM;
2051 		/* Check that both stack segments have the same anon_vma? */
2052 	}
2053 
2054 	if (next)
2055 		vma_iter_prev_range_limit(&vmi, address);
2056 
2057 	vma_iter_config(&vmi, vma->vm_start, address);
2058 	if (vma_iter_prealloc(&vmi, vma))
2059 		return -ENOMEM;
2060 
2061 	/* We must make sure the anon_vma is allocated. */
2062 	if (unlikely(anon_vma_prepare(vma))) {
2063 		vma_iter_free(&vmi);
2064 		return -ENOMEM;
2065 	}
2066 
2067 	/* Lock the VMA before expanding to prevent concurrent page faults */
2068 	vma_start_write(vma);
2069 	/*
2070 	 * vma->vm_start/vm_end cannot change under us because the caller
2071 	 * is required to hold the mmap_lock in read mode.  We need the
2072 	 * anon_vma lock to serialize against concurrent expand_stacks.
2073 	 */
2074 	anon_vma_lock_write(vma->anon_vma);
2075 
2076 	/* Somebody else might have raced and expanded it already */
2077 	if (address > vma->vm_end) {
2078 		unsigned long size, grow;
2079 
2080 		size = address - vma->vm_start;
2081 		grow = (address - vma->vm_end) >> PAGE_SHIFT;
2082 
2083 		error = -ENOMEM;
2084 		if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
2085 			error = acct_stack_growth(vma, size, grow);
2086 			if (!error) {
2087 				/*
2088 				 * We only hold a shared mmap_lock lock here, so
2089 				 * we need to protect against concurrent vma
2090 				 * expansions.  anon_vma_lock_write() doesn't
2091 				 * help here, as we don't guarantee that all
2092 				 * growable vmas in a mm share the same root
2093 				 * anon vma.  So, we reuse mm->page_table_lock
2094 				 * to guard against concurrent vma expansions.
2095 				 */
2096 				spin_lock(&mm->page_table_lock);
2097 				if (vma->vm_flags & VM_LOCKED)
2098 					mm->locked_vm += grow;
2099 				vm_stat_account(mm, vma->vm_flags, grow);
2100 				anon_vma_interval_tree_pre_update_vma(vma);
2101 				vma->vm_end = address;
2102 				/* Overwrite old entry in mtree. */
2103 				vma_iter_store(&vmi, vma);
2104 				anon_vma_interval_tree_post_update_vma(vma);
2105 				spin_unlock(&mm->page_table_lock);
2106 
2107 				perf_event_mmap(vma);
2108 			}
2109 		}
2110 	}
2111 	anon_vma_unlock_write(vma->anon_vma);
2112 	vma_iter_free(&vmi);
2113 	validate_mm(mm);
2114 	return error;
2115 }
2116 #endif /* CONFIG_STACK_GROWSUP */
2117 
2118 /*
2119  * vma is the first one with address < vma->vm_start.  Have to extend vma.
2120  * mmap_lock held for writing.
2121  */
expand_downwards(struct vm_area_struct * vma,unsigned long address)2122 int expand_downwards(struct vm_area_struct *vma, unsigned long address)
2123 {
2124 	struct mm_struct *mm = vma->vm_mm;
2125 	struct vm_area_struct *prev;
2126 	int error = 0;
2127 	VMA_ITERATOR(vmi, mm, vma->vm_start);
2128 
2129 	if (!(vma->vm_flags & VM_GROWSDOWN))
2130 		return -EFAULT;
2131 
2132 	address &= PAGE_MASK;
2133 	if (address < mmap_min_addr || address < FIRST_USER_ADDRESS)
2134 		return -EPERM;
2135 
2136 	/* Enforce stack_guard_gap */
2137 	prev = vma_prev(&vmi);
2138 	/* Check that both stack segments have the same anon_vma? */
2139 	if (prev) {
2140 		if (!(prev->vm_flags & VM_GROWSDOWN) &&
2141 		    vma_is_accessible(prev) &&
2142 		    (address - prev->vm_end < stack_guard_gap))
2143 			return -ENOMEM;
2144 	}
2145 
2146 	if (prev)
2147 		vma_iter_next_range_limit(&vmi, vma->vm_start);
2148 
2149 	vma_iter_config(&vmi, address, vma->vm_end);
2150 	if (vma_iter_prealloc(&vmi, vma))
2151 		return -ENOMEM;
2152 
2153 	/* We must make sure the anon_vma is allocated. */
2154 	if (unlikely(anon_vma_prepare(vma))) {
2155 		vma_iter_free(&vmi);
2156 		return -ENOMEM;
2157 	}
2158 
2159 	/* Lock the VMA before expanding to prevent concurrent page faults */
2160 	vma_start_write(vma);
2161 	/*
2162 	 * vma->vm_start/vm_end cannot change under us because the caller
2163 	 * is required to hold the mmap_lock in read mode.  We need the
2164 	 * anon_vma lock to serialize against concurrent expand_stacks.
2165 	 */
2166 	anon_vma_lock_write(vma->anon_vma);
2167 
2168 	/* Somebody else might have raced and expanded it already */
2169 	if (address < vma->vm_start) {
2170 		unsigned long size, grow;
2171 
2172 		size = vma->vm_end - address;
2173 		grow = (vma->vm_start - address) >> PAGE_SHIFT;
2174 
2175 		error = -ENOMEM;
2176 		if (grow <= vma->vm_pgoff) {
2177 			error = acct_stack_growth(vma, size, grow);
2178 			if (!error) {
2179 				/*
2180 				 * We only hold a shared mmap_lock lock here, so
2181 				 * we need to protect against concurrent vma
2182 				 * expansions.  anon_vma_lock_write() doesn't
2183 				 * help here, as we don't guarantee that all
2184 				 * growable vmas in a mm share the same root
2185 				 * anon vma.  So, we reuse mm->page_table_lock
2186 				 * to guard against concurrent vma expansions.
2187 				 */
2188 				spin_lock(&mm->page_table_lock);
2189 				if (vma->vm_flags & VM_LOCKED)
2190 					mm->locked_vm += grow;
2191 				vm_stat_account(mm, vma->vm_flags, grow);
2192 				anon_vma_interval_tree_pre_update_vma(vma);
2193 				vma->vm_start = address;
2194 				vma->vm_pgoff -= grow;
2195 				/* Overwrite old entry in mtree. */
2196 				vma_iter_store(&vmi, vma);
2197 				anon_vma_interval_tree_post_update_vma(vma);
2198 				spin_unlock(&mm->page_table_lock);
2199 
2200 				perf_event_mmap(vma);
2201 			}
2202 		}
2203 	}
2204 	anon_vma_unlock_write(vma->anon_vma);
2205 	vma_iter_free(&vmi);
2206 	validate_mm(mm);
2207 	return error;
2208 }
2209 
2210 /* enforced gap between the expanding stack and other mappings. */
2211 unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;
2212 
cmdline_parse_stack_guard_gap(char * p)2213 static int __init cmdline_parse_stack_guard_gap(char *p)
2214 {
2215 	unsigned long val;
2216 	char *endptr;
2217 
2218 	val = simple_strtoul(p, &endptr, 10);
2219 	if (!*endptr)
2220 		stack_guard_gap = val << PAGE_SHIFT;
2221 
2222 	return 1;
2223 }
2224 __setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
2225 
2226 #ifdef CONFIG_STACK_GROWSUP
expand_stack_locked(struct vm_area_struct * vma,unsigned long address)2227 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
2228 {
2229 	return expand_upwards(vma, address);
2230 }
2231 
find_extend_vma_locked(struct mm_struct * mm,unsigned long addr)2232 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
2233 {
2234 	struct vm_area_struct *vma, *prev;
2235 
2236 	addr &= PAGE_MASK;
2237 	vma = find_vma_prev(mm, addr, &prev);
2238 	if (vma && (vma->vm_start <= addr))
2239 		return vma;
2240 	if (!prev)
2241 		return NULL;
2242 	if (expand_stack_locked(prev, addr))
2243 		return NULL;
2244 	if (prev->vm_flags & VM_LOCKED)
2245 		populate_vma_page_range(prev, addr, prev->vm_end, NULL);
2246 	return prev;
2247 }
2248 #else
expand_stack_locked(struct vm_area_struct * vma,unsigned long address)2249 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
2250 {
2251 	return expand_downwards(vma, address);
2252 }
2253 
find_extend_vma_locked(struct mm_struct * mm,unsigned long addr)2254 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
2255 {
2256 	struct vm_area_struct *vma;
2257 	unsigned long start;
2258 
2259 	addr &= PAGE_MASK;
2260 	vma = find_vma(mm, addr);
2261 	if (!vma)
2262 		return NULL;
2263 	if (vma->vm_start <= addr)
2264 		return vma;
2265 	start = vma->vm_start;
2266 	if (expand_stack_locked(vma, addr))
2267 		return NULL;
2268 	if (vma->vm_flags & VM_LOCKED)
2269 		populate_vma_page_range(vma, addr, start, NULL);
2270 	return vma;
2271 }
2272 #endif
2273 
2274 #if defined(CONFIG_STACK_GROWSUP)
2275 
2276 #define vma_expand_up(vma,addr) expand_upwards(vma, addr)
2277 #define vma_expand_down(vma, addr) (-EFAULT)
2278 
2279 #else
2280 
2281 #define vma_expand_up(vma,addr) (-EFAULT)
2282 #define vma_expand_down(vma, addr) expand_downwards(vma, addr)
2283 
2284 #endif
2285 
2286 /*
2287  * expand_stack(): legacy interface for page faulting. Don't use unless
2288  * you have to.
2289  *
2290  * This is called with the mm locked for reading, drops the lock, takes
2291  * the lock for writing, tries to look up a vma again, expands it if
2292  * necessary, and downgrades the lock to reading again.
2293  *
2294  * If no vma is found or it can't be expanded, it returns NULL and has
2295  * dropped the lock.
2296  */
expand_stack(struct mm_struct * mm,unsigned long addr)2297 struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr)
2298 {
2299 	struct vm_area_struct *vma, *prev;
2300 
2301 	mmap_read_unlock(mm);
2302 	if (mmap_write_lock_killable(mm))
2303 		return NULL;
2304 
2305 	vma = find_vma_prev(mm, addr, &prev);
2306 	if (vma && vma->vm_start <= addr)
2307 		goto success;
2308 
2309 	if (prev && !vma_expand_up(prev, addr)) {
2310 		vma = prev;
2311 		goto success;
2312 	}
2313 
2314 	if (vma && !vma_expand_down(vma, addr))
2315 		goto success;
2316 
2317 	mmap_write_unlock(mm);
2318 	return NULL;
2319 
2320 success:
2321 	mmap_write_downgrade(mm);
2322 	return vma;
2323 }
2324 
2325 /*
2326  * Ok - we have the memory areas we should free on a maple tree so release them,
2327  * and do the vma updates.
2328  *
2329  * Called with the mm semaphore held.
2330  */
remove_mt(struct mm_struct * mm,struct ma_state * mas)2331 static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas)
2332 {
2333 	unsigned long nr_accounted = 0;
2334 	struct vm_area_struct *vma;
2335 
2336 	/* Update high watermark before we lower total_vm */
2337 	update_hiwater_vm(mm);
2338 	mas_for_each(mas, vma, ULONG_MAX) {
2339 		long nrpages = vma_pages(vma);
2340 
2341 		if (vma->vm_flags & VM_ACCOUNT)
2342 			nr_accounted += nrpages;
2343 		vm_stat_account(mm, vma->vm_flags, -nrpages);
2344 		remove_vma(vma, false);
2345 	}
2346 	vm_unacct_memory(nr_accounted);
2347 }
2348 
2349 /*
2350  * Get rid of page table information in the indicated region.
2351  *
2352  * Called with the mm semaphore held.
2353  */
unmap_region(struct mm_struct * mm,struct ma_state * mas,struct vm_area_struct * vma,struct vm_area_struct * prev,struct vm_area_struct * next,unsigned long start,unsigned long end,unsigned long tree_end,bool mm_wr_locked)2354 static void unmap_region(struct mm_struct *mm, struct ma_state *mas,
2355 		struct vm_area_struct *vma, struct vm_area_struct *prev,
2356 		struct vm_area_struct *next, unsigned long start,
2357 		unsigned long end, unsigned long tree_end, bool mm_wr_locked)
2358 {
2359 	struct mmu_gather tlb;
2360 	unsigned long mt_start = mas->index;
2361 
2362 	lru_add_drain();
2363 	tlb_gather_mmu(&tlb, mm);
2364 	update_hiwater_rss(mm);
2365 	unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked);
2366 	mas_set(mas, mt_start);
2367 	free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
2368 				 next ? next->vm_start : USER_PGTABLES_CEILING,
2369 				 mm_wr_locked);
2370 	tlb_finish_mmu(&tlb);
2371 }
2372 
2373 /*
2374  * __split_vma() bypasses sysctl_max_map_count checking.  We use this where it
2375  * has already been checked or doesn't make sense to fail.
2376  * VMA Iterator will point to the end VMA.
2377  */
__split_vma(struct vma_iterator * vmi,struct vm_area_struct * vma,unsigned long addr,int new_below)2378 static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
2379 		       unsigned long addr, int new_below)
2380 {
2381 	struct vma_prepare vp;
2382 	struct vm_area_struct *new;
2383 	int err;
2384 
2385 	WARN_ON(vma->vm_start >= addr);
2386 	WARN_ON(vma->vm_end <= addr);
2387 
2388 	if (vma->vm_ops && vma->vm_ops->may_split) {
2389 		err = vma->vm_ops->may_split(vma, addr);
2390 		if (err)
2391 			return err;
2392 	}
2393 
2394 	new = vm_area_dup(vma);
2395 	if (!new)
2396 		return -ENOMEM;
2397 
2398 	if (new_below) {
2399 		new->vm_end = addr;
2400 	} else {
2401 		new->vm_start = addr;
2402 		new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
2403 	}
2404 
2405 	err = -ENOMEM;
2406 	vma_iter_config(vmi, new->vm_start, new->vm_end);
2407 	if (vma_iter_prealloc(vmi, new))
2408 		goto out_free_vma;
2409 
2410 	err = vma_dup_policy(vma, new);
2411 	if (err)
2412 		goto out_free_vmi;
2413 
2414 	err = anon_vma_clone(new, vma);
2415 	if (err)
2416 		goto out_free_mpol;
2417 
2418 	if (new->vm_file)
2419 		get_file(new->vm_file);
2420 
2421 	if (new->vm_ops && new->vm_ops->open)
2422 		new->vm_ops->open(new);
2423 
2424 	vma_start_write(vma);
2425 	vma_start_write(new);
2426 
2427 	init_vma_prep(&vp, vma);
2428 	vp.insert = new;
2429 	vma_prepare(&vp);
2430 	vma_adjust_trans_huge(vma, vma->vm_start, addr, 0);
2431 
2432 	if (new_below) {
2433 		vma->vm_start = addr;
2434 		vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT;
2435 	} else {
2436 		vma->vm_end = addr;
2437 	}
2438 
2439 	/* vma_complete stores the new vma */
2440 	vma_complete(&vp, vmi, vma->vm_mm);
2441 
2442 	/* Success. */
2443 	if (new_below)
2444 		vma_next(vmi);
2445 	return 0;
2446 
2447 out_free_mpol:
2448 	mpol_put(vma_policy(new));
2449 out_free_vmi:
2450 	vma_iter_free(vmi);
2451 out_free_vma:
2452 	vm_area_free(new);
2453 	return err;
2454 }
2455 
2456 /*
2457  * Split a vma into two pieces at address 'addr', a new vma is allocated
2458  * either for the first part or the tail.
2459  */
split_vma(struct vma_iterator * vmi,struct vm_area_struct * vma,unsigned long addr,int new_below)2460 static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
2461 		     unsigned long addr, int new_below)
2462 {
2463 	if (vma->vm_mm->map_count >= sysctl_max_map_count)
2464 		return -ENOMEM;
2465 
2466 	return __split_vma(vmi, vma, addr, new_below);
2467 }
2468 
2469 /*
2470  * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd
2471  * context and anonymous VMA name within the range [start, end).
2472  *
2473  * As a result, we might be able to merge the newly modified VMA range with an
2474  * adjacent VMA with identical properties.
2475  *
2476  * If no merge is possible and the range does not span the entirety of the VMA,
2477  * we then need to split the VMA to accommodate the change.
2478  *
2479  * The function returns either the merged VMA, the original VMA if a split was
2480  * required instead, or an error if the split failed.
2481  */
vma_modify(struct vma_iterator * vmi,struct vm_area_struct * prev,struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long vm_flags,struct mempolicy * policy,struct vm_userfaultfd_ctx uffd_ctx,struct anon_vma_name * anon_name)2482 struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
2483 				  struct vm_area_struct *prev,
2484 				  struct vm_area_struct *vma,
2485 				  unsigned long start, unsigned long end,
2486 				  unsigned long vm_flags,
2487 				  struct mempolicy *policy,
2488 				  struct vm_userfaultfd_ctx uffd_ctx,
2489 				  struct anon_vma_name *anon_name)
2490 {
2491 	pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
2492 	struct vm_area_struct *merged;
2493 
2494 	merged = vma_merge(vmi, prev, vma, start, end, vm_flags,
2495 			   pgoff, policy, uffd_ctx, anon_name);
2496 	if (merged)
2497 		return merged;
2498 
2499 	if (vma->vm_start < start) {
2500 		int err = split_vma(vmi, vma, start, 1);
2501 
2502 		if (err)
2503 			return ERR_PTR(err);
2504 	}
2505 
2506 	if (vma->vm_end > end) {
2507 		int err = split_vma(vmi, vma, end, 0);
2508 
2509 		if (err)
2510 			return ERR_PTR(err);
2511 	}
2512 
2513 	return vma;
2514 }
2515 
2516 /*
2517  * Attempt to merge a newly mapped VMA with those adjacent to it. The caller
2518  * must ensure that [start, end) does not overlap any existing VMA.
2519  */
2520 static struct vm_area_struct
vma_merge_new_vma(struct vma_iterator * vmi,struct vm_area_struct * prev,struct vm_area_struct * vma,unsigned long start,unsigned long end,pgoff_t pgoff)2521 *vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev,
2522 		   struct vm_area_struct *vma, unsigned long start,
2523 		   unsigned long end, pgoff_t pgoff)
2524 {
2525 	return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff,
2526 			 vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma));
2527 }
2528 
2529 /*
2530  * Expand vma by delta bytes, potentially merging with an immediately adjacent
2531  * VMA with identical properties.
2532  */
vma_merge_extend(struct vma_iterator * vmi,struct vm_area_struct * vma,unsigned long delta)2533 struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
2534 					struct vm_area_struct *vma,
2535 					unsigned long delta)
2536 {
2537 	pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma);
2538 
2539 	/* vma is specified as prev, so case 1 or 2 will apply. */
2540 	return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta,
2541 			 vma->vm_flags, pgoff, vma_policy(vma),
2542 			 vma->vm_userfaultfd_ctx, anon_vma_name(vma));
2543 }
2544 
2545 /*
2546  * do_vmi_align_munmap() - munmap the aligned region from @start to @end.
2547  * @vmi: The vma iterator
2548  * @vma: The starting vm_area_struct
2549  * @mm: The mm_struct
2550  * @start: The aligned start address to munmap.
2551  * @end: The aligned end address to munmap.
2552  * @uf: The userfaultfd list_head
2553  * @unlock: Set to true to drop the mmap_lock.  unlocking only happens on
2554  * success.
2555  *
2556  * Return: 0 on success and drops the lock if so directed, error and leaves the
2557  * lock held otherwise.
2558  */
2559 static int
do_vmi_align_munmap(struct vma_iterator * vmi,struct vm_area_struct * vma,struct mm_struct * mm,unsigned long start,unsigned long end,struct list_head * uf,bool unlock)2560 do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
2561 		    struct mm_struct *mm, unsigned long start,
2562 		    unsigned long end, struct list_head *uf, bool unlock)
2563 {
2564 	struct vm_area_struct *prev, *next = NULL;
2565 	struct maple_tree mt_detach;
2566 	int count = 0;
2567 	int error = -ENOMEM;
2568 	unsigned long locked_vm = 0;
2569 	MA_STATE(mas_detach, &mt_detach, 0, 0);
2570 	mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
2571 	mt_on_stack(mt_detach);
2572 
2573 	/*
2574 	 * If we need to split any vma, do it now to save pain later.
2575 	 *
2576 	 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
2577 	 * unmapped vm_area_struct will remain in use: so lower split_vma
2578 	 * places tmp vma above, and higher split_vma places tmp vma below.
2579 	 */
2580 
2581 	/* Does it split the first one? */
2582 	if (start > vma->vm_start) {
2583 
2584 		/*
2585 		 * Make sure that map_count on return from munmap() will
2586 		 * not exceed its limit; but let map_count go just above
2587 		 * its limit temporarily, to help free resources as expected.
2588 		 */
2589 		if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
2590 			goto map_count_exceeded;
2591 
2592 		error = __split_vma(vmi, vma, start, 1);
2593 		if (error)
2594 			goto start_split_failed;
2595 	}
2596 
2597 	/*
2598 	 * Detach a range of VMAs from the mm. Using next as a temp variable as
2599 	 * it is always overwritten.
2600 	 */
2601 	next = vma;
2602 	do {
2603 		/* Does it split the end? */
2604 		if (next->vm_end > end) {
2605 			error = __split_vma(vmi, next, end, 0);
2606 			if (error)
2607 				goto end_split_failed;
2608 		}
2609 		vma_start_write(next);
2610 		mas_set(&mas_detach, count);
2611 		error = mas_store_gfp(&mas_detach, next, GFP_KERNEL);
2612 		if (error)
2613 			goto munmap_gather_failed;
2614 		vma_mark_detached(next, true);
2615 		if (next->vm_flags & VM_LOCKED)
2616 			locked_vm += vma_pages(next);
2617 
2618 		count++;
2619 		if (unlikely(uf)) {
2620 			/*
2621 			 * If userfaultfd_unmap_prep returns an error the vmas
2622 			 * will remain split, but userland will get a
2623 			 * highly unexpected error anyway. This is no
2624 			 * different than the case where the first of the two
2625 			 * __split_vma fails, but we don't undo the first
2626 			 * split, despite we could. This is unlikely enough
2627 			 * failure that it's not worth optimizing it for.
2628 			 */
2629 			error = userfaultfd_unmap_prep(next, start, end, uf);
2630 
2631 			if (error)
2632 				goto userfaultfd_error;
2633 		}
2634 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE
2635 		BUG_ON(next->vm_start < start);
2636 		BUG_ON(next->vm_start > end);
2637 #endif
2638 	} for_each_vma_range(*vmi, next, end);
2639 
2640 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
2641 	/* Make sure no VMAs are about to be lost. */
2642 	{
2643 		MA_STATE(test, &mt_detach, 0, 0);
2644 		struct vm_area_struct *vma_mas, *vma_test;
2645 		int test_count = 0;
2646 
2647 		vma_iter_set(vmi, start);
2648 		rcu_read_lock();
2649 		vma_test = mas_find(&test, count - 1);
2650 		for_each_vma_range(*vmi, vma_mas, end) {
2651 			BUG_ON(vma_mas != vma_test);
2652 			test_count++;
2653 			vma_test = mas_next(&test, count - 1);
2654 		}
2655 		rcu_read_unlock();
2656 		BUG_ON(count != test_count);
2657 	}
2658 #endif
2659 
2660 	while (vma_iter_addr(vmi) > start)
2661 		vma_iter_prev_range(vmi);
2662 
2663 	error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL);
2664 	if (error)
2665 		goto clear_tree_failed;
2666 
2667 	/* Point of no return */
2668 	mm->locked_vm -= locked_vm;
2669 	mm->map_count -= count;
2670 	if (unlock)
2671 		mmap_write_downgrade(mm);
2672 
2673 	prev = vma_iter_prev_range(vmi);
2674 	next = vma_next(vmi);
2675 	if (next)
2676 		vma_iter_prev_range(vmi);
2677 
2678 	/*
2679 	 * We can free page tables without write-locking mmap_lock because VMAs
2680 	 * were isolated before we downgraded mmap_lock.
2681 	 */
2682 	mas_set(&mas_detach, 1);
2683 	unmap_region(mm, &mas_detach, vma, prev, next, start, end, count,
2684 		     !unlock);
2685 	/* Statistics and freeing VMAs */
2686 	mas_set(&mas_detach, 0);
2687 	remove_mt(mm, &mas_detach);
2688 	validate_mm(mm);
2689 	if (unlock)
2690 		mmap_read_unlock(mm);
2691 
2692 	__mt_destroy(&mt_detach);
2693 	return 0;
2694 
2695 clear_tree_failed:
2696 userfaultfd_error:
2697 munmap_gather_failed:
2698 end_split_failed:
2699 	mas_set(&mas_detach, 0);
2700 	mas_for_each(&mas_detach, next, end)
2701 		vma_mark_detached(next, false);
2702 
2703 	__mt_destroy(&mt_detach);
2704 start_split_failed:
2705 map_count_exceeded:
2706 	validate_mm(mm);
2707 	return error;
2708 }
2709 
2710 /*
2711  * do_vmi_munmap() - munmap a given range.
2712  * @vmi: The vma iterator
2713  * @mm: The mm_struct
2714  * @start: The start address to munmap
2715  * @len: The length of the range to munmap
2716  * @uf: The userfaultfd list_head
2717  * @unlock: set to true if the user wants to drop the mmap_lock on success
2718  *
2719  * This function takes a @mas that is either pointing to the previous VMA or set
2720  * to MA_START and sets it up to remove the mapping(s).  The @len will be
2721  * aligned and any arch_unmap work will be preformed.
2722  *
2723  * Return: 0 on success and drops the lock if so directed, error and leaves the
2724  * lock held otherwise.
2725  */
do_vmi_munmap(struct vma_iterator * vmi,struct mm_struct * mm,unsigned long start,size_t len,struct list_head * uf,bool unlock)2726 int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
2727 		  unsigned long start, size_t len, struct list_head *uf,
2728 		  bool unlock)
2729 {
2730 	unsigned long end;
2731 	struct vm_area_struct *vma;
2732 
2733 	if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
2734 		return -EINVAL;
2735 
2736 	end = start + PAGE_ALIGN(len);
2737 	if (end == start)
2738 		return -EINVAL;
2739 
2740 	/*
2741 	 * Check if memory is sealed before arch_unmap.
2742 	 * Prevent unmapping a sealed VMA.
2743 	 * can_modify_mm assumes we have acquired the lock on MM.
2744 	 */
2745 	if (unlikely(!can_modify_mm(mm, start, end)))
2746 		return -EPERM;
2747 
2748 	 /* arch_unmap() might do unmaps itself.  */
2749 	arch_unmap(mm, start, end);
2750 
2751 	/* Find the first overlapping VMA */
2752 	vma = vma_find(vmi, end);
2753 	if (!vma) {
2754 		if (unlock)
2755 			mmap_write_unlock(mm);
2756 		return 0;
2757 	}
2758 
2759 	return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock);
2760 }
2761 
2762 /* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls.
2763  * @mm: The mm_struct
2764  * @start: The start address to munmap
2765  * @len: The length to be munmapped.
2766  * @uf: The userfaultfd list_head
2767  *
2768  * Return: 0 on success, error otherwise.
2769  */
do_munmap(struct mm_struct * mm,unsigned long start,size_t len,struct list_head * uf)2770 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
2771 	      struct list_head *uf)
2772 {
2773 	VMA_ITERATOR(vmi, mm, start);
2774 
2775 	return do_vmi_munmap(&vmi, mm, start, len, uf, false);
2776 }
2777 
mmap_region(struct file * file,unsigned long addr,unsigned long len,vm_flags_t vm_flags,unsigned long pgoff,struct list_head * uf)2778 unsigned long mmap_region(struct file *file, unsigned long addr,
2779 		unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
2780 		struct list_head *uf)
2781 {
2782 	struct mm_struct *mm = current->mm;
2783 	struct vm_area_struct *vma = NULL;
2784 	struct vm_area_struct *next, *prev, *merge;
2785 	pgoff_t pglen = len >> PAGE_SHIFT;
2786 	unsigned long charged = 0;
2787 	unsigned long end = addr + len;
2788 	unsigned long merge_start = addr, merge_end = end;
2789 	bool writable_file_mapping = false;
2790 	pgoff_t vm_pgoff;
2791 	int error;
2792 	VMA_ITERATOR(vmi, mm, addr);
2793 
2794 	/* Check against address space limit. */
2795 	if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) {
2796 		unsigned long nr_pages;
2797 
2798 		/*
2799 		 * MAP_FIXED may remove pages of mappings that intersects with
2800 		 * requested mapping. Account for the pages it would unmap.
2801 		 */
2802 		nr_pages = count_vma_pages_range(mm, addr, end);
2803 
2804 		if (!may_expand_vm(mm, vm_flags,
2805 					(len >> PAGE_SHIFT) - nr_pages))
2806 			return -ENOMEM;
2807 	}
2808 
2809 	/* Unmap any existing mapping in the area */
2810 	error = do_vmi_munmap(&vmi, mm, addr, len, uf, false);
2811 	if (error == -EPERM)
2812 		return error;
2813 	else if (error)
2814 		return -ENOMEM;
2815 
2816 	/*
2817 	 * Private writable mapping: check memory availability
2818 	 */
2819 	if (accountable_mapping(file, vm_flags)) {
2820 		charged = len >> PAGE_SHIFT;
2821 		if (security_vm_enough_memory_mm(mm, charged))
2822 			return -ENOMEM;
2823 		vm_flags |= VM_ACCOUNT;
2824 	}
2825 
2826 	next = vma_next(&vmi);
2827 	prev = vma_prev(&vmi);
2828 	if (vm_flags & VM_SPECIAL) {
2829 		if (prev)
2830 			vma_iter_next_range(&vmi);
2831 		goto cannot_expand;
2832 	}
2833 
2834 	/* Attempt to expand an old mapping */
2835 	/* Check next */
2836 	if (next && next->vm_start == end && !vma_policy(next) &&
2837 	    can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen,
2838 				 NULL_VM_UFFD_CTX, NULL)) {
2839 		merge_end = next->vm_end;
2840 		vma = next;
2841 		vm_pgoff = next->vm_pgoff - pglen;
2842 	}
2843 
2844 	/* Check prev */
2845 	if (prev && prev->vm_end == addr && !vma_policy(prev) &&
2846 	    (vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file,
2847 				       pgoff, vma->vm_userfaultfd_ctx, NULL) :
2848 		   can_vma_merge_after(prev, vm_flags, NULL, file, pgoff,
2849 				       NULL_VM_UFFD_CTX, NULL))) {
2850 		merge_start = prev->vm_start;
2851 		vma = prev;
2852 		vm_pgoff = prev->vm_pgoff;
2853 	} else if (prev) {
2854 		vma_iter_next_range(&vmi);
2855 	}
2856 
2857 	/* Actually expand, if possible */
2858 	if (vma &&
2859 	    !vma_expand(&vmi, vma, merge_start, merge_end, vm_pgoff, next)) {
2860 		khugepaged_enter_vma(vma, vm_flags);
2861 		goto expanded;
2862 	}
2863 
2864 	if (vma == prev)
2865 		vma_iter_set(&vmi, addr);
2866 cannot_expand:
2867 
2868 	/*
2869 	 * Determine the object being mapped and call the appropriate
2870 	 * specific mapper. the address has already been validated, but
2871 	 * not unmapped, but the maps are removed from the list.
2872 	 */
2873 	vma = vm_area_alloc(mm);
2874 	if (!vma) {
2875 		error = -ENOMEM;
2876 		goto unacct_error;
2877 	}
2878 
2879 	vma_iter_config(&vmi, addr, end);
2880 	vma_set_range(vma, addr, end, pgoff);
2881 	vm_flags_init(vma, vm_flags);
2882 	vma->vm_page_prot = vm_get_page_prot(vm_flags);
2883 
2884 	if (file) {
2885 		vma->vm_file = get_file(file);
2886 		error = call_mmap(file, vma);
2887 		if (error)
2888 			goto unmap_and_free_vma;
2889 
2890 		if (vma_is_shared_maywrite(vma)) {
2891 			error = mapping_map_writable(file->f_mapping);
2892 			if (error)
2893 				goto close_and_free_vma;
2894 
2895 			writable_file_mapping = true;
2896 		}
2897 
2898 		/*
2899 		 * Expansion is handled above, merging is handled below.
2900 		 * Drivers should not alter the address of the VMA.
2901 		 */
2902 		error = -EINVAL;
2903 		if (WARN_ON((addr != vma->vm_start)))
2904 			goto close_and_free_vma;
2905 
2906 		vma_iter_config(&vmi, addr, end);
2907 		/*
2908 		 * If vm_flags changed after call_mmap(), we should try merge
2909 		 * vma again as we may succeed this time.
2910 		 */
2911 		if (unlikely(vm_flags != vma->vm_flags && prev)) {
2912 			merge = vma_merge_new_vma(&vmi, prev, vma,
2913 						  vma->vm_start, vma->vm_end,
2914 						  vma->vm_pgoff);
2915 			if (merge) {
2916 				/*
2917 				 * ->mmap() can change vma->vm_file and fput
2918 				 * the original file. So fput the vma->vm_file
2919 				 * here or we would add an extra fput for file
2920 				 * and cause general protection fault
2921 				 * ultimately.
2922 				 */
2923 				fput(vma->vm_file);
2924 				vm_area_free(vma);
2925 				vma = merge;
2926 				/* Update vm_flags to pick up the change. */
2927 				vm_flags = vma->vm_flags;
2928 				goto unmap_writable;
2929 			}
2930 		}
2931 
2932 		vm_flags = vma->vm_flags;
2933 	} else if (vm_flags & VM_SHARED) {
2934 		error = shmem_zero_setup(vma);
2935 		if (error)
2936 			goto free_vma;
2937 	} else {
2938 		vma_set_anonymous(vma);
2939 	}
2940 
2941 	if (map_deny_write_exec(vma, vma->vm_flags)) {
2942 		error = -EACCES;
2943 		goto close_and_free_vma;
2944 	}
2945 
2946 	/* Allow architectures to sanity-check the vm_flags */
2947 	error = -EINVAL;
2948 	if (!arch_validate_flags(vma->vm_flags))
2949 		goto close_and_free_vma;
2950 
2951 	error = -ENOMEM;
2952 	if (vma_iter_prealloc(&vmi, vma))
2953 		goto close_and_free_vma;
2954 
2955 	/* Lock the VMA since it is modified after insertion into VMA tree */
2956 	vma_start_write(vma);
2957 	vma_iter_store(&vmi, vma);
2958 	mm->map_count++;
2959 	vma_link_file(vma);
2960 
2961 	/*
2962 	 * vma_merge() calls khugepaged_enter_vma() either, the below
2963 	 * call covers the non-merge case.
2964 	 */
2965 	khugepaged_enter_vma(vma, vma->vm_flags);
2966 
2967 	/* Once vma denies write, undo our temporary denial count */
2968 unmap_writable:
2969 	if (writable_file_mapping)
2970 		mapping_unmap_writable(file->f_mapping);
2971 	file = vma->vm_file;
2972 	ksm_add_vma(vma);
2973 expanded:
2974 	perf_event_mmap(vma);
2975 
2976 	vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
2977 	if (vm_flags & VM_LOCKED) {
2978 		if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) ||
2979 					is_vm_hugetlb_page(vma) ||
2980 					vma == get_gate_vma(current->mm))
2981 			vm_flags_clear(vma, VM_LOCKED_MASK);
2982 		else
2983 			mm->locked_vm += (len >> PAGE_SHIFT);
2984 	}
2985 
2986 	if (file)
2987 		uprobe_mmap(vma);
2988 
2989 	/*
2990 	 * New (or expanded) vma always get soft dirty status.
2991 	 * Otherwise user-space soft-dirty page tracker won't
2992 	 * be able to distinguish situation when vma area unmapped,
2993 	 * then new mapped in-place (which must be aimed as
2994 	 * a completely new data area).
2995 	 */
2996 	vm_flags_set(vma, VM_SOFTDIRTY);
2997 
2998 	vma_set_page_prot(vma);
2999 
3000 	validate_mm(mm);
3001 	return addr;
3002 
3003 close_and_free_vma:
3004 	if (file && vma->vm_ops && vma->vm_ops->close)
3005 		vma->vm_ops->close(vma);
3006 
3007 	if (file || vma->vm_file) {
3008 unmap_and_free_vma:
3009 		fput(vma->vm_file);
3010 		vma->vm_file = NULL;
3011 
3012 		vma_iter_set(&vmi, vma->vm_end);
3013 		/* Undo any partial mapping done by a device driver. */
3014 		unmap_region(mm, &vmi.mas, vma, prev, next, vma->vm_start,
3015 			     vma->vm_end, vma->vm_end, true);
3016 	}
3017 	if (writable_file_mapping)
3018 		mapping_unmap_writable(file->f_mapping);
3019 free_vma:
3020 	vm_area_free(vma);
3021 unacct_error:
3022 	if (charged)
3023 		vm_unacct_memory(charged);
3024 	validate_mm(mm);
3025 	return error;
3026 }
3027 
__vm_munmap(unsigned long start,size_t len,bool unlock)3028 static int __vm_munmap(unsigned long start, size_t len, bool unlock)
3029 {
3030 	int ret;
3031 	struct mm_struct *mm = current->mm;
3032 	LIST_HEAD(uf);
3033 	VMA_ITERATOR(vmi, mm, start);
3034 
3035 	if (mmap_write_lock_killable(mm))
3036 		return -EINTR;
3037 
3038 	ret = do_vmi_munmap(&vmi, mm, start, len, &uf, unlock);
3039 	if (ret || !unlock)
3040 		mmap_write_unlock(mm);
3041 
3042 	userfaultfd_unmap_complete(mm, &uf);
3043 	return ret;
3044 }
3045 
vm_munmap(unsigned long start,size_t len)3046 int vm_munmap(unsigned long start, size_t len)
3047 {
3048 	return __vm_munmap(start, len, false);
3049 }
3050 EXPORT_SYMBOL(vm_munmap);
3051 
SYSCALL_DEFINE2(munmap,unsigned long,addr,size_t,len)3052 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
3053 {
3054 	addr = untagged_addr(addr);
3055 	return __vm_munmap(addr, len, true);
3056 }
3057 
3058 
3059 /*
3060  * Emulation of deprecated remap_file_pages() syscall.
3061  */
SYSCALL_DEFINE5(remap_file_pages,unsigned long,start,unsigned long,size,unsigned long,prot,unsigned long,pgoff,unsigned long,flags)3062 SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
3063 		unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
3064 {
3065 
3066 	struct mm_struct *mm = current->mm;
3067 	struct vm_area_struct *vma;
3068 	unsigned long populate = 0;
3069 	unsigned long ret = -EINVAL;
3070 	struct file *file;
3071 
3072 	pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/mm/remap_file_pages.rst.\n",
3073 		     current->comm, current->pid);
3074 
3075 	if (prot)
3076 		return ret;
3077 	start = start & PAGE_MASK;
3078 	size = size & PAGE_MASK;
3079 
3080 	if (start + size <= start)
3081 		return ret;
3082 
3083 	/* Does pgoff wrap? */
3084 	if (pgoff + (size >> PAGE_SHIFT) < pgoff)
3085 		return ret;
3086 
3087 	if (mmap_write_lock_killable(mm))
3088 		return -EINTR;
3089 
3090 	vma = vma_lookup(mm, start);
3091 
3092 	if (!vma || !(vma->vm_flags & VM_SHARED))
3093 		goto out;
3094 
3095 	if (start + size > vma->vm_end) {
3096 		VMA_ITERATOR(vmi, mm, vma->vm_end);
3097 		struct vm_area_struct *next, *prev = vma;
3098 
3099 		for_each_vma_range(vmi, next, start + size) {
3100 			/* hole between vmas ? */
3101 			if (next->vm_start != prev->vm_end)
3102 				goto out;
3103 
3104 			if (next->vm_file != vma->vm_file)
3105 				goto out;
3106 
3107 			if (next->vm_flags != vma->vm_flags)
3108 				goto out;
3109 
3110 			if (start + size <= next->vm_end)
3111 				break;
3112 
3113 			prev = next;
3114 		}
3115 
3116 		if (!next)
3117 			goto out;
3118 	}
3119 
3120 	prot |= vma->vm_flags & VM_READ ? PROT_READ : 0;
3121 	prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0;
3122 	prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0;
3123 
3124 	flags &= MAP_NONBLOCK;
3125 	flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
3126 	if (vma->vm_flags & VM_LOCKED)
3127 		flags |= MAP_LOCKED;
3128 
3129 	file = get_file(vma->vm_file);
3130 	ret = do_mmap(vma->vm_file, start, size,
3131 			prot, flags, 0, pgoff, &populate, NULL);
3132 	fput(file);
3133 out:
3134 	mmap_write_unlock(mm);
3135 	if (populate)
3136 		mm_populate(ret, populate);
3137 	if (!IS_ERR_VALUE(ret))
3138 		ret = 0;
3139 	return ret;
3140 }
3141 
3142 /*
3143  * do_vma_munmap() - Unmap a full or partial vma.
3144  * @vmi: The vma iterator pointing at the vma
3145  * @vma: The first vma to be munmapped
3146  * @start: the start of the address to unmap
3147  * @end: The end of the address to unmap
3148  * @uf: The userfaultfd list_head
3149  * @unlock: Drop the lock on success
3150  *
3151  * unmaps a VMA mapping when the vma iterator is already in position.
3152  * Does not handle alignment.
3153  *
3154  * Return: 0 on success drops the lock of so directed, error on failure and will
3155  * still hold the lock.
3156  */
do_vma_munmap(struct vma_iterator * vmi,struct vm_area_struct * vma,unsigned long start,unsigned long end,struct list_head * uf,bool unlock)3157 int do_vma_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
3158 		unsigned long start, unsigned long end, struct list_head *uf,
3159 		bool unlock)
3160 {
3161 	struct mm_struct *mm = vma->vm_mm;
3162 
3163 	/*
3164 	 * Check if memory is sealed before arch_unmap.
3165 	 * Prevent unmapping a sealed VMA.
3166 	 * can_modify_mm assumes we have acquired the lock on MM.
3167 	 */
3168 	if (unlikely(!can_modify_mm(mm, start, end)))
3169 		return -EPERM;
3170 
3171 	arch_unmap(mm, start, end);
3172 	return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock);
3173 }
3174 
3175 /*
3176  * do_brk_flags() - Increase the brk vma if the flags match.
3177  * @vmi: The vma iterator
3178  * @addr: The start address
3179  * @len: The length of the increase
3180  * @vma: The vma,
3181  * @flags: The VMA Flags
3182  *
3183  * Extend the brk VMA from addr to addr + len.  If the VMA is NULL or the flags
3184  * do not match then create a new anonymous VMA.  Eventually we may be able to
3185  * do some brk-specific accounting here.
3186  */
do_brk_flags(struct vma_iterator * vmi,struct vm_area_struct * vma,unsigned long addr,unsigned long len,unsigned long flags)3187 static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma,
3188 		unsigned long addr, unsigned long len, unsigned long flags)
3189 {
3190 	struct mm_struct *mm = current->mm;
3191 	struct vma_prepare vp;
3192 
3193 	/*
3194 	 * Check against address space limits by the changed size
3195 	 * Note: This happens *after* clearing old mappings in some code paths.
3196 	 */
3197 	flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
3198 	if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT))
3199 		return -ENOMEM;
3200 
3201 	if (mm->map_count > sysctl_max_map_count)
3202 		return -ENOMEM;
3203 
3204 	if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
3205 		return -ENOMEM;
3206 
3207 	/*
3208 	 * Expand the existing vma if possible; Note that singular lists do not
3209 	 * occur after forking, so the expand will only happen on new VMAs.
3210 	 */
3211 	if (vma && vma->vm_end == addr && !vma_policy(vma) &&
3212 	    can_vma_merge_after(vma, flags, NULL, NULL,
3213 				addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL)) {
3214 		vma_iter_config(vmi, vma->vm_start, addr + len);
3215 		if (vma_iter_prealloc(vmi, vma))
3216 			goto unacct_fail;
3217 
3218 		vma_start_write(vma);
3219 
3220 		init_vma_prep(&vp, vma);
3221 		vma_prepare(&vp);
3222 		vma_adjust_trans_huge(vma, vma->vm_start, addr + len, 0);
3223 		vma->vm_end = addr + len;
3224 		vm_flags_set(vma, VM_SOFTDIRTY);
3225 		vma_iter_store(vmi, vma);
3226 
3227 		vma_complete(&vp, vmi, mm);
3228 		khugepaged_enter_vma(vma, flags);
3229 		goto out;
3230 	}
3231 
3232 	if (vma)
3233 		vma_iter_next_range(vmi);
3234 	/* create a vma struct for an anonymous mapping */
3235 	vma = vm_area_alloc(mm);
3236 	if (!vma)
3237 		goto unacct_fail;
3238 
3239 	vma_set_anonymous(vma);
3240 	vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT);
3241 	vm_flags_init(vma, flags);
3242 	vma->vm_page_prot = vm_get_page_prot(flags);
3243 	vma_start_write(vma);
3244 	if (vma_iter_store_gfp(vmi, vma, GFP_KERNEL))
3245 		goto mas_store_fail;
3246 
3247 	mm->map_count++;
3248 	validate_mm(mm);
3249 	ksm_add_vma(vma);
3250 out:
3251 	perf_event_mmap(vma);
3252 	mm->total_vm += len >> PAGE_SHIFT;
3253 	mm->data_vm += len >> PAGE_SHIFT;
3254 	if (flags & VM_LOCKED)
3255 		mm->locked_vm += (len >> PAGE_SHIFT);
3256 	vm_flags_set(vma, VM_SOFTDIRTY);
3257 	return 0;
3258 
3259 mas_store_fail:
3260 	vm_area_free(vma);
3261 unacct_fail:
3262 	vm_unacct_memory(len >> PAGE_SHIFT);
3263 	return -ENOMEM;
3264 }
3265 
vm_brk_flags(unsigned long addr,unsigned long request,unsigned long flags)3266 int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags)
3267 {
3268 	struct mm_struct *mm = current->mm;
3269 	struct vm_area_struct *vma = NULL;
3270 	unsigned long len;
3271 	int ret;
3272 	bool populate;
3273 	LIST_HEAD(uf);
3274 	VMA_ITERATOR(vmi, mm, addr);
3275 
3276 	len = PAGE_ALIGN(request);
3277 	if (len < request)
3278 		return -ENOMEM;
3279 	if (!len)
3280 		return 0;
3281 
3282 	/* Until we need other flags, refuse anything except VM_EXEC. */
3283 	if ((flags & (~VM_EXEC)) != 0)
3284 		return -EINVAL;
3285 
3286 	if (mmap_write_lock_killable(mm))
3287 		return -EINTR;
3288 
3289 	ret = check_brk_limits(addr, len);
3290 	if (ret)
3291 		goto limits_failed;
3292 
3293 	ret = do_vmi_munmap(&vmi, mm, addr, len, &uf, 0);
3294 	if (ret)
3295 		goto munmap_failed;
3296 
3297 	vma = vma_prev(&vmi);
3298 	ret = do_brk_flags(&vmi, vma, addr, len, flags);
3299 	populate = ((mm->def_flags & VM_LOCKED) != 0);
3300 	mmap_write_unlock(mm);
3301 	userfaultfd_unmap_complete(mm, &uf);
3302 	if (populate && !ret)
3303 		mm_populate(addr, len);
3304 	return ret;
3305 
3306 munmap_failed:
3307 limits_failed:
3308 	mmap_write_unlock(mm);
3309 	return ret;
3310 }
3311 EXPORT_SYMBOL(vm_brk_flags);
3312 
3313 /* Release all mmaps. */
exit_mmap(struct mm_struct * mm)3314 void exit_mmap(struct mm_struct *mm)
3315 {
3316 	struct mmu_gather tlb;
3317 	struct vm_area_struct *vma;
3318 	unsigned long nr_accounted = 0;
3319 	VMA_ITERATOR(vmi, mm, 0);
3320 	int count = 0;
3321 
3322 	/* mm's last user has gone, and its about to be pulled down */
3323 	mmu_notifier_release(mm);
3324 
3325 	mmap_read_lock(mm);
3326 	arch_exit_mmap(mm);
3327 
3328 	vma = vma_next(&vmi);
3329 	if (!vma || unlikely(xa_is_zero(vma))) {
3330 		/* Can happen if dup_mmap() received an OOM */
3331 		mmap_read_unlock(mm);
3332 		mmap_write_lock(mm);
3333 		goto destroy;
3334 	}
3335 
3336 	lru_add_drain();
3337 	flush_cache_mm(mm);
3338 	tlb_gather_mmu_fullmm(&tlb, mm);
3339 	/* update_hiwater_rss(mm) here? but nobody should be looking */
3340 	/* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */
3341 	unmap_vmas(&tlb, &vmi.mas, vma, 0, ULONG_MAX, ULONG_MAX, false);
3342 	mmap_read_unlock(mm);
3343 
3344 	/*
3345 	 * Set MMF_OOM_SKIP to hide this task from the oom killer/reaper
3346 	 * because the memory has been already freed.
3347 	 */
3348 	set_bit(MMF_OOM_SKIP, &mm->flags);
3349 	mmap_write_lock(mm);
3350 	mt_clear_in_rcu(&mm->mm_mt);
3351 	vma_iter_set(&vmi, vma->vm_end);
3352 	free_pgtables(&tlb, &vmi.mas, vma, FIRST_USER_ADDRESS,
3353 		      USER_PGTABLES_CEILING, true);
3354 	tlb_finish_mmu(&tlb);
3355 
3356 	/*
3357 	 * Walk the list again, actually closing and freeing it, with preemption
3358 	 * enabled, without holding any MM locks besides the unreachable
3359 	 * mmap_write_lock.
3360 	 */
3361 	vma_iter_set(&vmi, vma->vm_end);
3362 	do {
3363 		if (vma->vm_flags & VM_ACCOUNT)
3364 			nr_accounted += vma_pages(vma);
3365 		remove_vma(vma, true);
3366 		count++;
3367 		cond_resched();
3368 		vma = vma_next(&vmi);
3369 	} while (vma && likely(!xa_is_zero(vma)));
3370 
3371 	BUG_ON(count != mm->map_count);
3372 
3373 	trace_exit_mmap(mm);
3374 destroy:
3375 	__mt_destroy(&mm->mm_mt);
3376 	mmap_write_unlock(mm);
3377 	vm_unacct_memory(nr_accounted);
3378 }
3379 
3380 /* Insert vm structure into process list sorted by address
3381  * and into the inode's i_mmap tree.  If vm_file is non-NULL
3382  * then i_mmap_rwsem is taken here.
3383  */
insert_vm_struct(struct mm_struct * mm,struct vm_area_struct * vma)3384 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
3385 {
3386 	unsigned long charged = vma_pages(vma);
3387 
3388 
3389 	if (find_vma_intersection(mm, vma->vm_start, vma->vm_end))
3390 		return -ENOMEM;
3391 
3392 	if ((vma->vm_flags & VM_ACCOUNT) &&
3393 	     security_vm_enough_memory_mm(mm, charged))
3394 		return -ENOMEM;
3395 
3396 	/*
3397 	 * The vm_pgoff of a purely anonymous vma should be irrelevant
3398 	 * until its first write fault, when page's anon_vma and index
3399 	 * are set.  But now set the vm_pgoff it will almost certainly
3400 	 * end up with (unless mremap moves it elsewhere before that
3401 	 * first wfault), so /proc/pid/maps tells a consistent story.
3402 	 *
3403 	 * By setting it to reflect the virtual start address of the
3404 	 * vma, merges and splits can happen in a seamless way, just
3405 	 * using the existing file pgoff checks and manipulations.
3406 	 * Similarly in do_mmap and in do_brk_flags.
3407 	 */
3408 	if (vma_is_anonymous(vma)) {
3409 		BUG_ON(vma->anon_vma);
3410 		vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
3411 	}
3412 
3413 	if (vma_link(mm, vma)) {
3414 		if (vma->vm_flags & VM_ACCOUNT)
3415 			vm_unacct_memory(charged);
3416 		return -ENOMEM;
3417 	}
3418 
3419 	return 0;
3420 }
3421 
3422 /*
3423  * Copy the vma structure to a new location in the same mm,
3424  * prior to moving page table entries, to effect an mremap move.
3425  */
copy_vma(struct vm_area_struct ** vmap,unsigned long addr,unsigned long len,pgoff_t pgoff,bool * need_rmap_locks)3426 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
3427 	unsigned long addr, unsigned long len, pgoff_t pgoff,
3428 	bool *need_rmap_locks)
3429 {
3430 	struct vm_area_struct *vma = *vmap;
3431 	unsigned long vma_start = vma->vm_start;
3432 	struct mm_struct *mm = vma->vm_mm;
3433 	struct vm_area_struct *new_vma, *prev;
3434 	bool faulted_in_anon_vma = true;
3435 	VMA_ITERATOR(vmi, mm, addr);
3436 
3437 	/*
3438 	 * If anonymous vma has not yet been faulted, update new pgoff
3439 	 * to match new location, to increase its chance of merging.
3440 	 */
3441 	if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
3442 		pgoff = addr >> PAGE_SHIFT;
3443 		faulted_in_anon_vma = false;
3444 	}
3445 
3446 	new_vma = find_vma_prev(mm, addr, &prev);
3447 	if (new_vma && new_vma->vm_start < addr + len)
3448 		return NULL;	/* should never get here */
3449 
3450 	new_vma = vma_merge_new_vma(&vmi, prev, vma, addr, addr + len, pgoff);
3451 	if (new_vma) {
3452 		/*
3453 		 * Source vma may have been merged into new_vma
3454 		 */
3455 		if (unlikely(vma_start >= new_vma->vm_start &&
3456 			     vma_start < new_vma->vm_end)) {
3457 			/*
3458 			 * The only way we can get a vma_merge with
3459 			 * self during an mremap is if the vma hasn't
3460 			 * been faulted in yet and we were allowed to
3461 			 * reset the dst vma->vm_pgoff to the
3462 			 * destination address of the mremap to allow
3463 			 * the merge to happen. mremap must change the
3464 			 * vm_pgoff linearity between src and dst vmas
3465 			 * (in turn preventing a vma_merge) to be
3466 			 * safe. It is only safe to keep the vm_pgoff
3467 			 * linear if there are no pages mapped yet.
3468 			 */
3469 			VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
3470 			*vmap = vma = new_vma;
3471 		}
3472 		*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
3473 	} else {
3474 		new_vma = vm_area_dup(vma);
3475 		if (!new_vma)
3476 			goto out;
3477 		vma_set_range(new_vma, addr, addr + len, pgoff);
3478 		if (vma_dup_policy(vma, new_vma))
3479 			goto out_free_vma;
3480 		if (anon_vma_clone(new_vma, vma))
3481 			goto out_free_mempol;
3482 		if (new_vma->vm_file)
3483 			get_file(new_vma->vm_file);
3484 		if (new_vma->vm_ops && new_vma->vm_ops->open)
3485 			new_vma->vm_ops->open(new_vma);
3486 		if (vma_link(mm, new_vma))
3487 			goto out_vma_link;
3488 		*need_rmap_locks = false;
3489 	}
3490 	return new_vma;
3491 
3492 out_vma_link:
3493 	if (new_vma->vm_ops && new_vma->vm_ops->close)
3494 		new_vma->vm_ops->close(new_vma);
3495 
3496 	if (new_vma->vm_file)
3497 		fput(new_vma->vm_file);
3498 
3499 	unlink_anon_vmas(new_vma);
3500 out_free_mempol:
3501 	mpol_put(vma_policy(new_vma));
3502 out_free_vma:
3503 	vm_area_free(new_vma);
3504 out:
3505 	return NULL;
3506 }
3507 
3508 /*
3509  * Return true if the calling process may expand its vm space by the passed
3510  * number of pages
3511  */
may_expand_vm(struct mm_struct * mm,vm_flags_t flags,unsigned long npages)3512 bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages)
3513 {
3514 	if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT)
3515 		return false;
3516 
3517 	if (is_data_mapping(flags) &&
3518 	    mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {
3519 		/* Workaround for Valgrind */
3520 		if (rlimit(RLIMIT_DATA) == 0 &&
3521 		    mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT)
3522 			return true;
3523 
3524 		pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n",
3525 			     current->comm, current->pid,
3526 			     (mm->data_vm + npages) << PAGE_SHIFT,
3527 			     rlimit(RLIMIT_DATA),
3528 			     ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data");
3529 
3530 		if (!ignore_rlimit_data)
3531 			return false;
3532 	}
3533 
3534 	return true;
3535 }
3536 
vm_stat_account(struct mm_struct * mm,vm_flags_t flags,long npages)3537 void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages)
3538 {
3539 	WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm)+npages);
3540 
3541 	if (is_exec_mapping(flags))
3542 		mm->exec_vm += npages;
3543 	else if (is_stack_mapping(flags))
3544 		mm->stack_vm += npages;
3545 	else if (is_data_mapping(flags))
3546 		mm->data_vm += npages;
3547 }
3548 
3549 static vm_fault_t special_mapping_fault(struct vm_fault *vmf);
3550 
3551 /*
3552  * Having a close hook prevents vma merging regardless of flags.
3553  */
special_mapping_close(struct vm_area_struct * vma)3554 static void special_mapping_close(struct vm_area_struct *vma)
3555 {
3556 }
3557 
special_mapping_name(struct vm_area_struct * vma)3558 static const char *special_mapping_name(struct vm_area_struct *vma)
3559 {
3560 	return ((struct vm_special_mapping *)vma->vm_private_data)->name;
3561 }
3562 
special_mapping_mremap(struct vm_area_struct * new_vma)3563 static int special_mapping_mremap(struct vm_area_struct *new_vma)
3564 {
3565 	struct vm_special_mapping *sm = new_vma->vm_private_data;
3566 
3567 	if (WARN_ON_ONCE(current->mm != new_vma->vm_mm))
3568 		return -EFAULT;
3569 
3570 	if (sm->mremap)
3571 		return sm->mremap(sm, new_vma);
3572 
3573 	return 0;
3574 }
3575 
special_mapping_split(struct vm_area_struct * vma,unsigned long addr)3576 static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr)
3577 {
3578 	/*
3579 	 * Forbid splitting special mappings - kernel has expectations over
3580 	 * the number of pages in mapping. Together with VM_DONTEXPAND
3581 	 * the size of vma should stay the same over the special mapping's
3582 	 * lifetime.
3583 	 */
3584 	return -EINVAL;
3585 }
3586 
3587 static const struct vm_operations_struct special_mapping_vmops = {
3588 	.close = special_mapping_close,
3589 	.fault = special_mapping_fault,
3590 	.mremap = special_mapping_mremap,
3591 	.name = special_mapping_name,
3592 	/* vDSO code relies that VVAR can't be accessed remotely */
3593 	.access = NULL,
3594 	.may_split = special_mapping_split,
3595 };
3596 
3597 static const struct vm_operations_struct legacy_special_mapping_vmops = {
3598 	.close = special_mapping_close,
3599 	.fault = special_mapping_fault,
3600 };
3601 
special_mapping_fault(struct vm_fault * vmf)3602 static vm_fault_t special_mapping_fault(struct vm_fault *vmf)
3603 {
3604 	struct vm_area_struct *vma = vmf->vma;
3605 	pgoff_t pgoff;
3606 	struct page **pages;
3607 
3608 	if (vma->vm_ops == &legacy_special_mapping_vmops) {
3609 		pages = vma->vm_private_data;
3610 	} else {
3611 		struct vm_special_mapping *sm = vma->vm_private_data;
3612 
3613 		if (sm->fault)
3614 			return sm->fault(sm, vmf->vma, vmf);
3615 
3616 		pages = sm->pages;
3617 	}
3618 
3619 	for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
3620 		pgoff--;
3621 
3622 	if (*pages) {
3623 		struct page *page = *pages;
3624 		get_page(page);
3625 		vmf->page = page;
3626 		return 0;
3627 	}
3628 
3629 	return VM_FAULT_SIGBUS;
3630 }
3631 
__install_special_mapping(struct mm_struct * mm,unsigned long addr,unsigned long len,unsigned long vm_flags,void * priv,const struct vm_operations_struct * ops)3632 static struct vm_area_struct *__install_special_mapping(
3633 	struct mm_struct *mm,
3634 	unsigned long addr, unsigned long len,
3635 	unsigned long vm_flags, void *priv,
3636 	const struct vm_operations_struct *ops)
3637 {
3638 	int ret;
3639 	struct vm_area_struct *vma;
3640 
3641 	vma = vm_area_alloc(mm);
3642 	if (unlikely(vma == NULL))
3643 		return ERR_PTR(-ENOMEM);
3644 
3645 	vma_set_range(vma, addr, addr + len, 0);
3646 	vm_flags_init(vma, (vm_flags | mm->def_flags |
3647 		      VM_DONTEXPAND | VM_SOFTDIRTY) & ~VM_LOCKED_MASK);
3648 	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
3649 
3650 	vma->vm_ops = ops;
3651 	vma->vm_private_data = priv;
3652 
3653 	ret = insert_vm_struct(mm, vma);
3654 	if (ret)
3655 		goto out;
3656 
3657 	vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
3658 
3659 	perf_event_mmap(vma);
3660 
3661 	return vma;
3662 
3663 out:
3664 	vm_area_free(vma);
3665 	return ERR_PTR(ret);
3666 }
3667 
vma_is_special_mapping(const struct vm_area_struct * vma,const struct vm_special_mapping * sm)3668 bool vma_is_special_mapping(const struct vm_area_struct *vma,
3669 	const struct vm_special_mapping *sm)
3670 {
3671 	return vma->vm_private_data == sm &&
3672 		(vma->vm_ops == &special_mapping_vmops ||
3673 		 vma->vm_ops == &legacy_special_mapping_vmops);
3674 }
3675 
3676 /*
3677  * Called with mm->mmap_lock held for writing.
3678  * Insert a new vma covering the given region, with the given flags.
3679  * Its pages are supplied by the given array of struct page *.
3680  * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
3681  * The region past the last page supplied will always produce SIGBUS.
3682  * The array pointer and the pages it points to are assumed to stay alive
3683  * for as long as this mapping might exist.
3684  */
_install_special_mapping(struct mm_struct * mm,unsigned long addr,unsigned long len,unsigned long vm_flags,const struct vm_special_mapping * spec)3685 struct vm_area_struct *_install_special_mapping(
3686 	struct mm_struct *mm,
3687 	unsigned long addr, unsigned long len,
3688 	unsigned long vm_flags, const struct vm_special_mapping *spec)
3689 {
3690 	return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec,
3691 					&special_mapping_vmops);
3692 }
3693 
install_special_mapping(struct mm_struct * mm,unsigned long addr,unsigned long len,unsigned long vm_flags,struct page ** pages)3694 int install_special_mapping(struct mm_struct *mm,
3695 			    unsigned long addr, unsigned long len,
3696 			    unsigned long vm_flags, struct page **pages)
3697 {
3698 	struct vm_area_struct *vma = __install_special_mapping(
3699 		mm, addr, len, vm_flags, (void *)pages,
3700 		&legacy_special_mapping_vmops);
3701 
3702 	return PTR_ERR_OR_ZERO(vma);
3703 }
3704 
3705 static DEFINE_MUTEX(mm_all_locks_mutex);
3706 
vm_lock_anon_vma(struct mm_struct * mm,struct anon_vma * anon_vma)3707 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
3708 {
3709 	if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
3710 		/*
3711 		 * The LSB of head.next can't change from under us
3712 		 * because we hold the mm_all_locks_mutex.
3713 		 */
3714 		down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock);
3715 		/*
3716 		 * We can safely modify head.next after taking the
3717 		 * anon_vma->root->rwsem. If some other vma in this mm shares
3718 		 * the same anon_vma we won't take it again.
3719 		 *
3720 		 * No need of atomic instructions here, head.next
3721 		 * can't change from under us thanks to the
3722 		 * anon_vma->root->rwsem.
3723 		 */
3724 		if (__test_and_set_bit(0, (unsigned long *)
3725 				       &anon_vma->root->rb_root.rb_root.rb_node))
3726 			BUG();
3727 	}
3728 }
3729 
vm_lock_mapping(struct mm_struct * mm,struct address_space * mapping)3730 static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
3731 {
3732 	if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
3733 		/*
3734 		 * AS_MM_ALL_LOCKS can't change from under us because
3735 		 * we hold the mm_all_locks_mutex.
3736 		 *
3737 		 * Operations on ->flags have to be atomic because
3738 		 * even if AS_MM_ALL_LOCKS is stable thanks to the
3739 		 * mm_all_locks_mutex, there may be other cpus
3740 		 * changing other bitflags in parallel to us.
3741 		 */
3742 		if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
3743 			BUG();
3744 		down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock);
3745 	}
3746 }
3747 
3748 /*
3749  * This operation locks against the VM for all pte/vma/mm related
3750  * operations that could ever happen on a certain mm. This includes
3751  * vmtruncate, try_to_unmap, and all page faults.
3752  *
3753  * The caller must take the mmap_lock in write mode before calling
3754  * mm_take_all_locks(). The caller isn't allowed to release the
3755  * mmap_lock until mm_drop_all_locks() returns.
3756  *
3757  * mmap_lock in write mode is required in order to block all operations
3758  * that could modify pagetables and free pages without need of
3759  * altering the vma layout. It's also needed in write mode to avoid new
3760  * anon_vmas to be associated with existing vmas.
3761  *
3762  * A single task can't take more than one mm_take_all_locks() in a row
3763  * or it would deadlock.
3764  *
3765  * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
3766  * mapping->flags avoid to take the same lock twice, if more than one
3767  * vma in this mm is backed by the same anon_vma or address_space.
3768  *
3769  * We take locks in following order, accordingly to comment at beginning
3770  * of mm/rmap.c:
3771  *   - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for
3772  *     hugetlb mapping);
3773  *   - all vmas marked locked
3774  *   - all i_mmap_rwsem locks;
3775  *   - all anon_vma->rwseml
3776  *
3777  * We can take all locks within these types randomly because the VM code
3778  * doesn't nest them and we protected from parallel mm_take_all_locks() by
3779  * mm_all_locks_mutex.
3780  *
3781  * mm_take_all_locks() and mm_drop_all_locks are expensive operations
3782  * that may have to take thousand of locks.
3783  *
3784  * mm_take_all_locks() can fail if it's interrupted by signals.
3785  */
mm_take_all_locks(struct mm_struct * mm)3786 int mm_take_all_locks(struct mm_struct *mm)
3787 {
3788 	struct vm_area_struct *vma;
3789 	struct anon_vma_chain *avc;
3790 	VMA_ITERATOR(vmi, mm, 0);
3791 
3792 	mmap_assert_write_locked(mm);
3793 
3794 	mutex_lock(&mm_all_locks_mutex);
3795 
3796 	/*
3797 	 * vma_start_write() does not have a complement in mm_drop_all_locks()
3798 	 * because vma_start_write() is always asymmetrical; it marks a VMA as
3799 	 * being written to until mmap_write_unlock() or mmap_write_downgrade()
3800 	 * is reached.
3801 	 */
3802 	for_each_vma(vmi, vma) {
3803 		if (signal_pending(current))
3804 			goto out_unlock;
3805 		vma_start_write(vma);
3806 	}
3807 
3808 	vma_iter_init(&vmi, mm, 0);
3809 	for_each_vma(vmi, vma) {
3810 		if (signal_pending(current))
3811 			goto out_unlock;
3812 		if (vma->vm_file && vma->vm_file->f_mapping &&
3813 				is_vm_hugetlb_page(vma))
3814 			vm_lock_mapping(mm, vma->vm_file->f_mapping);
3815 	}
3816 
3817 	vma_iter_init(&vmi, mm, 0);
3818 	for_each_vma(vmi, vma) {
3819 		if (signal_pending(current))
3820 			goto out_unlock;
3821 		if (vma->vm_file && vma->vm_file->f_mapping &&
3822 				!is_vm_hugetlb_page(vma))
3823 			vm_lock_mapping(mm, vma->vm_file->f_mapping);
3824 	}
3825 
3826 	vma_iter_init(&vmi, mm, 0);
3827 	for_each_vma(vmi, vma) {
3828 		if (signal_pending(current))
3829 			goto out_unlock;
3830 		if (vma->anon_vma)
3831 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3832 				vm_lock_anon_vma(mm, avc->anon_vma);
3833 	}
3834 
3835 	return 0;
3836 
3837 out_unlock:
3838 	mm_drop_all_locks(mm);
3839 	return -EINTR;
3840 }
3841 
vm_unlock_anon_vma(struct anon_vma * anon_vma)3842 static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
3843 {
3844 	if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
3845 		/*
3846 		 * The LSB of head.next can't change to 0 from under
3847 		 * us because we hold the mm_all_locks_mutex.
3848 		 *
3849 		 * We must however clear the bitflag before unlocking
3850 		 * the vma so the users using the anon_vma->rb_root will
3851 		 * never see our bitflag.
3852 		 *
3853 		 * No need of atomic instructions here, head.next
3854 		 * can't change from under us until we release the
3855 		 * anon_vma->root->rwsem.
3856 		 */
3857 		if (!__test_and_clear_bit(0, (unsigned long *)
3858 					  &anon_vma->root->rb_root.rb_root.rb_node))
3859 			BUG();
3860 		anon_vma_unlock_write(anon_vma);
3861 	}
3862 }
3863 
vm_unlock_mapping(struct address_space * mapping)3864 static void vm_unlock_mapping(struct address_space *mapping)
3865 {
3866 	if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
3867 		/*
3868 		 * AS_MM_ALL_LOCKS can't change to 0 from under us
3869 		 * because we hold the mm_all_locks_mutex.
3870 		 */
3871 		i_mmap_unlock_write(mapping);
3872 		if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
3873 					&mapping->flags))
3874 			BUG();
3875 	}
3876 }
3877 
3878 /*
3879  * The mmap_lock cannot be released by the caller until
3880  * mm_drop_all_locks() returns.
3881  */
mm_drop_all_locks(struct mm_struct * mm)3882 void mm_drop_all_locks(struct mm_struct *mm)
3883 {
3884 	struct vm_area_struct *vma;
3885 	struct anon_vma_chain *avc;
3886 	VMA_ITERATOR(vmi, mm, 0);
3887 
3888 	mmap_assert_write_locked(mm);
3889 	BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
3890 
3891 	for_each_vma(vmi, vma) {
3892 		if (vma->anon_vma)
3893 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3894 				vm_unlock_anon_vma(avc->anon_vma);
3895 		if (vma->vm_file && vma->vm_file->f_mapping)
3896 			vm_unlock_mapping(vma->vm_file->f_mapping);
3897 	}
3898 
3899 	mutex_unlock(&mm_all_locks_mutex);
3900 }
3901 
3902 /*
3903  * initialise the percpu counter for VM
3904  */
mmap_init(void)3905 void __init mmap_init(void)
3906 {
3907 	int ret;
3908 
3909 	ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
3910 	VM_BUG_ON(ret);
3911 }
3912 
3913 /*
3914  * Initialise sysctl_user_reserve_kbytes.
3915  *
3916  * This is intended to prevent a user from starting a single memory hogging
3917  * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
3918  * mode.
3919  *
3920  * The default value is min(3% of free memory, 128MB)
3921  * 128MB is enough to recover with sshd/login, bash, and top/kill.
3922  */
init_user_reserve(void)3923 static int init_user_reserve(void)
3924 {
3925 	unsigned long free_kbytes;
3926 
3927 	free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
3928 
3929 	sysctl_user_reserve_kbytes = min(free_kbytes / 32, SZ_128K);
3930 	return 0;
3931 }
3932 subsys_initcall(init_user_reserve);
3933 
3934 /*
3935  * Initialise sysctl_admin_reserve_kbytes.
3936  *
3937  * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
3938  * to log in and kill a memory hogging process.
3939  *
3940  * Systems with more than 256MB will reserve 8MB, enough to recover
3941  * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
3942  * only reserve 3% of free pages by default.
3943  */
init_admin_reserve(void)3944 static int init_admin_reserve(void)
3945 {
3946 	unsigned long free_kbytes;
3947 
3948 	free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
3949 
3950 	sysctl_admin_reserve_kbytes = min(free_kbytes / 32, SZ_8K);
3951 	return 0;
3952 }
3953 subsys_initcall(init_admin_reserve);
3954 
3955 /*
3956  * Reinititalise user and admin reserves if memory is added or removed.
3957  *
3958  * The default user reserve max is 128MB, and the default max for the
3959  * admin reserve is 8MB. These are usually, but not always, enough to
3960  * enable recovery from a memory hogging process using login/sshd, a shell,
3961  * and tools like top. It may make sense to increase or even disable the
3962  * reserve depending on the existence of swap or variations in the recovery
3963  * tools. So, the admin may have changed them.
3964  *
3965  * If memory is added and the reserves have been eliminated or increased above
3966  * the default max, then we'll trust the admin.
3967  *
3968  * If memory is removed and there isn't enough free memory, then we
3969  * need to reset the reserves.
3970  *
3971  * Otherwise keep the reserve set by the admin.
3972  */
reserve_mem_notifier(struct notifier_block * nb,unsigned long action,void * data)3973 static int reserve_mem_notifier(struct notifier_block *nb,
3974 			     unsigned long action, void *data)
3975 {
3976 	unsigned long tmp, free_kbytes;
3977 
3978 	switch (action) {
3979 	case MEM_ONLINE:
3980 		/* Default max is 128MB. Leave alone if modified by operator. */
3981 		tmp = sysctl_user_reserve_kbytes;
3982 		if (tmp > 0 && tmp < SZ_128K)
3983 			init_user_reserve();
3984 
3985 		/* Default max is 8MB.  Leave alone if modified by operator. */
3986 		tmp = sysctl_admin_reserve_kbytes;
3987 		if (tmp > 0 && tmp < SZ_8K)
3988 			init_admin_reserve();
3989 
3990 		break;
3991 	case MEM_OFFLINE:
3992 		free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
3993 
3994 		if (sysctl_user_reserve_kbytes > free_kbytes) {
3995 			init_user_reserve();
3996 			pr_info("vm.user_reserve_kbytes reset to %lu\n",
3997 				sysctl_user_reserve_kbytes);
3998 		}
3999 
4000 		if (sysctl_admin_reserve_kbytes > free_kbytes) {
4001 			init_admin_reserve();
4002 			pr_info("vm.admin_reserve_kbytes reset to %lu\n",
4003 				sysctl_admin_reserve_kbytes);
4004 		}
4005 		break;
4006 	default:
4007 		break;
4008 	}
4009 	return NOTIFY_OK;
4010 }
4011 
init_reserve_notifier(void)4012 static int __meminit init_reserve_notifier(void)
4013 {
4014 	if (hotplug_memory_notifier(reserve_mem_notifier, DEFAULT_CALLBACK_PRI))
4015 		pr_err("Failed registering memory add/remove notifier for admin reserve\n");
4016 
4017 	return 0;
4018 }
4019 subsys_initcall(init_reserve_notifier);
4020