xref: /linux/mm/rmap.c (revision 2d9b9352ef885837f246225e9b814a05e9ed2a4a)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * mm/rmap.c - physical to virtual reverse mappings
4  *
5  * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
6  *
7  * Simple, low overhead reverse mapping scheme.
8  * Please try to keep this thing as modular as possible.
9  *
10  * Provides methods for unmapping each kind of mapped page:
11  * the anon methods track anonymous pages, and
12  * the file methods track pages belonging to an inode.
13  *
14  * Original design by Rik van Riel <riel@conectiva.com.br> 2001
15  * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
16  * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
17  * Contributions by Hugh Dickins 2003, 2004
18  */
19 
20 /*
21  * Lock ordering in mm:
22  *
23  * inode->i_rwsem	(while writing or truncating, not reading or faulting)
24  *   mm->mmap_lock
25  *     mapping->invalidate_lock (in filemap_fault)
26  *       folio_lock
27  *         hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below)
28  *           vma_start_write
29  *             mapping->i_mmap_rwsem
30  *               anon_vma->rwsem
31  *                 mm->page_table_lock or pte_lock
32  *                   swap_lock (in swap_duplicate, swap_info_get)
33  *                     mmlist_lock (in mmput, drain_mmlist and others)
34  *                     mapping->private_lock (in block_dirty_folio)
35  *                         i_pages lock (widely used)
36  *                           lruvec->lru_lock (in folio_lruvec_lock_irq)
37  *                     inode->i_lock (in set_page_dirty's __mark_inode_dirty)
38  *                     bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
39  *                       sb_lock (within inode_lock in fs/fs-writeback.c)
40  *                       i_pages lock (widely used, in set_page_dirty,
41  *                                 in arch-dependent flush_dcache_mmap_lock,
42  *                                 within bdi.wb->list_lock in __sync_single_inode)
43  *
44  * anon_vma->rwsem,mapping->i_mmap_rwsem   (memory_failure, collect_procs_anon)
45  *   ->tasklist_lock
46  *     pte map lock
47  *
48  * hugetlbfs PageHuge() take locks in this order:
49  *   hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
50  *     vma_lock (hugetlb specific lock for pmd_sharing)
51  *       mapping->i_mmap_rwsem (also used for hugetlb pmd sharing)
52  *         folio_lock
53  */
54 
55 #include <linux/mm.h>
56 #include <linux/sched/mm.h>
57 #include <linux/sched/task.h>
58 #include <linux/pagemap.h>
59 #include <linux/swap.h>
60 #include <linux/leafops.h>
61 #include <linux/slab.h>
62 #include <linux/init.h>
63 #include <linux/ksm.h>
64 #include <linux/rmap.h>
65 #include <linux/rcupdate.h>
66 #include <linux/export.h>
67 #include <linux/memcontrol.h>
68 #include <linux/mmu_notifier.h>
69 #include <linux/migrate.h>
70 #include <linux/hugetlb.h>
71 #include <linux/huge_mm.h>
72 #include <linux/backing-dev.h>
73 #include <linux/page_idle.h>
74 #include <linux/memremap.h>
75 #include <linux/userfaultfd_k.h>
76 #include <linux/mm_inline.h>
77 #include <linux/oom.h>
78 
79 #include <asm/tlb.h>
80 
81 #define CREATE_TRACE_POINTS
82 #include <trace/events/migrate.h>
83 
84 #include "internal.h"
85 #include "swap.h"
86 
87 static struct kmem_cache *anon_vma_cachep;
88 static struct kmem_cache *anon_vma_chain_cachep;
89 
90 static inline struct anon_vma *anon_vma_alloc(void)
91 {
92 	struct anon_vma *anon_vma;
93 
94 	anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
95 	if (anon_vma) {
96 		atomic_set(&anon_vma->refcount, 1);
97 		anon_vma->num_children = 0;
98 		anon_vma->num_active_vmas = 0;
99 		anon_vma->parent = anon_vma;
100 		/*
101 		 * Initialise the anon_vma root to point to itself. If called
102 		 * from fork, the root will be reset to the parents anon_vma.
103 		 */
104 		anon_vma->root = anon_vma;
105 	}
106 
107 	return anon_vma;
108 }
109 
110 static inline void anon_vma_free(struct anon_vma *anon_vma)
111 {
112 	VM_BUG_ON(atomic_read(&anon_vma->refcount));
113 
114 	/*
115 	 * Synchronize against folio_lock_anon_vma_read() such that
116 	 * we can safely hold the lock without the anon_vma getting
117 	 * freed.
118 	 *
119 	 * Relies on the full mb implied by the atomic_dec_and_test() from
120 	 * put_anon_vma() against the acquire barrier implied by
121 	 * down_read_trylock() from folio_lock_anon_vma_read(). This orders:
122 	 *
123 	 * folio_lock_anon_vma_read()	VS	put_anon_vma()
124 	 *   down_read_trylock()		  atomic_dec_and_test()
125 	 *   LOCK				  MB
126 	 *   atomic_read()			  rwsem_is_locked()
127 	 *
128 	 * LOCK should suffice since the actual taking of the lock must
129 	 * happen _before_ what follows.
130 	 */
131 	might_sleep();
132 	if (rwsem_is_locked(&anon_vma->root->rwsem)) {
133 		anon_vma_lock_write(anon_vma);
134 		anon_vma_unlock_write(anon_vma);
135 	}
136 
137 	kmem_cache_free(anon_vma_cachep, anon_vma);
138 }
139 
140 static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
141 {
142 	return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
143 }
144 
145 static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
146 {
147 	kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
148 }
149 
150 static void anon_vma_chain_assign(struct vm_area_struct *vma,
151 				  struct anon_vma_chain *avc,
152 				  struct anon_vma *anon_vma)
153 {
154 	avc->vma = vma;
155 	avc->anon_vma = anon_vma;
156 	list_add(&avc->same_vma, &vma->anon_vma_chain);
157 }
158 
159 /**
160  * __anon_vma_prepare - attach an anon_vma to a memory region
161  * @vma: the memory region in question
162  *
163  * This makes sure the memory mapping described by 'vma' has
164  * an 'anon_vma' attached to it, so that we can associate the
165  * anonymous pages mapped into it with that anon_vma.
166  *
167  * The common case will be that we already have one, which
168  * is handled inline by anon_vma_prepare(). But if
169  * not we either need to find an adjacent mapping that we
170  * can re-use the anon_vma from (very common when the only
171  * reason for splitting a vma has been mprotect()), or we
172  * allocate a new one.
173  *
174  * Anon-vma allocations are very subtle, because we may have
175  * optimistically looked up an anon_vma in folio_lock_anon_vma_read()
176  * and that may actually touch the rwsem even in the newly
177  * allocated vma (it depends on RCU to make sure that the
178  * anon_vma isn't actually destroyed).
179  *
180  * As a result, we need to do proper anon_vma locking even
181  * for the new allocation. At the same time, we do not want
182  * to do any locking for the common case of already having
183  * an anon_vma.
184  */
185 int __anon_vma_prepare(struct vm_area_struct *vma)
186 {
187 	struct mm_struct *mm = vma->vm_mm;
188 	struct anon_vma *anon_vma, *allocated;
189 	struct anon_vma_chain *avc;
190 
191 	mmap_assert_locked(mm);
192 	might_sleep();
193 
194 	avc = anon_vma_chain_alloc(GFP_KERNEL);
195 	if (!avc)
196 		goto out_enomem;
197 
198 	anon_vma = find_mergeable_anon_vma(vma);
199 	allocated = NULL;
200 	if (!anon_vma) {
201 		anon_vma = anon_vma_alloc();
202 		if (unlikely(!anon_vma))
203 			goto out_enomem_free_avc;
204 		anon_vma->num_children++; /* self-parent link for new root */
205 		allocated = anon_vma;
206 	}
207 
208 	anon_vma_lock_write(anon_vma);
209 	/* page_table_lock to protect against threads */
210 	spin_lock(&mm->page_table_lock);
211 	if (likely(!vma_has_anon_rmap(vma))) {
212 		/*
213 		 * Make anon_vma fields visible before anon_vma is published.
214 		 * Paired with an address dependency in reusable_anon_vma().
215 		 */
216 		smp_store_release(&vma->anon_vma, anon_vma);
217 		anon_vma_chain_assign(vma, avc, anon_vma);
218 		anon_rmap_tree_insert(avc, anon_vma);
219 		anon_vma->num_active_vmas++;
220 		allocated = NULL;
221 		avc = NULL;
222 	}
223 	spin_unlock(&mm->page_table_lock);
224 	anon_vma_unlock_write(anon_vma);
225 
226 	if (unlikely(allocated))
227 		put_anon_vma(allocated);
228 	if (unlikely(avc))
229 		anon_vma_chain_free(avc);
230 
231 	return 0;
232 
233  out_enomem_free_avc:
234 	anon_vma_chain_free(avc);
235  out_enomem:
236 	return -ENOMEM;
237 }
238 
239 static void check_anon_vma_clone(struct vm_area_struct *dst,
240 				 struct vm_area_struct *src,
241 				 enum vma_operation operation)
242 {
243 	/* The write lock must be held. */
244 	mmap_assert_write_locked(src->vm_mm);
245 	/* If not a fork then must be on same mm. */
246 	VM_WARN_ON_ONCE(operation != VMA_OP_FORK && dst->vm_mm != src->vm_mm);
247 
248 	/* If we have anything to do src->anon_vma must be provided. */
249 	VM_WARN_ON_ONCE(!vma_has_anon_rmap(src) && !list_empty(&src->anon_vma_chain));
250 	VM_WARN_ON_ONCE(!vma_has_anon_rmap(src) && vma_has_anon_rmap(dst));
251 	/* We are establishing a new anon_vma_chain. */
252 	VM_WARN_ON_ONCE(!list_empty(&dst->anon_vma_chain));
253 	/*
254 	 * On fork, dst->anon_vma is set NULL (temporarily). Otherwise, anon_vma
255 	 * must be the same across dst and src.
256 	 */
257 	VM_WARN_ON_ONCE(vma_has_anon_rmap(dst) && dst->anon_vma != src->anon_vma);
258 	/*
259 	 * Essentially equivalent to above - if not a no-op, we should expect
260 	 * dst->anon_vma to be set for everything except a fork.
261 	 */
262 	VM_WARN_ON_ONCE(operation != VMA_OP_FORK && vma_has_anon_rmap(src) &&
263 			!vma_has_anon_rmap(dst));
264 	/* For the anon_vma to be compatible, it can only be singular. */
265 	VM_WARN_ON_ONCE(operation == VMA_OP_MERGE_UNFAULTED &&
266 			!list_is_singular(&src->anon_vma_chain));
267 	/* Only merging an unfaulted VMA leaves the destination attached. */
268 	VM_WARN_ON_ONCE(operation != VMA_OP_MERGE_UNFAULTED &&
269 			vma_is_attached(dst));
270 }
271 
272 static void maybe_reuse_anon_vma(struct vm_area_struct *dst,
273 		struct anon_vma *anon_vma)
274 {
275 	/* If already populated, nothing to do.*/
276 	if (vma_has_anon_rmap(dst))
277 		return;
278 
279 	/*
280 	 * We reuse an anon_vma if any linking VMAs were unmapped and it has
281 	 * only a single child at most.
282 	 */
283 	if (anon_vma->num_active_vmas > 0)
284 		return;
285 	if (anon_vma->num_children > 1)
286 		return;
287 
288 	dst->anon_vma = anon_vma;
289 	anon_vma->num_active_vmas++;
290 }
291 
292 static void cleanup_partial_anon_vmas(struct vm_area_struct *vma);
293 
294 /**
295  * anon_vma_clone - Establishes new anon_vma_chain objects in @dst linking to
296  * all of the anon_vma objects contained within @src anon_vma_chain's.
297  * @dst: The destination VMA with an empty anon_vma_chain.
298  * @src: The source VMA we wish to duplicate.
299  * @operation: The type of operation which resulted in the clone.
300  *
301  * This is the heart of the VMA side of the anon_vma implementation - we invoke
302  * this function whenever we need to set up a new VMA's anon_vma state.
303  *
304  * This is invoked for:
305  *
306  * - VMA Merge, but only when @dst is unfaulted and @src is faulted - meaning we
307  *   clone @src into @dst.
308  * - VMA split.
309  * - VMA (m)remap.
310  * - Fork of faulted VMA.
311  *
312  * In all cases other than fork this is simply a duplication. Fork additionally
313  * adds a new active anon_vma.
314  *
315  * ONLY in the case of fork do we try to 'reuse' existing anon_vma's in an
316  * anon_vma hierarchy, reusing anon_vma's which have no VMA associated with them
317  * but do have a single child. This is to avoid waste of memory when repeatedly
318  * forking.
319  *
320  * Returns: 0 on success, -ENOMEM on failure.
321  */
322 int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src,
323 		   enum vma_operation operation)
324 {
325 	struct anon_vma_chain *avc, *pavc;
326 	struct anon_vma *active_anon_vma = src->anon_vma;
327 
328 	check_anon_vma_clone(dst, src, operation);
329 
330 	if (!vma_has_anon_rmap(src))
331 		return 0;
332 
333 	/*
334 	 * Allocate AVCs. We don't need an anon_vma lock for this as we
335 	 * are not updating the anon_vma rbtree nor are we changing
336 	 * anon_vma statistics.
337 	 *
338 	 * Either src, dst have the same mm for which we hold an exclusive mmap
339 	 * write lock, or we are forking and we hold it on src->vm_mm and dst is
340 	 * not yet accessible to other threads so there's no possibliity of the
341 	 * unlinked AVC's being observed yet.
342 	 */
343 	list_for_each_entry(pavc, &src->anon_vma_chain, same_vma) {
344 		avc = anon_vma_chain_alloc(GFP_KERNEL);
345 		if (!avc)
346 			goto enomem_failure;
347 
348 		anon_vma_chain_assign(dst, avc, pavc->anon_vma);
349 	}
350 
351 	/*
352 	 * Now link the anon_vma's back to the newly inserted AVCs.
353 	 * Note that all anon_vma's share the same root.
354 	 */
355 	anon_vma_lock_write(active_anon_vma);
356 	list_for_each_entry_reverse(avc, &dst->anon_vma_chain, same_vma) {
357 		struct anon_vma *anon_vma = avc->anon_vma;
358 
359 		anon_rmap_tree_insert(avc, anon_vma);
360 		if (operation == VMA_OP_FORK)
361 			maybe_reuse_anon_vma(dst, anon_vma);
362 	}
363 
364 	if (operation != VMA_OP_FORK)
365 		dst->anon_vma->num_active_vmas++;
366 
367 	anon_vma_unlock_write(active_anon_vma);
368 	return 0;
369 
370  enomem_failure:
371 	cleanup_partial_anon_vmas(dst);
372 	return -ENOMEM;
373 }
374 
375 /*
376  * Attach vma to its own anon_vma, as well as to the anon_vmas that
377  * the corresponding VMA in the parent process is attached to.
378  * Returns 0 on success, non-zero on failure.
379  */
380 int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
381 {
382 	struct anon_vma_chain *avc;
383 	struct anon_vma *anon_vma;
384 	int rc;
385 
386 	/* Don't bother if the parent process has no anon_vma here. */
387 	if (!vma_has_anon_rmap(pvma))
388 		return 0;
389 
390 	/* Drop inherited anon_vma, we'll reuse existing or allocate new. */
391 	vma->anon_vma = NULL;
392 
393 	anon_vma = anon_vma_alloc();
394 	if (!anon_vma)
395 		return -ENOMEM;
396 	avc = anon_vma_chain_alloc(GFP_KERNEL);
397 	if (!avc) {
398 		put_anon_vma(anon_vma);
399 		return -ENOMEM;
400 	}
401 
402 	/*
403 	 * First, attach the new VMA to the parent VMA's anon_vmas,
404 	 * so rmap can find non-COWed pages in child processes.
405 	 */
406 	rc = anon_vma_clone(vma, pvma, VMA_OP_FORK);
407 	/* An error arose or an existing anon_vma was reused, all done then. */
408 	if (rc || vma_has_anon_rmap(vma)) {
409 		put_anon_vma(anon_vma);
410 		anon_vma_chain_free(avc);
411 		return rc;
412 	}
413 
414 	/*
415 	 * OK no reuse, so add our own anon_vma.
416 	 *
417 	 * Since it is not linked anywhere we can safely manipulate anon_vma
418 	 * fields without a lock.
419 	 */
420 
421 	anon_vma->num_active_vmas = 1;
422 	/*
423 	 * The root anon_vma's rwsem is the lock actually used when we
424 	 * lock any of the anon_vmas in this anon_vma tree.
425 	 */
426 	anon_vma->root = pvma->anon_vma->root;
427 	anon_vma->parent = pvma->anon_vma;
428 	/*
429 	 * With refcounts, an anon_vma can stay around longer than the
430 	 * process it belongs to. The root anon_vma needs to be pinned until
431 	 * this anon_vma is freed, because the lock lives in the root.
432 	 */
433 	get_anon_vma(anon_vma->root);
434 	/* Mark this anon_vma as the one where our new (COWed) pages go. */
435 	vma->anon_vma = anon_vma;
436 	anon_vma_chain_assign(vma, avc, anon_vma);
437 	/* Now let rmap see it. */
438 	anon_vma_lock_write(anon_vma);
439 	anon_rmap_tree_insert(avc, anon_vma);
440 	anon_vma->parent->num_children++;
441 	anon_vma_unlock_write(anon_vma);
442 
443 	return 0;
444 }
445 
446 /*
447  * In the unfortunate case of anon_vma_clone() failing to allocate memory we
448  * have to clean things up.
449  *
450  * Since we allocate anon_vma_chain's before we insert them into the interval
451  * trees, we simply have to free up the AVC's and remove the entries from the
452  * VMA's anon_vma_chain.
453  */
454 static void cleanup_partial_anon_vmas(struct vm_area_struct *vma)
455 {
456 	struct anon_vma_chain *avc, *next;
457 
458 	list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
459 		list_del(&avc->same_vma);
460 		anon_vma_chain_free(avc);
461 	}
462 
463 	/*
464 	 * The anon_vma assigned to this VMA is no longer valid, as we were not
465 	 * able to correctly clone AVC state. Avoid inconsistent anon_vma tree
466 	 * state by resetting.
467 	 */
468 	vma->anon_vma = NULL;
469 }
470 
471 /**
472  * unlink_anon_vmas() - remove all links between a VMA and anon_vma's, freeing
473  * anon_vma_chain objects.
474  * @vma: The VMA whose links to anon_vma objects is to be severed.
475  *
476  * As part of the process anon_vma_chain's are freed,
477  * anon_vma->num_children,num_active_vmas is updated as required and, if the
478  * relevant anon_vma references no further VMAs, its reference count is
479  * decremented.
480  */
481 void unlink_anon_vmas(struct vm_area_struct *vma)
482 {
483 	struct anon_vma_chain *avc, *next;
484 	struct anon_vma *active_anon_vma = vma->anon_vma;
485 
486 	/* Always hold mmap lock, read-lock on unmap possibly. */
487 	mmap_assert_locked(vma->vm_mm);
488 
489 	/* Unfaulted is a no-op. */
490 	if (!active_anon_vma) {
491 		VM_WARN_ON_ONCE(!list_empty(&vma->anon_vma_chain));
492 		return;
493 	}
494 
495 	anon_vma_lock_write(active_anon_vma);
496 
497 	/*
498 	 * Unlink each anon_vma chained to the VMA.  This list is ordered
499 	 * from newest to oldest, ensuring the root anon_vma gets freed last.
500 	 */
501 	list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
502 		struct anon_vma *anon_vma = avc->anon_vma;
503 
504 		anon_rmap_tree_remove(avc, anon_vma);
505 
506 		/*
507 		 * Leave empty anon_vmas on the list - we'll need
508 		 * to free them outside the lock.
509 		 */
510 		if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) {
511 			anon_vma->parent->num_children--;
512 			continue;
513 		}
514 
515 		list_del(&avc->same_vma);
516 		anon_vma_chain_free(avc);
517 	}
518 
519 	active_anon_vma->num_active_vmas--;
520 	/*
521 	 * vma would still be needed after unlink, and anon_vma will be prepared
522 	 * when handle fault.
523 	 */
524 	vma->anon_vma = NULL;
525 	anon_vma_unlock_write(active_anon_vma);
526 
527 
528 	/*
529 	 * Iterate the list once more, it now only contains empty and unlinked
530 	 * anon_vmas, destroy them. Could not do before due to __put_anon_vma()
531 	 * needing to write-acquire the anon_vma->root->rwsem.
532 	 */
533 	list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
534 		struct anon_vma *anon_vma = avc->anon_vma;
535 
536 		VM_WARN_ON(anon_vma->num_children);
537 		VM_WARN_ON(anon_vma->num_active_vmas);
538 		put_anon_vma(anon_vma);
539 
540 		list_del(&avc->same_vma);
541 		anon_vma_chain_free(avc);
542 	}
543 }
544 
545 static void anon_vma_ctor(void *data)
546 {
547 	struct anon_vma *anon_vma = data;
548 
549 	init_rwsem(&anon_vma->rwsem);
550 	atomic_set(&anon_vma->refcount, 0);
551 	anon_vma->rb_root = RB_ROOT_CACHED;
552 }
553 
554 void __init anon_vma_init(void)
555 {
556 	anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
557 			0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT,
558 			anon_vma_ctor);
559 	anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain,
560 			SLAB_PANIC|SLAB_ACCOUNT);
561 }
562 
563 /*
564  * Getting a lock on a stable anon_vma from a page off the LRU is tricky!
565  *
566  * Since there is no serialization what so ever against folio_remove_rmap_*()
567  * the best this function can do is return a refcount increased anon_vma
568  * that might have been relevant to this page.
569  *
570  * The page might have been remapped to a different anon_vma or the anon_vma
571  * returned may already be freed (and even reused).
572  *
573  * In case it was remapped to a different anon_vma, the new anon_vma will be a
574  * child of the old anon_vma, and the anon_vma lifetime rules will therefore
575  * ensure that any anon_vma obtained from the page will still be valid for as
576  * long as we observe folio_mapped() [ hence all those folio_mapped() tests ].
577  *
578  * All users of this function must be very careful when walking the anon_vma
579  * chain and verify that the page in question is indeed mapped in it
580  * [ something equivalent to page_mapped_in_vma() ].
581  *
582  * Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from
583  * folio_remove_rmap_*() that the anon_vma pointer from page->mapping is valid
584  * if there is a mapcount, we can dereference the anon_vma after observing
585  * those.
586  *
587  * NOTE: the caller should hold folio lock when calling this.
588  */
589 struct anon_vma *folio_get_anon_vma(const struct folio *folio)
590 {
591 	struct anon_vma *anon_vma = NULL;
592 	unsigned long anon_mapping;
593 
594 	VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
595 
596 	rcu_read_lock();
597 	anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
598 	if ((anon_mapping & FOLIO_MAPPING_FLAGS) != FOLIO_MAPPING_ANON)
599 		goto out;
600 	if (!folio_mapped(folio))
601 		goto out;
602 
603 	anon_vma = (struct anon_vma *) (anon_mapping - FOLIO_MAPPING_ANON);
604 	if (!atomic_inc_not_zero(&anon_vma->refcount)) {
605 		anon_vma = NULL;
606 		goto out;
607 	}
608 
609 	/*
610 	 * If this folio is still mapped, then its anon_vma cannot have been
611 	 * freed.  But if it has been unmapped, we have no security against the
612 	 * anon_vma structure being freed and reused (for another anon_vma:
613 	 * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero()
614 	 * above cannot corrupt).
615 	 */
616 	if (!folio_mapped(folio)) {
617 		rcu_read_unlock();
618 		put_anon_vma(anon_vma);
619 		return NULL;
620 	}
621 out:
622 	rcu_read_unlock();
623 
624 	return anon_vma;
625 }
626 
627 /*
628  * Similar to folio_get_anon_vma() except it locks the anon_vma.
629  *
630  * Its a little more complex as it tries to keep the fast path to a single
631  * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
632  * reference like with folio_get_anon_vma() and then block on the mutex
633  * on !rwc->try_lock case.
634  */
635 struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio,
636 					  struct rmap_walk_control *rwc)
637 {
638 	struct anon_vma *anon_vma = NULL;
639 	struct anon_vma *root_anon_vma;
640 	unsigned long anon_mapping;
641 
642 	VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
643 
644 	rcu_read_lock();
645 	anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
646 	if ((anon_mapping & FOLIO_MAPPING_FLAGS) != FOLIO_MAPPING_ANON)
647 		goto out;
648 	if (!folio_mapped(folio))
649 		goto out;
650 
651 	anon_vma = (struct anon_vma *) (anon_mapping - FOLIO_MAPPING_ANON);
652 	root_anon_vma = READ_ONCE(anon_vma->root);
653 	if (down_read_trylock(&root_anon_vma->rwsem)) {
654 		/*
655 		 * If the folio is still mapped, then this anon_vma is still
656 		 * its anon_vma, and holding the mutex ensures that it will
657 		 * not go away, see anon_vma_free().
658 		 */
659 		if (!folio_mapped(folio)) {
660 			up_read(&root_anon_vma->rwsem);
661 			anon_vma = NULL;
662 		}
663 		goto out;
664 	}
665 
666 	if (rwc && rwc->try_lock) {
667 		anon_vma = NULL;
668 		rwc->contended = true;
669 		goto out;
670 	}
671 
672 	/* trylock failed, we got to sleep */
673 	if (!atomic_inc_not_zero(&anon_vma->refcount)) {
674 		anon_vma = NULL;
675 		goto out;
676 	}
677 
678 	if (!folio_mapped(folio)) {
679 		rcu_read_unlock();
680 		put_anon_vma(anon_vma);
681 		return NULL;
682 	}
683 
684 	/* we pinned the anon_vma, its safe to sleep */
685 	rcu_read_unlock();
686 	anon_vma_lock_read(anon_vma);
687 
688 	if (atomic_dec_and_test(&anon_vma->refcount)) {
689 		/*
690 		 * Oops, we held the last refcount, release the lock
691 		 * and bail -- can't simply use put_anon_vma() because
692 		 * we'll deadlock on the anon_vma_lock_write() recursion.
693 		 */
694 		anon_vma_unlock_read(anon_vma);
695 		__put_anon_vma(anon_vma);
696 		anon_vma = NULL;
697 	}
698 
699 	return anon_vma;
700 
701 out:
702 	rcu_read_unlock();
703 	return anon_vma;
704 }
705 
706 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
707 /*
708  * Flush TLB entries for recently unmapped pages from remote CPUs. It is
709  * important if a PTE was dirty when it was unmapped that it's flushed
710  * before any IO is initiated on the page to prevent lost writes. Similarly,
711  * it must be flushed before freeing to prevent data leakage.
712  */
713 void try_to_unmap_flush(void)
714 {
715 	struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
716 
717 	if (!tlb_ubc->flush_required)
718 		return;
719 
720 	arch_tlbbatch_flush(&tlb_ubc->arch);
721 	tlb_ubc->flush_required = false;
722 	tlb_ubc->writable = false;
723 }
724 
725 /* Flush iff there are potentially writable TLB entries that can race with IO */
726 void try_to_unmap_flush_dirty(void)
727 {
728 	struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
729 
730 	if (tlb_ubc->writable)
731 		try_to_unmap_flush();
732 }
733 
734 /*
735  * Bits 0-14 of mm->tlb_flush_batched record pending generations.
736  * Bits 16-30 of mm->tlb_flush_batched bit record flushed generations.
737  */
738 #define TLB_FLUSH_BATCH_FLUSHED_SHIFT	16
739 #define TLB_FLUSH_BATCH_PENDING_MASK			\
740 	((1 << (TLB_FLUSH_BATCH_FLUSHED_SHIFT - 1)) - 1)
741 #define TLB_FLUSH_BATCH_PENDING_LARGE			\
742 	(TLB_FLUSH_BATCH_PENDING_MASK / 2)
743 
744 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval,
745 		unsigned long start, unsigned long end)
746 {
747 	struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
748 	int batch;
749 	bool writable = pte_dirty(pteval);
750 
751 	if (!pte_accessible(mm, pteval))
752 		return;
753 
754 	arch_tlbbatch_add_pending(&tlb_ubc->arch, mm, start, end);
755 	tlb_ubc->flush_required = true;
756 
757 	/*
758 	 * Ensure compiler does not re-order the setting of tlb_flush_batched
759 	 * before the PTE is cleared.
760 	 */
761 	barrier();
762 	batch = atomic_read(&mm->tlb_flush_batched);
763 retry:
764 	if ((batch & TLB_FLUSH_BATCH_PENDING_MASK) > TLB_FLUSH_BATCH_PENDING_LARGE) {
765 		/*
766 		 * Prevent `pending' from catching up with `flushed' because of
767 		 * overflow.  Reset `pending' and `flushed' to be 1 and 0 if
768 		 * `pending' becomes large.
769 		 */
770 		if (!atomic_try_cmpxchg(&mm->tlb_flush_batched, &batch, 1))
771 			goto retry;
772 	} else {
773 		atomic_inc(&mm->tlb_flush_batched);
774 	}
775 
776 	/*
777 	 * If the PTE was dirty then it's best to assume it's writable. The
778 	 * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush()
779 	 * before the page is queued for IO.
780 	 */
781 	if (writable)
782 		tlb_ubc->writable = true;
783 }
784 
785 /*
786  * Returns true if the TLB flush should be deferred to the end of a batch of
787  * unmap operations to reduce IPIs.
788  */
789 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
790 {
791 	if (!(flags & TTU_BATCH_FLUSH))
792 		return false;
793 
794 	return arch_tlbbatch_should_defer(mm);
795 }
796 
797 /*
798  * Reclaim unmaps pages under the PTL but do not flush the TLB prior to
799  * releasing the PTL if TLB flushes are batched. It's possible for a parallel
800  * operation such as mprotect or munmap to race between reclaim unmapping
801  * the page and flushing the page. If this race occurs, it potentially allows
802  * access to data via a stale TLB entry. Tracking all mm's that have TLB
803  * batching in flight would be expensive during reclaim so instead track
804  * whether TLB batching occurred in the past and if so then do a flush here
805  * if required. This will cost one additional flush per reclaim cycle paid
806  * by the first operation at risk such as mprotect and mumap.
807  *
808  * This must be called under the PTL so that an access to tlb_flush_batched
809  * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise
810  * via the PTL.
811  */
812 void flush_tlb_batched_pending(struct mm_struct *mm)
813 {
814 	int batch = atomic_read(&mm->tlb_flush_batched);
815 	int pending = batch & TLB_FLUSH_BATCH_PENDING_MASK;
816 	int flushed = batch >> TLB_FLUSH_BATCH_FLUSHED_SHIFT;
817 
818 	if (pending != flushed) {
819 		flush_tlb_mm(mm);
820 		/*
821 		 * If the new TLB flushing is pending during flushing, leave
822 		 * mm->tlb_flush_batched as is, to avoid losing flushing.
823 		 */
824 		atomic_cmpxchg(&mm->tlb_flush_batched, batch,
825 			       pending | (pending << TLB_FLUSH_BATCH_FLUSHED_SHIFT));
826 	}
827 }
828 #else
829 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval,
830 		unsigned long start, unsigned long end)
831 {
832 }
833 
834 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
835 {
836 	return false;
837 }
838 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
839 
840 /**
841  * page_address_in_vma - The virtual address of a page in this VMA.
842  * @folio: The folio containing the page.
843  * @page: The page within the folio.
844  * @vma: The VMA we need to know the address in.
845  *
846  * Calculates the user virtual address of this page in the specified VMA.
847  * It is the caller's responsibility to check the page is actually
848  * within the VMA.  There may not currently be a PTE pointing at this
849  * page, but if a page fault occurs at this address, this is the page
850  * which will be accessed.
851  *
852  * Context: Caller should hold a reference to the folio.  Caller should
853  * hold a lock (eg the i_mmap_lock or the mmap_lock) which keeps the
854  * VMA from being altered.
855  *
856  * Return: The virtual address corresponding to this page in the VMA.
857  */
858 unsigned long page_address_in_vma(const struct folio *folio,
859 		const struct page *page, const struct vm_area_struct *vma)
860 {
861 	if (folio_test_anon(folio)) {
862 		struct anon_vma *anon_vma = folio_anon_vma(folio);
863 		/*
864 		 * Note: swapoff's unuse_vma() is more efficient with this
865 		 * check, and needs it to match anon_vma when KSM is active.
866 		 */
867 		if (!vma_has_anon_rmap(vma) || !anon_vma ||
868 		    vma->anon_vma->root != anon_vma->root)
869 			return -EFAULT;
870 		/* KSM folios don't reach here because of the !anon_vma check */
871 		return vma_anon_address(vma, page_pgoff(folio, page), 1);
872 	} else if (!vma->vm_file) {
873 		return -EFAULT;
874 	} else if (vma->vm_file->f_mapping != folio->mapping) {
875 		return -EFAULT;
876 	}
877 
878 	return vma_filebacked_address(vma, page_pgoff(folio, page), 1);
879 }
880 
881 /*
882  * Returns the actual pmd_t* where we expect 'address' to be mapped from, or
883  * NULL if it doesn't exist.  No guarantees / checks on what the pmd_t*
884  * represents.
885  */
886 pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
887 {
888 	pgd_t *pgd;
889 	p4d_t *p4d;
890 	pud_t *pud;
891 	pmd_t *pmd = NULL;
892 
893 	pgd = pgd_offset(mm, address);
894 	if (!pgd_present(*pgd))
895 		goto out;
896 
897 	p4d = p4d_offset(pgd, address);
898 	if (!p4d_present(*p4d))
899 		goto out;
900 
901 	pud = pud_offset(p4d, address);
902 	if (!pud_present(*pud))
903 		goto out;
904 
905 	pmd = pmd_offset(pud, address);
906 out:
907 	return pmd;
908 }
909 
910 struct folio_referenced_arg {
911 	int mapcount;
912 	int referenced;
913 	vma_flags_t vma_flags;
914 	struct mem_cgroup *memcg;
915 };
916 
917 /*
918  * arg: folio_referenced_arg will be passed
919  */
920 static bool folio_referenced_one(struct folio *folio,
921 		struct vm_area_struct *vma, unsigned long address, void *arg)
922 {
923 	struct folio_referenced_arg *pra = arg;
924 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
925 	int ptes = 0, referenced = 0;
926 	unsigned int nr;
927 	pte_t pteval;
928 
929 	while (page_vma_mapped_walk(&pvmw)) {
930 		address = pvmw.address;
931 		nr = 1;
932 
933 		if (vma_test(vma, VMA_LOCKED_BIT)) {
934 			ptes++;
935 			pra->mapcount--;
936 
937 			/* Only mlock fully mapped pages */
938 			if (pvmw.pte && ptes != pvmw.nr_pages)
939 				continue;
940 
941 			/*
942 			 * All PTEs must be protected by page table lock in
943 			 * order to mlock the page.
944 			 *
945 			 * If page table boundary has been cross, current ptl
946 			 * only protect part of ptes.
947 			 */
948 			if (pvmw.flags & PVMW_PGTABLE_CROSSED)
949 				continue;
950 
951 			/* Restore the mlock which got missed */
952 			mlock_vma_folio(folio, vma);
953 			page_vma_mapped_walk_done(&pvmw);
954 			vma_flags_set(&pra->vma_flags, VMA_LOCKED_BIT);
955 			return false; /* To break the loop */
956 		}
957 
958 		/*
959 		 * Skip the non-shared swapbacked folio mapped solely by
960 		 * the exiting or OOM-reaped process. This avoids redundant
961 		 * swap-out followed by an immediate unmap.
962 		 */
963 		if ((!atomic_read(&vma->vm_mm->mm_users) ||
964 		    check_stable_address_space(vma->vm_mm)) &&
965 		    folio_test_anon(folio) && folio_test_swapbacked(folio) &&
966 		    !folio_maybe_mapped_shared(folio)) {
967 			pra->referenced = -1;
968 			page_vma_mapped_walk_done(&pvmw);
969 			return false;
970 		}
971 
972 		if (pvmw.pte)
973 			pteval = ptep_get(pvmw.pte);
974 
975 		if (pvmw.pte && pte_present(pteval) && folio_test_large(folio)) {
976 			const unsigned long end_addr = pte_table_clamped_end(address, vma->vm_end);
977 			const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT;
978 
979 			nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr);
980 		}
981 
982 		/*
983 		 * When LRU is switching, we don’t know where the surrounding folios
984 		 * are. —they could be on active/inactive lists or on MGLRU. So the
985 		 * simplest approach is to disable this look-around optimization.
986 		 */
987 		if (lru_gen_enabled() && !lru_gen_switching() && pvmw.pte &&
988 		    pte_present(pteval)) {
989 			if (lru_gen_look_around(&pvmw, nr))
990 				referenced++;
991 		} else if (pvmw.pte) {
992 			bool young = false;
993 
994 			/*
995 			 * PFN swap PTEs, such as device-exclusive ones, that
996 			 * actually map pages are "old" from a CPU perspective.
997 			 * The MMU notifier takes care of any device aspects.
998 			 */
999 			if (pte_present(pteval))
1000 				young = clear_flush_young_ptes(vma, address, pvmw.pte, nr);
1001 			young |= mmu_notifier_clear_flush_young(vma->vm_mm, address,
1002 								address + nr * PAGE_SIZE);
1003 			if (young)
1004 				referenced++;
1005 		} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
1006 			if (pmdp_clear_flush_young_notify(vma, address,
1007 						pvmw.pmd))
1008 				referenced++;
1009 		} else {
1010 			/* unexpected pmd-mapped folio? */
1011 			WARN_ON_ONCE(1);
1012 		}
1013 
1014 		ptes += nr;
1015 		pra->mapcount -= nr;
1016 		/*
1017 		 * If we are sure that we batched the entire folio,
1018 		 * we can just optimize and stop right here.
1019 		 */
1020 		if (ptes == pvmw.nr_pages) {
1021 			page_vma_mapped_walk_done(&pvmw);
1022 			break;
1023 		}
1024 
1025 		/* Skip the batched PTEs */
1026 		pvmw.pte += nr - 1;
1027 		pvmw.address += (nr - 1) * PAGE_SIZE;
1028 	}
1029 
1030 	if (referenced)
1031 		folio_clear_idle(folio);
1032 	if (folio_test_clear_young(folio))
1033 		referenced++;
1034 
1035 	if (referenced) {
1036 		vma_flags_t vma_flags = vma->flags;
1037 
1038 		pra->referenced++;
1039 		vma_flags_clear(&vma_flags, VMA_LOCKED_BIT);
1040 		vma_flags_set_mask(&pra->vma_flags, vma_flags);
1041 	}
1042 
1043 	if (!pra->mapcount)
1044 		return false; /* To break the loop */
1045 
1046 	return true;
1047 }
1048 
1049 static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg)
1050 {
1051 	struct folio_referenced_arg *pra = arg;
1052 	struct mem_cgroup *memcg = pra->memcg;
1053 
1054 	/*
1055 	 * Ignore references from this mapping if it has no recency. If the
1056 	 * folio has been used in another mapping, we will catch it; if this
1057 	 * other mapping is already gone, the unmap path will have set the
1058 	 * referenced flag or activated the folio in zap_pte_range().
1059 	 */
1060 	if (!vma_has_recency(vma))
1061 		return true;
1062 
1063 	/*
1064 	 * If we are reclaiming on behalf of a cgroup, skip counting on behalf
1065 	 * of references from different cgroups.
1066 	 */
1067 	if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))
1068 		return true;
1069 
1070 	return false;
1071 }
1072 
1073 /**
1074  * folio_referenced() - Test if the folio was referenced.
1075  * @folio: The folio to test.
1076  * @is_locked: Caller holds lock on the folio.
1077  * @memcg: target memory cgroup
1078  * @vma_flags: A combination of all the vma->flags which referenced the folio.
1079  *
1080  * Quick test_and_clear_referenced for all mappings of a folio,
1081  *
1082  * Return: The number of mappings which referenced the folio. Return -1 if
1083  * the function bailed out due to rmap lock contention.
1084  */
1085 int folio_referenced(struct folio *folio, int is_locked,
1086 		struct mem_cgroup *memcg, vma_flags_t *vma_flags)
1087 {
1088 	bool we_locked = false;
1089 	struct folio_referenced_arg pra = {
1090 		.mapcount = folio_mapcount(folio),
1091 		.memcg = memcg,
1092 	};
1093 	struct rmap_walk_control rwc = {
1094 		.rmap_one = folio_referenced_one,
1095 		.arg = (void *)&pra,
1096 		.anon_lock = folio_lock_anon_vma_read,
1097 		.try_lock = true,
1098 		.invalid_vma = invalid_folio_referenced_vma,
1099 	};
1100 
1101 	VM_WARN_ON_ONCE_FOLIO(folio_is_zone_device(folio), folio);
1102 	vma_flags_clear_all(vma_flags);
1103 	if (!pra.mapcount)
1104 		return 0;
1105 
1106 	if (!folio_raw_mapping(folio))
1107 		return 0;
1108 
1109 	if (!is_locked) {
1110 		we_locked = folio_trylock(folio);
1111 		if (!we_locked)
1112 			return 1;
1113 	}
1114 
1115 	rmap_walk(folio, &rwc);
1116 	vma_flags_set_mask(vma_flags, pra.vma_flags);
1117 
1118 	if (we_locked)
1119 		folio_unlock(folio);
1120 
1121 	return rwc.contended ? -1 : pra.referenced;
1122 }
1123 
1124 static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw)
1125 {
1126 	int cleaned = 0;
1127 	struct vm_area_struct *vma = pvmw->vma;
1128 	struct mmu_notifier_range range;
1129 	unsigned long address = pvmw->address;
1130 
1131 	/*
1132 	 * We have to assume the worse case ie pmd for invalidation. Note that
1133 	 * the folio can not be freed from this function.
1134 	 */
1135 	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 0,
1136 				vma->vm_mm, address, vma_address_end(pvmw));
1137 	mmu_notifier_invalidate_range_start(&range);
1138 
1139 	while (page_vma_mapped_walk(pvmw)) {
1140 		int ret = 0;
1141 
1142 		address = pvmw->address;
1143 		if (pvmw->pte) {
1144 			pte_t *pte = pvmw->pte;
1145 			pte_t entry = ptep_get(pte);
1146 
1147 			/*
1148 			 * PFN swap PTEs, such as device-exclusive ones, that
1149 			 * actually map pages are clean and not writable from a
1150 			 * CPU perspective. The MMU notifier takes care of any
1151 			 * device aspects.
1152 			 */
1153 			if (!pte_present(entry))
1154 				continue;
1155 			if (!pte_dirty(entry) && !pte_write(entry))
1156 				continue;
1157 
1158 			flush_cache_page(vma, address, pte_pfn(entry));
1159 			entry = ptep_clear_flush(vma, address, pte);
1160 			entry = pte_wrprotect(entry);
1161 			entry = pte_mkclean(entry);
1162 			set_pte_at(vma->vm_mm, address, pte, entry);
1163 			ret = 1;
1164 		} else {
1165 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1166 			pmd_t *pmd = pvmw->pmd;
1167 			pmd_t entry = pmdp_get(pmd);
1168 
1169 			/*
1170 			 * Please see the comment above (!pte_present).
1171 			 * A non present PMD is not writable from a CPU
1172 			 * perspective.
1173 			 */
1174 			if (!pmd_present(entry))
1175 				continue;
1176 			if (!pmd_dirty(entry) && !pmd_write(entry))
1177 				continue;
1178 
1179 			flush_cache_range(vma, address,
1180 					  address + HPAGE_PMD_SIZE);
1181 			entry = pmdp_invalidate(vma, address, pmd);
1182 			entry = pmd_wrprotect(entry);
1183 			entry = pmd_mkclean(entry);
1184 			set_pmd_at(vma->vm_mm, address, pmd, entry);
1185 			ret = 1;
1186 #else
1187 			/* unexpected pmd-mapped folio? */
1188 			WARN_ON_ONCE(1);
1189 #endif
1190 		}
1191 
1192 		if (ret)
1193 			cleaned++;
1194 	}
1195 
1196 	mmu_notifier_invalidate_range_end(&range);
1197 
1198 	return cleaned;
1199 }
1200 
1201 static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma,
1202 			     unsigned long address, void *arg)
1203 {
1204 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC);
1205 	int *cleaned = arg;
1206 
1207 	*cleaned += page_vma_mkclean_one(&pvmw);
1208 
1209 	return true;
1210 }
1211 
1212 static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)
1213 {
1214 	if (vma->vm_flags & VM_SHARED)
1215 		return false;
1216 
1217 	return true;
1218 }
1219 
1220 int folio_mkclean(struct folio *folio)
1221 {
1222 	int cleaned = 0;
1223 	struct address_space *mapping;
1224 	struct rmap_walk_control rwc = {
1225 		.arg = (void *)&cleaned,
1226 		.rmap_one = page_mkclean_one,
1227 		.invalid_vma = invalid_mkclean_vma,
1228 	};
1229 
1230 	BUG_ON(!folio_test_locked(folio));
1231 
1232 	if (!folio_mapped(folio))
1233 		return 0;
1234 
1235 	mapping = folio_mapping(folio);
1236 	if (!mapping)
1237 		return 0;
1238 
1239 	rmap_walk(folio, &rwc);
1240 
1241 	return cleaned;
1242 }
1243 EXPORT_SYMBOL_GPL(folio_mkclean);
1244 
1245 struct wrprotect_file_state {
1246 	int cleaned;
1247 	pgoff_t pgoff;
1248 	unsigned long pfn;
1249 	unsigned long nr_pages;
1250 };
1251 
1252 static bool mapping_wrprotect_range_one(struct folio *folio,
1253 		struct vm_area_struct *vma, unsigned long address, void *arg)
1254 {
1255 	struct wrprotect_file_state *state = (struct wrprotect_file_state *)arg;
1256 	struct page_vma_mapped_walk pvmw = {
1257 		.pfn		= state->pfn,
1258 		.nr_pages	= state->nr_pages,
1259 		.pgoff		= state->pgoff,
1260 		.vma		= vma,
1261 		.address	= address,
1262 		.flags		= PVMW_SYNC,
1263 		.pgoff_is_anon	= false,
1264 	};
1265 
1266 	state->cleaned += page_vma_mkclean_one(&pvmw);
1267 
1268 	return true;
1269 }
1270 
1271 static void __rmap_walk_file(struct folio *folio, struct address_space *mapping,
1272 			     pgoff_t pgoff_start, unsigned long nr_pages,
1273 			     struct rmap_walk_control *rwc, bool locked);
1274 
1275 /**
1276  * mapping_wrprotect_range() - Write-protect all mappings in a specified range.
1277  *
1278  * @mapping:	The mapping whose reverse mapping should be traversed.
1279  * @pgoff:	The page offset at which @pfn is mapped within @mapping.
1280  * @pfn:	The PFN of the page mapped in @mapping at @pgoff.
1281  * @nr_pages:	The number of physically contiguous base pages spanned.
1282  *
1283  * Traverses the reverse mapping, finding all VMAs which contain a shared
1284  * mapping of the pages in the specified range in @mapping, and write-protects
1285  * them (that is, updates the page tables to mark the mappings read-only such
1286  * that a write protection fault arises when the mappings are written to).
1287  *
1288  * The @pfn value need not refer to a folio, but rather can reference a kernel
1289  * allocation which is mapped into userland. We therefore do not require that
1290  * the page maps to a folio with a valid mapping or index field, rather the
1291  * caller specifies these in @mapping and @pgoff.
1292  *
1293  * Return: the number of write-protected PTEs, or an error.
1294  */
1295 int mapping_wrprotect_range(struct address_space *mapping, pgoff_t pgoff,
1296 		unsigned long pfn, unsigned long nr_pages)
1297 {
1298 	struct wrprotect_file_state state = {
1299 		.cleaned = 0,
1300 		.pgoff = pgoff,
1301 		.pfn = pfn,
1302 		.nr_pages = nr_pages,
1303 	};
1304 	struct rmap_walk_control rwc = {
1305 		.arg = (void *)&state,
1306 		.rmap_one = mapping_wrprotect_range_one,
1307 		.invalid_vma = invalid_mkclean_vma,
1308 	};
1309 
1310 	if (!mapping)
1311 		return 0;
1312 
1313 	__rmap_walk_file(/* folio = */NULL, mapping, pgoff, nr_pages, &rwc,
1314 			 /* locked = */false);
1315 
1316 	return state.cleaned;
1317 }
1318 EXPORT_SYMBOL_GPL(mapping_wrprotect_range);
1319 
1320 /**
1321  * pfn_mkclean_range - Cleans the PTEs (including PMDs) mapped with range of
1322  *                     [@pfn, @pfn + @nr_pages) at the specific offset (@pgoff)
1323  *                     within the @vma of shared mappings. And since clean PTEs
1324  *                     should also be readonly, write protects them too.
1325  * @pfn: start pfn.
1326  * @nr_pages: number of physically contiguous pages srarting with @pfn.
1327  * @pgoff: page offset that the @pfn mapped with.
1328  * @vma: vma that @pfn mapped within.
1329  *
1330  * Returns the number of cleaned PTEs (including PMDs).
1331  */
1332 int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff,
1333 		      struct vm_area_struct *vma)
1334 {
1335 	struct page_vma_mapped_walk pvmw = {
1336 		.pfn		= pfn,
1337 		.nr_pages	= nr_pages,
1338 		.pgoff		= pgoff,
1339 		.vma		= vma,
1340 		.flags		= PVMW_SYNC,
1341 		.pgoff_is_anon	= false,
1342 	};
1343 
1344 	if (invalid_mkclean_vma(vma, NULL))
1345 		return 0;
1346 
1347 	pvmw.address = vma_filebacked_address(vma, pgoff, nr_pages);
1348 	VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma);
1349 
1350 	return page_vma_mkclean_one(&pvmw);
1351 }
1352 
1353 static void __folio_mod_stat(struct folio *folio, int nr, int nr_pmdmapped)
1354 {
1355 	int idx;
1356 
1357 	if (nr) {
1358 		idx = folio_test_anon(folio) ? NR_ANON_MAPPED : NR_FILE_MAPPED;
1359 		lruvec_stat_mod_folio(folio, idx, nr);
1360 	}
1361 	if (nr_pmdmapped) {
1362 		if (folio_test_anon(folio)) {
1363 			idx = NR_ANON_THPS;
1364 			lruvec_stat_mod_folio(folio, idx, nr_pmdmapped);
1365 		} else {
1366 			/* NR_*_PMDMAPPED are not maintained per-memcg */
1367 			idx = folio_test_swapbacked(folio) ?
1368 				NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED;
1369 			__mod_node_page_state(folio_pgdat(folio), idx,
1370 					      nr_pmdmapped);
1371 		}
1372 	}
1373 }
1374 
1375 static __always_inline void __folio_add_rmap(struct folio *folio,
1376 		struct page *page, int nr_pages, struct vm_area_struct *vma,
1377 		enum pgtable_level level)
1378 {
1379 	atomic_t *mapped = &folio->_nr_pages_mapped;
1380 	const int orig_nr_pages = nr_pages;
1381 	int first = 0, nr = 0, nr_pmdmapped = 0;
1382 
1383 	__folio_rmap_sanity_checks(folio, page, nr_pages, level);
1384 
1385 	switch (level) {
1386 	case PGTABLE_LEVEL_PTE:
1387 		if (!folio_test_large(folio)) {
1388 			nr = atomic_inc_and_test(&folio->_mapcount);
1389 			break;
1390 		}
1391 
1392 		if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) {
1393 			nr = folio_add_return_large_mapcount(folio, orig_nr_pages, vma);
1394 			if (nr == orig_nr_pages)
1395 				/* Was completely unmapped. */
1396 				nr = folio_large_nr_pages(folio);
1397 			else
1398 				nr = 0;
1399 			break;
1400 		}
1401 
1402 		do {
1403 			first += atomic_inc_and_test(&page->_mapcount);
1404 		} while (page++, --nr_pages > 0);
1405 
1406 		if (first &&
1407 		    atomic_add_return_relaxed(first, mapped) < ENTIRELY_MAPPED)
1408 			nr = first;
1409 
1410 		folio_add_large_mapcount(folio, orig_nr_pages, vma);
1411 		break;
1412 	case PGTABLE_LEVEL_PMD:
1413 	case PGTABLE_LEVEL_PUD:
1414 		first = atomic_inc_and_test(&folio->_entire_mapcount);
1415 		if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) {
1416 			if (level == PGTABLE_LEVEL_PMD && first)
1417 				nr_pmdmapped = folio_large_nr_pages(folio);
1418 			nr = folio_inc_return_large_mapcount(folio, vma);
1419 			if (nr == 1)
1420 				/* Was completely unmapped. */
1421 				nr = folio_large_nr_pages(folio);
1422 			else
1423 				nr = 0;
1424 			break;
1425 		}
1426 
1427 		if (first) {
1428 			nr = atomic_add_return_relaxed(ENTIRELY_MAPPED, mapped);
1429 			if (likely(nr < ENTIRELY_MAPPED + ENTIRELY_MAPPED)) {
1430 				nr_pages = folio_large_nr_pages(folio);
1431 				/*
1432 				 * We only track PMD mappings of PMD-sized
1433 				 * folios separately.
1434 				 */
1435 				if (level == PGTABLE_LEVEL_PMD)
1436 					nr_pmdmapped = nr_pages;
1437 				nr = nr_pages - (nr & FOLIO_PAGES_MAPPED);
1438 				/* Raced ahead of a remove and another add? */
1439 				if (unlikely(nr < 0))
1440 					nr = 0;
1441 			} else {
1442 				/* Raced ahead of a remove of ENTIRELY_MAPPED */
1443 				nr = 0;
1444 			}
1445 		}
1446 		folio_inc_large_mapcount(folio, vma);
1447 		break;
1448 	default:
1449 		BUILD_BUG();
1450 	}
1451 	__folio_mod_stat(folio, nr, nr_pmdmapped);
1452 }
1453 
1454 /**
1455  * folio_move_anon_rmap - move a folio to our anon_vma
1456  * @folio:	The folio to move to our anon_vma
1457  * @vma:	The vma the folio belongs to
1458  *
1459  * When a folio belongs exclusively to one process after a COW event,
1460  * that folio can be moved into the anon_vma that belongs to just that
1461  * process, so the rmap code will not search the parent or sibling processes.
1462  */
1463 void folio_move_anon_rmap(struct folio *folio, struct vm_area_struct *vma)
1464 {
1465 	void *anon_vma = vma->anon_vma;
1466 
1467 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
1468 	VM_BUG_ON_VMA(!anon_vma, vma);
1469 
1470 	anon_vma += FOLIO_MAPPING_ANON;
1471 	/*
1472 	 * Ensure that anon_vma and the FOLIO_MAPPING_ANON bit are written
1473 	 * simultaneously, so a concurrent reader (eg folio_referenced()'s
1474 	 * folio_test_anon()) will not see one without the other.
1475 	 */
1476 	WRITE_ONCE(folio->mapping, anon_vma);
1477 }
1478 
1479 /**
1480  * __folio_set_anon - set up a new anonymous rmap for a folio
1481  * @folio:	The folio to set up the new anonymous rmap for.
1482  * @vma:	VM area to add the folio to.
1483  * @address:	User virtual address of the mapping
1484  * @exclusive:	Whether the folio is exclusive to the process.
1485  */
1486 static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma,
1487 			     unsigned long address, bool exclusive)
1488 {
1489 	struct anon_vma *anon_vma = vma->anon_vma;
1490 
1491 	BUG_ON(!anon_vma);
1492 
1493 	/*
1494 	 * If the folio isn't exclusive to this vma, we must use the _oldest_
1495 	 * possible anon_vma for the folio mapping!
1496 	 */
1497 	if (!exclusive)
1498 		anon_vma = anon_vma->root;
1499 
1500 	/*
1501 	 * page_idle does a lockless/optimistic rmap scan on folio->mapping.
1502 	 * Make sure the compiler doesn't split the stores of anon_vma and
1503 	 * the FOLIO_MAPPING_ANON type identifier, otherwise the rmap code
1504 	 * could mistake the mapping for a struct address_space and crash.
1505 	 */
1506 	anon_vma = (void *) anon_vma + FOLIO_MAPPING_ANON;
1507 	WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma);
1508 	folio->index = linear_anon_page_index(vma, address);
1509 }
1510 
1511 /**
1512  * __page_check_anon_rmap - sanity check anonymous rmap addition
1513  * @folio:	The folio containing @page.
1514  * @page:	the page to check the mapping of
1515  * @vma:	the vm area in which the mapping is added
1516  * @address:	the user virtual address mapped
1517  */
1518 static void __page_check_anon_rmap(const struct folio *folio,
1519 		const struct page *page, struct vm_area_struct *vma,
1520 		unsigned long address)
1521 {
1522 	/*
1523 	 * The page's anon-rmap details (mapping and index) are guaranteed to
1524 	 * be set up correctly at this point.
1525 	 *
1526 	 * We have exclusion against folio_add_anon_rmap_*() because the caller
1527 	 * always holds the page locked.
1528 	 *
1529 	 * We have exclusion against folio_add_new_anon_rmap because those pages
1530 	 * are initially only visible via the pagetables, and the pte is locked
1531 	 * over the call to folio_add_new_anon_rmap.
1532 	 */
1533 	VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,
1534 			folio);
1535 	VM_BUG_ON_PAGE(page_pgoff(folio, page) !=
1536 		       linear_anon_page_index(vma, address), page);
1537 }
1538 
1539 static __always_inline void __folio_add_anon_rmap(struct folio *folio,
1540 		struct page *page, int nr_pages, struct vm_area_struct *vma,
1541 		unsigned long address, rmap_t flags, enum pgtable_level level)
1542 {
1543 	int i;
1544 
1545 	VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
1546 
1547 	__folio_add_rmap(folio, page, nr_pages, vma, level);
1548 
1549 	if (likely(!folio_test_ksm(folio)))
1550 		__page_check_anon_rmap(folio, page, vma, address);
1551 
1552 	if (flags & RMAP_EXCLUSIVE) {
1553 		switch (level) {
1554 		case PGTABLE_LEVEL_PTE:
1555 			for (i = 0; i < nr_pages; i++)
1556 				SetPageAnonExclusive(page + i);
1557 			break;
1558 		case PGTABLE_LEVEL_PMD:
1559 			SetPageAnonExclusive(page);
1560 			break;
1561 		case PGTABLE_LEVEL_PUD:
1562 			/*
1563 			 * Keep the compiler happy, we don't support anonymous
1564 			 * PUD mappings.
1565 			 */
1566 			WARN_ON_ONCE(1);
1567 			break;
1568 		default:
1569 			BUILD_BUG();
1570 		}
1571 	}
1572 
1573 	VM_WARN_ON_FOLIO(!folio_test_large(folio) && PageAnonExclusive(page) &&
1574 			 atomic_read(&folio->_mapcount) > 0, folio);
1575 	for (i = 0; i < nr_pages; i++) {
1576 		struct page *cur_page = page + i;
1577 
1578 		VM_WARN_ON_FOLIO(folio_test_large(folio) &&
1579 				 folio_entire_mapcount(folio) > 1 &&
1580 				 PageAnonExclusive(cur_page), folio);
1581 		if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT))
1582 			continue;
1583 
1584 		/*
1585 		 * While PTE-mapping a THP we have a PMD and a PTE
1586 		 * mapping.
1587 		 */
1588 		VM_WARN_ON_FOLIO(atomic_read(&cur_page->_mapcount) > 0 &&
1589 				 PageAnonExclusive(cur_page), folio);
1590 	}
1591 
1592 	/*
1593 	 * Only mlock it if the folio is fully mapped to the VMA.
1594 	 *
1595 	 * Partially mapped folios can be split on reclaim and part outside
1596 	 * of mlocked VMA can be evicted or freed.
1597 	 */
1598 	if (folio_nr_pages(folio) == nr_pages)
1599 		mlock_vma_folio(folio, vma);
1600 }
1601 
1602 /**
1603  * folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio
1604  * @folio:	The folio to add the mappings to
1605  * @page:	The first page to add
1606  * @nr_pages:	The number of pages which will be mapped
1607  * @vma:	The vm area in which the mappings are added
1608  * @address:	The user virtual address of the first page to map
1609  * @flags:	The rmap flags
1610  *
1611  * The page range of folio is defined by [first_page, first_page + nr_pages)
1612  *
1613  * The caller needs to hold the page table lock, and the page must be locked in
1614  * the anon_vma case: to serialize mapping,index checking after setting,
1615  * and to ensure that an anon folio is not being upgraded racily to a KSM folio
1616  * (but KSM folios are never downgraded).
1617  */
1618 void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page,
1619 		int nr_pages, struct vm_area_struct *vma, unsigned long address,
1620 		rmap_t flags)
1621 {
1622 	__folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags,
1623 			      PGTABLE_LEVEL_PTE);
1624 }
1625 
1626 /**
1627  * folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio
1628  * @folio:	The folio to add the mapping to
1629  * @page:	The first page to add
1630  * @vma:	The vm area in which the mapping is added
1631  * @address:	The user virtual address of the first page to map
1632  * @flags:	The rmap flags
1633  *
1634  * The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR)
1635  *
1636  * The caller needs to hold the page table lock, and the page must be locked in
1637  * the anon_vma case: to serialize mapping,index checking after setting.
1638  */
1639 void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page,
1640 		struct vm_area_struct *vma, unsigned long address, rmap_t flags)
1641 {
1642 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1643 	__folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags,
1644 			      PGTABLE_LEVEL_PMD);
1645 #else
1646 	WARN_ON_ONCE(true);
1647 #endif
1648 }
1649 
1650 /**
1651  * folio_add_new_anon_rmap - Add mapping to a new anonymous folio.
1652  * @folio:	The folio to add the mapping to.
1653  * @vma:	the vm area in which the mapping is added
1654  * @address:	the user virtual address mapped
1655  * @flags:	The rmap flags
1656  *
1657  * Like folio_add_anon_rmap_*() but must only be called on *new* folios.
1658  * This means the inc-and-test can be bypassed.
1659  * The folio doesn't necessarily need to be locked while it's exclusive
1660  * unless two threads map it concurrently. However, the folio must be
1661  * locked if it's shared.
1662  *
1663  * If the folio is pmd-mappable, it is accounted as a THP.
1664  */
1665 void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
1666 		unsigned long address, rmap_t flags)
1667 {
1668 	const bool exclusive = flags & RMAP_EXCLUSIVE;
1669 	int nr = 1, nr_pmdmapped = 0;
1670 
1671 	VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);
1672 	VM_WARN_ON_FOLIO(!exclusive && !folio_test_locked(folio), folio);
1673 
1674 	/*
1675 	 * VM_DROPPABLE mappings don't swap; instead they're just dropped when
1676 	 * under memory pressure.
1677 	 */
1678 	if (!folio_test_swapbacked(folio) && !(vma->vm_flags & VM_DROPPABLE))
1679 		__folio_set_swapbacked(folio);
1680 	__folio_set_anon(folio, vma, address, exclusive);
1681 
1682 	if (likely(!folio_test_large(folio))) {
1683 		/* increment count (starts at -1) */
1684 		atomic_set(&folio->_mapcount, 0);
1685 		if (exclusive)
1686 			SetPageAnonExclusive(&folio->page);
1687 	} else if (!folio_test_pmd_mappable(folio)) {
1688 		int i;
1689 
1690 		nr = folio_large_nr_pages(folio);
1691 		for (i = 0; i < nr; i++) {
1692 			struct page *page = folio_page(folio, i);
1693 
1694 			if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT))
1695 				/* increment count (starts at -1) */
1696 				atomic_set(&page->_mapcount, 0);
1697 			if (exclusive)
1698 				SetPageAnonExclusive(page);
1699 		}
1700 
1701 		folio_set_large_mapcount(folio, nr, vma);
1702 		if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT))
1703 			atomic_set(&folio->_nr_pages_mapped, nr);
1704 	} else {
1705 		nr = folio_large_nr_pages(folio);
1706 		/* increment count (starts at -1) */
1707 		atomic_set(&folio->_entire_mapcount, 0);
1708 		folio_set_large_mapcount(folio, 1, vma);
1709 		if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT))
1710 			atomic_set(&folio->_nr_pages_mapped, ENTIRELY_MAPPED);
1711 		if (exclusive)
1712 			SetPageAnonExclusive(&folio->page);
1713 		nr_pmdmapped = nr;
1714 	}
1715 
1716 	VM_WARN_ON_ONCE(address < vma->vm_start ||
1717 			address + (nr << PAGE_SHIFT) > vma->vm_end);
1718 
1719 	__folio_mod_stat(folio, nr, nr_pmdmapped);
1720 	mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1);
1721 }
1722 
1723 static __always_inline void __folio_add_file_rmap(struct folio *folio,
1724 		struct page *page, int nr_pages, struct vm_area_struct *vma,
1725 		enum pgtable_level level)
1726 {
1727 	VM_WARN_ON_FOLIO(folio_test_anon(folio), folio);
1728 
1729 	__folio_add_rmap(folio, page, nr_pages, vma, level);
1730 
1731 	/*
1732 	 * Only mlock it if the folio is fully mapped to the VMA.
1733 	 *
1734 	 * Partially mapped folios can be split on reclaim and part outside
1735 	 * of mlocked VMA can be evicted or freed.
1736 	 */
1737 	if (folio_nr_pages(folio) == nr_pages)
1738 		mlock_vma_folio(folio, vma);
1739 }
1740 
1741 /**
1742  * folio_add_file_rmap_ptes - add PTE mappings to a page range of a folio
1743  * @folio:	The folio to add the mappings to
1744  * @page:	The first page to add
1745  * @nr_pages:	The number of pages that will be mapped using PTEs
1746  * @vma:	The vm area in which the mappings are added
1747  *
1748  * The page range of the folio is defined by [page, page + nr_pages)
1749  *
1750  * The caller needs to hold the page table lock.
1751  */
1752 void folio_add_file_rmap_ptes(struct folio *folio, struct page *page,
1753 		int nr_pages, struct vm_area_struct *vma)
1754 {
1755 	__folio_add_file_rmap(folio, page, nr_pages, vma, PGTABLE_LEVEL_PTE);
1756 }
1757 
1758 /**
1759  * folio_add_file_rmap_pmd - add a PMD mapping to a page range of a folio
1760  * @folio:	The folio to add the mapping to
1761  * @page:	The first page to add
1762  * @vma:	The vm area in which the mapping is added
1763  *
1764  * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
1765  *
1766  * The caller needs to hold the page table lock.
1767  */
1768 void folio_add_file_rmap_pmd(struct folio *folio, struct page *page,
1769 		struct vm_area_struct *vma)
1770 {
1771 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1772 	__folio_add_file_rmap(folio, page, HPAGE_PMD_NR, vma, PGTABLE_LEVEL_PMD);
1773 #else
1774 	WARN_ON_ONCE(true);
1775 #endif
1776 }
1777 
1778 /**
1779  * folio_add_file_rmap_pud - add a PUD mapping to a page range of a folio
1780  * @folio:	The folio to add the mapping to
1781  * @page:	The first page to add
1782  * @vma:	The vm area in which the mapping is added
1783  *
1784  * The page range of the folio is defined by [page, page + HPAGE_PUD_NR)
1785  *
1786  * The caller needs to hold the page table lock.
1787  */
1788 void folio_add_file_rmap_pud(struct folio *folio, struct page *page,
1789 		struct vm_area_struct *vma)
1790 {
1791 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
1792 	defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
1793 	__folio_add_file_rmap(folio, page, HPAGE_PUD_NR, vma, PGTABLE_LEVEL_PUD);
1794 #else
1795 	WARN_ON_ONCE(true);
1796 #endif
1797 }
1798 
1799 static __always_inline void __folio_remove_rmap(struct folio *folio,
1800 		struct page *page, int nr_pages, struct vm_area_struct *vma,
1801 		enum pgtable_level level)
1802 {
1803 	atomic_t *mapped = &folio->_nr_pages_mapped;
1804 	int last = 0, nr = 0, nr_pmdmapped = 0;
1805 	bool partially_mapped = false;
1806 
1807 	__folio_rmap_sanity_checks(folio, page, nr_pages, level);
1808 
1809 	switch (level) {
1810 	case PGTABLE_LEVEL_PTE:
1811 		if (!folio_test_large(folio)) {
1812 			nr = atomic_add_negative(-1, &folio->_mapcount);
1813 			break;
1814 		}
1815 
1816 		if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) {
1817 			nr = folio_sub_return_large_mapcount(folio, nr_pages, vma);
1818 			if (!nr) {
1819 				/* Now completely unmapped. */
1820 				nr = folio_large_nr_pages(folio);
1821 			} else {
1822 				partially_mapped = nr < folio_large_nr_pages(folio) &&
1823 						   !folio_entire_mapcount(folio);
1824 				nr = 0;
1825 			}
1826 			break;
1827 		}
1828 
1829 		folio_sub_large_mapcount(folio, nr_pages, vma);
1830 		do {
1831 			last += atomic_add_negative(-1, &page->_mapcount);
1832 		} while (page++, --nr_pages > 0);
1833 
1834 		if (last &&
1835 		    atomic_sub_return_relaxed(last, mapped) < ENTIRELY_MAPPED)
1836 			nr = last;
1837 
1838 		partially_mapped = nr && atomic_read(mapped);
1839 		break;
1840 	case PGTABLE_LEVEL_PMD:
1841 	case PGTABLE_LEVEL_PUD:
1842 		if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) {
1843 			last = atomic_add_negative(-1, &folio->_entire_mapcount);
1844 			if (level == PGTABLE_LEVEL_PMD && last)
1845 				nr_pmdmapped = folio_large_nr_pages(folio);
1846 			nr = folio_dec_return_large_mapcount(folio, vma);
1847 			if (!nr) {
1848 				/* Now completely unmapped. */
1849 				nr = folio_large_nr_pages(folio);
1850 			} else {
1851 				partially_mapped = last &&
1852 						   nr < folio_large_nr_pages(folio);
1853 				nr = 0;
1854 			}
1855 			break;
1856 		}
1857 
1858 		folio_dec_large_mapcount(folio, vma);
1859 		last = atomic_add_negative(-1, &folio->_entire_mapcount);
1860 		if (last) {
1861 			nr = atomic_sub_return_relaxed(ENTIRELY_MAPPED, mapped);
1862 			if (likely(nr < ENTIRELY_MAPPED)) {
1863 				nr_pages = folio_large_nr_pages(folio);
1864 				if (level == PGTABLE_LEVEL_PMD)
1865 					nr_pmdmapped = nr_pages;
1866 				nr = nr_pages - nr;
1867 				/* Raced ahead of another remove and an add? */
1868 				if (unlikely(nr < 0))
1869 					nr = 0;
1870 			} else {
1871 				/* An add of ENTIRELY_MAPPED raced ahead */
1872 				nr = 0;
1873 			}
1874 		}
1875 
1876 		partially_mapped = nr && nr < nr_pmdmapped;
1877 		break;
1878 	default:
1879 		BUILD_BUG();
1880 	}
1881 
1882 	/*
1883 	 * Queue anon large folio for deferred split if at least one page of
1884 	 * the folio is unmapped and at least one page is still mapped.
1885 	 *
1886 	 * Check partially_mapped first to ensure it is a large folio.
1887 	 *
1888 	 * Device private folios do not support deferred splitting and
1889 	 * shrinker based scanning of the folios to free.
1890 	 */
1891 	if (partially_mapped && folio_test_anon(folio) &&
1892 	    !folio_test_partially_mapped(folio) &&
1893 	    !folio_is_device_private(folio))
1894 		deferred_split_folio(folio, true);
1895 
1896 	__folio_mod_stat(folio, -nr, -nr_pmdmapped);
1897 
1898 	/*
1899 	 * It would be tidy to reset folio_test_anon mapping when fully
1900 	 * unmapped, but that might overwrite a racing folio_add_anon_rmap_*()
1901 	 * which increments mapcount after us but sets mapping before us:
1902 	 * so leave the reset to free_pages_prepare, and remember that
1903 	 * it's only reliable while mapped.
1904 	 */
1905 
1906 	munlock_vma_folio(folio, vma);
1907 }
1908 
1909 /**
1910  * folio_remove_rmap_ptes - remove PTE mappings from a page range of a folio
1911  * @folio:	The folio to remove the mappings from
1912  * @page:	The first page to remove
1913  * @nr_pages:	The number of pages that will be removed from the mapping
1914  * @vma:	The vm area from which the mappings are removed
1915  *
1916  * The page range of the folio is defined by [page, page + nr_pages)
1917  *
1918  * The caller needs to hold the page table lock.
1919  */
1920 void folio_remove_rmap_ptes(struct folio *folio, struct page *page,
1921 		int nr_pages, struct vm_area_struct *vma)
1922 {
1923 	__folio_remove_rmap(folio, page, nr_pages, vma, PGTABLE_LEVEL_PTE);
1924 }
1925 
1926 /**
1927  * folio_remove_rmap_pmd - remove a PMD mapping from a page range of a folio
1928  * @folio:	The folio to remove the mapping from
1929  * @page:	The first page to remove
1930  * @vma:	The vm area from which the mapping is removed
1931  *
1932  * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
1933  *
1934  * The caller needs to hold the page table lock.
1935  */
1936 void folio_remove_rmap_pmd(struct folio *folio, struct page *page,
1937 		struct vm_area_struct *vma)
1938 {
1939 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1940 	__folio_remove_rmap(folio, page, HPAGE_PMD_NR, vma, PGTABLE_LEVEL_PMD);
1941 #else
1942 	WARN_ON_ONCE(true);
1943 #endif
1944 }
1945 
1946 /**
1947  * folio_remove_rmap_pud - remove a PUD mapping from a page range of a folio
1948  * @folio:	The folio to remove the mapping from
1949  * @page:	The first page to remove
1950  * @vma:	The vm area from which the mapping is removed
1951  *
1952  * The page range of the folio is defined by [page, page + HPAGE_PUD_NR)
1953  *
1954  * The caller needs to hold the page table lock.
1955  */
1956 void folio_remove_rmap_pud(struct folio *folio, struct page *page,
1957 		struct vm_area_struct *vma)
1958 {
1959 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
1960 	defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
1961 	__folio_remove_rmap(folio, page, HPAGE_PUD_NR, vma, PGTABLE_LEVEL_PUD);
1962 #else
1963 	WARN_ON_ONCE(true);
1964 #endif
1965 }
1966 
1967 static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
1968 			struct page_vma_mapped_walk *pvmw,
1969 			enum ttu_flags flags, pte_t pte)
1970 {
1971 	unsigned long end_addr, addr = pvmw->address;
1972 	struct vm_area_struct *vma = pvmw->vma;
1973 	unsigned int max_nr;
1974 
1975 	if (flags & TTU_HWPOISON)
1976 		return 1;
1977 	if (!folio_test_large(folio))
1978 		return 1;
1979 
1980 	/* We may only batch within a single VMA and a single page table. */
1981 	end_addr = pte_table_clamped_end(addr, vma->vm_end);
1982 	max_nr = (end_addr - addr) >> PAGE_SHIFT;
1983 
1984 	/* We only support lazyfree or file folios batching for now ... */
1985 	if (folio_test_anon(folio) && folio_test_swapbacked(folio))
1986 		return 1;
1987 
1988 	if (pte_unused(pte))
1989 		return 1;
1990 
1991 	/*
1992 	 * If unmap fails, we need to restore the ptes. To avoid accidentally
1993 	 * upgrading write permissions for ptes that were not originally
1994 	 * writable, and to avoid losing the soft-dirty bit, use the
1995 	 * appropriate FPB flags.
1996 	 */
1997 	return folio_pte_batch_flags(folio, vma, pvmw->pte, &pte, max_nr,
1998 				     FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY);
1999 }
2000 
2001 static bool try_to_unmap_poisoned_hugetlb_one(struct folio *folio,
2002 		struct vm_area_struct *vma, unsigned long address, void *arg)
2003 {
2004 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
2005 	const unsigned long hsz = huge_page_size(hstate_vma(vma));
2006 	const enum ttu_flags flags = (enum ttu_flags)(long)arg;
2007 	struct mm_struct *mm = vma->vm_mm;
2008 	struct mmu_notifier_range range;
2009 	bool ret = true;
2010 	pte_t pteval;
2011 
2012 	/*
2013 	 * The try_to_unmap() is only passed a hugetlb folio in the case
2014 	 * where the hugetlb folio is poisoned.
2015 	 */
2016 	VM_WARN_ON_ONCE_FOLIO(!folio_test_hwpoison(folio), folio);
2017 	VM_WARN_ON_ONCE(!(flags & TTU_HWPOISON));
2018 
2019 	range.end = vma_address_end(&pvmw);
2020 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
2021 				address, range.end);
2022 	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
2023 	mmu_notifier_invalidate_range_start(&range);
2024 
2025 	/* There is only a single mapping in a VMA. */
2026 	if (!page_vma_mapped_walk(&pvmw))
2027 		goto range_end;
2028 
2029 	VM_WARN_ON_ONCE(address != pvmw.address);
2030 
2031 	pteval = huge_ptep_get(mm, address, pvmw.pte);
2032 	VM_WARN_ON_ONCE(!pte_present(pteval));
2033 	VM_WARN_ON_ONCE(pte_pfn(pteval) != folio_pfn(folio));
2034 
2035 	/*
2036 	 * huge_pmd_unshare may unmap an entire PMD page. There is no way of
2037 	 * knowing exactly which PMDs may be cached for this mm, so we must
2038 	 * flush them all. start/end were already adjusted above to cover this
2039 	 * range.
2040 	 */
2041 	flush_cache_range(vma, range.start, range.end);
2042 
2043 	/*
2044 	 * To call huge_pmd_unshare, i_mmap_rwsem must be held in write mode.
2045 	 * Caller needs to explicitly do this outside rmap routines.
2046 	 *
2047 	 * We also must hold hugetlb vma_lock in write mode. Lock order dictates
2048 	 * acquiring vma_lock BEFORE i_mmap_rwsem. We can only try lock here and
2049 	 * fail if unsuccessful.
2050 	 */
2051 	if (!folio_test_anon(folio)) {
2052 		struct mmu_gather tlb;
2053 
2054 		VM_WARN_ON_ONCE(!(flags & TTU_RMAP_LOCKED));
2055 		if (!hugetlb_vma_trylock_write(vma)) {
2056 			ret = false;
2057 			goto walk_done;
2058 		}
2059 
2060 		tlb_gather_mmu_vma(&tlb, vma);
2061 		if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) {
2062 			hugetlb_vma_unlock_write(vma);
2063 			huge_pmd_unshare_flush(&tlb, vma);
2064 			tlb_finish_mmu(&tlb);
2065 			/*
2066 			 * The PMD table was unmapped, consequently unmapping
2067 			 * the folio.
2068 			 */
2069 			goto walk_done;
2070 		}
2071 		hugetlb_vma_unlock_write(vma);
2072 		tlb_finish_mmu(&tlb);
2073 	}
2074 	pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
2075 	if (huge_pte_dirty(pteval))
2076 		folio_mark_dirty(folio);
2077 
2078 	pteval = swp_entry_to_pte(make_hwpoison_entry(folio_page(folio, 0)));
2079 	hugetlb_count_sub(folio_nr_pages(folio), mm);
2080 	set_huge_pte_at(mm, address, pvmw.pte, pteval, hsz);
2081 	hugetlb_remove_rmap(folio);
2082 	folio_put_refs(folio, 1);
2083 
2084 walk_done:
2085 	page_vma_mapped_walk_done(&pvmw);
2086 range_end:
2087 	mmu_notifier_invalidate_range_end(&range);
2088 	return ret;
2089 }
2090 
2091 static bool ttu_anon_lazyfree_folio(struct vm_area_struct *vma,
2092 		struct folio *folio, unsigned long nr_pages)
2093 {
2094 	int ref_count, map_count;
2095 
2096 	/*
2097 	 * Synchronize with gup_pte_range():
2098 	 * - clear PTE; barrier; read refcount
2099 	 * - inc refcount; barrier; read PTE
2100 	 */
2101 	smp_mb();
2102 
2103 	ref_count = folio_ref_count(folio);
2104 	map_count = folio_mapcount(folio);
2105 
2106 	/*
2107 	 * Order reads for page refcount and dirty flag
2108 	 * (see comments in __remove_mapping()).
2109 	 */
2110 	smp_rmb();
2111 
2112 	if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) {
2113 		/*
2114 		 * redirtied either using the page table or a previously
2115 		 * obtained GUP reference.
2116 		 */
2117 		folio_set_swapbacked(folio);
2118 		return false;
2119 	}
2120 
2121 	/*
2122 	 * Additional references could be due to GUP or speculative lookups.
2123 	 * GUP users must mark the folio dirty if there was a modification.
2124 	 * This folio cannot be reclaimed right now either way, so act just
2125 	 * like nothing happened. We'll come back here later and detect if the
2126 	 * folio was dirtied when the additional reference is gone.
2127 	 */
2128 	if (ref_count != 1 + map_count)
2129 		return false;
2130 
2131 	add_mm_counter(vma->vm_mm, MM_ANONPAGES, -nr_pages);
2132 	return true;
2133 }
2134 
2135 static pte_t swp_pte_prepare(swp_entry_t entry, pte_t old_pte,
2136 		bool anon_exclusive)
2137 {
2138 	pte_t swp_pte = swp_entry_to_pte(entry);
2139 
2140 	if (anon_exclusive)
2141 		swp_pte = pte_swp_mkexclusive(swp_pte);
2142 
2143 	if (likely(pte_present(old_pte))) {
2144 		if (pte_soft_dirty(old_pte))
2145 			swp_pte = pte_swp_mksoft_dirty(swp_pte);
2146 		if (pte_uffd(old_pte))
2147 			swp_pte = pte_swp_mkuffd(swp_pte);
2148 	} else {
2149 		/* Device-exclusive entry */
2150 		if (pte_swp_soft_dirty(old_pte))
2151 			swp_pte = pte_swp_mksoft_dirty(swp_pte);
2152 		if (pte_swp_uffd(old_pte))
2153 			swp_pte = pte_swp_mkuffd(swp_pte);
2154 	}
2155 
2156 	return swp_pte;
2157 }
2158 
2159 static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma,
2160 		struct folio *folio, struct page *page, unsigned long address,
2161 		pte_t *ptep, pte_t pteval)
2162 {
2163 	const bool anon_exclusive = folio_test_anon(folio) &&
2164 				    PageAnonExclusive(page);
2165 	swp_entry_t entry = folio_page_swap_entry(folio, page);
2166 	struct mm_struct *mm = vma->vm_mm;
2167 
2168 	if (folio_dup_swap(folio, page) < 0)
2169 		return false;
2170 
2171 	/*
2172 	 * arch_unmap_one() is expected to be a NOP on
2173 	 * architectures where we could have PFN swap PTEs,
2174 	 * so we'll not check/care.
2175 	 */
2176 	if (arch_unmap_one(mm, vma, address, pteval) < 0) {
2177 		folio_put_swap(folio, page);
2178 		return false;
2179 	}
2180 
2181 	/* See folio_try_share_anon_rmap(): clear PTE first. */
2182 	if (anon_exclusive && folio_try_share_anon_rmap_pte(folio, page)) {
2183 		folio_put_swap(folio, page);
2184 		return false;
2185 	}
2186 
2187 	mm_prepare_for_swap_entries(mm);
2188 	dec_mm_counter(mm, MM_ANONPAGES);
2189 	inc_mm_counter(mm, MM_SWAPENTS);
2190 	set_pte_at(mm, address, ptep,
2191 		   swp_pte_prepare(entry, pteval, anon_exclusive));
2192 	return true;
2193 }
2194 
2195 static bool ttu_anon_folio(struct vm_area_struct *vma, struct folio *folio,
2196 		struct page *page, unsigned long address, pte_t *ptep,
2197 		pte_t pteval, unsigned long nr_pages)
2198 {
2199 	/*
2200 	 * Store the swap location in the pte.
2201 	 * See handle_pte_fault() ...
2202 	 */
2203 	if (WARN_ON_ONCE(folio_test_swapbacked(folio) !=
2204 			 folio_test_swapcache(folio)))
2205 		return false;
2206 
2207 	if (!folio_test_swapbacked(folio))
2208 		return ttu_anon_lazyfree_folio(vma, folio, nr_pages);
2209 
2210 	/* nr_pages > 1 not supported yet */
2211 	return ttu_anon_swapbacked_folio(vma, folio, page, address, ptep,
2212 					 pteval);
2213 }
2214 
2215 /*
2216  * @arg: enum ttu_flags will be passed to this argument
2217  */
2218 static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
2219 		     unsigned long address, void *arg)
2220 {
2221 	struct mm_struct *mm = vma->vm_mm;
2222 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
2223 	bool ret = true;
2224 	pte_t pteval;
2225 	struct page *page;
2226 	struct mmu_notifier_range range;
2227 	enum ttu_flags flags = (enum ttu_flags)(long)arg;
2228 	unsigned long nr_pages = 1, end_addr;
2229 	unsigned long pfn;
2230 	int ptes = 0;
2231 
2232 	/*
2233 	 * When racing against e.g. zap_pte_range() on another cpu,
2234 	 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
2235 	 * try_to_unmap() may return before folio_mapped() has become false,
2236 	 * if page table locking is skipped: use TTU_SYNC to wait for that.
2237 	 */
2238 	if (flags & TTU_SYNC)
2239 		pvmw.flags = PVMW_SYNC;
2240 
2241 	/*
2242 	 * For THP, we have to assume the worse case ie pmd for invalidation.
2243 	 *
2244 	 * Note that the folio can not be freed in this function as call of
2245 	 * try_to_unmap() must hold a reference on the folio.
2246 	 */
2247 	range.end = vma_address_end(&pvmw);
2248 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
2249 				address, range.end);
2250 	mmu_notifier_invalidate_range_start(&range);
2251 
2252 	while (page_vma_mapped_walk(&pvmw)) {
2253 		nr_pages = 1;
2254 
2255 		/*
2256 		 * If the folio is in an mlock()d vma, we must not swap it out.
2257 		 * VMA_LOCKONFAULT_BIT alone marks an mlock walk in progress, see
2258 		 * mlock_vma_pages_range().
2259 		 */
2260 		if (!(flags & TTU_IGNORE_MLOCK) &&
2261 		    vma_test_any_mask(vma, VMA_LOCKED_MASK)) {
2262 			ptes++;
2263 
2264 			/*
2265 			 * Set 'ret' to indicate the page cannot be unmapped.
2266 			 *
2267 			 * Do not jump to walk_abort immediately as additional
2268 			 * iteration might be required to detect fully mapped
2269 			 * folio an mlock it.
2270 			 */
2271 			ret = false;
2272 
2273 			/* Only mlock fully mapped pages */
2274 			if (pvmw.pte && ptes != pvmw.nr_pages)
2275 				continue;
2276 
2277 			/*
2278 			 * All PTEs must be protected by page table lock in
2279 			 * order to mlock the page.
2280 			 *
2281 			 * If page table boundary has been cross, current ptl
2282 			 * only protect part of ptes.
2283 			 */
2284 			if (pvmw.flags & PVMW_PGTABLE_CROSSED)
2285 				goto walk_done;
2286 
2287 			/* Restore the mlock which got missed */
2288 			mlock_vma_folio(folio, vma);
2289 			goto walk_done;
2290 		}
2291 
2292 		if (!pvmw.pte) {
2293 			if (flags & TTU_SPLIT_HUGE_PMD) {
2294 				/*
2295 				 * We temporarily have to drop the PTL and
2296 				 * restart so we can process the PTE-mapped THP.
2297 				 */
2298 				split_huge_pmd_locked(vma, pvmw.address,
2299 						      pvmw.pmd, false);
2300 				flags &= ~TTU_SPLIT_HUGE_PMD;
2301 				page_vma_mapped_walk_restart(&pvmw);
2302 				continue;
2303 			}
2304 		}
2305 
2306 		/* Unexpected PMD-mapped THP? */
2307 		VM_BUG_ON_FOLIO(!pvmw.pte, folio);
2308 
2309 		address = pvmw.address;
2310 		pteval = ptep_get(pvmw.pte);
2311 
2312 		if (likely(pte_present(pteval))) {
2313 			pfn = pte_pfn(pteval);
2314 		} else {
2315 			/*
2316 			 * Handle PFN swap PTEs, such as device-exclusive ones,
2317 			 * that actually map pages.
2318 			 */
2319 			const softleaf_t entry = softleaf_from_pte(pteval);
2320 
2321 			pfn = softleaf_to_pfn(entry);
2322 		}
2323 
2324 		page = folio_page(folio, pfn - folio_pfn(folio));
2325 
2326 		if (likely(pte_present(pteval))) {
2327 			nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval);
2328 			end_addr = address + nr_pages * PAGE_SIZE;
2329 			flush_cache_range(vma, address, end_addr);
2330 
2331 			/* Nuke the page table entry. */
2332 			pteval = get_and_clear_ptes(mm, address, pvmw.pte, nr_pages);
2333 			/*
2334 			 * We clear the PTE but do not flush so potentially
2335 			 * a remote CPU could still be writing to the folio.
2336 			 * If the entry was previously clean then the
2337 			 * architecture must guarantee that a clear->dirty
2338 			 * transition on a cached TLB entry is written through
2339 			 * and traps if the PTE is unmapped.
2340 			 */
2341 			if (should_defer_flush(mm, flags))
2342 				set_tlb_ubc_flush_pending(mm, pteval, address, end_addr);
2343 			else
2344 				flush_tlb_range(vma, address, end_addr);
2345 			if (pte_dirty(pteval))
2346 				folio_mark_dirty(folio);
2347 		} else {
2348 			pte_clear(mm, address, pvmw.pte);
2349 		}
2350 
2351 		/*
2352 		 * Now the pte is cleared. If this pte was uffd-wp armed,
2353 		 * we may want to replace a none pte with a marker pte if
2354 		 * it's file-backed, so we don't lose the tracking info.
2355 		 */
2356 		cond_install_uffd_wp_ptes(vma, address, pvmw.pte, pteval,
2357 					  nr_pages);
2358 
2359 		/* Update high watermark before we lower rss */
2360 		update_hiwater_rss(mm);
2361 
2362 		/* unmap_poisoned_folio() only refs order-0 folios */
2363 		if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) {
2364 			pteval = swp_entry_to_pte(make_hwpoison_entry(page));
2365 			dec_mm_counter(mm, mm_counter(folio));
2366 			set_pte_at(mm, address, pvmw.pte, pteval);
2367 		} else if (likely(pte_present(pteval)) && pte_unused(pteval) &&
2368 			   !userfaultfd_armed(vma)) {
2369 			/*
2370 			 * The guest indicated that the page content is of no
2371 			 * interest anymore. Simply discard the pte, vmscan
2372 			 * will take care of the rest.
2373 			 * A future reference will then fault in a new zero
2374 			 * page. When userfaultfd is active, we must not drop
2375 			 * this page though, as its main user (postcopy
2376 			 * migration) will not expect userfaults on already
2377 			 * copied pages.
2378 			 */
2379 			dec_mm_counter(mm, mm_counter(folio));
2380 		} else if (folio_test_anon(folio)) {
2381 			if (!ttu_anon_folio(vma, folio, page, address,
2382 					    pvmw.pte, pteval, nr_pages)) {
2383 				set_ptes(mm, address, pvmw.pte, pteval, nr_pages);
2384 				goto walk_abort;
2385 			}
2386 
2387 			goto finish_unmap;
2388 		} else {
2389 			/*
2390 			 * This is a locked file-backed folio,
2391 			 * so it cannot be removed from the page
2392 			 * cache and replaced by a new folio before
2393 			 * mmu_notifier_invalidate_range_end, so no
2394 			 * concurrent thread might update its page table
2395 			 * to point at a new folio while a device is
2396 			 * still using this folio.
2397 			 *
2398 			 * See Documentation/mm/mmu_notifier.rst
2399 			 */
2400 			add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
2401 		}
2402 finish_unmap:
2403 		folio_remove_rmap_ptes(folio, page, nr_pages, vma);
2404 		if (vma_test_any_mask(vma, VMA_LOCKED_MASK))
2405 			mlock_drain_local();
2406 		folio_put_refs(folio, nr_pages);
2407 
2408 		/*
2409 		 * If we are sure that we batched the entire folio and cleared
2410 		 * all PTEs, we can just optimize and stop right here.
2411 		 */
2412 		if (nr_pages == folio_nr_pages(folio))
2413 			goto walk_done;
2414 		continue;
2415 walk_abort:
2416 		ret = false;
2417 walk_done:
2418 		page_vma_mapped_walk_done(&pvmw);
2419 		break;
2420 	}
2421 
2422 	mmu_notifier_invalidate_range_end(&range);
2423 
2424 	return ret;
2425 }
2426 
2427 static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)
2428 {
2429 	return vma_is_temporary_stack(vma);
2430 }
2431 
2432 static int folio_not_mapped(struct folio *folio)
2433 {
2434 	return !folio_mapped(folio);
2435 }
2436 
2437 /**
2438  * try_to_unmap - Try to remove all page table mappings to a folio.
2439  * @folio: The folio to unmap.
2440  * @flags: action and flags
2441  *
2442  * Tries to remove all the page table entries which are mapping this
2443  * folio.  It is the caller's responsibility to check if the folio is
2444  * still mapped if needed (use TTU_SYNC to prevent accounting races).
2445  *
2446  * Context: Caller must hold the folio lock.
2447  */
2448 void try_to_unmap(struct folio *folio, enum ttu_flags flags)
2449 {
2450 	struct rmap_walk_control rwc = {
2451 		.rmap_one = folio_test_hugetlb(folio) ?
2452 				try_to_unmap_poisoned_hugetlb_one : try_to_unmap_one,
2453 		.arg = (void *)flags,
2454 		.done = folio_not_mapped,
2455 		.anon_lock = folio_lock_anon_vma_read,
2456 	};
2457 
2458 	if (flags & TTU_RMAP_LOCKED)
2459 		rmap_walk_locked(folio, &rwc);
2460 	else
2461 		rmap_walk(folio, &rwc);
2462 }
2463 
2464 /*
2465  * @arg: enum ttu_flags will be passed to this argument.
2466  *
2467  * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs
2468  * containing migration entries.
2469  */
2470 static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
2471 		     unsigned long address, void *arg)
2472 {
2473 	struct mm_struct *mm = vma->vm_mm;
2474 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
2475 	bool anon_exclusive, writable, ret = true;
2476 	pte_t pteval;
2477 	struct page *subpage;
2478 	struct mmu_notifier_range range;
2479 	enum ttu_flags flags = (enum ttu_flags)(long)arg;
2480 	unsigned long pfn;
2481 	unsigned long hsz = 0;
2482 
2483 	/*
2484 	 * When racing against e.g. zap_pte_range() on another cpu,
2485 	 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
2486 	 * try_to_migrate() may return before folio_mapped() has become false,
2487 	 * if page table locking is skipped: use TTU_SYNC to wait for that.
2488 	 */
2489 	if (flags & TTU_SYNC)
2490 		pvmw.flags = PVMW_SYNC;
2491 
2492 	/*
2493 	 * For THP, we have to assume the worse case ie pmd for invalidation.
2494 	 * For hugetlb, it could be much worse if we need to do pud
2495 	 * invalidation in the case of pmd sharing.
2496 	 *
2497 	 * Note that the page can not be free in this function as call of
2498 	 * try_to_unmap() must hold a reference on the page.
2499 	 */
2500 	range.end = vma_address_end(&pvmw);
2501 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
2502 				address, range.end);
2503 	if (folio_test_hugetlb(folio)) {
2504 		/*
2505 		 * If sharing is possible, start and end will be adjusted
2506 		 * accordingly.
2507 		 */
2508 		adjust_range_if_pmd_sharing_possible(vma, &range.start,
2509 						     &range.end);
2510 
2511 		/* We need the huge page size for set_huge_pte_at() */
2512 		hsz = huge_page_size(hstate_vma(vma));
2513 	}
2514 	mmu_notifier_invalidate_range_start(&range);
2515 
2516 	while (page_vma_mapped_walk(&pvmw)) {
2517 		/* PMD-mapped THP migration entry */
2518 		if (!pvmw.pte) {
2519 			__maybe_unused unsigned long pfn;
2520 			__maybe_unused pmd_t pmdval;
2521 
2522 			if (flags & TTU_SPLIT_HUGE_PMD) {
2523 				/*
2524 				 * split_huge_pmd_locked() might leave the
2525 				 * folio mapped through PTEs. Retry the walk
2526 				 * so we can detect this scenario and properly
2527 				 * abort the walk.
2528 				 */
2529 				split_huge_pmd_locked(vma, pvmw.address,
2530 						      pvmw.pmd, true);
2531 				flags &= ~TTU_SPLIT_HUGE_PMD;
2532 				page_vma_mapped_walk_restart(&pvmw);
2533 				continue;
2534 			}
2535 #ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
2536 			pmdval = pmdp_get(pvmw.pmd);
2537 			if (likely(pmd_present(pmdval)))
2538 				pfn = pmd_pfn(pmdval);
2539 			else
2540 				pfn = softleaf_to_pfn(softleaf_from_pmd(pmdval));
2541 
2542 			subpage = folio_page(folio, pfn - folio_pfn(folio));
2543 
2544 			VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
2545 					!folio_test_pmd_mappable(folio), folio);
2546 
2547 			if (set_pmd_migration_entry(&pvmw, subpage)) {
2548 				ret = false;
2549 				page_vma_mapped_walk_done(&pvmw);
2550 				break;
2551 			}
2552 			continue;
2553 #endif
2554 		}
2555 
2556 		/* Unexpected PMD-mapped THP? */
2557 		VM_BUG_ON_FOLIO(!pvmw.pte, folio);
2558 
2559 		address = pvmw.address;
2560 		if (folio_test_hugetlb(folio))
2561 			pteval = huge_ptep_get(mm, address, pvmw.pte);
2562 		else
2563 			pteval = ptep_get(pvmw.pte);
2564 		if (likely(pte_present(pteval))) {
2565 			pfn = pte_pfn(pteval);
2566 		} else {
2567 			/*
2568 			 * Handle PFN swap PTEs, such as device-exclusive ones,
2569 			 * that actually map pages.
2570 			 */
2571 			const softleaf_t entry = softleaf_from_pte(pteval);
2572 
2573 			pfn = softleaf_to_pfn(entry);
2574 			VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);
2575 		}
2576 
2577 		subpage = folio_page(folio, pfn - folio_pfn(folio));
2578 		anon_exclusive = folio_test_anon(folio) &&
2579 				 PageAnonExclusive(subpage);
2580 
2581 		if (folio_test_hugetlb(folio)) {
2582 			bool anon = folio_test_anon(folio);
2583 
2584 			/*
2585 			 * huge_pmd_unshare may unmap an entire PMD page.
2586 			 * There is no way of knowing exactly which PMDs may
2587 			 * be cached for this mm, so we must flush them all.
2588 			 * start/end were already adjusted above to cover this
2589 			 * range.
2590 			 */
2591 			flush_cache_range(vma, range.start, range.end);
2592 
2593 			/*
2594 			 * To call huge_pmd_unshare, i_mmap_rwsem must be
2595 			 * held in write mode.  Caller needs to explicitly
2596 			 * do this outside rmap routines.
2597 			 *
2598 			 * We also must hold hugetlb vma_lock in write mode.
2599 			 * Lock order dictates acquiring vma_lock BEFORE
2600 			 * i_mmap_rwsem.  We can only try lock here and
2601 			 * fail if unsuccessful.
2602 			 */
2603 			if (!anon) {
2604 				struct mmu_gather tlb;
2605 
2606 				VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
2607 				if (!hugetlb_vma_trylock_write(vma)) {
2608 					page_vma_mapped_walk_done(&pvmw);
2609 					ret = false;
2610 					break;
2611 				}
2612 
2613 				tlb_gather_mmu_vma(&tlb, vma);
2614 				if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) {
2615 					hugetlb_vma_unlock_write(vma);
2616 					huge_pmd_unshare_flush(&tlb, vma);
2617 					tlb_finish_mmu(&tlb);
2618 					/*
2619 					 * The PMD table was unmapped,
2620 					 * consequently unmapping the folio.
2621 					 */
2622 					page_vma_mapped_walk_done(&pvmw);
2623 					break;
2624 				}
2625 				hugetlb_vma_unlock_write(vma);
2626 				tlb_finish_mmu(&tlb);
2627 			}
2628 			/* Nuke the hugetlb page table entry */
2629 			pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
2630 			if (pte_dirty(pteval))
2631 				folio_mark_dirty(folio);
2632 			writable = pte_write(pteval);
2633 		} else if (likely(pte_present(pteval))) {
2634 			flush_cache_page(vma, address, pfn);
2635 			/* Nuke the page table entry. */
2636 			if (should_defer_flush(mm, flags)) {
2637 				/*
2638 				 * We clear the PTE but do not flush so potentially
2639 				 * a remote CPU could still be writing to the folio.
2640 				 * If the entry was previously clean then the
2641 				 * architecture must guarantee that a clear->dirty
2642 				 * transition on a cached TLB entry is written through
2643 				 * and traps if the PTE is unmapped.
2644 				 */
2645 				pteval = ptep_get_and_clear(mm, address, pvmw.pte);
2646 
2647 				set_tlb_ubc_flush_pending(mm, pteval, address, address + PAGE_SIZE);
2648 			} else {
2649 				pteval = ptep_clear_flush(vma, address, pvmw.pte);
2650 			}
2651 			if (pte_dirty(pteval))
2652 				folio_mark_dirty(folio);
2653 			writable = pte_write(pteval);
2654 		} else {
2655 			const softleaf_t entry = softleaf_from_pte(pteval);
2656 
2657 			pte_clear(mm, address, pvmw.pte);
2658 
2659 			writable = softleaf_is_device_private_write(entry);
2660 		}
2661 
2662 		VM_WARN_ON_FOLIO(writable && folio_test_anon(folio) &&
2663 				!anon_exclusive, folio);
2664 
2665 		/* Update high watermark before we lower rss */
2666 		update_hiwater_rss(mm);
2667 
2668 		if (PageHWPoison(subpage)) {
2669 			VM_WARN_ON_FOLIO(folio_is_device_private(folio), folio);
2670 
2671 			pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
2672 			if (folio_test_hugetlb(folio)) {
2673 				hugetlb_count_sub(folio_nr_pages(folio), mm);
2674 				set_huge_pte_at(mm, address, pvmw.pte, pteval,
2675 						hsz);
2676 			} else {
2677 				dec_mm_counter(mm, mm_counter(folio));
2678 				set_pte_at(mm, address, pvmw.pte, pteval);
2679 			}
2680 		} else if (likely(pte_present(pteval)) && pte_unused(pteval) &&
2681 			   !userfaultfd_armed(vma)) {
2682 			/*
2683 			 * The guest indicated that the page content is of no
2684 			 * interest anymore. Simply discard the pte, vmscan
2685 			 * will take care of the rest.
2686 			 * A future reference will then fault in a new zero
2687 			 * page. When userfaultfd is active, we must not drop
2688 			 * this page though, as its main user (postcopy
2689 			 * migration) will not expect userfaults on already
2690 			 * copied pages.
2691 			 */
2692 			dec_mm_counter(mm, mm_counter(folio));
2693 		} else {
2694 			swp_entry_t entry;
2695 			pte_t swp_pte;
2696 
2697 			/*
2698 			 * arch_unmap_one() is expected to be a NOP on
2699 			 * architectures where we could have PFN swap PTEs,
2700 			 * so we'll not check/care.
2701 			 */
2702 			if (arch_unmap_one(mm, vma, address, pteval) < 0) {
2703 				if (folio_test_hugetlb(folio))
2704 					set_huge_pte_at(mm, address, pvmw.pte,
2705 							pteval, hsz);
2706 				else
2707 					set_pte_at(mm, address, pvmw.pte, pteval);
2708 				ret = false;
2709 				page_vma_mapped_walk_done(&pvmw);
2710 				break;
2711 			}
2712 
2713 			/* See folio_try_share_anon_rmap_pte(): clear PTE first. */
2714 			if (folio_test_hugetlb(folio)) {
2715 				if (anon_exclusive &&
2716 				    hugetlb_try_share_anon_rmap(folio)) {
2717 					set_huge_pte_at(mm, address, pvmw.pte,
2718 							pteval, hsz);
2719 					ret = false;
2720 					page_vma_mapped_walk_done(&pvmw);
2721 					break;
2722 				}
2723 			} else if (anon_exclusive &&
2724 				   folio_try_share_anon_rmap_pte(folio, subpage)) {
2725 				set_pte_at(mm, address, pvmw.pte, pteval);
2726 				ret = false;
2727 				page_vma_mapped_walk_done(&pvmw);
2728 				break;
2729 			}
2730 
2731 			/*
2732 			 * Store the pfn of the page in a special migration
2733 			 * pte. do_swap_page() will wait until the migration
2734 			 * pte is removed and then restart fault handling.
2735 			 */
2736 			if (writable)
2737 				entry = make_writable_migration_entry(
2738 							page_to_pfn(subpage));
2739 			else if (anon_exclusive)
2740 				entry = make_readable_exclusive_migration_entry(
2741 							page_to_pfn(subpage));
2742 			else
2743 				entry = make_readable_migration_entry(
2744 							page_to_pfn(subpage));
2745 			if (likely(pte_present(pteval))) {
2746 				if (pte_young(pteval))
2747 					entry = make_migration_entry_young(entry);
2748 				if (pte_dirty(pteval))
2749 					entry = make_migration_entry_dirty(entry);
2750 				swp_pte = swp_entry_to_pte(entry);
2751 				if (pte_soft_dirty(pteval))
2752 					swp_pte = pte_swp_mksoft_dirty(swp_pte);
2753 				if (pte_uffd(pteval))
2754 					swp_pte = pte_swp_mkuffd(swp_pte);
2755 			} else {
2756 				swp_pte = swp_entry_to_pte(entry);
2757 				if (pte_swp_soft_dirty(pteval))
2758 					swp_pte = pte_swp_mksoft_dirty(swp_pte);
2759 				if (pte_swp_uffd(pteval))
2760 					swp_pte = pte_swp_mkuffd(swp_pte);
2761 			}
2762 			if (folio_test_hugetlb(folio))
2763 				set_huge_pte_at(mm, address, pvmw.pte, swp_pte,
2764 						hsz);
2765 			else
2766 				set_pte_at(mm, address, pvmw.pte, swp_pte);
2767 			trace_set_migration_pte(address, pte_val(swp_pte),
2768 						folio_order(folio));
2769 			/*
2770 			 * No need to invalidate here it will synchronize on
2771 			 * against the special swap migration pte.
2772 			 */
2773 		}
2774 
2775 		if (unlikely(folio_test_hugetlb(folio)))
2776 			hugetlb_remove_rmap(folio);
2777 		else
2778 			folio_remove_rmap_pte(folio, subpage, vma);
2779 		if (vma_test_any_mask(vma, VMA_LOCKED_MASK))
2780 			mlock_drain_local();
2781 		folio_put(folio);
2782 	}
2783 
2784 	mmu_notifier_invalidate_range_end(&range);
2785 
2786 	return ret;
2787 }
2788 
2789 /**
2790  * try_to_migrate - try to replace all page table mappings with swap entries
2791  * @folio: the folio to replace page table entries for
2792  * @flags: action and flags
2793  *
2794  * Tries to remove all the page table entries which are mapping this folio and
2795  * replace them with special swap entries. Caller must hold the folio lock.
2796  */
2797 void try_to_migrate(struct folio *folio, enum ttu_flags flags)
2798 {
2799 	struct rmap_walk_control rwc = {
2800 		.rmap_one = try_to_migrate_one,
2801 		.arg = (void *)flags,
2802 		.done = folio_not_mapped,
2803 		.anon_lock = folio_lock_anon_vma_read,
2804 	};
2805 
2806 	/*
2807 	 * Migration always ignores mlock and only supports TTU_RMAP_LOCKED and
2808 	 * TTU_SPLIT_HUGE_PMD, TTU_SYNC, and TTU_BATCH_FLUSH flags.
2809 	 */
2810 	if (WARN_ON_ONCE(flags & ~(TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
2811 					TTU_SYNC | TTU_BATCH_FLUSH)))
2812 		return;
2813 
2814 	if (folio_is_zone_device(folio) &&
2815 	    (!folio_is_device_private(folio) && !folio_is_device_coherent(folio)))
2816 		return;
2817 
2818 	/*
2819 	 * During exec, a temporary VMA is setup and later moved.
2820 	 * The VMA is moved under the anon_vma lock but not the
2821 	 * page tables leading to a race where migration cannot
2822 	 * find the migration ptes. Rather than increasing the
2823 	 * locking requirements of exec(), migration skips
2824 	 * temporary VMAs until after exec() completes.
2825 	 */
2826 	if (!folio_test_ksm(folio) && folio_test_anon(folio))
2827 		rwc.invalid_vma = invalid_migration_vma;
2828 
2829 	if (flags & TTU_RMAP_LOCKED)
2830 		rmap_walk_locked(folio, &rwc);
2831 	else
2832 		rmap_walk(folio, &rwc);
2833 }
2834 
2835 #ifdef CONFIG_DEVICE_PRIVATE
2836 /**
2837  * make_device_exclusive() - Mark a page for exclusive use by a device
2838  * @mm: mm_struct of associated target process
2839  * @addr: the virtual address to mark for exclusive device access
2840  * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier to allow filtering
2841  * @foliop: folio pointer will be stored here on success.
2842  *
2843  * This function looks up the page mapped at the given address, grabs a
2844  * folio reference, locks the folio and replaces the PTE with special
2845  * device-exclusive PFN swap entry, preventing access through the process
2846  * page tables. The function will return with the folio locked and referenced.
2847  *
2848  * On fault, the device-exclusive entries are replaced with the original PTE
2849  * under folio lock, after calling MMU notifiers.
2850  *
2851  * Only anonymous non-hugetlb folios are supported and the VMA must have
2852  * write permissions such that we can fault in the anonymous page writable
2853  * in order to mark it exclusive. The caller must hold the mmap_lock in read
2854  * mode.
2855  *
2856  * A driver using this to program access from a device must use a mmu notifier
2857  * critical section to hold a device specific lock during programming. Once
2858  * programming is complete it should drop the folio lock and reference after
2859  * which point CPU access to the page will revoke the exclusive access.
2860  *
2861  * Notes:
2862  *   #. This function always operates on individual PTEs mapping individual
2863  *      pages. PMD-sized THPs are first remapped to be mapped by PTEs before
2864  *      the conversion happens on a single PTE corresponding to @addr.
2865  *   #. While concurrent access through the process page tables is prevented,
2866  *      concurrent access through other page references (e.g., earlier GUP
2867  *      invocation) is not handled and not supported.
2868  *   #. device-exclusive entries are considered "clean" and "old" by core-mm.
2869  *      Device drivers must update the folio state when informed by MMU
2870  *      notifiers.
2871  *
2872  * Returns: pointer to mapped page on success, otherwise a negative error.
2873  */
2874 struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr,
2875 		void *owner, struct folio **foliop)
2876 {
2877 	struct mmu_notifier_range range;
2878 	struct folio *folio, *fw_folio;
2879 	struct vm_area_struct *vma;
2880 	struct folio_walk fw;
2881 	struct page *page;
2882 	swp_entry_t entry;
2883 	pte_t swp_pte;
2884 	int ret;
2885 
2886 	mmap_assert_locked(mm);
2887 	addr = PAGE_ALIGN_DOWN(addr);
2888 
2889 	/*
2890 	 * Fault in the page writable and try to lock it; note that if the
2891 	 * address would already be marked for exclusive use by a device,
2892 	 * the GUP call would undo that first by triggering a fault.
2893 	 *
2894 	 * If any other device would already map this page exclusively, the
2895 	 * fault will trigger a conversion to an ordinary
2896 	 * (non-device-exclusive) PTE and issue a MMU_NOTIFY_EXCLUSIVE.
2897 	 */
2898 retry:
2899 	page = get_user_page_vma_remote(mm, addr,
2900 					FOLL_GET | FOLL_WRITE | FOLL_SPLIT_PMD,
2901 					&vma);
2902 	if (IS_ERR(page))
2903 		return page;
2904 	folio = page_folio(page);
2905 
2906 	if (!folio_test_anon(folio) || folio_test_hugetlb(folio)) {
2907 		folio_put(folio);
2908 		return ERR_PTR(-EOPNOTSUPP);
2909 	}
2910 
2911 	ret = folio_lock_killable(folio);
2912 	if (ret) {
2913 		folio_put(folio);
2914 		return ERR_PTR(ret);
2915 	}
2916 
2917 	/*
2918 	 * Inform secondary MMUs that we are going to convert this PTE to
2919 	 * device-exclusive, such that they unmap it now. Note that the
2920 	 * caller must filter this event out to prevent livelocks.
2921 	 */
2922 	mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
2923 				      mm, addr, addr + PAGE_SIZE, owner);
2924 	mmu_notifier_invalidate_range_start(&range);
2925 
2926 	/*
2927 	 * Let's do a second walk and make sure we still find the same page
2928 	 * mapped writable. Note that any page of an anonymous folio can
2929 	 * only be mapped writable using exactly one PTE ("exclusive"), so
2930 	 * there cannot be other mappings.
2931 	 */
2932 	fw_folio = folio_walk_start(&fw, vma, addr, 0);
2933 	if (fw_folio != folio || fw.page != page ||
2934 	    fw.level != FW_LEVEL_PTE || !pte_write(fw.pte)) {
2935 		if (fw_folio)
2936 			folio_walk_end(&fw, vma);
2937 		mmu_notifier_invalidate_range_end(&range);
2938 		folio_unlock(folio);
2939 		folio_put(folio);
2940 		goto retry;
2941 	}
2942 
2943 	/* Nuke the page table entry so we get the uptodate dirty bit. */
2944 	flush_cache_page(vma, addr, page_to_pfn(page));
2945 	fw.pte = ptep_clear_flush(vma, addr, fw.ptep);
2946 
2947 	/* Set the dirty flag on the folio now the PTE is gone. */
2948 	if (pte_dirty(fw.pte))
2949 		folio_mark_dirty(folio);
2950 
2951 	/*
2952 	 * Store the pfn of the page in a special device-exclusive PFN swap PTE.
2953 	 * do_swap_page() will trigger the conversion back while holding the
2954 	 * folio lock.
2955 	 */
2956 	entry = make_device_exclusive_entry(page_to_pfn(page));
2957 	swp_pte = swp_entry_to_pte(entry);
2958 	if (pte_soft_dirty(fw.pte))
2959 		swp_pte = pte_swp_mksoft_dirty(swp_pte);
2960 	/* The pte is writable, uffd-wp does not apply. */
2961 	set_pte_at(mm, addr, fw.ptep, swp_pte);
2962 
2963 	folio_walk_end(&fw, vma);
2964 	mmu_notifier_invalidate_range_end(&range);
2965 	*foliop = folio;
2966 	return page;
2967 }
2968 EXPORT_SYMBOL_GPL(make_device_exclusive);
2969 #endif
2970 
2971 void __put_anon_vma(struct anon_vma *anon_vma)
2972 {
2973 	struct anon_vma *root = anon_vma->root;
2974 
2975 	anon_vma_free(anon_vma);
2976 	if (root != anon_vma && atomic_dec_and_test(&root->refcount))
2977 		anon_vma_free(root);
2978 }
2979 
2980 static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio,
2981 					    struct rmap_walk_control *rwc)
2982 {
2983 	struct anon_vma *anon_vma;
2984 
2985 	if (rwc->anon_lock)
2986 		return rwc->anon_lock(folio, rwc);
2987 
2988 	/*
2989 	 * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read()
2990 	 * because that depends on folio_mapped(); but not all its usages
2991 	 * are holding mmap_lock. Users without mmap_lock are required to
2992 	 * take a reference count to prevent the anon_vma disappearing
2993 	 */
2994 	anon_vma = folio_anon_vma(folio);
2995 	if (!anon_vma)
2996 		return NULL;
2997 
2998 	if (anon_vma_trylock_read(anon_vma))
2999 		goto out;
3000 
3001 	if (rwc->try_lock) {
3002 		anon_vma = NULL;
3003 		rwc->contended = true;
3004 		goto out;
3005 	}
3006 
3007 	anon_vma_lock_read(anon_vma);
3008 out:
3009 	return anon_vma;
3010 }
3011 
3012 /*
3013  * rmap_walk_anon - do something to anonymous page using the object-based
3014  * rmap method
3015  * @folio: the folio to be handled
3016  * @rwc: control variable according to each walk type
3017  * @locked: caller holds relevant rmap lock
3018  *
3019  * Find all the mappings of a folio using the mapping pointer and the vma
3020  * chains contained in the anon_vma struct it points to.
3021  */
3022 static void rmap_walk_anon(struct folio *folio,
3023 		struct rmap_walk_control *rwc, bool locked)
3024 {
3025 	struct anon_vma *anon_vma;
3026 	pgoff_t pgoff_start, pgoff_end;
3027 	struct anon_vma_chain *avc;
3028 
3029 	/*
3030 	 * The folio lock ensures that folio->mapping can't be changed under us
3031 	 * to an anon_vma with different root.
3032 	 */
3033 	VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
3034 
3035 	if (locked) {
3036 		anon_vma = folio_anon_vma(folio);
3037 		/* anon_vma disappear under us? */
3038 		VM_BUG_ON_FOLIO(!anon_vma, folio);
3039 	} else {
3040 		anon_vma = rmap_walk_anon_lock(folio, rwc);
3041 	}
3042 	if (!anon_vma)
3043 		return;
3044 
3045 	pgoff_start = folio_pgoff(folio);
3046 	pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
3047 	anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) {
3048 		struct vm_area_struct *vma = avc->vma;
3049 		const unsigned long address = vma_anon_address(vma, pgoff_start,
3050 				folio_nr_pages(folio));
3051 
3052 		VM_WARN_ON_ONCE_VMA(address == -EFAULT, vma);
3053 		cond_resched();
3054 
3055 		if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
3056 			continue;
3057 
3058 		if (!rwc->rmap_one(folio, vma, address, rwc->arg))
3059 			break;
3060 		if (rwc->done && rwc->done(folio))
3061 			break;
3062 	}
3063 
3064 	if (!locked)
3065 		anon_vma_unlock_read(anon_vma);
3066 }
3067 
3068 /**
3069  * __rmap_walk_file() - Traverse the reverse mapping for a file-backed mapping
3070  * of a page mapped within a specified page cache object at a specified offset.
3071  *
3072  * @folio: 		Either the folio whose mappings to traverse, or if NULL,
3073  * 			the callbacks specified in @rwc will be configured such
3074  * 			as to be able to look up mappings correctly.
3075  * @mapping: 		The page cache object whose mapping VMAs we intend to
3076  * 			traverse. If @folio is non-NULL, this should be equal to
3077  *			folio_mapping(folio).
3078  * @pgoff_start:	The offset within @mapping of the page which we are
3079  * 			looking up. If @folio is non-NULL, this should be equal
3080  * 			to folio_pgoff(folio).
3081  * @nr_pages:		The number of pages mapped by the mapping. If @folio is
3082  *			non-NULL, this should be equal to folio_nr_pages(folio).
3083  * @rwc:		The reverse mapping walk control object describing how
3084  *			the traversal should proceed.
3085  * @locked:		Is the @mapping already locked? If not, we acquire the
3086  *			lock.
3087  */
3088 static void __rmap_walk_file(struct folio *folio, struct address_space *mapping,
3089 			     pgoff_t pgoff_start, unsigned long nr_pages,
3090 			     struct rmap_walk_control *rwc, bool locked)
3091 {
3092 	pgoff_t pgoff_end = pgoff_start + nr_pages - 1;
3093 	struct vm_area_struct *vma;
3094 
3095 	VM_WARN_ON_FOLIO(folio && mapping != folio_mapping(folio), folio);
3096 	VM_WARN_ON_FOLIO(folio && pgoff_start != folio_pgoff(folio), folio);
3097 	VM_WARN_ON_FOLIO(folio && nr_pages != folio_nr_pages(folio), folio);
3098 
3099 	if (!locked) {
3100 		if (i_mmap_trylock_read(mapping))
3101 			goto lookup;
3102 
3103 		if (rwc->try_lock) {
3104 			rwc->contended = true;
3105 			return;
3106 		}
3107 
3108 		i_mmap_lock_read(mapping);
3109 	}
3110 lookup:
3111 	mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_end) {
3112 		unsigned long address = vma_filebacked_address(vma, pgoff_start,
3113 							       nr_pages);
3114 
3115 		VM_BUG_ON_VMA(address == -EFAULT, vma);
3116 		cond_resched();
3117 
3118 		if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
3119 			continue;
3120 
3121 		if (!rwc->rmap_one(folio, vma, address, rwc->arg))
3122 			goto done;
3123 		if (rwc->done && rwc->done(folio))
3124 			goto done;
3125 	}
3126 done:
3127 	if (!locked)
3128 		i_mmap_unlock_read(mapping);
3129 }
3130 
3131 /*
3132  * rmap_walk_file - do something to file page using the object-based rmap method
3133  * @folio: the folio to be handled
3134  * @rwc: control variable according to each walk type
3135  * @locked: caller holds relevant rmap lock
3136  *
3137  * Find all the mappings of a folio using the mapping pointer and the vma chains
3138  * contained in the address_space struct it points to.
3139  */
3140 static void rmap_walk_file(struct folio *folio,
3141 		struct rmap_walk_control *rwc, bool locked)
3142 {
3143 	/*
3144 	 * The folio lock not only makes sure that folio->mapping cannot
3145 	 * suddenly be NULLified by truncation, it makes sure that the structure
3146 	 * at mapping cannot be freed and reused yet, so we can safely take
3147 	 * mapping->i_mmap_rwsem.
3148 	 */
3149 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
3150 
3151 	if (!folio->mapping)
3152 		return;
3153 
3154 	__rmap_walk_file(folio, folio->mapping, folio->index,
3155 			 folio_nr_pages(folio), rwc, locked);
3156 }
3157 
3158 void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc)
3159 {
3160 	if (unlikely(folio_test_ksm(folio)))
3161 		rmap_walk_ksm(folio, rwc);
3162 	else if (folio_test_anon(folio))
3163 		rmap_walk_anon(folio, rwc, false);
3164 	else
3165 		rmap_walk_file(folio, rwc, false);
3166 }
3167 
3168 /* Like rmap_walk, but caller holds relevant rmap lock */
3169 void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc)
3170 {
3171 	/* no ksm support for now */
3172 	VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio);
3173 	if (folio_test_anon(folio))
3174 		rmap_walk_anon(folio, rwc, true);
3175 	else
3176 		rmap_walk_file(folio, rwc, true);
3177 }
3178 
3179 #ifdef CONFIG_HUGETLB_PAGE
3180 /*
3181  * The following two functions are for anonymous (private mapped) hugepages.
3182  * Unlike common anonymous pages, anonymous hugepages have no accounting code
3183  * and no lru code, because we handle hugepages differently from common pages.
3184  */
3185 void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
3186 		unsigned long address, rmap_t flags)
3187 {
3188 	VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
3189 	VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
3190 
3191 	atomic_inc(&folio->_entire_mapcount);
3192 	atomic_inc(&folio->_large_mapcount);
3193 	if (flags & RMAP_EXCLUSIVE)
3194 		SetPageAnonExclusive(&folio->page);
3195 	VM_WARN_ON_FOLIO(folio_entire_mapcount(folio) > 1 &&
3196 			 PageAnonExclusive(&folio->page), folio);
3197 }
3198 
3199 void hugetlb_add_new_anon_rmap(struct folio *folio,
3200 		struct vm_area_struct *vma, unsigned long address)
3201 {
3202 	VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
3203 
3204 	BUG_ON(address < vma->vm_start || address >= vma->vm_end);
3205 	/* increment count (starts at -1) */
3206 	atomic_set(&folio->_entire_mapcount, 0);
3207 	atomic_set(&folio->_large_mapcount, 0);
3208 	folio_clear_hugetlb_restore_reserve(folio);
3209 	__folio_set_anon(folio, vma, address, true);
3210 	SetPageAnonExclusive(&folio->page);
3211 }
3212 #endif /* CONFIG_HUGETLB_PAGE */
3213