xref: /linux/mm/mempolicy.c (revision 2c1ed907520c50326b8f604907a8478b27881a2e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Simple NUMA memory policy for the Linux kernel.
4  *
5  * Copyright 2003,2004 Andi Kleen, SuSE Labs.
6  * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
7  *
8  * NUMA policy allows the user to give hints in which node(s) memory should
9  * be allocated.
10  *
11  * Support six policies per VMA and per process:
12  *
13  * The VMA policy has priority over the process policy for a page fault.
14  *
15  * interleave     Allocate memory interleaved over a set of nodes,
16  *                with normal fallback if it fails.
17  *                For VMA based allocations this interleaves based on the
18  *                offset into the backing object or offset into the mapping
19  *                for anonymous memory. For process policy an process counter
20  *                is used.
21  *
22  * weighted interleave
23  *                Allocate memory interleaved over a set of nodes based on
24  *                a set of weights (per-node), with normal fallback if it
25  *                fails.  Otherwise operates the same as interleave.
26  *                Example: nodeset(0,1) & weights (2,1) - 2 pages allocated
27  *                on node 0 for every 1 page allocated on node 1.
28  *
29  * bind           Only allocate memory on a specific set of nodes,
30  *                no fallback.
31  *                FIXME: memory is allocated starting with the first node
32  *                to the last. It would be better if bind would truly restrict
33  *                the allocation to memory nodes instead
34  *
35  * preferred      Try a specific node first before normal fallback.
36  *                As a special case NUMA_NO_NODE here means do the allocation
37  *                on the local CPU. This is normally identical to default,
38  *                but useful to set in a VMA when you have a non default
39  *                process policy.
40  *
41  * preferred many Try a set of nodes first before normal fallback. This is
42  *                similar to preferred without the special case.
43  *
44  * default        Allocate on the local node first, or when on a VMA
45  *                use the process policy. This is what Linux always did
46  *		  in a NUMA aware kernel and still does by, ahem, default.
47  *
48  * The process policy is applied for most non interrupt memory allocations
49  * in that process' context. Interrupts ignore the policies and always
50  * try to allocate on the local CPU. The VMA policy is only applied for memory
51  * allocations for a VMA in the VM.
52  *
53  * Currently there are a few corner cases in swapping where the policy
54  * is not applied, but the majority should be handled. When process policy
55  * is used it is not remembered over swap outs/swap ins.
56  *
57  * Only the highest zone in the zone hierarchy gets policied. Allocations
58  * requesting a lower zone just use default policy. This implies that
59  * on systems with highmem kernel lowmem allocation don't get policied.
60  * Same with GFP_DMA allocations.
61  *
62  * For shmem/tmpfs shared memory the policy is shared between
63  * all users and remembered even when nobody has memory mapped.
64  */
65 
66 /* Notebook:
67    fix mmap readahead to honour policy and enable policy for any page cache
68    object
69    statistics for bigpages
70    global policy for page cache? currently it uses process policy. Requires
71    first item above.
72    handle mremap for shared memory (currently ignored for the policy)
73    grows down?
74    make bind policy root only? It can trigger oom much faster and the
75    kernel is not always grateful with that.
76 */
77 
78 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
79 
80 #include <linux/mempolicy.h>
81 #include <linux/pagewalk.h>
82 #include <linux/highmem.h>
83 #include <linux/hugetlb.h>
84 #include <linux/kernel.h>
85 #include <linux/sched.h>
86 #include <linux/sched/mm.h>
87 #include <linux/sched/numa_balancing.h>
88 #include <linux/sched/task.h>
89 #include <linux/nodemask.h>
90 #include <linux/cpuset.h>
91 #include <linux/slab.h>
92 #include <linux/string.h>
93 #include <linux/export.h>
94 #include <linux/nsproxy.h>
95 #include <linux/interrupt.h>
96 #include <linux/init.h>
97 #include <linux/compat.h>
98 #include <linux/ptrace.h>
99 #include <linux/swap.h>
100 #include <linux/seq_file.h>
101 #include <linux/proc_fs.h>
102 #include <linux/migrate.h>
103 #include <linux/ksm.h>
104 #include <linux/rmap.h>
105 #include <linux/security.h>
106 #include <linux/syscalls.h>
107 #include <linux/ctype.h>
108 #include <linux/mm_inline.h>
109 #include <linux/mmu_notifier.h>
110 #include <linux/printk.h>
111 #include <linux/swapops.h>
112 
113 #include <asm/tlbflush.h>
114 #include <asm/tlb.h>
115 #include <linux/uaccess.h>
116 
117 #include "internal.h"
118 
119 /* Internal flags */
120 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
121 #define MPOL_MF_INVERT       (MPOL_MF_INTERNAL << 1)	/* Invert check for nodemask */
122 #define MPOL_MF_WRLOCK       (MPOL_MF_INTERNAL << 2)	/* Write-lock walked vmas */
123 
124 static struct kmem_cache *policy_cache;
125 static struct kmem_cache *sn_cache;
126 
127 /* Highest zone. An specific allocation for a zone below that is not
128    policied. */
129 enum zone_type policy_zone = 0;
130 
131 /*
132  * run-time system-wide default policy => local allocation
133  */
134 static struct mempolicy default_policy = {
135 	.refcnt = ATOMIC_INIT(1), /* never free it */
136 	.mode = MPOL_LOCAL,
137 };
138 
139 static struct mempolicy preferred_node_policy[MAX_NUMNODES];
140 
141 /*
142  * iw_table is the sysfs-set interleave weight table, a value of 0 denotes
143  * system-default value should be used. A NULL iw_table also denotes that
144  * system-default values should be used. Until the system-default table
145  * is implemented, the system-default is always 1.
146  *
147  * iw_table is RCU protected
148  */
149 static u8 __rcu *iw_table;
150 static DEFINE_MUTEX(iw_table_lock);
151 
get_il_weight(int node)152 static u8 get_il_weight(int node)
153 {
154 	u8 *table;
155 	u8 weight;
156 
157 	rcu_read_lock();
158 	table = rcu_dereference(iw_table);
159 	/* if no iw_table, use system default */
160 	weight = table ? table[node] : 1;
161 	/* if value in iw_table is 0, use system default */
162 	weight = weight ? weight : 1;
163 	rcu_read_unlock();
164 	return weight;
165 }
166 
167 /**
168  * numa_nearest_node - Find nearest node by state
169  * @node: Node id to start the search
170  * @state: State to filter the search
171  *
172  * Lookup the closest node by distance if @nid is not in state.
173  *
174  * Return: this @node if it is in state, otherwise the closest node by distance
175  */
numa_nearest_node(int node,unsigned int state)176 int numa_nearest_node(int node, unsigned int state)
177 {
178 	int min_dist = INT_MAX, dist, n, min_node;
179 
180 	if (state >= NR_NODE_STATES)
181 		return -EINVAL;
182 
183 	if (node == NUMA_NO_NODE || node_state(node, state))
184 		return node;
185 
186 	min_node = node;
187 	for_each_node_state(n, state) {
188 		dist = node_distance(node, n);
189 		if (dist < min_dist) {
190 			min_dist = dist;
191 			min_node = n;
192 		}
193 	}
194 
195 	return min_node;
196 }
197 EXPORT_SYMBOL_GPL(numa_nearest_node);
198 
get_task_policy(struct task_struct * p)199 struct mempolicy *get_task_policy(struct task_struct *p)
200 {
201 	struct mempolicy *pol = p->mempolicy;
202 	int node;
203 
204 	if (pol)
205 		return pol;
206 
207 	node = numa_node_id();
208 	if (node != NUMA_NO_NODE) {
209 		pol = &preferred_node_policy[node];
210 		/* preferred_node_policy is not initialised early in boot */
211 		if (pol->mode)
212 			return pol;
213 	}
214 
215 	return &default_policy;
216 }
217 
218 static const struct mempolicy_operations {
219 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
220 	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
221 } mpol_ops[MPOL_MAX];
222 
mpol_store_user_nodemask(const struct mempolicy * pol)223 static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
224 {
225 	return pol->flags & MPOL_MODE_FLAGS;
226 }
227 
mpol_relative_nodemask(nodemask_t * ret,const nodemask_t * orig,const nodemask_t * rel)228 static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
229 				   const nodemask_t *rel)
230 {
231 	nodemask_t tmp;
232 	nodes_fold(tmp, *orig, nodes_weight(*rel));
233 	nodes_onto(*ret, tmp, *rel);
234 }
235 
mpol_new_nodemask(struct mempolicy * pol,const nodemask_t * nodes)236 static int mpol_new_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
237 {
238 	if (nodes_empty(*nodes))
239 		return -EINVAL;
240 	pol->nodes = *nodes;
241 	return 0;
242 }
243 
mpol_new_preferred(struct mempolicy * pol,const nodemask_t * nodes)244 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
245 {
246 	if (nodes_empty(*nodes))
247 		return -EINVAL;
248 
249 	nodes_clear(pol->nodes);
250 	node_set(first_node(*nodes), pol->nodes);
251 	return 0;
252 }
253 
254 /*
255  * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
256  * any, for the new policy.  mpol_new() has already validated the nodes
257  * parameter with respect to the policy mode and flags.
258  *
259  * Must be called holding task's alloc_lock to protect task's mems_allowed
260  * and mempolicy.  May also be called holding the mmap_lock for write.
261  */
mpol_set_nodemask(struct mempolicy * pol,const nodemask_t * nodes,struct nodemask_scratch * nsc)262 static int mpol_set_nodemask(struct mempolicy *pol,
263 		     const nodemask_t *nodes, struct nodemask_scratch *nsc)
264 {
265 	int ret;
266 
267 	/*
268 	 * Default (pol==NULL) resp. local memory policies are not a
269 	 * subject of any remapping. They also do not need any special
270 	 * constructor.
271 	 */
272 	if (!pol || pol->mode == MPOL_LOCAL)
273 		return 0;
274 
275 	/* Check N_MEMORY */
276 	nodes_and(nsc->mask1,
277 		  cpuset_current_mems_allowed, node_states[N_MEMORY]);
278 
279 	VM_BUG_ON(!nodes);
280 
281 	if (pol->flags & MPOL_F_RELATIVE_NODES)
282 		mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);
283 	else
284 		nodes_and(nsc->mask2, *nodes, nsc->mask1);
285 
286 	if (mpol_store_user_nodemask(pol))
287 		pol->w.user_nodemask = *nodes;
288 	else
289 		pol->w.cpuset_mems_allowed = cpuset_current_mems_allowed;
290 
291 	ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
292 	return ret;
293 }
294 
295 /*
296  * This function just creates a new policy, does some check and simple
297  * initialization. You must invoke mpol_set_nodemask() to set nodes.
298  */
mpol_new(unsigned short mode,unsigned short flags,nodemask_t * nodes)299 static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
300 				  nodemask_t *nodes)
301 {
302 	struct mempolicy *policy;
303 
304 	if (mode == MPOL_DEFAULT) {
305 		if (nodes && !nodes_empty(*nodes))
306 			return ERR_PTR(-EINVAL);
307 		return NULL;
308 	}
309 	VM_BUG_ON(!nodes);
310 
311 	/*
312 	 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
313 	 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
314 	 * All other modes require a valid pointer to a non-empty nodemask.
315 	 */
316 	if (mode == MPOL_PREFERRED) {
317 		if (nodes_empty(*nodes)) {
318 			if (((flags & MPOL_F_STATIC_NODES) ||
319 			     (flags & MPOL_F_RELATIVE_NODES)))
320 				return ERR_PTR(-EINVAL);
321 
322 			mode = MPOL_LOCAL;
323 		}
324 	} else if (mode == MPOL_LOCAL) {
325 		if (!nodes_empty(*nodes) ||
326 		    (flags & MPOL_F_STATIC_NODES) ||
327 		    (flags & MPOL_F_RELATIVE_NODES))
328 			return ERR_PTR(-EINVAL);
329 	} else if (nodes_empty(*nodes))
330 		return ERR_PTR(-EINVAL);
331 
332 	policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
333 	if (!policy)
334 		return ERR_PTR(-ENOMEM);
335 	atomic_set(&policy->refcnt, 1);
336 	policy->mode = mode;
337 	policy->flags = flags;
338 	policy->home_node = NUMA_NO_NODE;
339 
340 	return policy;
341 }
342 
343 /* Slow path of a mpol destructor. */
__mpol_put(struct mempolicy * pol)344 void __mpol_put(struct mempolicy *pol)
345 {
346 	if (!atomic_dec_and_test(&pol->refcnt))
347 		return;
348 	kmem_cache_free(policy_cache, pol);
349 }
350 
mpol_rebind_default(struct mempolicy * pol,const nodemask_t * nodes)351 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
352 {
353 }
354 
mpol_rebind_nodemask(struct mempolicy * pol,const nodemask_t * nodes)355 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
356 {
357 	nodemask_t tmp;
358 
359 	if (pol->flags & MPOL_F_STATIC_NODES)
360 		nodes_and(tmp, pol->w.user_nodemask, *nodes);
361 	else if (pol->flags & MPOL_F_RELATIVE_NODES)
362 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
363 	else {
364 		nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed,
365 								*nodes);
366 		pol->w.cpuset_mems_allowed = *nodes;
367 	}
368 
369 	if (nodes_empty(tmp))
370 		tmp = *nodes;
371 
372 	pol->nodes = tmp;
373 }
374 
mpol_rebind_preferred(struct mempolicy * pol,const nodemask_t * nodes)375 static void mpol_rebind_preferred(struct mempolicy *pol,
376 						const nodemask_t *nodes)
377 {
378 	pol->w.cpuset_mems_allowed = *nodes;
379 }
380 
381 /*
382  * mpol_rebind_policy - Migrate a policy to a different set of nodes
383  *
384  * Per-vma policies are protected by mmap_lock. Allocations using per-task
385  * policies are protected by task->mems_allowed_seq to prevent a premature
386  * OOM/allocation failure due to parallel nodemask modification.
387  */
mpol_rebind_policy(struct mempolicy * pol,const nodemask_t * newmask)388 static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
389 {
390 	if (!pol || pol->mode == MPOL_LOCAL)
391 		return;
392 	if (!mpol_store_user_nodemask(pol) &&
393 	    nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
394 		return;
395 
396 	mpol_ops[pol->mode].rebind(pol, newmask);
397 }
398 
399 /*
400  * Wrapper for mpol_rebind_policy() that just requires task
401  * pointer, and updates task mempolicy.
402  *
403  * Called with task's alloc_lock held.
404  */
mpol_rebind_task(struct task_struct * tsk,const nodemask_t * new)405 void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
406 {
407 	mpol_rebind_policy(tsk->mempolicy, new);
408 }
409 
410 /*
411  * Rebind each vma in mm to new nodemask.
412  *
413  * Call holding a reference to mm.  Takes mm->mmap_lock during call.
414  */
mpol_rebind_mm(struct mm_struct * mm,nodemask_t * new)415 void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
416 {
417 	struct vm_area_struct *vma;
418 	VMA_ITERATOR(vmi, mm, 0);
419 
420 	mmap_write_lock(mm);
421 	for_each_vma(vmi, vma) {
422 		vma_start_write(vma);
423 		mpol_rebind_policy(vma->vm_policy, new);
424 	}
425 	mmap_write_unlock(mm);
426 }
427 
428 static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
429 	[MPOL_DEFAULT] = {
430 		.rebind = mpol_rebind_default,
431 	},
432 	[MPOL_INTERLEAVE] = {
433 		.create = mpol_new_nodemask,
434 		.rebind = mpol_rebind_nodemask,
435 	},
436 	[MPOL_PREFERRED] = {
437 		.create = mpol_new_preferred,
438 		.rebind = mpol_rebind_preferred,
439 	},
440 	[MPOL_BIND] = {
441 		.create = mpol_new_nodemask,
442 		.rebind = mpol_rebind_nodemask,
443 	},
444 	[MPOL_LOCAL] = {
445 		.rebind = mpol_rebind_default,
446 	},
447 	[MPOL_PREFERRED_MANY] = {
448 		.create = mpol_new_nodemask,
449 		.rebind = mpol_rebind_preferred,
450 	},
451 	[MPOL_WEIGHTED_INTERLEAVE] = {
452 		.create = mpol_new_nodemask,
453 		.rebind = mpol_rebind_nodemask,
454 	},
455 };
456 
457 static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist,
458 				unsigned long flags);
459 static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *pol,
460 				pgoff_t ilx, int *nid);
461 
strictly_unmovable(unsigned long flags)462 static bool strictly_unmovable(unsigned long flags)
463 {
464 	/*
465 	 * STRICT without MOVE flags lets do_mbind() fail immediately with -EIO
466 	 * if any misplaced page is found.
467 	 */
468 	return (flags & (MPOL_MF_STRICT | MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ==
469 			 MPOL_MF_STRICT;
470 }
471 
472 struct migration_mpol {		/* for alloc_migration_target_by_mpol() */
473 	struct mempolicy *pol;
474 	pgoff_t ilx;
475 };
476 
477 struct queue_pages {
478 	struct list_head *pagelist;
479 	unsigned long flags;
480 	nodemask_t *nmask;
481 	unsigned long start;
482 	unsigned long end;
483 	struct vm_area_struct *first;
484 	struct folio *large;		/* note last large folio encountered */
485 	long nr_failed;			/* could not be isolated at this time */
486 };
487 
488 /*
489  * Check if the folio's nid is in qp->nmask.
490  *
491  * If MPOL_MF_INVERT is set in qp->flags, check if the nid is
492  * in the invert of qp->nmask.
493  */
queue_folio_required(struct folio * folio,struct queue_pages * qp)494 static inline bool queue_folio_required(struct folio *folio,
495 					struct queue_pages *qp)
496 {
497 	int nid = folio_nid(folio);
498 	unsigned long flags = qp->flags;
499 
500 	return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT);
501 }
502 
queue_folios_pmd(pmd_t * pmd,struct mm_walk * walk)503 static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk)
504 {
505 	struct folio *folio;
506 	struct queue_pages *qp = walk->private;
507 
508 	if (unlikely(is_pmd_migration_entry(*pmd))) {
509 		qp->nr_failed++;
510 		return;
511 	}
512 	folio = pmd_folio(*pmd);
513 	if (is_huge_zero_folio(folio)) {
514 		walk->action = ACTION_CONTINUE;
515 		return;
516 	}
517 	if (!queue_folio_required(folio, qp))
518 		return;
519 	if (!(qp->flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
520 	    !vma_migratable(walk->vma) ||
521 	    !migrate_folio_add(folio, qp->pagelist, qp->flags))
522 		qp->nr_failed++;
523 }
524 
525 /*
526  * Scan through folios, checking if they satisfy the required conditions,
527  * moving them from LRU to local pagelist for migration if they do (or not).
528  *
529  * queue_folios_pte_range() has two possible return values:
530  * 0 - continue walking to scan for more, even if an existing folio on the
531  *     wrong node could not be isolated and queued for migration.
532  * -EIO - only MPOL_MF_STRICT was specified, without MPOL_MF_MOVE or ..._ALL,
533  *        and an existing folio was on a node that does not follow the policy.
534  */
queue_folios_pte_range(pmd_t * pmd,unsigned long addr,unsigned long end,struct mm_walk * walk)535 static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
536 			unsigned long end, struct mm_walk *walk)
537 {
538 	struct vm_area_struct *vma = walk->vma;
539 	struct folio *folio;
540 	struct queue_pages *qp = walk->private;
541 	unsigned long flags = qp->flags;
542 	pte_t *pte, *mapped_pte;
543 	pte_t ptent;
544 	spinlock_t *ptl;
545 
546 	ptl = pmd_trans_huge_lock(pmd, vma);
547 	if (ptl) {
548 		queue_folios_pmd(pmd, walk);
549 		spin_unlock(ptl);
550 		goto out;
551 	}
552 
553 	mapped_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
554 	if (!pte) {
555 		walk->action = ACTION_AGAIN;
556 		return 0;
557 	}
558 	for (; addr != end; pte++, addr += PAGE_SIZE) {
559 		ptent = ptep_get(pte);
560 		if (pte_none(ptent))
561 			continue;
562 		if (!pte_present(ptent)) {
563 			if (is_migration_entry(pte_to_swp_entry(ptent)))
564 				qp->nr_failed++;
565 			continue;
566 		}
567 		folio = vm_normal_folio(vma, addr, ptent);
568 		if (!folio || folio_is_zone_device(folio))
569 			continue;
570 		/*
571 		 * vm_normal_folio() filters out zero pages, but there might
572 		 * still be reserved folios to skip, perhaps in a VDSO.
573 		 */
574 		if (folio_test_reserved(folio))
575 			continue;
576 		if (!queue_folio_required(folio, qp))
577 			continue;
578 		if (folio_test_large(folio)) {
579 			/*
580 			 * A large folio can only be isolated from LRU once,
581 			 * but may be mapped by many PTEs (and Copy-On-Write may
582 			 * intersperse PTEs of other, order 0, folios).  This is
583 			 * a common case, so don't mistake it for failure (but
584 			 * there can be other cases of multi-mapped pages which
585 			 * this quick check does not help to filter out - and a
586 			 * search of the pagelist might grow to be prohibitive).
587 			 *
588 			 * migrate_pages(&pagelist) returns nr_failed folios, so
589 			 * check "large" now so that queue_pages_range() returns
590 			 * a comparable nr_failed folios.  This does imply that
591 			 * if folio could not be isolated for some racy reason
592 			 * at its first PTE, later PTEs will not give it another
593 			 * chance of isolation; but keeps the accounting simple.
594 			 */
595 			if (folio == qp->large)
596 				continue;
597 			qp->large = folio;
598 		}
599 		if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
600 		    !vma_migratable(vma) ||
601 		    !migrate_folio_add(folio, qp->pagelist, flags)) {
602 			qp->nr_failed++;
603 			if (strictly_unmovable(flags))
604 				break;
605 		}
606 	}
607 	pte_unmap_unlock(mapped_pte, ptl);
608 	cond_resched();
609 out:
610 	if (qp->nr_failed && strictly_unmovable(flags))
611 		return -EIO;
612 	return 0;
613 }
614 
queue_folios_hugetlb(pte_t * pte,unsigned long hmask,unsigned long addr,unsigned long end,struct mm_walk * walk)615 static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask,
616 			       unsigned long addr, unsigned long end,
617 			       struct mm_walk *walk)
618 {
619 #ifdef CONFIG_HUGETLB_PAGE
620 	struct queue_pages *qp = walk->private;
621 	unsigned long flags = qp->flags;
622 	struct folio *folio;
623 	spinlock_t *ptl;
624 	pte_t entry;
625 
626 	ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
627 	entry = huge_ptep_get(walk->mm, addr, pte);
628 	if (!pte_present(entry)) {
629 		if (unlikely(is_hugetlb_entry_migration(entry)))
630 			qp->nr_failed++;
631 		goto unlock;
632 	}
633 	folio = pfn_folio(pte_pfn(entry));
634 	if (!queue_folio_required(folio, qp))
635 		goto unlock;
636 	if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
637 	    !vma_migratable(walk->vma)) {
638 		qp->nr_failed++;
639 		goto unlock;
640 	}
641 	/*
642 	 * Unless MPOL_MF_MOVE_ALL, we try to avoid migrating a shared folio.
643 	 * Choosing not to migrate a shared folio is not counted as a failure.
644 	 *
645 	 * See folio_likely_mapped_shared() on possible imprecision when we
646 	 * cannot easily detect if a folio is shared.
647 	 */
648 	if ((flags & MPOL_MF_MOVE_ALL) ||
649 	    (!folio_likely_mapped_shared(folio) && !hugetlb_pmd_shared(pte)))
650 		if (!folio_isolate_hugetlb(folio, qp->pagelist))
651 			qp->nr_failed++;
652 unlock:
653 	spin_unlock(ptl);
654 	if (qp->nr_failed && strictly_unmovable(flags))
655 		return -EIO;
656 #endif
657 	return 0;
658 }
659 
660 #ifdef CONFIG_NUMA_BALANCING
661 /*
662  * This is used to mark a range of virtual addresses to be inaccessible.
663  * These are later cleared by a NUMA hinting fault. Depending on these
664  * faults, pages may be migrated for better NUMA placement.
665  *
666  * This is assuming that NUMA faults are handled using PROT_NONE. If
667  * an architecture makes a different choice, it will need further
668  * changes to the core.
669  */
change_prot_numa(struct vm_area_struct * vma,unsigned long addr,unsigned long end)670 unsigned long change_prot_numa(struct vm_area_struct *vma,
671 			unsigned long addr, unsigned long end)
672 {
673 	struct mmu_gather tlb;
674 	long nr_updated;
675 
676 	tlb_gather_mmu(&tlb, vma->vm_mm);
677 
678 	nr_updated = change_protection(&tlb, vma, addr, end, MM_CP_PROT_NUMA);
679 	if (nr_updated > 0) {
680 		count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
681 		count_memcg_events_mm(vma->vm_mm, NUMA_PTE_UPDATES, nr_updated);
682 	}
683 
684 	tlb_finish_mmu(&tlb);
685 
686 	return nr_updated;
687 }
688 #endif /* CONFIG_NUMA_BALANCING */
689 
queue_pages_test_walk(unsigned long start,unsigned long end,struct mm_walk * walk)690 static int queue_pages_test_walk(unsigned long start, unsigned long end,
691 				struct mm_walk *walk)
692 {
693 	struct vm_area_struct *next, *vma = walk->vma;
694 	struct queue_pages *qp = walk->private;
695 	unsigned long flags = qp->flags;
696 
697 	/* range check first */
698 	VM_BUG_ON_VMA(!range_in_vma(vma, start, end), vma);
699 
700 	if (!qp->first) {
701 		qp->first = vma;
702 		if (!(flags & MPOL_MF_DISCONTIG_OK) &&
703 			(qp->start < vma->vm_start))
704 			/* hole at head side of range */
705 			return -EFAULT;
706 	}
707 	next = find_vma(vma->vm_mm, vma->vm_end);
708 	if (!(flags & MPOL_MF_DISCONTIG_OK) &&
709 		((vma->vm_end < qp->end) &&
710 		(!next || vma->vm_end < next->vm_start)))
711 		/* hole at middle or tail of range */
712 		return -EFAULT;
713 
714 	/*
715 	 * Need check MPOL_MF_STRICT to return -EIO if possible
716 	 * regardless of vma_migratable
717 	 */
718 	if (!vma_migratable(vma) &&
719 	    !(flags & MPOL_MF_STRICT))
720 		return 1;
721 
722 	/*
723 	 * Check page nodes, and queue pages to move, in the current vma.
724 	 * But if no moving, and no strict checking, the scan can be skipped.
725 	 */
726 	if (flags & (MPOL_MF_STRICT | MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
727 		return 0;
728 	return 1;
729 }
730 
731 static const struct mm_walk_ops queue_pages_walk_ops = {
732 	.hugetlb_entry		= queue_folios_hugetlb,
733 	.pmd_entry		= queue_folios_pte_range,
734 	.test_walk		= queue_pages_test_walk,
735 	.walk_lock		= PGWALK_RDLOCK,
736 };
737 
738 static const struct mm_walk_ops queue_pages_lock_vma_walk_ops = {
739 	.hugetlb_entry		= queue_folios_hugetlb,
740 	.pmd_entry		= queue_folios_pte_range,
741 	.test_walk		= queue_pages_test_walk,
742 	.walk_lock		= PGWALK_WRLOCK,
743 };
744 
745 /*
746  * Walk through page tables and collect pages to be migrated.
747  *
748  * If pages found in a given range are not on the required set of @nodes,
749  * and migration is allowed, they are isolated and queued to @pagelist.
750  *
751  * queue_pages_range() may return:
752  * 0 - all pages already on the right node, or successfully queued for moving
753  *     (or neither strict checking nor moving requested: only range checking).
754  * >0 - this number of misplaced folios could not be queued for moving
755  *      (a hugetlbfs page or a transparent huge page being counted as 1).
756  * -EIO - a misplaced page found, when MPOL_MF_STRICT specified without MOVEs.
757  * -EFAULT - a hole in the memory range, when MPOL_MF_DISCONTIG_OK unspecified.
758  */
759 static long
queue_pages_range(struct mm_struct * mm,unsigned long start,unsigned long end,nodemask_t * nodes,unsigned long flags,struct list_head * pagelist)760 queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
761 		nodemask_t *nodes, unsigned long flags,
762 		struct list_head *pagelist)
763 {
764 	int err;
765 	struct queue_pages qp = {
766 		.pagelist = pagelist,
767 		.flags = flags,
768 		.nmask = nodes,
769 		.start = start,
770 		.end = end,
771 		.first = NULL,
772 	};
773 	const struct mm_walk_ops *ops = (flags & MPOL_MF_WRLOCK) ?
774 			&queue_pages_lock_vma_walk_ops : &queue_pages_walk_ops;
775 
776 	err = walk_page_range(mm, start, end, ops, &qp);
777 
778 	if (!qp.first)
779 		/* whole range in hole */
780 		err = -EFAULT;
781 
782 	return err ? : qp.nr_failed;
783 }
784 
785 /*
786  * Apply policy to a single VMA
787  * This must be called with the mmap_lock held for writing.
788  */
vma_replace_policy(struct vm_area_struct * vma,struct mempolicy * pol)789 static int vma_replace_policy(struct vm_area_struct *vma,
790 				struct mempolicy *pol)
791 {
792 	int err;
793 	struct mempolicy *old;
794 	struct mempolicy *new;
795 
796 	vma_assert_write_locked(vma);
797 
798 	new = mpol_dup(pol);
799 	if (IS_ERR(new))
800 		return PTR_ERR(new);
801 
802 	if (vma->vm_ops && vma->vm_ops->set_policy) {
803 		err = vma->vm_ops->set_policy(vma, new);
804 		if (err)
805 			goto err_out;
806 	}
807 
808 	old = vma->vm_policy;
809 	vma->vm_policy = new; /* protected by mmap_lock */
810 	mpol_put(old);
811 
812 	return 0;
813  err_out:
814 	mpol_put(new);
815 	return err;
816 }
817 
818 /* Split or merge the VMA (if required) and apply the new policy */
mbind_range(struct vma_iterator * vmi,struct vm_area_struct * vma,struct vm_area_struct ** prev,unsigned long start,unsigned long end,struct mempolicy * new_pol)819 static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma,
820 		struct vm_area_struct **prev, unsigned long start,
821 		unsigned long end, struct mempolicy *new_pol)
822 {
823 	unsigned long vmstart, vmend;
824 
825 	vmend = min(end, vma->vm_end);
826 	if (start > vma->vm_start) {
827 		*prev = vma;
828 		vmstart = start;
829 	} else {
830 		vmstart = vma->vm_start;
831 	}
832 
833 	if (mpol_equal(vma->vm_policy, new_pol)) {
834 		*prev = vma;
835 		return 0;
836 	}
837 
838 	vma =  vma_modify_policy(vmi, *prev, vma, vmstart, vmend, new_pol);
839 	if (IS_ERR(vma))
840 		return PTR_ERR(vma);
841 
842 	*prev = vma;
843 	return vma_replace_policy(vma, new_pol);
844 }
845 
846 /* Set the process memory policy */
do_set_mempolicy(unsigned short mode,unsigned short flags,nodemask_t * nodes)847 static long do_set_mempolicy(unsigned short mode, unsigned short flags,
848 			     nodemask_t *nodes)
849 {
850 	struct mempolicy *new, *old;
851 	NODEMASK_SCRATCH(scratch);
852 	int ret;
853 
854 	if (!scratch)
855 		return -ENOMEM;
856 
857 	new = mpol_new(mode, flags, nodes);
858 	if (IS_ERR(new)) {
859 		ret = PTR_ERR(new);
860 		goto out;
861 	}
862 
863 	task_lock(current);
864 	ret = mpol_set_nodemask(new, nodes, scratch);
865 	if (ret) {
866 		task_unlock(current);
867 		mpol_put(new);
868 		goto out;
869 	}
870 
871 	old = current->mempolicy;
872 	current->mempolicy = new;
873 	if (new && (new->mode == MPOL_INTERLEAVE ||
874 		    new->mode == MPOL_WEIGHTED_INTERLEAVE)) {
875 		current->il_prev = MAX_NUMNODES-1;
876 		current->il_weight = 0;
877 	}
878 	task_unlock(current);
879 	mpol_put(old);
880 	ret = 0;
881 out:
882 	NODEMASK_SCRATCH_FREE(scratch);
883 	return ret;
884 }
885 
886 /*
887  * Return nodemask for policy for get_mempolicy() query
888  *
889  * Called with task's alloc_lock held
890  */
get_policy_nodemask(struct mempolicy * pol,nodemask_t * nodes)891 static void get_policy_nodemask(struct mempolicy *pol, nodemask_t *nodes)
892 {
893 	nodes_clear(*nodes);
894 	if (pol == &default_policy)
895 		return;
896 
897 	switch (pol->mode) {
898 	case MPOL_BIND:
899 	case MPOL_INTERLEAVE:
900 	case MPOL_PREFERRED:
901 	case MPOL_PREFERRED_MANY:
902 	case MPOL_WEIGHTED_INTERLEAVE:
903 		*nodes = pol->nodes;
904 		break;
905 	case MPOL_LOCAL:
906 		/* return empty node mask for local allocation */
907 		break;
908 	default:
909 		BUG();
910 	}
911 }
912 
lookup_node(struct mm_struct * mm,unsigned long addr)913 static int lookup_node(struct mm_struct *mm, unsigned long addr)
914 {
915 	struct page *p = NULL;
916 	int ret;
917 
918 	ret = get_user_pages_fast(addr & PAGE_MASK, 1, 0, &p);
919 	if (ret > 0) {
920 		ret = page_to_nid(p);
921 		put_page(p);
922 	}
923 	return ret;
924 }
925 
926 /* Retrieve NUMA policy */
do_get_mempolicy(int * policy,nodemask_t * nmask,unsigned long addr,unsigned long flags)927 static long do_get_mempolicy(int *policy, nodemask_t *nmask,
928 			     unsigned long addr, unsigned long flags)
929 {
930 	int err;
931 	struct mm_struct *mm = current->mm;
932 	struct vm_area_struct *vma = NULL;
933 	struct mempolicy *pol = current->mempolicy, *pol_refcount = NULL;
934 
935 	if (flags &
936 		~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
937 		return -EINVAL;
938 
939 	if (flags & MPOL_F_MEMS_ALLOWED) {
940 		if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
941 			return -EINVAL;
942 		*policy = 0;	/* just so it's initialized */
943 		task_lock(current);
944 		*nmask  = cpuset_current_mems_allowed;
945 		task_unlock(current);
946 		return 0;
947 	}
948 
949 	if (flags & MPOL_F_ADDR) {
950 		pgoff_t ilx;		/* ignored here */
951 		/*
952 		 * Do NOT fall back to task policy if the
953 		 * vma/shared policy at addr is NULL.  We
954 		 * want to return MPOL_DEFAULT in this case.
955 		 */
956 		mmap_read_lock(mm);
957 		vma = vma_lookup(mm, addr);
958 		if (!vma) {
959 			mmap_read_unlock(mm);
960 			return -EFAULT;
961 		}
962 		pol = __get_vma_policy(vma, addr, &ilx);
963 	} else if (addr)
964 		return -EINVAL;
965 
966 	if (!pol)
967 		pol = &default_policy;	/* indicates default behavior */
968 
969 	if (flags & MPOL_F_NODE) {
970 		if (flags & MPOL_F_ADDR) {
971 			/*
972 			 * Take a refcount on the mpol, because we are about to
973 			 * drop the mmap_lock, after which only "pol" remains
974 			 * valid, "vma" is stale.
975 			 */
976 			pol_refcount = pol;
977 			vma = NULL;
978 			mpol_get(pol);
979 			mmap_read_unlock(mm);
980 			err = lookup_node(mm, addr);
981 			if (err < 0)
982 				goto out;
983 			*policy = err;
984 		} else if (pol == current->mempolicy &&
985 				pol->mode == MPOL_INTERLEAVE) {
986 			*policy = next_node_in(current->il_prev, pol->nodes);
987 		} else if (pol == current->mempolicy &&
988 				pol->mode == MPOL_WEIGHTED_INTERLEAVE) {
989 			if (current->il_weight)
990 				*policy = current->il_prev;
991 			else
992 				*policy = next_node_in(current->il_prev,
993 						       pol->nodes);
994 		} else {
995 			err = -EINVAL;
996 			goto out;
997 		}
998 	} else {
999 		*policy = pol == &default_policy ? MPOL_DEFAULT :
1000 						pol->mode;
1001 		/*
1002 		 * Internal mempolicy flags must be masked off before exposing
1003 		 * the policy to userspace.
1004 		 */
1005 		*policy |= (pol->flags & MPOL_MODE_FLAGS);
1006 	}
1007 
1008 	err = 0;
1009 	if (nmask) {
1010 		if (mpol_store_user_nodemask(pol)) {
1011 			*nmask = pol->w.user_nodemask;
1012 		} else {
1013 			task_lock(current);
1014 			get_policy_nodemask(pol, nmask);
1015 			task_unlock(current);
1016 		}
1017 	}
1018 
1019  out:
1020 	mpol_cond_put(pol);
1021 	if (vma)
1022 		mmap_read_unlock(mm);
1023 	if (pol_refcount)
1024 		mpol_put(pol_refcount);
1025 	return err;
1026 }
1027 
1028 #ifdef CONFIG_MIGRATION
migrate_folio_add(struct folio * folio,struct list_head * foliolist,unsigned long flags)1029 static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist,
1030 				unsigned long flags)
1031 {
1032 	/*
1033 	 * Unless MPOL_MF_MOVE_ALL, we try to avoid migrating a shared folio.
1034 	 * Choosing not to migrate a shared folio is not counted as a failure.
1035 	 *
1036 	 * See folio_likely_mapped_shared() on possible imprecision when we
1037 	 * cannot easily detect if a folio is shared.
1038 	 */
1039 	if ((flags & MPOL_MF_MOVE_ALL) || !folio_likely_mapped_shared(folio)) {
1040 		if (folio_isolate_lru(folio)) {
1041 			list_add_tail(&folio->lru, foliolist);
1042 			node_stat_mod_folio(folio,
1043 				NR_ISOLATED_ANON + folio_is_file_lru(folio),
1044 				folio_nr_pages(folio));
1045 		} else {
1046 			/*
1047 			 * Non-movable folio may reach here.  And, there may be
1048 			 * temporary off LRU folios or non-LRU movable folios.
1049 			 * Treat them as unmovable folios since they can't be
1050 			 * isolated, so they can't be moved at the moment.
1051 			 */
1052 			return false;
1053 		}
1054 	}
1055 	return true;
1056 }
1057 
1058 /*
1059  * Migrate pages from one node to a target node.
1060  * Returns error or the number of pages not migrated.
1061  */
migrate_to_node(struct mm_struct * mm,int source,int dest,int flags)1062 static long migrate_to_node(struct mm_struct *mm, int source, int dest,
1063 			    int flags)
1064 {
1065 	nodemask_t nmask;
1066 	struct vm_area_struct *vma;
1067 	LIST_HEAD(pagelist);
1068 	long nr_failed;
1069 	long err = 0;
1070 	struct migration_target_control mtc = {
1071 		.nid = dest,
1072 		.gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
1073 		.reason = MR_SYSCALL,
1074 	};
1075 
1076 	nodes_clear(nmask);
1077 	node_set(source, nmask);
1078 
1079 	VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
1080 
1081 	mmap_read_lock(mm);
1082 	vma = find_vma(mm, 0);
1083 	if (unlikely(!vma)) {
1084 		mmap_read_unlock(mm);
1085 		return 0;
1086 	}
1087 
1088 	/*
1089 	 * This does not migrate the range, but isolates all pages that
1090 	 * need migration.  Between passing in the full user address
1091 	 * space range and MPOL_MF_DISCONTIG_OK, this call cannot fail,
1092 	 * but passes back the count of pages which could not be isolated.
1093 	 */
1094 	nr_failed = queue_pages_range(mm, vma->vm_start, mm->task_size, &nmask,
1095 				      flags | MPOL_MF_DISCONTIG_OK, &pagelist);
1096 	mmap_read_unlock(mm);
1097 
1098 	if (!list_empty(&pagelist)) {
1099 		err = migrate_pages(&pagelist, alloc_migration_target, NULL,
1100 			(unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL);
1101 		if (err)
1102 			putback_movable_pages(&pagelist);
1103 	}
1104 
1105 	if (err >= 0)
1106 		err += nr_failed;
1107 	return err;
1108 }
1109 
1110 /*
1111  * Move pages between the two nodesets so as to preserve the physical
1112  * layout as much as possible.
1113  *
1114  * Returns the number of page that could not be moved.
1115  */
do_migrate_pages(struct mm_struct * mm,const nodemask_t * from,const nodemask_t * to,int flags)1116 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1117 		     const nodemask_t *to, int flags)
1118 {
1119 	long nr_failed = 0;
1120 	long err = 0;
1121 	nodemask_t tmp;
1122 
1123 	lru_cache_disable();
1124 
1125 	/*
1126 	 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
1127 	 * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
1128 	 * bit in 'tmp', and return that <source, dest> pair for migration.
1129 	 * The pair of nodemasks 'to' and 'from' define the map.
1130 	 *
1131 	 * If no pair of bits is found that way, fallback to picking some
1132 	 * pair of 'source' and 'dest' bits that are not the same.  If the
1133 	 * 'source' and 'dest' bits are the same, this represents a node
1134 	 * that will be migrating to itself, so no pages need move.
1135 	 *
1136 	 * If no bits are left in 'tmp', or if all remaining bits left
1137 	 * in 'tmp' correspond to the same bit in 'to', return false
1138 	 * (nothing left to migrate).
1139 	 *
1140 	 * This lets us pick a pair of nodes to migrate between, such that
1141 	 * if possible the dest node is not already occupied by some other
1142 	 * source node, minimizing the risk of overloading the memory on a
1143 	 * node that would happen if we migrated incoming memory to a node
1144 	 * before migrating outgoing memory source that same node.
1145 	 *
1146 	 * A single scan of tmp is sufficient.  As we go, we remember the
1147 	 * most recent <s, d> pair that moved (s != d).  If we find a pair
1148 	 * that not only moved, but what's better, moved to an empty slot
1149 	 * (d is not set in tmp), then we break out then, with that pair.
1150 	 * Otherwise when we finish scanning from_tmp, we at least have the
1151 	 * most recent <s, d> pair that moved.  If we get all the way through
1152 	 * the scan of tmp without finding any node that moved, much less
1153 	 * moved to an empty node, then there is nothing left worth migrating.
1154 	 */
1155 
1156 	tmp = *from;
1157 	while (!nodes_empty(tmp)) {
1158 		int s, d;
1159 		int source = NUMA_NO_NODE;
1160 		int dest = 0;
1161 
1162 		for_each_node_mask(s, tmp) {
1163 
1164 			/*
1165 			 * do_migrate_pages() tries to maintain the relative
1166 			 * node relationship of the pages established between
1167 			 * threads and memory areas.
1168                          *
1169 			 * However if the number of source nodes is not equal to
1170 			 * the number of destination nodes we can not preserve
1171 			 * this node relative relationship.  In that case, skip
1172 			 * copying memory from a node that is in the destination
1173 			 * mask.
1174 			 *
1175 			 * Example: [2,3,4] -> [3,4,5] moves everything.
1176 			 *          [0-7] - > [3,4,5] moves only 0,1,2,6,7.
1177 			 */
1178 
1179 			if ((nodes_weight(*from) != nodes_weight(*to)) &&
1180 						(node_isset(s, *to)))
1181 				continue;
1182 
1183 			d = node_remap(s, *from, *to);
1184 			if (s == d)
1185 				continue;
1186 
1187 			source = s;	/* Node moved. Memorize */
1188 			dest = d;
1189 
1190 			/* dest not in remaining from nodes? */
1191 			if (!node_isset(dest, tmp))
1192 				break;
1193 		}
1194 		if (source == NUMA_NO_NODE)
1195 			break;
1196 
1197 		node_clear(source, tmp);
1198 		err = migrate_to_node(mm, source, dest, flags);
1199 		if (err > 0)
1200 			nr_failed += err;
1201 		if (err < 0)
1202 			break;
1203 	}
1204 
1205 	lru_cache_enable();
1206 	if (err < 0)
1207 		return err;
1208 	return (nr_failed < INT_MAX) ? nr_failed : INT_MAX;
1209 }
1210 
1211 /*
1212  * Allocate a new folio for page migration, according to NUMA mempolicy.
1213  */
alloc_migration_target_by_mpol(struct folio * src,unsigned long private)1214 static struct folio *alloc_migration_target_by_mpol(struct folio *src,
1215 						    unsigned long private)
1216 {
1217 	struct migration_mpol *mmpol = (struct migration_mpol *)private;
1218 	struct mempolicy *pol = mmpol->pol;
1219 	pgoff_t ilx = mmpol->ilx;
1220 	unsigned int order;
1221 	int nid = numa_node_id();
1222 	gfp_t gfp;
1223 
1224 	order = folio_order(src);
1225 	ilx += src->index >> order;
1226 
1227 	if (folio_test_hugetlb(src)) {
1228 		nodemask_t *nodemask;
1229 		struct hstate *h;
1230 
1231 		h = folio_hstate(src);
1232 		gfp = htlb_alloc_mask(h);
1233 		nodemask = policy_nodemask(gfp, pol, ilx, &nid);
1234 		return alloc_hugetlb_folio_nodemask(h, nid, nodemask, gfp,
1235 				htlb_allow_alloc_fallback(MR_MEMPOLICY_MBIND));
1236 	}
1237 
1238 	if (folio_test_large(src))
1239 		gfp = GFP_TRANSHUGE;
1240 	else
1241 		gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL | __GFP_COMP;
1242 
1243 	return folio_alloc_mpol(gfp, order, pol, ilx, nid);
1244 }
1245 #else
1246 
migrate_folio_add(struct folio * folio,struct list_head * foliolist,unsigned long flags)1247 static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist,
1248 				unsigned long flags)
1249 {
1250 	return false;
1251 }
1252 
do_migrate_pages(struct mm_struct * mm,const nodemask_t * from,const nodemask_t * to,int flags)1253 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1254 		     const nodemask_t *to, int flags)
1255 {
1256 	return -ENOSYS;
1257 }
1258 
alloc_migration_target_by_mpol(struct folio * src,unsigned long private)1259 static struct folio *alloc_migration_target_by_mpol(struct folio *src,
1260 						    unsigned long private)
1261 {
1262 	return NULL;
1263 }
1264 #endif
1265 
do_mbind(unsigned long start,unsigned long len,unsigned short mode,unsigned short mode_flags,nodemask_t * nmask,unsigned long flags)1266 static long do_mbind(unsigned long start, unsigned long len,
1267 		     unsigned short mode, unsigned short mode_flags,
1268 		     nodemask_t *nmask, unsigned long flags)
1269 {
1270 	struct mm_struct *mm = current->mm;
1271 	struct vm_area_struct *vma, *prev;
1272 	struct vma_iterator vmi;
1273 	struct migration_mpol mmpol;
1274 	struct mempolicy *new;
1275 	unsigned long end;
1276 	long err;
1277 	long nr_failed;
1278 	LIST_HEAD(pagelist);
1279 
1280 	if (flags & ~(unsigned long)MPOL_MF_VALID)
1281 		return -EINVAL;
1282 	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1283 		return -EPERM;
1284 
1285 	if (start & ~PAGE_MASK)
1286 		return -EINVAL;
1287 
1288 	if (mode == MPOL_DEFAULT)
1289 		flags &= ~MPOL_MF_STRICT;
1290 
1291 	len = PAGE_ALIGN(len);
1292 	end = start + len;
1293 
1294 	if (end < start)
1295 		return -EINVAL;
1296 	if (end == start)
1297 		return 0;
1298 
1299 	new = mpol_new(mode, mode_flags, nmask);
1300 	if (IS_ERR(new))
1301 		return PTR_ERR(new);
1302 
1303 	/*
1304 	 * If we are using the default policy then operation
1305 	 * on discontinuous address spaces is okay after all
1306 	 */
1307 	if (!new)
1308 		flags |= MPOL_MF_DISCONTIG_OK;
1309 
1310 	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
1311 		lru_cache_disable();
1312 	{
1313 		NODEMASK_SCRATCH(scratch);
1314 		if (scratch) {
1315 			mmap_write_lock(mm);
1316 			err = mpol_set_nodemask(new, nmask, scratch);
1317 			if (err)
1318 				mmap_write_unlock(mm);
1319 		} else
1320 			err = -ENOMEM;
1321 		NODEMASK_SCRATCH_FREE(scratch);
1322 	}
1323 	if (err)
1324 		goto mpol_out;
1325 
1326 	/*
1327 	 * Lock the VMAs before scanning for pages to migrate,
1328 	 * to ensure we don't miss a concurrently inserted page.
1329 	 */
1330 	nr_failed = queue_pages_range(mm, start, end, nmask,
1331 			flags | MPOL_MF_INVERT | MPOL_MF_WRLOCK, &pagelist);
1332 
1333 	if (nr_failed < 0) {
1334 		err = nr_failed;
1335 		nr_failed = 0;
1336 	} else {
1337 		vma_iter_init(&vmi, mm, start);
1338 		prev = vma_prev(&vmi);
1339 		for_each_vma_range(vmi, vma, end) {
1340 			err = mbind_range(&vmi, vma, &prev, start, end, new);
1341 			if (err)
1342 				break;
1343 		}
1344 	}
1345 
1346 	if (!err && !list_empty(&pagelist)) {
1347 		/* Convert MPOL_DEFAULT's NULL to task or default policy */
1348 		if (!new) {
1349 			new = get_task_policy(current);
1350 			mpol_get(new);
1351 		}
1352 		mmpol.pol = new;
1353 		mmpol.ilx = 0;
1354 
1355 		/*
1356 		 * In the interleaved case, attempt to allocate on exactly the
1357 		 * targeted nodes, for the first VMA to be migrated; for later
1358 		 * VMAs, the nodes will still be interleaved from the targeted
1359 		 * nodemask, but one by one may be selected differently.
1360 		 */
1361 		if (new->mode == MPOL_INTERLEAVE ||
1362 		    new->mode == MPOL_WEIGHTED_INTERLEAVE) {
1363 			struct folio *folio;
1364 			unsigned int order;
1365 			unsigned long addr = -EFAULT;
1366 
1367 			list_for_each_entry(folio, &pagelist, lru) {
1368 				if (!folio_test_ksm(folio))
1369 					break;
1370 			}
1371 			if (!list_entry_is_head(folio, &pagelist, lru)) {
1372 				vma_iter_init(&vmi, mm, start);
1373 				for_each_vma_range(vmi, vma, end) {
1374 					addr = page_address_in_vma(folio,
1375 						folio_page(folio, 0), vma);
1376 					if (addr != -EFAULT)
1377 						break;
1378 				}
1379 			}
1380 			if (addr != -EFAULT) {
1381 				order = folio_order(folio);
1382 				/* We already know the pol, but not the ilx */
1383 				mpol_cond_put(get_vma_policy(vma, addr, order,
1384 							     &mmpol.ilx));
1385 				/* Set base from which to increment by index */
1386 				mmpol.ilx -= folio->index >> order;
1387 			}
1388 		}
1389 	}
1390 
1391 	mmap_write_unlock(mm);
1392 
1393 	if (!err && !list_empty(&pagelist)) {
1394 		nr_failed |= migrate_pages(&pagelist,
1395 				alloc_migration_target_by_mpol, NULL,
1396 				(unsigned long)&mmpol, MIGRATE_SYNC,
1397 				MR_MEMPOLICY_MBIND, NULL);
1398 	}
1399 
1400 	if (nr_failed && (flags & MPOL_MF_STRICT))
1401 		err = -EIO;
1402 	if (!list_empty(&pagelist))
1403 		putback_movable_pages(&pagelist);
1404 mpol_out:
1405 	mpol_put(new);
1406 	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
1407 		lru_cache_enable();
1408 	return err;
1409 }
1410 
1411 /*
1412  * User space interface with variable sized bitmaps for nodelists.
1413  */
get_bitmap(unsigned long * mask,const unsigned long __user * nmask,unsigned long maxnode)1414 static int get_bitmap(unsigned long *mask, const unsigned long __user *nmask,
1415 		      unsigned long maxnode)
1416 {
1417 	unsigned long nlongs = BITS_TO_LONGS(maxnode);
1418 	int ret;
1419 
1420 	if (in_compat_syscall())
1421 		ret = compat_get_bitmap(mask,
1422 					(const compat_ulong_t __user *)nmask,
1423 					maxnode);
1424 	else
1425 		ret = copy_from_user(mask, nmask,
1426 				     nlongs * sizeof(unsigned long));
1427 
1428 	if (ret)
1429 		return -EFAULT;
1430 
1431 	if (maxnode % BITS_PER_LONG)
1432 		mask[nlongs - 1] &= (1UL << (maxnode % BITS_PER_LONG)) - 1;
1433 
1434 	return 0;
1435 }
1436 
1437 /* Copy a node mask from user space. */
get_nodes(nodemask_t * nodes,const unsigned long __user * nmask,unsigned long maxnode)1438 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
1439 		     unsigned long maxnode)
1440 {
1441 	--maxnode;
1442 	nodes_clear(*nodes);
1443 	if (maxnode == 0 || !nmask)
1444 		return 0;
1445 	if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
1446 		return -EINVAL;
1447 
1448 	/*
1449 	 * When the user specified more nodes than supported just check
1450 	 * if the non supported part is all zero, one word at a time,
1451 	 * starting at the end.
1452 	 */
1453 	while (maxnode > MAX_NUMNODES) {
1454 		unsigned long bits = min_t(unsigned long, maxnode, BITS_PER_LONG);
1455 		unsigned long t;
1456 
1457 		if (get_bitmap(&t, &nmask[(maxnode - 1) / BITS_PER_LONG], bits))
1458 			return -EFAULT;
1459 
1460 		if (maxnode - bits >= MAX_NUMNODES) {
1461 			maxnode -= bits;
1462 		} else {
1463 			maxnode = MAX_NUMNODES;
1464 			t &= ~((1UL << (MAX_NUMNODES % BITS_PER_LONG)) - 1);
1465 		}
1466 		if (t)
1467 			return -EINVAL;
1468 	}
1469 
1470 	return get_bitmap(nodes_addr(*nodes), nmask, maxnode);
1471 }
1472 
1473 /* Copy a kernel node mask to user space */
copy_nodes_to_user(unsigned long __user * mask,unsigned long maxnode,nodemask_t * nodes)1474 static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1475 			      nodemask_t *nodes)
1476 {
1477 	unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1478 	unsigned int nbytes = BITS_TO_LONGS(nr_node_ids) * sizeof(long);
1479 	bool compat = in_compat_syscall();
1480 
1481 	if (compat)
1482 		nbytes = BITS_TO_COMPAT_LONGS(nr_node_ids) * sizeof(compat_long_t);
1483 
1484 	if (copy > nbytes) {
1485 		if (copy > PAGE_SIZE)
1486 			return -EINVAL;
1487 		if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1488 			return -EFAULT;
1489 		copy = nbytes;
1490 		maxnode = nr_node_ids;
1491 	}
1492 
1493 	if (compat)
1494 		return compat_put_bitmap((compat_ulong_t __user *)mask,
1495 					 nodes_addr(*nodes), maxnode);
1496 
1497 	return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1498 }
1499 
1500 /* Basic parameter sanity check used by both mbind() and set_mempolicy() */
sanitize_mpol_flags(int * mode,unsigned short * flags)1501 static inline int sanitize_mpol_flags(int *mode, unsigned short *flags)
1502 {
1503 	*flags = *mode & MPOL_MODE_FLAGS;
1504 	*mode &= ~MPOL_MODE_FLAGS;
1505 
1506 	if ((unsigned int)(*mode) >=  MPOL_MAX)
1507 		return -EINVAL;
1508 	if ((*flags & MPOL_F_STATIC_NODES) && (*flags & MPOL_F_RELATIVE_NODES))
1509 		return -EINVAL;
1510 	if (*flags & MPOL_F_NUMA_BALANCING) {
1511 		if (*mode == MPOL_BIND || *mode == MPOL_PREFERRED_MANY)
1512 			*flags |= (MPOL_F_MOF | MPOL_F_MORON);
1513 		else
1514 			return -EINVAL;
1515 	}
1516 	return 0;
1517 }
1518 
kernel_mbind(unsigned long start,unsigned long len,unsigned long mode,const unsigned long __user * nmask,unsigned long maxnode,unsigned int flags)1519 static long kernel_mbind(unsigned long start, unsigned long len,
1520 			 unsigned long mode, const unsigned long __user *nmask,
1521 			 unsigned long maxnode, unsigned int flags)
1522 {
1523 	unsigned short mode_flags;
1524 	nodemask_t nodes;
1525 	int lmode = mode;
1526 	int err;
1527 
1528 	start = untagged_addr(start);
1529 	err = sanitize_mpol_flags(&lmode, &mode_flags);
1530 	if (err)
1531 		return err;
1532 
1533 	err = get_nodes(&nodes, nmask, maxnode);
1534 	if (err)
1535 		return err;
1536 
1537 	return do_mbind(start, len, lmode, mode_flags, &nodes, flags);
1538 }
1539 
SYSCALL_DEFINE4(set_mempolicy_home_node,unsigned long,start,unsigned long,len,unsigned long,home_node,unsigned long,flags)1540 SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, len,
1541 		unsigned long, home_node, unsigned long, flags)
1542 {
1543 	struct mm_struct *mm = current->mm;
1544 	struct vm_area_struct *vma, *prev;
1545 	struct mempolicy *new, *old;
1546 	unsigned long end;
1547 	int err = -ENOENT;
1548 	VMA_ITERATOR(vmi, mm, start);
1549 
1550 	start = untagged_addr(start);
1551 	if (start & ~PAGE_MASK)
1552 		return -EINVAL;
1553 	/*
1554 	 * flags is used for future extension if any.
1555 	 */
1556 	if (flags != 0)
1557 		return -EINVAL;
1558 
1559 	/*
1560 	 * Check home_node is online to avoid accessing uninitialized
1561 	 * NODE_DATA.
1562 	 */
1563 	if (home_node >= MAX_NUMNODES || !node_online(home_node))
1564 		return -EINVAL;
1565 
1566 	len = PAGE_ALIGN(len);
1567 	end = start + len;
1568 
1569 	if (end < start)
1570 		return -EINVAL;
1571 	if (end == start)
1572 		return 0;
1573 	mmap_write_lock(mm);
1574 	prev = vma_prev(&vmi);
1575 	for_each_vma_range(vmi, vma, end) {
1576 		/*
1577 		 * If any vma in the range got policy other than MPOL_BIND
1578 		 * or MPOL_PREFERRED_MANY we return error. We don't reset
1579 		 * the home node for vmas we already updated before.
1580 		 */
1581 		old = vma_policy(vma);
1582 		if (!old) {
1583 			prev = vma;
1584 			continue;
1585 		}
1586 		if (old->mode != MPOL_BIND && old->mode != MPOL_PREFERRED_MANY) {
1587 			err = -EOPNOTSUPP;
1588 			break;
1589 		}
1590 		new = mpol_dup(old);
1591 		if (IS_ERR(new)) {
1592 			err = PTR_ERR(new);
1593 			break;
1594 		}
1595 
1596 		vma_start_write(vma);
1597 		new->home_node = home_node;
1598 		err = mbind_range(&vmi, vma, &prev, start, end, new);
1599 		mpol_put(new);
1600 		if (err)
1601 			break;
1602 	}
1603 	mmap_write_unlock(mm);
1604 	return err;
1605 }
1606 
SYSCALL_DEFINE6(mbind,unsigned long,start,unsigned long,len,unsigned long,mode,const unsigned long __user *,nmask,unsigned long,maxnode,unsigned int,flags)1607 SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
1608 		unsigned long, mode, const unsigned long __user *, nmask,
1609 		unsigned long, maxnode, unsigned int, flags)
1610 {
1611 	return kernel_mbind(start, len, mode, nmask, maxnode, flags);
1612 }
1613 
1614 /* Set the process memory policy */
kernel_set_mempolicy(int mode,const unsigned long __user * nmask,unsigned long maxnode)1615 static long kernel_set_mempolicy(int mode, const unsigned long __user *nmask,
1616 				 unsigned long maxnode)
1617 {
1618 	unsigned short mode_flags;
1619 	nodemask_t nodes;
1620 	int lmode = mode;
1621 	int err;
1622 
1623 	err = sanitize_mpol_flags(&lmode, &mode_flags);
1624 	if (err)
1625 		return err;
1626 
1627 	err = get_nodes(&nodes, nmask, maxnode);
1628 	if (err)
1629 		return err;
1630 
1631 	return do_set_mempolicy(lmode, mode_flags, &nodes);
1632 }
1633 
SYSCALL_DEFINE3(set_mempolicy,int,mode,const unsigned long __user *,nmask,unsigned long,maxnode)1634 SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask,
1635 		unsigned long, maxnode)
1636 {
1637 	return kernel_set_mempolicy(mode, nmask, maxnode);
1638 }
1639 
kernel_migrate_pages(pid_t pid,unsigned long maxnode,const unsigned long __user * old_nodes,const unsigned long __user * new_nodes)1640 static int kernel_migrate_pages(pid_t pid, unsigned long maxnode,
1641 				const unsigned long __user *old_nodes,
1642 				const unsigned long __user *new_nodes)
1643 {
1644 	struct mm_struct *mm = NULL;
1645 	struct task_struct *task;
1646 	nodemask_t task_nodes;
1647 	int err;
1648 	nodemask_t *old;
1649 	nodemask_t *new;
1650 	NODEMASK_SCRATCH(scratch);
1651 
1652 	if (!scratch)
1653 		return -ENOMEM;
1654 
1655 	old = &scratch->mask1;
1656 	new = &scratch->mask2;
1657 
1658 	err = get_nodes(old, old_nodes, maxnode);
1659 	if (err)
1660 		goto out;
1661 
1662 	err = get_nodes(new, new_nodes, maxnode);
1663 	if (err)
1664 		goto out;
1665 
1666 	/* Find the mm_struct */
1667 	rcu_read_lock();
1668 	task = pid ? find_task_by_vpid(pid) : current;
1669 	if (!task) {
1670 		rcu_read_unlock();
1671 		err = -ESRCH;
1672 		goto out;
1673 	}
1674 	get_task_struct(task);
1675 
1676 	err = -EINVAL;
1677 
1678 	/*
1679 	 * Check if this process has the right to modify the specified process.
1680 	 * Use the regular "ptrace_may_access()" checks.
1681 	 */
1682 	if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
1683 		rcu_read_unlock();
1684 		err = -EPERM;
1685 		goto out_put;
1686 	}
1687 	rcu_read_unlock();
1688 
1689 	task_nodes = cpuset_mems_allowed(task);
1690 	/* Is the user allowed to access the target nodes? */
1691 	if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
1692 		err = -EPERM;
1693 		goto out_put;
1694 	}
1695 
1696 	task_nodes = cpuset_mems_allowed(current);
1697 	nodes_and(*new, *new, task_nodes);
1698 	if (nodes_empty(*new))
1699 		goto out_put;
1700 
1701 	err = security_task_movememory(task);
1702 	if (err)
1703 		goto out_put;
1704 
1705 	mm = get_task_mm(task);
1706 	put_task_struct(task);
1707 
1708 	if (!mm) {
1709 		err = -EINVAL;
1710 		goto out;
1711 	}
1712 
1713 	err = do_migrate_pages(mm, old, new,
1714 		capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
1715 
1716 	mmput(mm);
1717 out:
1718 	NODEMASK_SCRATCH_FREE(scratch);
1719 
1720 	return err;
1721 
1722 out_put:
1723 	put_task_struct(task);
1724 	goto out;
1725 }
1726 
SYSCALL_DEFINE4(migrate_pages,pid_t,pid,unsigned long,maxnode,const unsigned long __user *,old_nodes,const unsigned long __user *,new_nodes)1727 SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1728 		const unsigned long __user *, old_nodes,
1729 		const unsigned long __user *, new_nodes)
1730 {
1731 	return kernel_migrate_pages(pid, maxnode, old_nodes, new_nodes);
1732 }
1733 
1734 /* Retrieve NUMA policy */
kernel_get_mempolicy(int __user * policy,unsigned long __user * nmask,unsigned long maxnode,unsigned long addr,unsigned long flags)1735 static int kernel_get_mempolicy(int __user *policy,
1736 				unsigned long __user *nmask,
1737 				unsigned long maxnode,
1738 				unsigned long addr,
1739 				unsigned long flags)
1740 {
1741 	int err;
1742 	int pval;
1743 	nodemask_t nodes;
1744 
1745 	if (nmask != NULL && maxnode < nr_node_ids)
1746 		return -EINVAL;
1747 
1748 	addr = untagged_addr(addr);
1749 
1750 	err = do_get_mempolicy(&pval, &nodes, addr, flags);
1751 
1752 	if (err)
1753 		return err;
1754 
1755 	if (policy && put_user(pval, policy))
1756 		return -EFAULT;
1757 
1758 	if (nmask)
1759 		err = copy_nodes_to_user(nmask, maxnode, &nodes);
1760 
1761 	return err;
1762 }
1763 
SYSCALL_DEFINE5(get_mempolicy,int __user *,policy,unsigned long __user *,nmask,unsigned long,maxnode,unsigned long,addr,unsigned long,flags)1764 SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1765 		unsigned long __user *, nmask, unsigned long, maxnode,
1766 		unsigned long, addr, unsigned long, flags)
1767 {
1768 	return kernel_get_mempolicy(policy, nmask, maxnode, addr, flags);
1769 }
1770 
vma_migratable(struct vm_area_struct * vma)1771 bool vma_migratable(struct vm_area_struct *vma)
1772 {
1773 	if (vma->vm_flags & (VM_IO | VM_PFNMAP))
1774 		return false;
1775 
1776 	/*
1777 	 * DAX device mappings require predictable access latency, so avoid
1778 	 * incurring periodic faults.
1779 	 */
1780 	if (vma_is_dax(vma))
1781 		return false;
1782 
1783 	if (is_vm_hugetlb_page(vma) &&
1784 		!hugepage_migration_supported(hstate_vma(vma)))
1785 		return false;
1786 
1787 	/*
1788 	 * Migration allocates pages in the highest zone. If we cannot
1789 	 * do so then migration (at least from node to node) is not
1790 	 * possible.
1791 	 */
1792 	if (vma->vm_file &&
1793 		gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping))
1794 			< policy_zone)
1795 		return false;
1796 	return true;
1797 }
1798 
__get_vma_policy(struct vm_area_struct * vma,unsigned long addr,pgoff_t * ilx)1799 struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
1800 				   unsigned long addr, pgoff_t *ilx)
1801 {
1802 	*ilx = 0;
1803 	return (vma->vm_ops && vma->vm_ops->get_policy) ?
1804 		vma->vm_ops->get_policy(vma, addr, ilx) : vma->vm_policy;
1805 }
1806 
1807 /*
1808  * get_vma_policy(@vma, @addr, @order, @ilx)
1809  * @vma: virtual memory area whose policy is sought
1810  * @addr: address in @vma for shared policy lookup
1811  * @order: 0, or appropriate huge_page_order for interleaving
1812  * @ilx: interleave index (output), for use only when MPOL_INTERLEAVE or
1813  *       MPOL_WEIGHTED_INTERLEAVE
1814  *
1815  * Returns effective policy for a VMA at specified address.
1816  * Falls back to current->mempolicy or system default policy, as necessary.
1817  * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1818  * count--added by the get_policy() vm_op, as appropriate--to protect against
1819  * freeing by another task.  It is the caller's responsibility to free the
1820  * extra reference for shared policies.
1821  */
get_vma_policy(struct vm_area_struct * vma,unsigned long addr,int order,pgoff_t * ilx)1822 struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
1823 				 unsigned long addr, int order, pgoff_t *ilx)
1824 {
1825 	struct mempolicy *pol;
1826 
1827 	pol = __get_vma_policy(vma, addr, ilx);
1828 	if (!pol)
1829 		pol = get_task_policy(current);
1830 	if (pol->mode == MPOL_INTERLEAVE ||
1831 	    pol->mode == MPOL_WEIGHTED_INTERLEAVE) {
1832 		*ilx += vma->vm_pgoff >> order;
1833 		*ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order);
1834 	}
1835 	return pol;
1836 }
1837 
vma_policy_mof(struct vm_area_struct * vma)1838 bool vma_policy_mof(struct vm_area_struct *vma)
1839 {
1840 	struct mempolicy *pol;
1841 
1842 	if (vma->vm_ops && vma->vm_ops->get_policy) {
1843 		bool ret = false;
1844 		pgoff_t ilx;		/* ignored here */
1845 
1846 		pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx);
1847 		if (pol && (pol->flags & MPOL_F_MOF))
1848 			ret = true;
1849 		mpol_cond_put(pol);
1850 
1851 		return ret;
1852 	}
1853 
1854 	pol = vma->vm_policy;
1855 	if (!pol)
1856 		pol = get_task_policy(current);
1857 
1858 	return pol->flags & MPOL_F_MOF;
1859 }
1860 
apply_policy_zone(struct mempolicy * policy,enum zone_type zone)1861 bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
1862 {
1863 	enum zone_type dynamic_policy_zone = policy_zone;
1864 
1865 	BUG_ON(dynamic_policy_zone == ZONE_MOVABLE);
1866 
1867 	/*
1868 	 * if policy->nodes has movable memory only,
1869 	 * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
1870 	 *
1871 	 * policy->nodes is intersect with node_states[N_MEMORY].
1872 	 * so if the following test fails, it implies
1873 	 * policy->nodes has movable memory only.
1874 	 */
1875 	if (!nodes_intersects(policy->nodes, node_states[N_HIGH_MEMORY]))
1876 		dynamic_policy_zone = ZONE_MOVABLE;
1877 
1878 	return zone >= dynamic_policy_zone;
1879 }
1880 
weighted_interleave_nodes(struct mempolicy * policy)1881 static unsigned int weighted_interleave_nodes(struct mempolicy *policy)
1882 {
1883 	unsigned int node;
1884 	unsigned int cpuset_mems_cookie;
1885 
1886 retry:
1887 	/* to prevent miscount use tsk->mems_allowed_seq to detect rebind */
1888 	cpuset_mems_cookie = read_mems_allowed_begin();
1889 	node = current->il_prev;
1890 	if (!current->il_weight || !node_isset(node, policy->nodes)) {
1891 		node = next_node_in(node, policy->nodes);
1892 		if (read_mems_allowed_retry(cpuset_mems_cookie))
1893 			goto retry;
1894 		if (node == MAX_NUMNODES)
1895 			return node;
1896 		current->il_prev = node;
1897 		current->il_weight = get_il_weight(node);
1898 	}
1899 	current->il_weight--;
1900 	return node;
1901 }
1902 
1903 /* Do dynamic interleaving for a process */
interleave_nodes(struct mempolicy * policy)1904 static unsigned int interleave_nodes(struct mempolicy *policy)
1905 {
1906 	unsigned int nid;
1907 	unsigned int cpuset_mems_cookie;
1908 
1909 	/* to prevent miscount, use tsk->mems_allowed_seq to detect rebind */
1910 	do {
1911 		cpuset_mems_cookie = read_mems_allowed_begin();
1912 		nid = next_node_in(current->il_prev, policy->nodes);
1913 	} while (read_mems_allowed_retry(cpuset_mems_cookie));
1914 
1915 	if (nid < MAX_NUMNODES)
1916 		current->il_prev = nid;
1917 	return nid;
1918 }
1919 
1920 /*
1921  * Depending on the memory policy provide a node from which to allocate the
1922  * next slab entry.
1923  */
mempolicy_slab_node(void)1924 unsigned int mempolicy_slab_node(void)
1925 {
1926 	struct mempolicy *policy;
1927 	int node = numa_mem_id();
1928 
1929 	if (!in_task())
1930 		return node;
1931 
1932 	policy = current->mempolicy;
1933 	if (!policy)
1934 		return node;
1935 
1936 	switch (policy->mode) {
1937 	case MPOL_PREFERRED:
1938 		return first_node(policy->nodes);
1939 
1940 	case MPOL_INTERLEAVE:
1941 		return interleave_nodes(policy);
1942 
1943 	case MPOL_WEIGHTED_INTERLEAVE:
1944 		return weighted_interleave_nodes(policy);
1945 
1946 	case MPOL_BIND:
1947 	case MPOL_PREFERRED_MANY:
1948 	{
1949 		struct zoneref *z;
1950 
1951 		/*
1952 		 * Follow bind policy behavior and start allocation at the
1953 		 * first node.
1954 		 */
1955 		struct zonelist *zonelist;
1956 		enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
1957 		zonelist = &NODE_DATA(node)->node_zonelists[ZONELIST_FALLBACK];
1958 		z = first_zones_zonelist(zonelist, highest_zoneidx,
1959 							&policy->nodes);
1960 		return zonelist_zone(z) ? zonelist_node_idx(z) : node;
1961 	}
1962 	case MPOL_LOCAL:
1963 		return node;
1964 
1965 	default:
1966 		BUG();
1967 	}
1968 }
1969 
read_once_policy_nodemask(struct mempolicy * pol,nodemask_t * mask)1970 static unsigned int read_once_policy_nodemask(struct mempolicy *pol,
1971 					      nodemask_t *mask)
1972 {
1973 	/*
1974 	 * barrier stabilizes the nodemask locally so that it can be iterated
1975 	 * over safely without concern for changes. Allocators validate node
1976 	 * selection does not violate mems_allowed, so this is safe.
1977 	 */
1978 	barrier();
1979 	memcpy(mask, &pol->nodes, sizeof(nodemask_t));
1980 	barrier();
1981 	return nodes_weight(*mask);
1982 }
1983 
weighted_interleave_nid(struct mempolicy * pol,pgoff_t ilx)1984 static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx)
1985 {
1986 	nodemask_t nodemask;
1987 	unsigned int target, nr_nodes;
1988 	u8 *table;
1989 	unsigned int weight_total = 0;
1990 	u8 weight;
1991 	int nid;
1992 
1993 	nr_nodes = read_once_policy_nodemask(pol, &nodemask);
1994 	if (!nr_nodes)
1995 		return numa_node_id();
1996 
1997 	rcu_read_lock();
1998 	table = rcu_dereference(iw_table);
1999 	/* calculate the total weight */
2000 	for_each_node_mask(nid, nodemask) {
2001 		/* detect system default usage */
2002 		weight = table ? table[nid] : 1;
2003 		weight = weight ? weight : 1;
2004 		weight_total += weight;
2005 	}
2006 
2007 	/* Calculate the node offset based on totals */
2008 	target = ilx % weight_total;
2009 	nid = first_node(nodemask);
2010 	while (target) {
2011 		/* detect system default usage */
2012 		weight = table ? table[nid] : 1;
2013 		weight = weight ? weight : 1;
2014 		if (target < weight)
2015 			break;
2016 		target -= weight;
2017 		nid = next_node_in(nid, nodemask);
2018 	}
2019 	rcu_read_unlock();
2020 	return nid;
2021 }
2022 
2023 /*
2024  * Do static interleaving for interleave index @ilx.  Returns the ilx'th
2025  * node in pol->nodes (starting from ilx=0), wrapping around if ilx
2026  * exceeds the number of present nodes.
2027  */
interleave_nid(struct mempolicy * pol,pgoff_t ilx)2028 static unsigned int interleave_nid(struct mempolicy *pol, pgoff_t ilx)
2029 {
2030 	nodemask_t nodemask;
2031 	unsigned int target, nnodes;
2032 	int i;
2033 	int nid;
2034 
2035 	nnodes = read_once_policy_nodemask(pol, &nodemask);
2036 	if (!nnodes)
2037 		return numa_node_id();
2038 	target = ilx % nnodes;
2039 	nid = first_node(nodemask);
2040 	for (i = 0; i < target; i++)
2041 		nid = next_node(nid, nodemask);
2042 	return nid;
2043 }
2044 
2045 /*
2046  * Return a nodemask representing a mempolicy for filtering nodes for
2047  * page allocation, together with preferred node id (or the input node id).
2048  */
policy_nodemask(gfp_t gfp,struct mempolicy * pol,pgoff_t ilx,int * nid)2049 static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *pol,
2050 				   pgoff_t ilx, int *nid)
2051 {
2052 	nodemask_t *nodemask = NULL;
2053 
2054 	switch (pol->mode) {
2055 	case MPOL_PREFERRED:
2056 		/* Override input node id */
2057 		*nid = first_node(pol->nodes);
2058 		break;
2059 	case MPOL_PREFERRED_MANY:
2060 		nodemask = &pol->nodes;
2061 		if (pol->home_node != NUMA_NO_NODE)
2062 			*nid = pol->home_node;
2063 		break;
2064 	case MPOL_BIND:
2065 		/* Restrict to nodemask (but not on lower zones) */
2066 		if (apply_policy_zone(pol, gfp_zone(gfp)) &&
2067 		    cpuset_nodemask_valid_mems_allowed(&pol->nodes))
2068 			nodemask = &pol->nodes;
2069 		if (pol->home_node != NUMA_NO_NODE)
2070 			*nid = pol->home_node;
2071 		/*
2072 		 * __GFP_THISNODE shouldn't even be used with the bind policy
2073 		 * because we might easily break the expectation to stay on the
2074 		 * requested node and not break the policy.
2075 		 */
2076 		WARN_ON_ONCE(gfp & __GFP_THISNODE);
2077 		break;
2078 	case MPOL_INTERLEAVE:
2079 		/* Override input node id */
2080 		*nid = (ilx == NO_INTERLEAVE_INDEX) ?
2081 			interleave_nodes(pol) : interleave_nid(pol, ilx);
2082 		break;
2083 	case MPOL_WEIGHTED_INTERLEAVE:
2084 		*nid = (ilx == NO_INTERLEAVE_INDEX) ?
2085 			weighted_interleave_nodes(pol) :
2086 			weighted_interleave_nid(pol, ilx);
2087 		break;
2088 	}
2089 
2090 	return nodemask;
2091 }
2092 
2093 #ifdef CONFIG_HUGETLBFS
2094 /*
2095  * huge_node(@vma, @addr, @gfp_flags, @mpol)
2096  * @vma: virtual memory area whose policy is sought
2097  * @addr: address in @vma for shared policy lookup and interleave policy
2098  * @gfp_flags: for requested zone
2099  * @mpol: pointer to mempolicy pointer for reference counted mempolicy
2100  * @nodemask: pointer to nodemask pointer for 'bind' and 'prefer-many' policy
2101  *
2102  * Returns a nid suitable for a huge page allocation and a pointer
2103  * to the struct mempolicy for conditional unref after allocation.
2104  * If the effective policy is 'bind' or 'prefer-many', returns a pointer
2105  * to the mempolicy's @nodemask for filtering the zonelist.
2106  */
huge_node(struct vm_area_struct * vma,unsigned long addr,gfp_t gfp_flags,struct mempolicy ** mpol,nodemask_t ** nodemask)2107 int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags,
2108 		struct mempolicy **mpol, nodemask_t **nodemask)
2109 {
2110 	pgoff_t ilx;
2111 	int nid;
2112 
2113 	nid = numa_node_id();
2114 	*mpol = get_vma_policy(vma, addr, hstate_vma(vma)->order, &ilx);
2115 	*nodemask = policy_nodemask(gfp_flags, *mpol, ilx, &nid);
2116 	return nid;
2117 }
2118 
2119 /*
2120  * init_nodemask_of_mempolicy
2121  *
2122  * If the current task's mempolicy is "default" [NULL], return 'false'
2123  * to indicate default policy.  Otherwise, extract the policy nodemask
2124  * for 'bind' or 'interleave' policy into the argument nodemask, or
2125  * initialize the argument nodemask to contain the single node for
2126  * 'preferred' or 'local' policy and return 'true' to indicate presence
2127  * of non-default mempolicy.
2128  *
2129  * We don't bother with reference counting the mempolicy [mpol_get/put]
2130  * because the current task is examining it's own mempolicy and a task's
2131  * mempolicy is only ever changed by the task itself.
2132  *
2133  * N.B., it is the caller's responsibility to free a returned nodemask.
2134  */
init_nodemask_of_mempolicy(nodemask_t * mask)2135 bool init_nodemask_of_mempolicy(nodemask_t *mask)
2136 {
2137 	struct mempolicy *mempolicy;
2138 
2139 	if (!(mask && current->mempolicy))
2140 		return false;
2141 
2142 	task_lock(current);
2143 	mempolicy = current->mempolicy;
2144 	switch (mempolicy->mode) {
2145 	case MPOL_PREFERRED:
2146 	case MPOL_PREFERRED_MANY:
2147 	case MPOL_BIND:
2148 	case MPOL_INTERLEAVE:
2149 	case MPOL_WEIGHTED_INTERLEAVE:
2150 		*mask = mempolicy->nodes;
2151 		break;
2152 
2153 	case MPOL_LOCAL:
2154 		init_nodemask_of_node(mask, numa_node_id());
2155 		break;
2156 
2157 	default:
2158 		BUG();
2159 	}
2160 	task_unlock(current);
2161 
2162 	return true;
2163 }
2164 #endif
2165 
2166 /*
2167  * mempolicy_in_oom_domain
2168  *
2169  * If tsk's mempolicy is "bind", check for intersection between mask and
2170  * the policy nodemask. Otherwise, return true for all other policies
2171  * including "interleave", as a tsk with "interleave" policy may have
2172  * memory allocated from all nodes in system.
2173  *
2174  * Takes task_lock(tsk) to prevent freeing of its mempolicy.
2175  */
mempolicy_in_oom_domain(struct task_struct * tsk,const nodemask_t * mask)2176 bool mempolicy_in_oom_domain(struct task_struct *tsk,
2177 					const nodemask_t *mask)
2178 {
2179 	struct mempolicy *mempolicy;
2180 	bool ret = true;
2181 
2182 	if (!mask)
2183 		return ret;
2184 
2185 	task_lock(tsk);
2186 	mempolicy = tsk->mempolicy;
2187 	if (mempolicy && mempolicy->mode == MPOL_BIND)
2188 		ret = nodes_intersects(mempolicy->nodes, *mask);
2189 	task_unlock(tsk);
2190 
2191 	return ret;
2192 }
2193 
alloc_pages_preferred_many(gfp_t gfp,unsigned int order,int nid,nodemask_t * nodemask)2194 static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order,
2195 						int nid, nodemask_t *nodemask)
2196 {
2197 	struct page *page;
2198 	gfp_t preferred_gfp;
2199 
2200 	/*
2201 	 * This is a two pass approach. The first pass will only try the
2202 	 * preferred nodes but skip the direct reclaim and allow the
2203 	 * allocation to fail, while the second pass will try all the
2204 	 * nodes in system.
2205 	 */
2206 	preferred_gfp = gfp | __GFP_NOWARN;
2207 	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2208 	page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask);
2209 	if (!page)
2210 		page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL);
2211 
2212 	return page;
2213 }
2214 
2215 /**
2216  * alloc_pages_mpol - Allocate pages according to NUMA mempolicy.
2217  * @gfp: GFP flags.
2218  * @order: Order of the page allocation.
2219  * @pol: Pointer to the NUMA mempolicy.
2220  * @ilx: Index for interleave mempolicy (also distinguishes alloc_pages()).
2221  * @nid: Preferred node (usually numa_node_id() but @mpol may override it).
2222  *
2223  * Return: The page on success or NULL if allocation fails.
2224  */
alloc_pages_mpol(gfp_t gfp,unsigned int order,struct mempolicy * pol,pgoff_t ilx,int nid)2225 static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
2226 		struct mempolicy *pol, pgoff_t ilx, int nid)
2227 {
2228 	nodemask_t *nodemask;
2229 	struct page *page;
2230 
2231 	nodemask = policy_nodemask(gfp, pol, ilx, &nid);
2232 
2233 	if (pol->mode == MPOL_PREFERRED_MANY)
2234 		return alloc_pages_preferred_many(gfp, order, nid, nodemask);
2235 
2236 	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
2237 	    /* filter "hugepage" allocation, unless from alloc_pages() */
2238 	    order == HPAGE_PMD_ORDER && ilx != NO_INTERLEAVE_INDEX) {
2239 		/*
2240 		 * For hugepage allocation and non-interleave policy which
2241 		 * allows the current node (or other explicitly preferred
2242 		 * node) we only try to allocate from the current/preferred
2243 		 * node and don't fall back to other nodes, as the cost of
2244 		 * remote accesses would likely offset THP benefits.
2245 		 *
2246 		 * If the policy is interleave or does not allow the current
2247 		 * node in its nodemask, we allocate the standard way.
2248 		 */
2249 		if (pol->mode != MPOL_INTERLEAVE &&
2250 		    pol->mode != MPOL_WEIGHTED_INTERLEAVE &&
2251 		    (!nodemask || node_isset(nid, *nodemask))) {
2252 			/*
2253 			 * First, try to allocate THP only on local node, but
2254 			 * don't reclaim unnecessarily, just compact.
2255 			 */
2256 			page = __alloc_frozen_pages_noprof(
2257 				gfp | __GFP_THISNODE | __GFP_NORETRY, order,
2258 				nid, NULL);
2259 			if (page || !(gfp & __GFP_DIRECT_RECLAIM))
2260 				return page;
2261 			/*
2262 			 * If hugepage allocations are configured to always
2263 			 * synchronous compact or the vma has been madvised
2264 			 * to prefer hugepage backing, retry allowing remote
2265 			 * memory with both reclaim and compact as well.
2266 			 */
2267 		}
2268 	}
2269 
2270 	page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask);
2271 
2272 	if (unlikely(pol->mode == MPOL_INTERLEAVE ||
2273 		     pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) {
2274 		/* skip NUMA_INTERLEAVE_HIT update if numa stats is disabled */
2275 		if (static_branch_likely(&vm_numa_stat_key) &&
2276 		    page_to_nid(page) == nid) {
2277 			preempt_disable();
2278 			__count_numa_event(page_zone(page), NUMA_INTERLEAVE_HIT);
2279 			preempt_enable();
2280 		}
2281 	}
2282 
2283 	return page;
2284 }
2285 
folio_alloc_mpol_noprof(gfp_t gfp,unsigned int order,struct mempolicy * pol,pgoff_t ilx,int nid)2286 struct folio *folio_alloc_mpol_noprof(gfp_t gfp, unsigned int order,
2287 		struct mempolicy *pol, pgoff_t ilx, int nid)
2288 {
2289 	struct page *page = alloc_pages_mpol(gfp | __GFP_COMP, order, pol,
2290 			ilx, nid);
2291 	if (!page)
2292 		return NULL;
2293 
2294 	set_page_refcounted(page);
2295 	return page_rmappable_folio(page);
2296 }
2297 
2298 /**
2299  * vma_alloc_folio - Allocate a folio for a VMA.
2300  * @gfp: GFP flags.
2301  * @order: Order of the folio.
2302  * @vma: Pointer to VMA.
2303  * @addr: Virtual address of the allocation.  Must be inside @vma.
2304  *
2305  * Allocate a folio for a specific address in @vma, using the appropriate
2306  * NUMA policy.  The caller must hold the mmap_lock of the mm_struct of the
2307  * VMA to prevent it from going away.  Should be used for all allocations
2308  * for folios that will be mapped into user space, excepting hugetlbfs, and
2309  * excepting where direct use of folio_alloc_mpol() is more appropriate.
2310  *
2311  * Return: The folio on success or NULL if allocation fails.
2312  */
vma_alloc_folio_noprof(gfp_t gfp,int order,struct vm_area_struct * vma,unsigned long addr)2313 struct folio *vma_alloc_folio_noprof(gfp_t gfp, int order, struct vm_area_struct *vma,
2314 		unsigned long addr)
2315 {
2316 	struct mempolicy *pol;
2317 	pgoff_t ilx;
2318 	struct folio *folio;
2319 
2320 	if (vma->vm_flags & VM_DROPPABLE)
2321 		gfp |= __GFP_NOWARN;
2322 
2323 	pol = get_vma_policy(vma, addr, order, &ilx);
2324 	folio = folio_alloc_mpol_noprof(gfp, order, pol, ilx, numa_node_id());
2325 	mpol_cond_put(pol);
2326 	return folio;
2327 }
2328 EXPORT_SYMBOL(vma_alloc_folio_noprof);
2329 
alloc_frozen_pages_noprof(gfp_t gfp,unsigned order)2330 struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned order)
2331 {
2332 	struct mempolicy *pol = &default_policy;
2333 
2334 	/*
2335 	 * No reference counting needed for current->mempolicy
2336 	 * nor system default_policy
2337 	 */
2338 	if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2339 		pol = get_task_policy(current);
2340 
2341 	return alloc_pages_mpol(gfp, order, pol, NO_INTERLEAVE_INDEX,
2342 				       numa_node_id());
2343 }
2344 
2345 /**
2346  * alloc_pages - Allocate pages.
2347  * @gfp: GFP flags.
2348  * @order: Power of two of number of pages to allocate.
2349  *
2350  * Allocate 1 << @order contiguous pages.  The physical address of the
2351  * first page is naturally aligned (eg an order-3 allocation will be aligned
2352  * to a multiple of 8 * PAGE_SIZE bytes).  The NUMA policy of the current
2353  * process is honoured when in process context.
2354  *
2355  * Context: Can be called from any context, providing the appropriate GFP
2356  * flags are used.
2357  * Return: The page on success or NULL if allocation fails.
2358  */
alloc_pages_noprof(gfp_t gfp,unsigned int order)2359 struct page *alloc_pages_noprof(gfp_t gfp, unsigned int order)
2360 {
2361 	struct page *page = alloc_frozen_pages_noprof(gfp, order);
2362 
2363 	if (page)
2364 		set_page_refcounted(page);
2365 	return page;
2366 }
2367 EXPORT_SYMBOL(alloc_pages_noprof);
2368 
folio_alloc_noprof(gfp_t gfp,unsigned int order)2369 struct folio *folio_alloc_noprof(gfp_t gfp, unsigned int order)
2370 {
2371 	return page_rmappable_folio(alloc_pages_noprof(gfp | __GFP_COMP, order));
2372 }
2373 EXPORT_SYMBOL(folio_alloc_noprof);
2374 
alloc_pages_bulk_interleave(gfp_t gfp,struct mempolicy * pol,unsigned long nr_pages,struct page ** page_array)2375 static unsigned long alloc_pages_bulk_interleave(gfp_t gfp,
2376 		struct mempolicy *pol, unsigned long nr_pages,
2377 		struct page **page_array)
2378 {
2379 	int nodes;
2380 	unsigned long nr_pages_per_node;
2381 	int delta;
2382 	int i;
2383 	unsigned long nr_allocated;
2384 	unsigned long total_allocated = 0;
2385 
2386 	nodes = nodes_weight(pol->nodes);
2387 	nr_pages_per_node = nr_pages / nodes;
2388 	delta = nr_pages - nodes * nr_pages_per_node;
2389 
2390 	for (i = 0; i < nodes; i++) {
2391 		if (delta) {
2392 			nr_allocated = alloc_pages_bulk_noprof(gfp,
2393 					interleave_nodes(pol), NULL,
2394 					nr_pages_per_node + 1,
2395 					page_array);
2396 			delta--;
2397 		} else {
2398 			nr_allocated = alloc_pages_bulk_noprof(gfp,
2399 					interleave_nodes(pol), NULL,
2400 					nr_pages_per_node, page_array);
2401 		}
2402 
2403 		page_array += nr_allocated;
2404 		total_allocated += nr_allocated;
2405 	}
2406 
2407 	return total_allocated;
2408 }
2409 
alloc_pages_bulk_weighted_interleave(gfp_t gfp,struct mempolicy * pol,unsigned long nr_pages,struct page ** page_array)2410 static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp,
2411 		struct mempolicy *pol, unsigned long nr_pages,
2412 		struct page **page_array)
2413 {
2414 	struct task_struct *me = current;
2415 	unsigned int cpuset_mems_cookie;
2416 	unsigned long total_allocated = 0;
2417 	unsigned long nr_allocated = 0;
2418 	unsigned long rounds;
2419 	unsigned long node_pages, delta;
2420 	u8 *table, *weights, weight;
2421 	unsigned int weight_total = 0;
2422 	unsigned long rem_pages = nr_pages;
2423 	nodemask_t nodes;
2424 	int nnodes, node;
2425 	int resume_node = MAX_NUMNODES - 1;
2426 	u8 resume_weight = 0;
2427 	int prev_node;
2428 	int i;
2429 
2430 	if (!nr_pages)
2431 		return 0;
2432 
2433 	/* read the nodes onto the stack, retry if done during rebind */
2434 	do {
2435 		cpuset_mems_cookie = read_mems_allowed_begin();
2436 		nnodes = read_once_policy_nodemask(pol, &nodes);
2437 	} while (read_mems_allowed_retry(cpuset_mems_cookie));
2438 
2439 	/* if the nodemask has become invalid, we cannot do anything */
2440 	if (!nnodes)
2441 		return 0;
2442 
2443 	/* Continue allocating from most recent node and adjust the nr_pages */
2444 	node = me->il_prev;
2445 	weight = me->il_weight;
2446 	if (weight && node_isset(node, nodes)) {
2447 		node_pages = min(rem_pages, weight);
2448 		nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages,
2449 						  page_array);
2450 		page_array += nr_allocated;
2451 		total_allocated += nr_allocated;
2452 		/* if that's all the pages, no need to interleave */
2453 		if (rem_pages <= weight) {
2454 			me->il_weight -= rem_pages;
2455 			return total_allocated;
2456 		}
2457 		/* Otherwise we adjust remaining pages, continue from there */
2458 		rem_pages -= weight;
2459 	}
2460 	/* clear active weight in case of an allocation failure */
2461 	me->il_weight = 0;
2462 	prev_node = node;
2463 
2464 	/* create a local copy of node weights to operate on outside rcu */
2465 	weights = kzalloc(nr_node_ids, GFP_KERNEL);
2466 	if (!weights)
2467 		return total_allocated;
2468 
2469 	rcu_read_lock();
2470 	table = rcu_dereference(iw_table);
2471 	if (table)
2472 		memcpy(weights, table, nr_node_ids);
2473 	rcu_read_unlock();
2474 
2475 	/* calculate total, detect system default usage */
2476 	for_each_node_mask(node, nodes) {
2477 		if (!weights[node])
2478 			weights[node] = 1;
2479 		weight_total += weights[node];
2480 	}
2481 
2482 	/*
2483 	 * Calculate rounds/partial rounds to minimize __alloc_pages_bulk calls.
2484 	 * Track which node weighted interleave should resume from.
2485 	 *
2486 	 * if (rounds > 0) and (delta == 0), resume_node will always be
2487 	 * the node following prev_node and its weight.
2488 	 */
2489 	rounds = rem_pages / weight_total;
2490 	delta = rem_pages % weight_total;
2491 	resume_node = next_node_in(prev_node, nodes);
2492 	resume_weight = weights[resume_node];
2493 	for (i = 0; i < nnodes; i++) {
2494 		node = next_node_in(prev_node, nodes);
2495 		weight = weights[node];
2496 		node_pages = weight * rounds;
2497 		/* If a delta exists, add this node's portion of the delta */
2498 		if (delta > weight) {
2499 			node_pages += weight;
2500 			delta -= weight;
2501 		} else if (delta) {
2502 			/* when delta is depleted, resume from that node */
2503 			node_pages += delta;
2504 			resume_node = node;
2505 			resume_weight = weight - delta;
2506 			delta = 0;
2507 		}
2508 		/* node_pages can be 0 if an allocation fails and rounds == 0 */
2509 		if (!node_pages)
2510 			break;
2511 		nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages,
2512 						  page_array);
2513 		page_array += nr_allocated;
2514 		total_allocated += nr_allocated;
2515 		if (total_allocated == nr_pages)
2516 			break;
2517 		prev_node = node;
2518 	}
2519 	me->il_prev = resume_node;
2520 	me->il_weight = resume_weight;
2521 	kfree(weights);
2522 	return total_allocated;
2523 }
2524 
alloc_pages_bulk_preferred_many(gfp_t gfp,int nid,struct mempolicy * pol,unsigned long nr_pages,struct page ** page_array)2525 static unsigned long alloc_pages_bulk_preferred_many(gfp_t gfp, int nid,
2526 		struct mempolicy *pol, unsigned long nr_pages,
2527 		struct page **page_array)
2528 {
2529 	gfp_t preferred_gfp;
2530 	unsigned long nr_allocated = 0;
2531 
2532 	preferred_gfp = gfp | __GFP_NOWARN;
2533 	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2534 
2535 	nr_allocated  = alloc_pages_bulk_noprof(preferred_gfp, nid, &pol->nodes,
2536 					   nr_pages, page_array);
2537 
2538 	if (nr_allocated < nr_pages)
2539 		nr_allocated += alloc_pages_bulk_noprof(gfp, numa_node_id(), NULL,
2540 				nr_pages - nr_allocated,
2541 				page_array + nr_allocated);
2542 	return nr_allocated;
2543 }
2544 
2545 /* alloc pages bulk and mempolicy should be considered at the
2546  * same time in some situation such as vmalloc.
2547  *
2548  * It can accelerate memory allocation especially interleaving
2549  * allocate memory.
2550  */
alloc_pages_bulk_mempolicy_noprof(gfp_t gfp,unsigned long nr_pages,struct page ** page_array)2551 unsigned long alloc_pages_bulk_mempolicy_noprof(gfp_t gfp,
2552 		unsigned long nr_pages, struct page **page_array)
2553 {
2554 	struct mempolicy *pol = &default_policy;
2555 	nodemask_t *nodemask;
2556 	int nid;
2557 
2558 	if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2559 		pol = get_task_policy(current);
2560 
2561 	if (pol->mode == MPOL_INTERLEAVE)
2562 		return alloc_pages_bulk_interleave(gfp, pol,
2563 							 nr_pages, page_array);
2564 
2565 	if (pol->mode == MPOL_WEIGHTED_INTERLEAVE)
2566 		return alloc_pages_bulk_weighted_interleave(
2567 				  gfp, pol, nr_pages, page_array);
2568 
2569 	if (pol->mode == MPOL_PREFERRED_MANY)
2570 		return alloc_pages_bulk_preferred_many(gfp,
2571 				numa_node_id(), pol, nr_pages, page_array);
2572 
2573 	nid = numa_node_id();
2574 	nodemask = policy_nodemask(gfp, pol, NO_INTERLEAVE_INDEX, &nid);
2575 	return alloc_pages_bulk_noprof(gfp, nid, nodemask,
2576 				       nr_pages, page_array);
2577 }
2578 
vma_dup_policy(struct vm_area_struct * src,struct vm_area_struct * dst)2579 int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2580 {
2581 	struct mempolicy *pol = mpol_dup(src->vm_policy);
2582 
2583 	if (IS_ERR(pol))
2584 		return PTR_ERR(pol);
2585 	dst->vm_policy = pol;
2586 	return 0;
2587 }
2588 
2589 /*
2590  * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
2591  * rebinds the mempolicy its copying by calling mpol_rebind_policy()
2592  * with the mems_allowed returned by cpuset_mems_allowed().  This
2593  * keeps mempolicies cpuset relative after its cpuset moves.  See
2594  * further kernel/cpuset.c update_nodemask().
2595  *
2596  * current's mempolicy may be rebinded by the other task(the task that changes
2597  * cpuset's mems), so we needn't do rebind work for current task.
2598  */
2599 
2600 /* Slow path of a mempolicy duplicate */
__mpol_dup(struct mempolicy * old)2601 struct mempolicy *__mpol_dup(struct mempolicy *old)
2602 {
2603 	struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2604 
2605 	if (!new)
2606 		return ERR_PTR(-ENOMEM);
2607 
2608 	/* task's mempolicy is protected by alloc_lock */
2609 	if (old == current->mempolicy) {
2610 		task_lock(current);
2611 		*new = *old;
2612 		task_unlock(current);
2613 	} else
2614 		*new = *old;
2615 
2616 	if (current_cpuset_is_being_rebound()) {
2617 		nodemask_t mems = cpuset_mems_allowed(current);
2618 		mpol_rebind_policy(new, &mems);
2619 	}
2620 	atomic_set(&new->refcnt, 1);
2621 	return new;
2622 }
2623 
2624 /* Slow path of a mempolicy comparison */
__mpol_equal(struct mempolicy * a,struct mempolicy * b)2625 bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
2626 {
2627 	if (!a || !b)
2628 		return false;
2629 	if (a->mode != b->mode)
2630 		return false;
2631 	if (a->flags != b->flags)
2632 		return false;
2633 	if (a->home_node != b->home_node)
2634 		return false;
2635 	if (mpol_store_user_nodemask(a))
2636 		if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
2637 			return false;
2638 
2639 	switch (a->mode) {
2640 	case MPOL_BIND:
2641 	case MPOL_INTERLEAVE:
2642 	case MPOL_PREFERRED:
2643 	case MPOL_PREFERRED_MANY:
2644 	case MPOL_WEIGHTED_INTERLEAVE:
2645 		return !!nodes_equal(a->nodes, b->nodes);
2646 	case MPOL_LOCAL:
2647 		return true;
2648 	default:
2649 		BUG();
2650 		return false;
2651 	}
2652 }
2653 
2654 /*
2655  * Shared memory backing store policy support.
2656  *
2657  * Remember policies even when nobody has shared memory mapped.
2658  * The policies are kept in Red-Black tree linked from the inode.
2659  * They are protected by the sp->lock rwlock, which should be held
2660  * for any accesses to the tree.
2661  */
2662 
2663 /*
2664  * lookup first element intersecting start-end.  Caller holds sp->lock for
2665  * reading or for writing
2666  */
sp_lookup(struct shared_policy * sp,pgoff_t start,pgoff_t end)2667 static struct sp_node *sp_lookup(struct shared_policy *sp,
2668 					pgoff_t start, pgoff_t end)
2669 {
2670 	struct rb_node *n = sp->root.rb_node;
2671 
2672 	while (n) {
2673 		struct sp_node *p = rb_entry(n, struct sp_node, nd);
2674 
2675 		if (start >= p->end)
2676 			n = n->rb_right;
2677 		else if (end <= p->start)
2678 			n = n->rb_left;
2679 		else
2680 			break;
2681 	}
2682 	if (!n)
2683 		return NULL;
2684 	for (;;) {
2685 		struct sp_node *w = NULL;
2686 		struct rb_node *prev = rb_prev(n);
2687 		if (!prev)
2688 			break;
2689 		w = rb_entry(prev, struct sp_node, nd);
2690 		if (w->end <= start)
2691 			break;
2692 		n = prev;
2693 	}
2694 	return rb_entry(n, struct sp_node, nd);
2695 }
2696 
2697 /*
2698  * Insert a new shared policy into the list.  Caller holds sp->lock for
2699  * writing.
2700  */
sp_insert(struct shared_policy * sp,struct sp_node * new)2701 static void sp_insert(struct shared_policy *sp, struct sp_node *new)
2702 {
2703 	struct rb_node **p = &sp->root.rb_node;
2704 	struct rb_node *parent = NULL;
2705 	struct sp_node *nd;
2706 
2707 	while (*p) {
2708 		parent = *p;
2709 		nd = rb_entry(parent, struct sp_node, nd);
2710 		if (new->start < nd->start)
2711 			p = &(*p)->rb_left;
2712 		else if (new->end > nd->end)
2713 			p = &(*p)->rb_right;
2714 		else
2715 			BUG();
2716 	}
2717 	rb_link_node(&new->nd, parent, p);
2718 	rb_insert_color(&new->nd, &sp->root);
2719 }
2720 
2721 /* Find shared policy intersecting idx */
mpol_shared_policy_lookup(struct shared_policy * sp,pgoff_t idx)2722 struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp,
2723 						pgoff_t idx)
2724 {
2725 	struct mempolicy *pol = NULL;
2726 	struct sp_node *sn;
2727 
2728 	if (!sp->root.rb_node)
2729 		return NULL;
2730 	read_lock(&sp->lock);
2731 	sn = sp_lookup(sp, idx, idx+1);
2732 	if (sn) {
2733 		mpol_get(sn->policy);
2734 		pol = sn->policy;
2735 	}
2736 	read_unlock(&sp->lock);
2737 	return pol;
2738 }
2739 
sp_free(struct sp_node * n)2740 static void sp_free(struct sp_node *n)
2741 {
2742 	mpol_put(n->policy);
2743 	kmem_cache_free(sn_cache, n);
2744 }
2745 
2746 /**
2747  * mpol_misplaced - check whether current folio node is valid in policy
2748  *
2749  * @folio: folio to be checked
2750  * @vmf: structure describing the fault
2751  * @addr: virtual address in @vma for shared policy lookup and interleave policy
2752  *
2753  * Lookup current policy node id for vma,addr and "compare to" folio's
2754  * node id.  Policy determination "mimics" alloc_page_vma().
2755  * Called from fault path where we know the vma and faulting address.
2756  *
2757  * Return: NUMA_NO_NODE if the page is in a node that is valid for this
2758  * policy, or a suitable node ID to allocate a replacement folio from.
2759  */
mpol_misplaced(struct folio * folio,struct vm_fault * vmf,unsigned long addr)2760 int mpol_misplaced(struct folio *folio, struct vm_fault *vmf,
2761 		   unsigned long addr)
2762 {
2763 	struct mempolicy *pol;
2764 	pgoff_t ilx;
2765 	struct zoneref *z;
2766 	int curnid = folio_nid(folio);
2767 	struct vm_area_struct *vma = vmf->vma;
2768 	int thiscpu = raw_smp_processor_id();
2769 	int thisnid = numa_node_id();
2770 	int polnid = NUMA_NO_NODE;
2771 	int ret = NUMA_NO_NODE;
2772 
2773 	/*
2774 	 * Make sure ptl is held so that we don't preempt and we
2775 	 * have a stable smp processor id
2776 	 */
2777 	lockdep_assert_held(vmf->ptl);
2778 	pol = get_vma_policy(vma, addr, folio_order(folio), &ilx);
2779 	if (!(pol->flags & MPOL_F_MOF))
2780 		goto out;
2781 
2782 	switch (pol->mode) {
2783 	case MPOL_INTERLEAVE:
2784 		polnid = interleave_nid(pol, ilx);
2785 		break;
2786 
2787 	case MPOL_WEIGHTED_INTERLEAVE:
2788 		polnid = weighted_interleave_nid(pol, ilx);
2789 		break;
2790 
2791 	case MPOL_PREFERRED:
2792 		if (node_isset(curnid, pol->nodes))
2793 			goto out;
2794 		polnid = first_node(pol->nodes);
2795 		break;
2796 
2797 	case MPOL_LOCAL:
2798 		polnid = numa_node_id();
2799 		break;
2800 
2801 	case MPOL_BIND:
2802 	case MPOL_PREFERRED_MANY:
2803 		/*
2804 		 * Even though MPOL_PREFERRED_MANY can allocate pages outside
2805 		 * policy nodemask we don't allow numa migration to nodes
2806 		 * outside policy nodemask for now. This is done so that if we
2807 		 * want demotion to slow memory to happen, before allocating
2808 		 * from some DRAM node say 'x', we will end up using a
2809 		 * MPOL_PREFERRED_MANY mask excluding node 'x'. In such scenario
2810 		 * we should not promote to node 'x' from slow memory node.
2811 		 */
2812 		if (pol->flags & MPOL_F_MORON) {
2813 			/*
2814 			 * Optimize placement among multiple nodes
2815 			 * via NUMA balancing
2816 			 */
2817 			if (node_isset(thisnid, pol->nodes))
2818 				break;
2819 			goto out;
2820 		}
2821 
2822 		/*
2823 		 * use current page if in policy nodemask,
2824 		 * else select nearest allowed node, if any.
2825 		 * If no allowed nodes, use current [!misplaced].
2826 		 */
2827 		if (node_isset(curnid, pol->nodes))
2828 			goto out;
2829 		z = first_zones_zonelist(
2830 				node_zonelist(thisnid, GFP_HIGHUSER),
2831 				gfp_zone(GFP_HIGHUSER),
2832 				&pol->nodes);
2833 		polnid = zonelist_node_idx(z);
2834 		break;
2835 
2836 	default:
2837 		BUG();
2838 	}
2839 
2840 	/* Migrate the folio towards the node whose CPU is referencing it */
2841 	if (pol->flags & MPOL_F_MORON) {
2842 		polnid = thisnid;
2843 
2844 		if (!should_numa_migrate_memory(current, folio, curnid,
2845 						thiscpu))
2846 			goto out;
2847 	}
2848 
2849 	if (curnid != polnid)
2850 		ret = polnid;
2851 out:
2852 	mpol_cond_put(pol);
2853 
2854 	return ret;
2855 }
2856 
2857 /*
2858  * Drop the (possibly final) reference to task->mempolicy.  It needs to be
2859  * dropped after task->mempolicy is set to NULL so that any allocation done as
2860  * part of its kmem_cache_free(), such as by KASAN, doesn't reference a freed
2861  * policy.
2862  */
mpol_put_task_policy(struct task_struct * task)2863 void mpol_put_task_policy(struct task_struct *task)
2864 {
2865 	struct mempolicy *pol;
2866 
2867 	task_lock(task);
2868 	pol = task->mempolicy;
2869 	task->mempolicy = NULL;
2870 	task_unlock(task);
2871 	mpol_put(pol);
2872 }
2873 
sp_delete(struct shared_policy * sp,struct sp_node * n)2874 static void sp_delete(struct shared_policy *sp, struct sp_node *n)
2875 {
2876 	rb_erase(&n->nd, &sp->root);
2877 	sp_free(n);
2878 }
2879 
sp_node_init(struct sp_node * node,unsigned long start,unsigned long end,struct mempolicy * pol)2880 static void sp_node_init(struct sp_node *node, unsigned long start,
2881 			unsigned long end, struct mempolicy *pol)
2882 {
2883 	node->start = start;
2884 	node->end = end;
2885 	node->policy = pol;
2886 }
2887 
sp_alloc(unsigned long start,unsigned long end,struct mempolicy * pol)2888 static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
2889 				struct mempolicy *pol)
2890 {
2891 	struct sp_node *n;
2892 	struct mempolicy *newpol;
2893 
2894 	n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2895 	if (!n)
2896 		return NULL;
2897 
2898 	newpol = mpol_dup(pol);
2899 	if (IS_ERR(newpol)) {
2900 		kmem_cache_free(sn_cache, n);
2901 		return NULL;
2902 	}
2903 	newpol->flags |= MPOL_F_SHARED;
2904 	sp_node_init(n, start, end, newpol);
2905 
2906 	return n;
2907 }
2908 
2909 /* Replace a policy range. */
shared_policy_replace(struct shared_policy * sp,pgoff_t start,pgoff_t end,struct sp_node * new)2910 static int shared_policy_replace(struct shared_policy *sp, pgoff_t start,
2911 				 pgoff_t end, struct sp_node *new)
2912 {
2913 	struct sp_node *n;
2914 	struct sp_node *n_new = NULL;
2915 	struct mempolicy *mpol_new = NULL;
2916 	int ret = 0;
2917 
2918 restart:
2919 	write_lock(&sp->lock);
2920 	n = sp_lookup(sp, start, end);
2921 	/* Take care of old policies in the same range. */
2922 	while (n && n->start < end) {
2923 		struct rb_node *next = rb_next(&n->nd);
2924 		if (n->start >= start) {
2925 			if (n->end <= end)
2926 				sp_delete(sp, n);
2927 			else
2928 				n->start = end;
2929 		} else {
2930 			/* Old policy spanning whole new range. */
2931 			if (n->end > end) {
2932 				if (!n_new)
2933 					goto alloc_new;
2934 
2935 				*mpol_new = *n->policy;
2936 				atomic_set(&mpol_new->refcnt, 1);
2937 				sp_node_init(n_new, end, n->end, mpol_new);
2938 				n->end = start;
2939 				sp_insert(sp, n_new);
2940 				n_new = NULL;
2941 				mpol_new = NULL;
2942 				break;
2943 			} else
2944 				n->end = start;
2945 		}
2946 		if (!next)
2947 			break;
2948 		n = rb_entry(next, struct sp_node, nd);
2949 	}
2950 	if (new)
2951 		sp_insert(sp, new);
2952 	write_unlock(&sp->lock);
2953 	ret = 0;
2954 
2955 err_out:
2956 	if (mpol_new)
2957 		mpol_put(mpol_new);
2958 	if (n_new)
2959 		kmem_cache_free(sn_cache, n_new);
2960 
2961 	return ret;
2962 
2963 alloc_new:
2964 	write_unlock(&sp->lock);
2965 	ret = -ENOMEM;
2966 	n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2967 	if (!n_new)
2968 		goto err_out;
2969 	mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2970 	if (!mpol_new)
2971 		goto err_out;
2972 	atomic_set(&mpol_new->refcnt, 1);
2973 	goto restart;
2974 }
2975 
2976 /**
2977  * mpol_shared_policy_init - initialize shared policy for inode
2978  * @sp: pointer to inode shared policy
2979  * @mpol:  struct mempolicy to install
2980  *
2981  * Install non-NULL @mpol in inode's shared policy rb-tree.
2982  * On entry, the current task has a reference on a non-NULL @mpol.
2983  * This must be released on exit.
2984  * This is called at get_inode() calls and we can use GFP_KERNEL.
2985  */
mpol_shared_policy_init(struct shared_policy * sp,struct mempolicy * mpol)2986 void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
2987 {
2988 	int ret;
2989 
2990 	sp->root = RB_ROOT;		/* empty tree == default mempolicy */
2991 	rwlock_init(&sp->lock);
2992 
2993 	if (mpol) {
2994 		struct sp_node *sn;
2995 		struct mempolicy *npol;
2996 		NODEMASK_SCRATCH(scratch);
2997 
2998 		if (!scratch)
2999 			goto put_mpol;
3000 
3001 		/* contextualize the tmpfs mount point mempolicy to this file */
3002 		npol = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
3003 		if (IS_ERR(npol))
3004 			goto free_scratch; /* no valid nodemask intersection */
3005 
3006 		task_lock(current);
3007 		ret = mpol_set_nodemask(npol, &mpol->w.user_nodemask, scratch);
3008 		task_unlock(current);
3009 		if (ret)
3010 			goto put_npol;
3011 
3012 		/* alloc node covering entire file; adds ref to file's npol */
3013 		sn = sp_alloc(0, MAX_LFS_FILESIZE >> PAGE_SHIFT, npol);
3014 		if (sn)
3015 			sp_insert(sp, sn);
3016 put_npol:
3017 		mpol_put(npol);	/* drop initial ref on file's npol */
3018 free_scratch:
3019 		NODEMASK_SCRATCH_FREE(scratch);
3020 put_mpol:
3021 		mpol_put(mpol);	/* drop our incoming ref on sb mpol */
3022 	}
3023 }
3024 
mpol_set_shared_policy(struct shared_policy * sp,struct vm_area_struct * vma,struct mempolicy * pol)3025 int mpol_set_shared_policy(struct shared_policy *sp,
3026 			struct vm_area_struct *vma, struct mempolicy *pol)
3027 {
3028 	int err;
3029 	struct sp_node *new = NULL;
3030 	unsigned long sz = vma_pages(vma);
3031 
3032 	if (pol) {
3033 		new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol);
3034 		if (!new)
3035 			return -ENOMEM;
3036 	}
3037 	err = shared_policy_replace(sp, vma->vm_pgoff, vma->vm_pgoff + sz, new);
3038 	if (err && new)
3039 		sp_free(new);
3040 	return err;
3041 }
3042 
3043 /* Free a backing policy store on inode delete. */
mpol_free_shared_policy(struct shared_policy * sp)3044 void mpol_free_shared_policy(struct shared_policy *sp)
3045 {
3046 	struct sp_node *n;
3047 	struct rb_node *next;
3048 
3049 	if (!sp->root.rb_node)
3050 		return;
3051 	write_lock(&sp->lock);
3052 	next = rb_first(&sp->root);
3053 	while (next) {
3054 		n = rb_entry(next, struct sp_node, nd);
3055 		next = rb_next(&n->nd);
3056 		sp_delete(sp, n);
3057 	}
3058 	write_unlock(&sp->lock);
3059 }
3060 
3061 #ifdef CONFIG_NUMA_BALANCING
3062 static int __initdata numabalancing_override;
3063 
check_numabalancing_enable(void)3064 static void __init check_numabalancing_enable(void)
3065 {
3066 	bool numabalancing_default = false;
3067 
3068 	if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED))
3069 		numabalancing_default = true;
3070 
3071 	/* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
3072 	if (numabalancing_override)
3073 		set_numabalancing_state(numabalancing_override == 1);
3074 
3075 	if (num_online_nodes() > 1 && !numabalancing_override) {
3076 		pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n",
3077 			numabalancing_default ? "Enabling" : "Disabling");
3078 		set_numabalancing_state(numabalancing_default);
3079 	}
3080 }
3081 
setup_numabalancing(char * str)3082 static int __init setup_numabalancing(char *str)
3083 {
3084 	int ret = 0;
3085 	if (!str)
3086 		goto out;
3087 
3088 	if (!strcmp(str, "enable")) {
3089 		numabalancing_override = 1;
3090 		ret = 1;
3091 	} else if (!strcmp(str, "disable")) {
3092 		numabalancing_override = -1;
3093 		ret = 1;
3094 	}
3095 out:
3096 	if (!ret)
3097 		pr_warn("Unable to parse numa_balancing=\n");
3098 
3099 	return ret;
3100 }
3101 __setup("numa_balancing=", setup_numabalancing);
3102 #else
check_numabalancing_enable(void)3103 static inline void __init check_numabalancing_enable(void)
3104 {
3105 }
3106 #endif /* CONFIG_NUMA_BALANCING */
3107 
numa_policy_init(void)3108 void __init numa_policy_init(void)
3109 {
3110 	nodemask_t interleave_nodes;
3111 	unsigned long largest = 0;
3112 	int nid, prefer = 0;
3113 
3114 	policy_cache = kmem_cache_create("numa_policy",
3115 					 sizeof(struct mempolicy),
3116 					 0, SLAB_PANIC, NULL);
3117 
3118 	sn_cache = kmem_cache_create("shared_policy_node",
3119 				     sizeof(struct sp_node),
3120 				     0, SLAB_PANIC, NULL);
3121 
3122 	for_each_node(nid) {
3123 		preferred_node_policy[nid] = (struct mempolicy) {
3124 			.refcnt = ATOMIC_INIT(1),
3125 			.mode = MPOL_PREFERRED,
3126 			.flags = MPOL_F_MOF | MPOL_F_MORON,
3127 			.nodes = nodemask_of_node(nid),
3128 		};
3129 	}
3130 
3131 	/*
3132 	 * Set interleaving policy for system init. Interleaving is only
3133 	 * enabled across suitably sized nodes (default is >= 16MB), or
3134 	 * fall back to the largest node if they're all smaller.
3135 	 */
3136 	nodes_clear(interleave_nodes);
3137 	for_each_node_state(nid, N_MEMORY) {
3138 		unsigned long total_pages = node_present_pages(nid);
3139 
3140 		/* Preserve the largest node */
3141 		if (largest < total_pages) {
3142 			largest = total_pages;
3143 			prefer = nid;
3144 		}
3145 
3146 		/* Interleave this node? */
3147 		if ((total_pages << PAGE_SHIFT) >= (16 << 20))
3148 			node_set(nid, interleave_nodes);
3149 	}
3150 
3151 	/* All too small, use the largest */
3152 	if (unlikely(nodes_empty(interleave_nodes)))
3153 		node_set(prefer, interleave_nodes);
3154 
3155 	if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
3156 		pr_err("%s: interleaving failed\n", __func__);
3157 
3158 	check_numabalancing_enable();
3159 }
3160 
3161 /* Reset policy of current process to default */
numa_default_policy(void)3162 void numa_default_policy(void)
3163 {
3164 	do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
3165 }
3166 
3167 /*
3168  * Parse and format mempolicy from/to strings
3169  */
3170 static const char * const policy_modes[] =
3171 {
3172 	[MPOL_DEFAULT]    = "default",
3173 	[MPOL_PREFERRED]  = "prefer",
3174 	[MPOL_BIND]       = "bind",
3175 	[MPOL_INTERLEAVE] = "interleave",
3176 	[MPOL_WEIGHTED_INTERLEAVE] = "weighted interleave",
3177 	[MPOL_LOCAL]      = "local",
3178 	[MPOL_PREFERRED_MANY]  = "prefer (many)",
3179 };
3180 
3181 #ifdef CONFIG_TMPFS
3182 /**
3183  * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
3184  * @str:  string containing mempolicy to parse
3185  * @mpol:  pointer to struct mempolicy pointer, returned on success.
3186  *
3187  * Format of input:
3188  *	<mode>[=<flags>][:<nodelist>]
3189  *
3190  * Return: %0 on success, else %1
3191  */
mpol_parse_str(char * str,struct mempolicy ** mpol)3192 int mpol_parse_str(char *str, struct mempolicy **mpol)
3193 {
3194 	struct mempolicy *new = NULL;
3195 	unsigned short mode_flags;
3196 	nodemask_t nodes;
3197 	char *nodelist = strchr(str, ':');
3198 	char *flags = strchr(str, '=');
3199 	int err = 1, mode;
3200 
3201 	if (flags)
3202 		*flags++ = '\0';	/* terminate mode string */
3203 
3204 	if (nodelist) {
3205 		/* NUL-terminate mode or flags string */
3206 		*nodelist++ = '\0';
3207 		if (nodelist_parse(nodelist, nodes))
3208 			goto out;
3209 		if (!nodes_subset(nodes, node_states[N_MEMORY]))
3210 			goto out;
3211 	} else
3212 		nodes_clear(nodes);
3213 
3214 	mode = match_string(policy_modes, MPOL_MAX, str);
3215 	if (mode < 0)
3216 		goto out;
3217 
3218 	switch (mode) {
3219 	case MPOL_PREFERRED:
3220 		/*
3221 		 * Insist on a nodelist of one node only, although later
3222 		 * we use first_node(nodes) to grab a single node, so here
3223 		 * nodelist (or nodes) cannot be empty.
3224 		 */
3225 		if (nodelist) {
3226 			char *rest = nodelist;
3227 			while (isdigit(*rest))
3228 				rest++;
3229 			if (*rest)
3230 				goto out;
3231 			if (nodes_empty(nodes))
3232 				goto out;
3233 		}
3234 		break;
3235 	case MPOL_INTERLEAVE:
3236 	case MPOL_WEIGHTED_INTERLEAVE:
3237 		/*
3238 		 * Default to online nodes with memory if no nodelist
3239 		 */
3240 		if (!nodelist)
3241 			nodes = node_states[N_MEMORY];
3242 		break;
3243 	case MPOL_LOCAL:
3244 		/*
3245 		 * Don't allow a nodelist;  mpol_new() checks flags
3246 		 */
3247 		if (nodelist)
3248 			goto out;
3249 		break;
3250 	case MPOL_DEFAULT:
3251 		/*
3252 		 * Insist on a empty nodelist
3253 		 */
3254 		if (!nodelist)
3255 			err = 0;
3256 		goto out;
3257 	case MPOL_PREFERRED_MANY:
3258 	case MPOL_BIND:
3259 		/*
3260 		 * Insist on a nodelist
3261 		 */
3262 		if (!nodelist)
3263 			goto out;
3264 	}
3265 
3266 	mode_flags = 0;
3267 	if (flags) {
3268 		/*
3269 		 * Currently, we only support two mutually exclusive
3270 		 * mode flags.
3271 		 */
3272 		if (!strcmp(flags, "static"))
3273 			mode_flags |= MPOL_F_STATIC_NODES;
3274 		else if (!strcmp(flags, "relative"))
3275 			mode_flags |= MPOL_F_RELATIVE_NODES;
3276 		else
3277 			goto out;
3278 	}
3279 
3280 	new = mpol_new(mode, mode_flags, &nodes);
3281 	if (IS_ERR(new))
3282 		goto out;
3283 
3284 	/*
3285 	 * Save nodes for mpol_to_str() to show the tmpfs mount options
3286 	 * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
3287 	 */
3288 	if (mode != MPOL_PREFERRED) {
3289 		new->nodes = nodes;
3290 	} else if (nodelist) {
3291 		nodes_clear(new->nodes);
3292 		node_set(first_node(nodes), new->nodes);
3293 	} else {
3294 		new->mode = MPOL_LOCAL;
3295 	}
3296 
3297 	/*
3298 	 * Save nodes for contextualization: this will be used to "clone"
3299 	 * the mempolicy in a specific context [cpuset] at a later time.
3300 	 */
3301 	new->w.user_nodemask = nodes;
3302 
3303 	err = 0;
3304 
3305 out:
3306 	/* Restore string for error message */
3307 	if (nodelist)
3308 		*--nodelist = ':';
3309 	if (flags)
3310 		*--flags = '=';
3311 	if (!err)
3312 		*mpol = new;
3313 	return err;
3314 }
3315 #endif /* CONFIG_TMPFS */
3316 
3317 /**
3318  * mpol_to_str - format a mempolicy structure for printing
3319  * @buffer:  to contain formatted mempolicy string
3320  * @maxlen:  length of @buffer
3321  * @pol:  pointer to mempolicy to be formatted
3322  *
3323  * Convert @pol into a string.  If @buffer is too short, truncate the string.
3324  * Recommend a @maxlen of at least 51 for the longest mode, "weighted
3325  * interleave", plus the longest flag flags, "relative|balancing", and to
3326  * display at least a few node ids.
3327  */
mpol_to_str(char * buffer,int maxlen,struct mempolicy * pol)3328 void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
3329 {
3330 	char *p = buffer;
3331 	nodemask_t nodes = NODE_MASK_NONE;
3332 	unsigned short mode = MPOL_DEFAULT;
3333 	unsigned short flags = 0;
3334 
3335 	if (pol &&
3336 	    pol != &default_policy &&
3337 	    !(pol >= &preferred_node_policy[0] &&
3338 	      pol <= &preferred_node_policy[ARRAY_SIZE(preferred_node_policy) - 1])) {
3339 		mode = pol->mode;
3340 		flags = pol->flags;
3341 	}
3342 
3343 	switch (mode) {
3344 	case MPOL_DEFAULT:
3345 	case MPOL_LOCAL:
3346 		break;
3347 	case MPOL_PREFERRED:
3348 	case MPOL_PREFERRED_MANY:
3349 	case MPOL_BIND:
3350 	case MPOL_INTERLEAVE:
3351 	case MPOL_WEIGHTED_INTERLEAVE:
3352 		nodes = pol->nodes;
3353 		break;
3354 	default:
3355 		WARN_ON_ONCE(1);
3356 		snprintf(p, maxlen, "unknown");
3357 		return;
3358 	}
3359 
3360 	p += snprintf(p, maxlen, "%s", policy_modes[mode]);
3361 
3362 	if (flags & MPOL_MODE_FLAGS) {
3363 		p += snprintf(p, buffer + maxlen - p, "=");
3364 
3365 		/*
3366 		 * Static and relative are mutually exclusive.
3367 		 */
3368 		if (flags & MPOL_F_STATIC_NODES)
3369 			p += snprintf(p, buffer + maxlen - p, "static");
3370 		else if (flags & MPOL_F_RELATIVE_NODES)
3371 			p += snprintf(p, buffer + maxlen - p, "relative");
3372 
3373 		if (flags & MPOL_F_NUMA_BALANCING) {
3374 			if (!is_power_of_2(flags & MPOL_MODE_FLAGS))
3375 				p += snprintf(p, buffer + maxlen - p, "|");
3376 			p += snprintf(p, buffer + maxlen - p, "balancing");
3377 		}
3378 	}
3379 
3380 	if (!nodes_empty(nodes))
3381 		p += scnprintf(p, buffer + maxlen - p, ":%*pbl",
3382 			       nodemask_pr_args(&nodes));
3383 }
3384 
3385 #ifdef CONFIG_SYSFS
3386 struct iw_node_attr {
3387 	struct kobj_attribute kobj_attr;
3388 	int nid;
3389 };
3390 
node_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)3391 static ssize_t node_show(struct kobject *kobj, struct kobj_attribute *attr,
3392 			 char *buf)
3393 {
3394 	struct iw_node_attr *node_attr;
3395 	u8 weight;
3396 
3397 	node_attr = container_of(attr, struct iw_node_attr, kobj_attr);
3398 	weight = get_il_weight(node_attr->nid);
3399 	return sysfs_emit(buf, "%d\n", weight);
3400 }
3401 
node_store(struct kobject * kobj,struct kobj_attribute * attr,const char * buf,size_t count)3402 static ssize_t node_store(struct kobject *kobj, struct kobj_attribute *attr,
3403 			  const char *buf, size_t count)
3404 {
3405 	struct iw_node_attr *node_attr;
3406 	u8 *new;
3407 	u8 *old;
3408 	u8 weight = 0;
3409 
3410 	node_attr = container_of(attr, struct iw_node_attr, kobj_attr);
3411 	if (count == 0 || sysfs_streq(buf, ""))
3412 		weight = 0;
3413 	else if (kstrtou8(buf, 0, &weight))
3414 		return -EINVAL;
3415 
3416 	new = kzalloc(nr_node_ids, GFP_KERNEL);
3417 	if (!new)
3418 		return -ENOMEM;
3419 
3420 	mutex_lock(&iw_table_lock);
3421 	old = rcu_dereference_protected(iw_table,
3422 					lockdep_is_held(&iw_table_lock));
3423 	if (old)
3424 		memcpy(new, old, nr_node_ids);
3425 	new[node_attr->nid] = weight;
3426 	rcu_assign_pointer(iw_table, new);
3427 	mutex_unlock(&iw_table_lock);
3428 	synchronize_rcu();
3429 	kfree(old);
3430 	return count;
3431 }
3432 
3433 static struct iw_node_attr **node_attrs;
3434 
sysfs_wi_node_release(struct iw_node_attr * node_attr,struct kobject * parent)3435 static void sysfs_wi_node_release(struct iw_node_attr *node_attr,
3436 				  struct kobject *parent)
3437 {
3438 	if (!node_attr)
3439 		return;
3440 	sysfs_remove_file(parent, &node_attr->kobj_attr.attr);
3441 	kfree(node_attr->kobj_attr.attr.name);
3442 	kfree(node_attr);
3443 }
3444 
sysfs_wi_release(struct kobject * wi_kobj)3445 static void sysfs_wi_release(struct kobject *wi_kobj)
3446 {
3447 	int i;
3448 
3449 	for (i = 0; i < nr_node_ids; i++)
3450 		sysfs_wi_node_release(node_attrs[i], wi_kobj);
3451 	kobject_put(wi_kobj);
3452 }
3453 
3454 static const struct kobj_type wi_ktype = {
3455 	.sysfs_ops = &kobj_sysfs_ops,
3456 	.release = sysfs_wi_release,
3457 };
3458 
add_weight_node(int nid,struct kobject * wi_kobj)3459 static int add_weight_node(int nid, struct kobject *wi_kobj)
3460 {
3461 	struct iw_node_attr *node_attr;
3462 	char *name;
3463 
3464 	node_attr = kzalloc(sizeof(*node_attr), GFP_KERNEL);
3465 	if (!node_attr)
3466 		return -ENOMEM;
3467 
3468 	name = kasprintf(GFP_KERNEL, "node%d", nid);
3469 	if (!name) {
3470 		kfree(node_attr);
3471 		return -ENOMEM;
3472 	}
3473 
3474 	sysfs_attr_init(&node_attr->kobj_attr.attr);
3475 	node_attr->kobj_attr.attr.name = name;
3476 	node_attr->kobj_attr.attr.mode = 0644;
3477 	node_attr->kobj_attr.show = node_show;
3478 	node_attr->kobj_attr.store = node_store;
3479 	node_attr->nid = nid;
3480 
3481 	if (sysfs_create_file(wi_kobj, &node_attr->kobj_attr.attr)) {
3482 		kfree(node_attr->kobj_attr.attr.name);
3483 		kfree(node_attr);
3484 		pr_err("failed to add attribute to weighted_interleave\n");
3485 		return -ENOMEM;
3486 	}
3487 
3488 	node_attrs[nid] = node_attr;
3489 	return 0;
3490 }
3491 
add_weighted_interleave_group(struct kobject * root_kobj)3492 static int add_weighted_interleave_group(struct kobject *root_kobj)
3493 {
3494 	struct kobject *wi_kobj;
3495 	int nid, err;
3496 
3497 	wi_kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
3498 	if (!wi_kobj)
3499 		return -ENOMEM;
3500 
3501 	err = kobject_init_and_add(wi_kobj, &wi_ktype, root_kobj,
3502 				   "weighted_interleave");
3503 	if (err) {
3504 		kfree(wi_kobj);
3505 		return err;
3506 	}
3507 
3508 	for_each_node_state(nid, N_POSSIBLE) {
3509 		err = add_weight_node(nid, wi_kobj);
3510 		if (err) {
3511 			pr_err("failed to add sysfs [node%d]\n", nid);
3512 			break;
3513 		}
3514 	}
3515 	if (err)
3516 		kobject_put(wi_kobj);
3517 	return 0;
3518 }
3519 
mempolicy_kobj_release(struct kobject * kobj)3520 static void mempolicy_kobj_release(struct kobject *kobj)
3521 {
3522 	u8 *old;
3523 
3524 	mutex_lock(&iw_table_lock);
3525 	old = rcu_dereference_protected(iw_table,
3526 					lockdep_is_held(&iw_table_lock));
3527 	rcu_assign_pointer(iw_table, NULL);
3528 	mutex_unlock(&iw_table_lock);
3529 	synchronize_rcu();
3530 	kfree(old);
3531 	kfree(node_attrs);
3532 	kfree(kobj);
3533 }
3534 
3535 static const struct kobj_type mempolicy_ktype = {
3536 	.release = mempolicy_kobj_release
3537 };
3538 
mempolicy_sysfs_init(void)3539 static int __init mempolicy_sysfs_init(void)
3540 {
3541 	int err;
3542 	static struct kobject *mempolicy_kobj;
3543 
3544 	mempolicy_kobj = kzalloc(sizeof(*mempolicy_kobj), GFP_KERNEL);
3545 	if (!mempolicy_kobj) {
3546 		err = -ENOMEM;
3547 		goto err_out;
3548 	}
3549 
3550 	node_attrs = kcalloc(nr_node_ids, sizeof(struct iw_node_attr *),
3551 			     GFP_KERNEL);
3552 	if (!node_attrs) {
3553 		err = -ENOMEM;
3554 		goto mempol_out;
3555 	}
3556 
3557 	err = kobject_init_and_add(mempolicy_kobj, &mempolicy_ktype, mm_kobj,
3558 				   "mempolicy");
3559 	if (err)
3560 		goto node_out;
3561 
3562 	err = add_weighted_interleave_group(mempolicy_kobj);
3563 	if (err) {
3564 		pr_err("mempolicy sysfs structure failed to initialize\n");
3565 		kobject_put(mempolicy_kobj);
3566 		return err;
3567 	}
3568 
3569 	return err;
3570 node_out:
3571 	kfree(node_attrs);
3572 mempol_out:
3573 	kfree(mempolicy_kobj);
3574 err_out:
3575 	pr_err("failed to add mempolicy kobject to the system\n");
3576 	return err;
3577 }
3578 
3579 late_initcall(mempolicy_sysfs_init);
3580 #endif /* CONFIG_SYSFS */
3581