xref: /linux/kernel/cgroup/cpuset.c (revision acec9637e03ca54bd3c3a0c1a6ad5c6dfa07c0df)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  kernel/cpuset.c
4  *
5  *  Processor and Memory placement constraints for sets of tasks.
6  *
7  *  Copyright (C) 2003 BULL SA.
8  *  Copyright (C) 2004-2007 Silicon Graphics, Inc.
9  *  Copyright (C) 2006 Google, Inc
10  *
11  *  Portions derived from Patrick Mochel's sysfs code.
12  *  sysfs is Copyright (c) 2001-3 Patrick Mochel
13  *
14  *  2003-10-10 Written by Simon Derr.
15  *  2003-10-22 Updates by Stephen Hemminger.
16  *  2004 May-July Rework by Paul Jackson.
17  *  2006 Rework by Paul Menage to use generic cgroups
18  *  2008 Rework of the scheduler domains and CPU hotplug handling
19  *       by Max Krasnyansky
20  */
21 #include "cpuset-internal.h"
22 
23 #include <linux/init.h>
24 #include <linux/interrupt.h>
25 #include <linux/kernel.h>
26 #include <linux/mempolicy.h>
27 #include <linux/mm.h>
28 #include <linux/memory.h>
29 #include <linux/rcupdate.h>
30 #include <linux/sched.h>
31 #include <linux/sched/deadline.h>
32 #include <linux/sched/mm.h>
33 #include <linux/sched/task.h>
34 #include <linux/security.h>
35 #include <linux/oom.h>
36 #include <linux/sched/isolation.h>
37 #include <linux/wait.h>
38 #include <linux/workqueue.h>
39 #include <linux/task_work.h>
40 #include <linux/llist.h>
41 
42 DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key);
43 DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key);
44 
45 /*
46  * There could be abnormal cpuset configurations for cpu or memory
47  * node binding, add this key to provide a quick low-cost judgment
48  * of the situation.
49  */
50 DEFINE_STATIC_KEY_FALSE(cpusets_insane_config_key);
51 
52 static const char * const perr_strings[] = {
53 	[PERR_INVCPUS]   = "Invalid cpu list in cpuset.cpus.exclusive",
54 	[PERR_INVPARENT] = "Parent is an invalid partition root",
55 	[PERR_NOTPART]   = "Parent is not a partition root",
56 	[PERR_NOTEXCL]   = "Cpu list in cpuset.cpus not exclusive",
57 	[PERR_NOCPUS]    = "Parent unable to distribute cpu downstream",
58 	[PERR_HOTPLUG]   = "No cpu available due to hotplug",
59 	[PERR_CPUSEMPTY] = "cpuset.cpus and cpuset.cpus.exclusive are empty",
60 	[PERR_HKEEPING]  = "partition config conflicts with housekeeping setup",
61 	[PERR_ACCESS]    = "Enable partition not permitted",
62 	[PERR_REMOTE]    = "Have remote partition underneath",
63 	[PERR_ISOPARENT] = "Root partition is not allowed with isolated parent",
64 };
65 
66 /*
67  * CPUSET Locking Convention
68  * -------------------------
69  *
70  * Below are the four global/local locks guarding cpuset structures in lock
71  * acquisition order:
72  *  - cpuset_top_mutex
73  *  - cpu_hotplug_lock (cpus_read_lock/cpus_write_lock)
74  *  - cpuset_mutex
75  *  - callback_lock (raw spinlock)
76  *
77  * As cpuset will now indirectly flush a number of different workqueues in
78  * housekeeping_update() to update housekeeping cpumasks when the set of
79  * isolated CPUs is going to be changed, it may be vulnerable to deadlock
80  * if we hold cpus_read_lock while calling into housekeeping_update().
81  *
82  * The first cpuset_top_mutex will be held except when calling into
83  * cpuset_handle_hotplug() from the CPU hotplug code where cpus_write_lock
84  * and cpuset_mutex will be held instead. The main purpose of this mutex
85  * is to prevent regular cpuset control file write actions from interfering
86  * with the call to housekeeping_update(), though CPU hotplug operation can
87  * still happen in parallel. This mutex also provides protection for some
88  * internal variables.
89  *
90  * A task must hold all the remaining three locks to modify externally visible
91  * or used fields of cpusets, though some of the internally used cpuset fields
92  * and internal variables can be modified without holding callback_lock. If only
93  * reliable read access of the externally used fields are needed, a task can
94  * hold either cpuset_mutex or callback_lock which are exposed to other
95  * external subsystems.
96  *
97  * If a task holds cpu_hotplug_lock and cpuset_mutex, it blocks others,
98  * ensuring that it is the only task able to also acquire callback_lock and
99  * be able to modify cpusets.  It can perform various checks on the cpuset
100  * structure first, knowing nothing will change. It can also allocate memory
101  * without holding callback_lock. While it is performing these checks, various
102  * callback routines can briefly acquire callback_lock to query cpusets.  Once
103  * it is ready to make the changes, it takes callback_lock, blocking everyone
104  * else.
105  *
106  * Calls to the kernel memory allocator cannot be made while holding
107  * callback_lock which is a spinlock, as the memory allocator may sleep or
108  * call back into cpuset code and acquire callback_lock.
109  *
110  * Now, the task_struct fields mems_allowed and mempolicy may be changed
111  * by other task, we use alloc_lock in the task_struct fields to protect
112  * them.
113  *
114  * The cpuset_common_seq_show() handlers only hold callback_lock across
115  * small pieces of code, such as when reading out possibly multi-word
116  * cpumasks and nodemasks.
117  */
118 
119 static DEFINE_MUTEX(cpuset_top_mutex);
120 static DEFINE_MUTEX(cpuset_mutex);
121 
122 /*
123  * File level internal variables below follow one of the following exclusion
124  * rules.
125  *
126  * RWCS: Read/write-able by holding either cpus_write_lock (and optionally
127  *	 cpuset_mutex) or both cpus_read_lock and cpuset_mutex.
128  *
129  * CSCB: Readable by holding either cpuset_mutex or callback_lock. Writable
130  *	 by holding both cpuset_mutex and callback_lock.
131  *
132  * T:	 Read/write-able by holding the cpuset_top_mutex.
133  */
134 
135 /*
136  * For local partitions, update to subpartitions_cpus & isolated_cpus is done
137  * in update_parent_effective_cpumask(). For remote partitions, it is done in
138  * the remote_partition_*() and remote_cpus_update() helpers.
139  */
140 /*
141  * Exclusive CPUs distributed out to local or remote sub-partitions of
142  * top_cpuset
143  */
144 static cpumask_var_t	subpartitions_cpus;	/* RWCS */
145 
146 /*
147  * Exclusive CPUs in isolated partitions (shown in cpuset.cpus.isolated)
148  */
149 static cpumask_var_t	isolated_cpus;		/* CSCB */
150 
151 /*
152  * Set if housekeeping cpumasks are to be updated.
153  */
154 static bool		update_housekeeping;	/* RWCS */
155 
156 /*
157  * Set if "cpuset_v2_mode" mount option is used
158  * Cached at bind time and not lock protected; accessed via {READ,WRITE}_ONCE
159  */
160 static bool		cpuset_v2_mode;
161 
162 /*
163  * Copy of isolated_cpus to be passed to housekeeping_update()
164  */
165 static cpumask_var_t	isolated_hk_cpus;	/* T */
166 
167 /*
168  * A flag to force sched domain rebuild at the end of an operation.
169  * It can be set in
170  *  - update_partition_sd_lb()
171  *  - update_cpumasks_hier()
172  *  - cpuset_hotplug_update_tasks()
173  *  - cpuset_handle_hotplug()
174  *
175  * Protected by cpuset_mutex (with cpus_read_lock held) or cpus_write_lock.
176  *
177  * Note that update_relax_domain_level() in cpuset-v1.c can still call
178  * rebuild_sched_domains_locked() directly without using this flag.
179  */
180 static bool force_sd_rebuild;			/* RWCS */
181 
182 /*
183  * Partition root states:
184  *
185  *   0 - member (not a partition root)
186  *   1 - partition root
187  *   2 - partition root without load balancing (isolated)
188  *  -1 - invalid partition root
189  *  -2 - invalid isolated partition root
190  *
191  *  There are 2 types of partitions - local or remote. Local partitions are
192  *  those whose parents are partition root themselves. Setting of
193  *  cpuset.cpus.exclusive are optional in setting up local partitions.
194  *  Remote partitions are those whose parents are not partition roots. Passing
195  *  down exclusive CPUs by setting cpuset.cpus.exclusive along its ancestor
196  *  nodes are mandatory in creating a remote partition.
197  *
198  *  For simplicity, a local partition can be created under a local or remote
199  *  partition but a remote partition cannot have any partition root in its
200  *  ancestor chain except the cgroup root.
201  *
202  *  A valid partition can be formed by setting exclusive_cpus or cpus_allowed
203  *  if exclusive_cpus is not set. In the case of partition with empty
204  *  exclusive_cpus, all the conflicting exclusive CPUs specified in the
205  *  following cpumasks of sibling cpusets will be removed from its
206  *  cpus_allowed in determining its effective_xcpus.
207  *  - effective_xcpus
208  *  - exclusive_cpus
209  *
210  *  The "cpuset.cpus.exclusive" control file should be used for setting up
211  *  partition if the users want to get as many CPUs as possible.
212  */
213 #define PRS_MEMBER		0
214 #define PRS_ROOT		1
215 #define PRS_ISOLATED		2
216 #define PRS_INVALID_ROOT	-1
217 #define PRS_INVALID_ISOLATED	-2
218 
219 /*
220  * Temporary cpumasks for working with partitions that are passed among
221  * functions to avoid memory allocation in inner functions.
222  */
223 struct tmpmasks {
224 	cpumask_var_t addmask, delmask;	/* For partition root */
225 	cpumask_var_t new_cpus;		/* For update_cpumasks_hier() */
226 };
227 
228 void inc_dl_tasks_cs(struct task_struct *p)
229 {
230 	struct cpuset *cs = task_cs(p);
231 
232 	atomic_inc(&cs->nr_deadline_tasks);
233 }
234 
235 void dec_dl_tasks_cs(struct task_struct *p)
236 {
237 	struct cpuset *cs = task_cs(p);
238 
239 	atomic_dec(&cs->nr_deadline_tasks);
240 }
241 
242 static inline bool is_partition_valid(const struct cpuset *cs)
243 {
244 	return cs->partition_root_state > 0;
245 }
246 
247 static inline bool is_partition_invalid(const struct cpuset *cs)
248 {
249 	return cs->partition_root_state < 0;
250 }
251 
252 static inline bool cs_is_member(const struct cpuset *cs)
253 {
254 	return cs->partition_root_state == PRS_MEMBER;
255 }
256 
257 /*
258  * Send notification event of whenever partition_root_state changes.
259  */
260 static inline void notify_partition_change(struct cpuset *cs, int old_prs)
261 {
262 	if (old_prs == cs->partition_root_state)
263 		return;
264 	cgroup_file_notify(&cs->partition_file);
265 
266 	/* Reset prs_err if not invalid */
267 	if (is_partition_valid(cs))
268 		WRITE_ONCE(cs->prs_err, PERR_NONE);
269 }
270 
271 /*
272  * The top_cpuset is always synchronized to cpu_active_mask and we should avoid
273  * using cpu_online_mask as much as possible. An active CPU is always an online
274  * CPU, but not vice versa. cpu_active_mask and cpu_online_mask can differ
275  * during hotplug operations. A CPU is marked active at the last stage of CPU
276  * bringup (CPUHP_AP_ACTIVE). It is also the stage where cpuset hotplug code
277  * will be called to update the sched domains so that the scheduler can move
278  * a normal task to a newly active CPU or remove tasks away from a newly
279  * inactivated CPU. The online bit is set much earlier in the CPU bringup
280  * process and cleared much later in CPU teardown.
281  *
282  * If cpu_online_mask is used while a hotunplug operation is happening in
283  * parallel, we may leave an offline CPU in cpu_allowed or some other masks.
284  */
285 struct cpuset top_cpuset = {
286 	.flags = BIT(CS_CPU_EXCLUSIVE) |
287 		 BIT(CS_MEM_EXCLUSIVE) | BIT(CS_SCHED_LOAD_BALANCE),
288 	.partition_root_state = PRS_ROOT,
289 	.dl_bw_cpu = -1,
290 };
291 
292 /**
293  * cpuset_lock - Acquire the global cpuset mutex
294  *
295  * This locks the global cpuset mutex to prevent modifications to cpuset
296  * hierarchy and configurations. This helper is not enough to make modification.
297  */
298 void cpuset_lock(void)
299 {
300 	mutex_lock(&cpuset_mutex);
301 }
302 
303 void cpuset_unlock(void)
304 {
305 	mutex_unlock(&cpuset_mutex);
306 }
307 
308 void lockdep_assert_cpuset_lock_held(void)
309 {
310 	lockdep_assert_held(&cpuset_mutex);
311 }
312 
313 /**
314  * cpuset_full_lock - Acquire full protection for cpuset modification
315  *
316  * Takes both CPU hotplug read lock (cpus_read_lock()) and cpuset mutex
317  * to safely modify cpuset data.
318  */
319 void cpuset_full_lock(void)
320 {
321 	mutex_lock(&cpuset_top_mutex);
322 	cpus_read_lock();
323 	mutex_lock(&cpuset_mutex);
324 }
325 
326 void cpuset_full_unlock(void)
327 {
328 	mutex_unlock(&cpuset_mutex);
329 	cpus_read_unlock();
330 	mutex_unlock(&cpuset_top_mutex);
331 }
332 
333 #ifdef CONFIG_LOCKDEP
334 bool lockdep_is_cpuset_held(void)
335 {
336 	return lockdep_is_held(&cpuset_mutex) ||
337 	       lockdep_is_held(&cpuset_top_mutex);
338 }
339 #endif
340 
341 static DEFINE_SPINLOCK(callback_lock);
342 
343 void cpuset_callback_lock_irq(void)
344 {
345 	spin_lock_irq(&callback_lock);
346 }
347 
348 void cpuset_callback_unlock_irq(void)
349 {
350 	spin_unlock_irq(&callback_lock);
351 }
352 
353 static struct workqueue_struct *cpuset_migrate_mm_wq;
354 
355 static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq);
356 
357 /*
358  * Cpuset task attach context
359  * Protected by cpuset_mutex
360  */
361 static struct {
362 	int in_progress;
363 	bool cpus_updated;
364 	bool mems_updated;
365 	bool task_work_queued;
366 	bool many_dest_cs;	/* Have many destination cpusets */
367 	struct cpuset *old_cs;	/* Source cpuset */
368 	nodemask_t nodemask_to;
369 } attach_ctx;
370 static LLIST_HEAD(src_cs_head);
371 static LLIST_HEAD(dst_cs_head);
372 
373 /*
374  * Wait if task attach is in progress until it is done and then acquire
375  * cpuset_mutex before returning.
376  */
377 static void wait_attach_done_lock(void)
378 	    __acquires(&cpuset_mutex)
379 {
380 	for (;;) {
381 		mutex_lock(&cpuset_mutex);
382 		if (!attach_ctx.in_progress)
383 			return;
384 
385 		mutex_unlock(&cpuset_mutex);
386 
387 		/* Wait until attach operation is done to prevent racing */
388 		wait_event(cpuset_attach_wq, attach_ctx.in_progress == 0);
389 	}
390 }
391 
392 static inline void check_insane_mems_config(nodemask_t *nodes)
393 {
394 	if (!cpusets_insane_config() &&
395 		movable_only_nodes(nodes)) {
396 		static_branch_enable_cpuslocked(&cpusets_insane_config_key);
397 		pr_info("Unsupported (movable nodes only) cpuset configuration detected (nmask=%*pbl)!\n"
398 			"Cpuset allocations might fail even with a lot of memory available.\n",
399 			nodemask_pr_args(nodes));
400 	}
401 }
402 
403 /*
404  * decrease attach_ctx.in_progress.
405  * wake_up cpuset_attach_wq if attach_ctx.in_progress==0.
406  */
407 static inline void dec_attach_in_progress_locked(void)
408 {
409 	lockdep_assert_cpuset_lock_held();
410 
411 	attach_ctx.in_progress--;
412 	if (!attach_ctx.in_progress)
413 		wake_up(&cpuset_attach_wq);
414 }
415 
416 static inline void dec_attach_in_progress(void)
417 {
418 	mutex_lock(&cpuset_mutex);
419 	dec_attach_in_progress_locked();
420 	mutex_unlock(&cpuset_mutex);
421 }
422 
423 static inline bool cpuset_v2(void)
424 {
425 	return !IS_ENABLED(CONFIG_CPUSETS_V1) ||
426 		cgroup_subsys_on_dfl(cpuset_cgrp_subsys);
427 }
428 
429 /*
430  * Cgroup v2 behavior is used on the "cpus" and "mems" control files when
431  * on default hierarchy or when the cpuset_v2_mode flag is set by mounting
432  * the v1 cpuset cgroup filesystem with the "cpuset_v2_mode" mount option.
433  * With v2 behavior, "cpus" and "mems" are always what the users have
434  * requested and won't be changed by hotplug events. Only the effective
435  * cpus or mems will be affected.
436  */
437 static inline bool is_in_v2_mode(void)
438 {
439 	return cpuset_v2() || READ_ONCE(cpuset_v2_mode);
440 }
441 
442 /**
443  * partition_is_populated - check if partition has tasks
444  * @cs: partition root to be checked
445  * @excluded_child: a child cpuset to be excluded in task checking
446  * Return: true if there are tasks, false otherwise
447  *
448  * @cs should be a valid partition root or going to become a partition root.
449  * @excluded_child should be non-NULL when this cpuset is going to become a
450  * partition itself.
451  *
452  * Note that a remote partition is not allowed underneath a valid local
453  * or remote partition. So if a non-partition root child is populated,
454  * the whole partition is considered populated.
455  */
456 static inline bool partition_is_populated(struct cpuset *cs,
457 					  struct cpuset *excluded_child)
458 {
459 	struct cpuset *cp;
460 	struct cgroup_subsys_state *pos_css;
461 
462 	/*
463 	 * We cannot call cs_is_populated(cs) directly, as
464 	 * nr_populated_domain_children may include populated
465 	 * csets from descendants that are partitions.
466 	 */
467 	if (cgroup_has_tasks(cs->css.cgroup))
468 		return true;
469 
470 	rcu_read_lock();
471 	cpuset_for_each_descendant_pre(cp, pos_css, cs) {
472 		if (cp == cs)
473 			continue;
474 
475 		if (cp == excluded_child || is_partition_valid(cp)) {
476 			pos_css = css_rightmost_descendant(pos_css);
477 			continue;
478 		}
479 
480 		if (cpuset_is_populated(cp)) {
481 			rcu_read_unlock();
482 			return true;
483 		}
484 	}
485 	rcu_read_unlock();
486 	return false;
487 }
488 
489 /*
490  * Return in pmask the portion of a task's cpusets's cpus_allowed that
491  * are online and are capable of running the task.  If none are found,
492  * walk up the cpuset hierarchy until we find one that does have some
493  * appropriate cpus.
494  *
495  * One way or another, we guarantee to return some non-empty subset
496  * of cpu_active_mask.
497  *
498  * Call with callback_lock or cpuset_mutex held.
499  */
500 static void guarantee_active_cpus(struct task_struct *tsk,
501 				  struct cpumask *pmask)
502 {
503 	const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
504 	struct cpuset *cs;
505 
506 	if (WARN_ON(!cpumask_and(pmask, possible_mask, cpu_active_mask)))
507 		cpumask_copy(pmask, cpu_active_mask);
508 
509 	rcu_read_lock();
510 	cs = task_cs(tsk);
511 
512 	while (!cpumask_intersects(cs->effective_cpus, pmask)) {
513 		cs = parent_cs(cs);
514 		if (unlikely(!cs)) {
515 			/*
516 			 * The top cpuset doesn't have any active cpu as a
517 			 * consequence of a race between its caller and the cpu
518 			 * hotplug operation where cpu_active_mask is updated
519 			 * asynchronously before cpuset_handle_hotplug() is
520 			 * being called to adjust the effective_cpus of the
521 			 * affected cpusets. But we know the top cpuset's
522 			 * effective_cpus is on its way to be identical to
523 			 * cpu_active_mask minus the exclusive CPUs dedicated
524 			 * to other valid cpuset partitions. Just pass back
525 			 * the filtered cpu_active_mask in this case.
526 			 */
527 			goto out_unlock;
528 		}
529 	}
530 	cpumask_and(pmask, pmask, cs->effective_cpus);
531 out_unlock:
532 	rcu_read_unlock();
533 }
534 
535 /*
536  * Return in *pmask the portion of a cpusets's mems_allowed that
537  * are online, with memory.  If none are online with memory, walk
538  * up the cpuset hierarchy until we find one that does have some
539  * online mems.  The top cpuset always has some mems online. With v2,
540  * effective_mems should always contain online memory nodes except
541  * during the transition period where a memory node hotunplug operation
542  * is in progress.
543  *
544  * One way or another, we guarantee to return some non-empty subset
545  * of node_states[N_MEMORY].
546  *
547  * Call with callback_lock or cpuset_mutex held.
548  */
549 static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
550 {
551 	while (!nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]))
552 		cs = parent_cs(cs);
553 }
554 
555 /**
556  * alloc_cpumasks - Allocate an array of cpumask variables
557  * @pmasks: Pointer to array of cpumask_var_t pointers
558  * @size: Number of cpumasks to allocate
559  * Return: 0 if successful, -ENOMEM otherwise.
560  *
561  * Allocates @size cpumasks and initializes them to empty. Returns 0 on
562  * success, -ENOMEM on allocation failure. On failure, any previously
563  * allocated cpumasks are freed.
564  */
565 static inline int alloc_cpumasks(cpumask_var_t *pmasks[], u32 size)
566 {
567 	int i;
568 
569 	for (i = 0; i < size; i++) {
570 		if (!zalloc_cpumask_var(pmasks[i], GFP_KERNEL)) {
571 			while (--i >= 0)
572 				free_cpumask_var(*pmasks[i]);
573 			return -ENOMEM;
574 		}
575 	}
576 	return 0;
577 }
578 
579 /**
580  * alloc_tmpmasks - Allocate temporary cpumasks for cpuset operations.
581  * @tmp: Pointer to tmpmasks structure to populate
582  * Return: 0 on success, -ENOMEM on allocation failure
583  */
584 static inline int alloc_tmpmasks(struct tmpmasks *tmp)
585 {
586 	/*
587 	 * Array of pointers to the three cpumask_var_t fields in tmpmasks.
588 	 * Note: Array size must match actual number of masks (3)
589 	 */
590 	cpumask_var_t *pmask[3] = {
591 		&tmp->new_cpus,
592 		&tmp->addmask,
593 		&tmp->delmask
594 	};
595 
596 	return alloc_cpumasks(pmask, ARRAY_SIZE(pmask));
597 }
598 
599 /**
600  * free_tmpmasks - free cpumasks in a tmpmasks structure
601  * @tmp: the tmpmasks structure pointer
602  */
603 static inline void free_tmpmasks(struct tmpmasks *tmp)
604 {
605 	if (!tmp)
606 		return;
607 
608 	free_cpumask_var(tmp->new_cpus);
609 	free_cpumask_var(tmp->addmask);
610 	free_cpumask_var(tmp->delmask);
611 }
612 
613 /**
614  * dup_or_alloc_cpuset - Duplicate or allocate a new cpuset
615  * @cs: Source cpuset to duplicate (NULL for a fresh allocation)
616  *
617  * Creates a new cpuset by either:
618  * 1. Duplicating an existing cpuset (if @cs is non-NULL), or
619  * 2. Allocating a fresh cpuset with zero-initialized masks (if @cs is NULL)
620  *
621  * Return: Pointer to newly allocated cpuset on success, NULL on failure
622  */
623 struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs)
624 {
625 	struct cpuset *trial;
626 
627 	/* Allocate base structure */
628 	trial = cs ? kmemdup(cs, sizeof(*cs), GFP_KERNEL) :
629 		     kzalloc_obj(*cs);
630 	if (!trial)
631 		return NULL;
632 
633 	trial->dl_bw_cpu = -1;
634 	init_llist_node(&trial->attach_node);
635 
636 	/* Setup cpumask pointer array */
637 	cpumask_var_t *pmask[4] = {
638 		&trial->cpus_allowed,
639 		&trial->effective_cpus,
640 		&trial->effective_xcpus,
641 		&trial->exclusive_cpus
642 	};
643 
644 	if (alloc_cpumasks(pmask, ARRAY_SIZE(pmask))) {
645 		kfree(trial);
646 		return NULL;
647 	}
648 
649 	/* Copy masks if duplicating */
650 	if (cs) {
651 		cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);
652 		cpumask_copy(trial->effective_cpus, cs->effective_cpus);
653 		cpumask_copy(trial->effective_xcpus, cs->effective_xcpus);
654 		cpumask_copy(trial->exclusive_cpus, cs->exclusive_cpus);
655 	}
656 
657 	return trial;
658 }
659 
660 /**
661  * free_cpuset - free the cpuset
662  * @cs: the cpuset to be freed
663  */
664 void free_cpuset(struct cpuset *cs)
665 {
666 	free_cpumask_var(cs->cpus_allowed);
667 	free_cpumask_var(cs->effective_cpus);
668 	free_cpumask_var(cs->effective_xcpus);
669 	free_cpumask_var(cs->exclusive_cpus);
670 	kfree(cs);
671 }
672 
673 /* Return user specified exclusive CPUs */
674 static inline struct cpumask *user_xcpus(struct cpuset *cs)
675 {
676 	return cpumask_empty(cs->exclusive_cpus) ? cs->cpus_allowed
677 						 : cs->exclusive_cpus;
678 }
679 
680 static inline bool xcpus_empty(struct cpuset *cs)
681 {
682 	return cpumask_empty(cs->cpus_allowed) &&
683 	       cpumask_empty(cs->exclusive_cpus);
684 }
685 
686 /*
687  * cpusets_are_exclusive() - check if two cpusets are exclusive
688  *
689  * Return true if exclusive, false if not
690  */
691 static inline bool cpusets_are_exclusive(struct cpuset *cs1, struct cpuset *cs2)
692 {
693 	struct cpumask *xcpus1 = user_xcpus(cs1);
694 	struct cpumask *xcpus2 = user_xcpus(cs2);
695 
696 	if (cpumask_intersects(xcpus1, xcpus2))
697 		return false;
698 	return true;
699 }
700 
701 /**
702  * cpus_excl_conflict - Check if two cpusets have exclusive CPU conflicts
703  * @trial:	the trial cpuset to be checked
704  * @sibling:	a sibling cpuset to be checked against
705  * @xcpus_changed: set if exclusive_cpus has been set
706  *
707  * Returns: true if CPU exclusivity conflict exists, false otherwise
708  *
709  * Conflict detection rules:
710  *  o cgroup v1
711  *    See cpuset1_cpus_excl_conflict()
712  *  o cgroup v2
713  *    - The exclusive_cpus values cannot overlap.
714  *    - New exclusive_cpus cannot be a superset of a sibling's cpus_allowed.
715  */
716 static inline bool cpus_excl_conflict(struct cpuset *trial, struct cpuset *sibling,
717 				      bool xcpus_changed)
718 {
719 	if (!cpuset_v2())
720 		return cpuset1_cpus_excl_conflict(trial, sibling);
721 
722 	/* The cpus_allowed of a sibling cpuset cannot be a subset of the new exclusive_cpus */
723 	if (xcpus_changed && !cpumask_empty(sibling->cpus_allowed) &&
724 	    cpumask_subset(sibling->cpus_allowed, trial->exclusive_cpus))
725 		return true;
726 
727 	/* Exclusive_cpus cannot intersect */
728 	return cpumask_intersects(trial->exclusive_cpus, sibling->exclusive_cpus);
729 }
730 
731 static inline bool mems_excl_conflict(struct cpuset *cs1, struct cpuset *cs2)
732 {
733 	if ((is_mem_exclusive(cs1) || is_mem_exclusive(cs2)))
734 		return nodes_intersects(cs1->mems_allowed, cs2->mems_allowed);
735 	return false;
736 }
737 
738 /*
739  * validate_change() - Used to validate that any proposed cpuset change
740  *		       follows the structural rules for cpusets.
741  *
742  * If we replaced the flag and mask values of the current cpuset
743  * (cur) with those values in the trial cpuset (trial), would
744  * our various subset and exclusive rules still be valid?  Presumes
745  * cpuset_mutex held.
746  *
747  * 'cur' is the address of an actual, in-use cpuset.  Operations
748  * such as list traversal that depend on the actual address of the
749  * cpuset in the list must use cur below, not trial.
750  *
751  * 'trial' is the address of bulk structure copy of cur, with
752  * perhaps one or more of the fields cpus_allowed, mems_allowed,
753  * or flags changed to new, trial values.
754  *
755  * Return 0 if valid, -errno if not.
756  */
757 
758 int validate_change(struct cpuset *cur, struct cpuset *trial)
759 {
760 	struct cgroup_subsys_state *css;
761 	struct cpuset *c, *par;
762 	bool xcpus_changed;
763 	int ret = 0;
764 
765 	rcu_read_lock();
766 
767 	if (!is_in_v2_mode())
768 		ret = cpuset1_validate_change(cur, trial);
769 	if (ret)
770 		goto out;
771 
772 	/* Remaining checks don't apply to root cpuset */
773 	if (cur == &top_cpuset)
774 		goto out;
775 
776 	par = parent_cs(cur);
777 
778 	/*
779 	 * We can't shrink if we won't have enough room for SCHED_DEADLINE
780 	 * tasks. This check is not done when scheduling is disabled as the
781 	 * users should know what they are doing.
782 	 *
783 	 * For v1, effective_cpus == cpus_allowed & user_xcpus() returns
784 	 * cpus_allowed.
785 	 *
786 	 * For v2, is_partition_valid(cur) & is_sched_load_balance() are true
787 	 * only for non-isolated partition root. At this point, the target
788 	 * effective_cpus isn't computed yet. user_xcpus() is the best
789 	 * approximation.
790 	 *
791 	 * TBD: May need to precompute the real effective_cpus here in case
792 	 * incorrect scheduling of SCHED_DEADLINE tasks in a partition
793 	 * becomes an issue.
794 	 */
795 	ret = -EBUSY;
796 	if ((is_partition_valid(cur) || (!cpuset_v2() && is_cpu_exclusive(cur))) &&
797 	    is_sched_load_balance(cur) &&
798 	    !cpuset_cpumask_can_shrink(cur->effective_cpus, user_xcpus(trial)))
799 		goto out;
800 
801 	/*
802 	 * If either I or some sibling (!= me) is exclusive, we can't
803 	 * overlap. exclusive_cpus cannot overlap with each other if set.
804 	 */
805 	ret = -EINVAL;
806 	xcpus_changed = !cpumask_equal(cur->exclusive_cpus, trial->exclusive_cpus);
807 	cpuset_for_each_child(c, css, par) {
808 		if (c == cur)
809 			continue;
810 		if (cpus_excl_conflict(trial, c, xcpus_changed))
811 			goto out;
812 		if (mems_excl_conflict(trial, c))
813 			goto out;
814 	}
815 
816 	ret = 0;
817 out:
818 	rcu_read_unlock();
819 	return ret;
820 }
821 
822 #ifdef CONFIG_SMP
823 
824 /*
825  * generate_sched_domains()
826  *
827  * This function builds a partial partition of the systems CPUs
828  * A 'partial partition' is a set of non-overlapping subsets whose
829  * union is a subset of that set.
830  * The output of this function needs to be passed to kernel/sched/core.c
831  * partition_sched_domains() routine, which will rebuild the scheduler's
832  * load balancing domains (sched domains) as specified by that partial
833  * partition.
834  *
835  * See "What is sched_load_balance" in Documentation/admin-guide/cgroup-v1/cpusets.rst
836  * for a background explanation of this.
837  *
838  * Does not return errors, on the theory that the callers of this
839  * routine would rather not worry about failures to rebuild sched
840  * domains when operating in the severe memory shortage situations
841  * that could cause allocation failures below.
842  *
843  * Must be called with cpuset_mutex held.
844  *
845  * The three key local variables below are:
846  *    cp - cpuset pointer, used (together with pos_css) to perform a
847  *	   top-down scan of all cpusets. For our purposes, rebuilding
848  *	   the schedulers sched domains, we can ignore !is_sched_load_
849  *	   balance cpusets.
850  *  csa  - (for CpuSet Array) Array of pointers to all the cpusets
851  *	   that need to be load balanced, for convenient iterative
852  *	   access by the subsequent code that finds the best partition,
853  *	   i.e the set of domains (subsets) of CPUs such that the
854  *	   cpus_allowed of every cpuset marked is_sched_load_balance
855  *	   is a subset of one of these domains, while there are as
856  *	   many such domains as possible, each as small as possible.
857  * doms  - Conversion of 'csa' to an array of cpumasks, for passing to
858  *	   the kernel/sched/core.c routine partition_sched_domains() in a
859  *	   convenient format, that can be easily compared to the prior
860  *	   value to determine what partition elements (sched domains)
861  *	   were changed (added or removed.)
862  */
863 static int generate_sched_domains(cpumask_var_t **domains,
864 			struct sched_domain_attr **attributes)
865 {
866 	struct cpuset *cp;	/* top-down scan of cpusets */
867 	struct cpuset **csa;	/* array of all cpuset ptrs */
868 	int i, j;		/* indices for partition finding loops */
869 	cpumask_var_t *doms;	/* resulting partition; i.e. sched domains */
870 	struct sched_domain_attr *dattr;  /* attributes for custom domains */
871 	int ndoms = 0;		/* number of sched domains in result */
872 	struct cgroup_subsys_state *pos_css;
873 
874 	if (!cpuset_v2())
875 		return cpuset1_generate_sched_domains(domains, attributes);
876 
877 	doms = NULL;
878 	dattr = NULL;
879 	csa = NULL;
880 
881 	/* Special case for the 99% of systems with one, full, sched domain */
882 	if (cpumask_empty(subpartitions_cpus)) {
883 		ndoms = 1;
884 		/* !csa will be checked and can be correctly handled */
885 		goto generate_doms;
886 	}
887 
888 	csa = kmalloc_objs(cp, nr_cpusets());
889 	if (!csa)
890 		goto done;
891 
892 	/* Find how many partitions and cache them to csa[] */
893 	rcu_read_lock();
894 	cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
895 		/*
896 		 * Only valid partition roots that are not isolated and with
897 		 * non-empty effective_cpus will be saved into csa[].
898 		 */
899 		if ((cp->partition_root_state == PRS_ROOT) &&
900 		    !cpumask_empty(cp->effective_cpus))
901 			csa[ndoms++] = cp;
902 
903 		/*
904 		 * Skip @cp's subtree if not a partition root and has no
905 		 * exclusive CPUs to be granted to child cpusets.
906 		 */
907 		if (!is_partition_valid(cp) && cpumask_empty(cp->exclusive_cpus))
908 			pos_css = css_rightmost_descendant(pos_css);
909 	}
910 	rcu_read_unlock();
911 
912 	for (i = 0; i < ndoms; i++) {
913 		for (j = i + 1; j < ndoms; j++) {
914 			if (cpusets_overlap(csa[i], csa[j]))
915 				/*
916 				 * Cgroup v2 shouldn't pass down overlapping
917 				 * partition root cpusets.
918 				 */
919 				WARN_ON_ONCE(1);
920 		}
921 	}
922 
923 generate_doms:
924 	doms = alloc_sched_domains(ndoms);
925 	if (!doms)
926 		goto done;
927 
928 	/*
929 	 * The rest of the code, including the scheduler, can deal with
930 	 * dattr==NULL case. No need to abort if alloc fails.
931 	 */
932 	dattr = kmalloc_objs(struct sched_domain_attr, ndoms);
933 
934 	/*
935 	 * Cgroup v2 doesn't support domain attributes, just set all of them
936 	 * to SD_ATTR_INIT. Also non-isolating partition root CPUs are a
937 	 * subset of HK_TYPE_DOMAIN_BOOT housekeeping CPUs.
938 	 */
939 	for (i = 0; i < ndoms; i++) {
940 		/*
941 		 * The top cpuset may contain some boot time isolated
942 		 * CPUs that need to be excluded from the sched domain.
943 		 */
944 		if (!csa || csa[i] == &top_cpuset)
945 			cpumask_and(doms[i], top_cpuset.effective_cpus,
946 				    housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT));
947 		else
948 			cpumask_copy(doms[i], csa[i]->effective_cpus);
949 		if (dattr)
950 			dattr[i] = SD_ATTR_INIT;
951 	}
952 
953 done:
954 	kfree(csa);
955 
956 	/*
957 	 * Fallback to the default domain if kmalloc() failed.
958 	 * See comments in partition_sched_domains().
959 	 */
960 	if (doms == NULL)
961 		ndoms = 1;
962 
963 	*domains    = doms;
964 	*attributes = dattr;
965 	return ndoms;
966 }
967 
968 static void dl_update_tasks_root_domain(struct cpuset *cs)
969 {
970 	struct css_task_iter it;
971 	struct task_struct *task;
972 
973 	if (atomic_read(&cs->nr_deadline_tasks) == 0)
974 		return;
975 
976 	css_task_iter_start(&cs->css, 0, &it);
977 
978 	while ((task = css_task_iter_next(&it)))
979 		dl_add_task_root_domain(task);
980 
981 	css_task_iter_end(&it);
982 }
983 
984 void dl_rebuild_rd_accounting(void)
985 {
986 	struct cpuset *cs = NULL;
987 	struct cgroup_subsys_state *pos_css;
988 	int cpu;
989 	u64 cookie = ++dl_cookie;
990 
991 	lockdep_assert_cpuset_lock_held();
992 	lockdep_assert_cpus_held();
993 	lockdep_assert_held(&sched_domains_mutex);
994 
995 	rcu_read_lock();
996 
997 	for_each_possible_cpu(cpu) {
998 		if (dl_bw_visited(cpu, cookie))
999 			continue;
1000 
1001 		dl_clear_root_domain_cpu(cpu);
1002 	}
1003 
1004 	cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
1005 
1006 		if (cpumask_empty(cs->effective_cpus)) {
1007 			pos_css = css_rightmost_descendant(pos_css);
1008 			continue;
1009 		}
1010 
1011 		css_get(&cs->css);
1012 
1013 		rcu_read_unlock();
1014 
1015 		dl_update_tasks_root_domain(cs);
1016 
1017 		rcu_read_lock();
1018 		css_put(&cs->css);
1019 	}
1020 	rcu_read_unlock();
1021 }
1022 
1023 /*
1024  * Rebuild scheduler domains.
1025  *
1026  * If the flag 'sched_load_balance' of any cpuset with non-empty
1027  * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
1028  * which has that flag enabled, or if any cpuset with a non-empty
1029  * 'cpus' is removed, then call this routine to rebuild the
1030  * scheduler's dynamic sched domains.
1031  *
1032  * Call with cpuset_mutex held.  Takes cpus_read_lock().
1033  */
1034 void rebuild_sched_domains_locked(void)
1035 {
1036 	struct sched_domain_attr *attr;
1037 	cpumask_var_t *doms;
1038 	int ndoms;
1039 	int i;
1040 
1041 	lockdep_assert_cpus_held();
1042 	lockdep_assert_cpuset_lock_held();
1043 	force_sd_rebuild = false;
1044 
1045 	/* Generate domain masks and attrs */
1046 	ndoms = generate_sched_domains(&doms, &attr);
1047 
1048 	/*
1049 	* cpuset_hotplug_workfn is invoked synchronously now, thus this
1050 	* function should not race with CPU hotplug. And the effective CPUs
1051 	* must not include any offline CPUs. Passing an offline CPU in the
1052 	* doms to partition_sched_domains() will trigger a kernel panic.
1053 	*
1054 	* We perform a final check here: if the doms contains any
1055 	* offline CPUs, a warning is emitted and we return directly to
1056 	* prevent the panic.
1057 	*/
1058 	for (i = 0; doms && i < ndoms; i++) {
1059 		if (WARN_ON_ONCE(!cpumask_subset(doms[i], cpu_active_mask))) {
1060 			free_sched_domains(doms, ndoms);
1061 			kfree(attr);
1062 			return;
1063 		}
1064 	}
1065 
1066 	/* Have scheduler rebuild the domains */
1067 	partition_sched_domains(ndoms, doms, attr);
1068 }
1069 #else /* !CONFIG_SMP */
1070 void rebuild_sched_domains_locked(void)
1071 {
1072 }
1073 #endif /* CONFIG_SMP */
1074 
1075 static void rebuild_sched_domains_cpuslocked(void)
1076 {
1077 	mutex_lock(&cpuset_mutex);
1078 	rebuild_sched_domains_locked();
1079 	mutex_unlock(&cpuset_mutex);
1080 }
1081 
1082 void rebuild_sched_domains(void)
1083 {
1084 	cpus_read_lock();
1085 	rebuild_sched_domains_cpuslocked();
1086 	cpus_read_unlock();
1087 }
1088 
1089 void cpuset_reset_sched_domains(void)
1090 {
1091 	mutex_lock(&cpuset_mutex);
1092 	partition_sched_domains(1, NULL, NULL);
1093 	mutex_unlock(&cpuset_mutex);
1094 }
1095 
1096 /**
1097  * cpuset_update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
1098  * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
1099  * @new_cpus: the temp variable for the new effective_cpus mask
1100  *
1101  * Iterate through each task of @cs updating its cpus_allowed to the
1102  * effective cpuset's.  As this function is called with cpuset_mutex held,
1103  * cpuset membership stays stable.
1104  *
1105  * For top_cpuset, task_cpu_possible_mask() is used instead of effective_cpus
1106  * to make sure all offline CPUs are also included as hotplug code won't
1107  * update cpumasks for tasks in top_cpuset.
1108  *
1109  * As task_cpu_possible_mask() can be task dependent in arm64, we have to
1110  * do cpu masking per task instead of doing it once for all.
1111  */
1112 void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus)
1113 {
1114 	struct css_task_iter it;
1115 	struct task_struct *task;
1116 	bool top_cs = cs == &top_cpuset;
1117 
1118 	css_task_iter_start(&cs->css, 0, &it);
1119 	while ((task = css_task_iter_next(&it))) {
1120 		const struct cpumask *possible_mask = task_cpu_possible_mask(task);
1121 
1122 		if (top_cs) {
1123 			/*
1124 			 * PF_KTHREAD tasks are handled by housekeeping.
1125 			 * PF_NO_SETAFFINITY tasks are ignored.
1126 			 */
1127 			if (task->flags & (PF_KTHREAD | PF_NO_SETAFFINITY))
1128 				continue;
1129 			cpumask_andnot(new_cpus, possible_mask, subpartitions_cpus);
1130 		} else {
1131 			cpumask_and(new_cpus, possible_mask, cs->effective_cpus);
1132 		}
1133 		set_cpus_allowed_ptr(task, new_cpus);
1134 	}
1135 	css_task_iter_end(&it);
1136 }
1137 
1138 /*
1139  * Commands for update_parent_effective_cpumask
1140  */
1141 enum partition_cmd {
1142 	partcmd_enable,		/* Enable partition root	  */
1143 	partcmd_enablei,	/* Enable isolated partition root */
1144 	partcmd_disable,	/* Disable partition root	  */
1145 	partcmd_update,		/* Update parent's effective_cpus */
1146 	partcmd_invalidate,	/* Make partition invalid	  */
1147 };
1148 
1149 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
1150 				    struct tmpmasks *tmp);
1151 
1152 /*
1153  * Update partition load balance flag and/or rebuild sched domain
1154  *
1155  * Changing load balance flag will automatically call
1156  * rebuild_sched_domains_locked().
1157  * This function is for cgroup v2 only.
1158  */
1159 static void update_partition_sd_lb(struct cpuset *cs, int old_prs)
1160 {
1161 	int new_prs = cs->partition_root_state;
1162 	bool rebuild_domains = (new_prs > 0) || (old_prs > 0);
1163 	bool new_lb;
1164 
1165 	/*
1166 	 * If cs is not a valid partition root, the load balance state
1167 	 * will follow its parent.
1168 	 */
1169 	if (new_prs > 0) {
1170 		new_lb = (new_prs != PRS_ISOLATED);
1171 	} else {
1172 		new_lb = is_sched_load_balance(parent_cs(cs));
1173 	}
1174 	if (new_lb != !!is_sched_load_balance(cs)) {
1175 		rebuild_domains = true;
1176 		assign_bit(CS_SCHED_LOAD_BALANCE, &cs->flags, new_lb);
1177 	}
1178 
1179 	if (rebuild_domains)
1180 		cpuset_force_rebuild();
1181 }
1182 
1183 /*
1184  * tasks_nocpu_error - Return true if tasks will have no effective_cpus
1185  */
1186 static bool tasks_nocpu_error(struct cpuset *parent, struct cpuset *cs,
1187 			      struct cpumask *xcpus)
1188 {
1189 	/*
1190 	 * A populated partition (cs or parent) can't have empty effective_cpus
1191 	 */
1192 	return (cpumask_subset(parent->effective_cpus, xcpus) &&
1193 		partition_is_populated(parent, cs)) ||
1194 	       (!cpumask_intersects(xcpus, cpu_active_mask) &&
1195 		partition_is_populated(cs, NULL));
1196 }
1197 
1198 static void reset_partition_data(struct cpuset *cs)
1199 {
1200 	struct cpuset *parent = parent_cs(cs);
1201 
1202 	if (!cpuset_v2())
1203 		return;
1204 
1205 	lockdep_assert_held(&callback_lock);
1206 
1207 	if (cpumask_empty(cs->exclusive_cpus))
1208 		cpumask_clear(cs->effective_xcpus);
1209 
1210 	if (!cpumask_and(cs->effective_cpus, parent->effective_cpus, cs->cpus_allowed))
1211 		cpumask_copy(cs->effective_cpus, parent->effective_cpus);
1212 }
1213 
1214 static inline bool is_remote_partition(struct cpuset *cs)
1215 {
1216 	return cs->remote_partition;
1217 }
1218 
1219 static inline bool is_local_partition(struct cpuset *cs)
1220 {
1221 	return is_partition_valid(cs) && !is_remote_partition(cs);
1222 }
1223 
1224 /* Return true if isolated_cpus changes. */
1225 static bool isolated_cpu_update(int new_prs, int cpu)
1226 {
1227 	lockdep_assert_held(&callback_lock);
1228 	lockdep_assert_held(&cpuset_mutex);
1229 
1230 	if (new_prs == PRS_ISOLATED) {
1231 		if (cpumask_test_cpu(cpu, isolated_cpus))
1232 			return false;
1233 		cpumask_set_cpu(cpu, isolated_cpus);
1234 		return true;
1235 	}
1236 
1237 	/* CPUs isolated at boot must remain isolated. */
1238 	if (!cpumask_test_cpu(cpu,
1239 			      housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)) ||
1240 	    !cpumask_test_cpu(cpu, isolated_cpus))
1241 		return false;
1242 	cpumask_clear_cpu(cpu, isolated_cpus);
1243 	return true;
1244 }
1245 
1246 /*
1247  * isolated_cpus_update - Set the isolation state of @xcpus
1248  * @new_prs: partition root state that @xcpus will belong to
1249  * @xcpus: exclusive CPUs whose isolation state may change
1250  *
1251  * CPUs isolated at boot time are kept isolated.
1252  */
1253 static void isolated_cpus_update(int new_prs, struct cpumask *xcpus)
1254 {
1255 	bool updated = false;
1256 	int cpu;
1257 
1258 	lockdep_assert_held(&callback_lock);
1259 	lockdep_assert_held(&cpuset_mutex);
1260 	for_each_cpu(cpu, xcpus)
1261 		updated |= isolated_cpu_update(new_prs, cpu);
1262 	if (updated)
1263 		update_housekeeping = true;
1264 }
1265 
1266 /*
1267  * partition_xcpus_add - Add new exclusive CPUs to partition
1268  * @new_prs: new partition_root_state
1269  * @parent: parent cpuset
1270  * @xcpus: exclusive CPUs to be added
1271  *
1272  * Remote partition if parent == NULL
1273  */
1274 static void partition_xcpus_add(int new_prs, struct cpuset *parent,
1275 				struct cpumask *xcpus)
1276 {
1277 	WARN_ON_ONCE(new_prs < 0);
1278 	lockdep_assert_held(&callback_lock);
1279 
1280 	if (cpumask_empty(xcpus))
1281 		return;
1282 
1283 	if (!parent)
1284 		parent = &top_cpuset;
1285 
1286 	if (parent == &top_cpuset)
1287 		cpumask_or(subpartitions_cpus, subpartitions_cpus, xcpus);
1288 
1289 	isolated_cpus_update(new_prs, xcpus);
1290 
1291 	cpumask_andnot(parent->effective_cpus, parent->effective_cpus, xcpus);
1292 }
1293 
1294 /*
1295  * partition_xcpus_del - Remove exclusive CPUs from partition
1296  * @old_prs: old partition_root_state
1297  * @parent: parent cpuset
1298  * @xcpus: exclusive CPUs to be removed
1299  *
1300  * Remote partition if parent == NULL
1301  */
1302 static void partition_xcpus_del(int old_prs, struct cpuset *parent,
1303 				struct cpumask *xcpus)
1304 {
1305 	WARN_ON_ONCE(old_prs < 0);
1306 	lockdep_assert_held(&callback_lock);
1307 
1308 	if (cpumask_empty(xcpus))
1309 		return;
1310 
1311 	if (!parent)
1312 		parent = &top_cpuset;
1313 
1314 	if (parent == &top_cpuset)
1315 		cpumask_andnot(subpartitions_cpus, subpartitions_cpus, xcpus);
1316 
1317 	/* CPUs given back, including those of descendants, take parent's state */
1318 	isolated_cpus_update(parent->partition_root_state, xcpus);
1319 
1320 	cpumask_or(parent->effective_cpus, parent->effective_cpus, xcpus);
1321 	cpumask_and(parent->effective_cpus, parent->effective_cpus, cpu_active_mask);
1322 }
1323 
1324 /*
1325  * prstate_housekeeping_conflict - check for partition & housekeeping conflicts
1326  * @new_prs: new partition root state to be checked
1327  * @parent_prs: Parent partition root state
1328  * @new_cpus: the set of new effective xcpus in the current cpuset
1329  * @del_cpus: old CPUs going to be removed from current cpuset, can be NULL
1330  * Return: true if there is conflict, false otherwise
1331  *
1332  * There are two different housekeeping conflicts to be checked:
1333  * 1) If new_prs is PRS_ROOT, none of the @new_cpus can be a boot-time isolated
1334  *    CPU. IOW, the whole @new_cpus must be a subset of HK_TYPE_DOMAIN_BOOT.
1335  * 2) If nohz_full is enabled and we have isolated CPUs, their combination must
1336  *    still leave housekeeping CPUs. This check is only needed if new_prs
1337  *    differs from parent_prs and one of them is PRS_ISOLATED.
1338  */
1339 static bool prstate_housekeeping_conflict(int new_prs, int parent_prs,
1340 					  struct cpumask *new_cpus,
1341 					  struct cpumask *del_cpus)
1342 {
1343 	cpumask_var_t full_hk_cpus;
1344 	int res;
1345 
1346 	WARN_ON_ONCE(parent_prs <= 0);
1347 	if (!new_prs)
1348 		return false;
1349 
1350 	/*
1351 	 * For a currently invalid partition root state, do the check assuming
1352 	 * that it may be converted back to valid.
1353 	 *
1354 	 * There are code paths where an invalid partition may be reverted back
1355 	 * to a valid partition but we still need to run the housekeeping check
1356 	 * before the conversion happens to make sure the switch is valid. We
1357 	 * can either do the negative to positive PRS switch individually in all
1358 	 * these code paths or do it here which I think will make the code a bit
1359 	 * easier to read.
1360 	 */
1361 	if (new_prs < 0)
1362 		new_prs = -new_prs;
1363 
1364 	if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT) && (new_prs == PRS_ROOT) &&
1365 	    !cpumask_subset(new_cpus, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)))
1366 		return true;
1367 
1368 	if (del_cpus && cpumask_empty(del_cpus))
1369 		del_cpus = NULL;
1370 
1371 	/*
1372 	 * Exit 2nd check early and return false if
1373 	 * 1) there is no boot time isolated nohz_full CPU
1374 	 * 2) parent and child has the same partition state
1375 	 * 3) new_prs is PRS_ROOT, parent_prs is PRS_ISOLATED and no CPU will
1376 	 *    be returned back to the parent.
1377 	 */
1378 	if (!housekeeping_enabled(HK_TYPE_KERNEL_NOISE) ||
1379 	    (new_prs == parent_prs) || (!del_cpus && (new_prs == PRS_ROOT)))
1380 		return false;
1381 
1382 	/*
1383 	 * Now make sure that @new_cpus contains new CPUs to be isolated and
1384 	 * @del_cpus contains isolated CPUs to be un-isolated.
1385 	 * With the guards above, @new_cpus can't be NULL after the swap.
1386 	 */
1387 	if (parent_prs == PRS_ISOLATED)
1388 		swap(new_cpus, del_cpus);
1389 
1390 	if (!alloc_cpumask_var(&full_hk_cpus, GFP_KERNEL))
1391 		return true;
1392 
1393 	/*
1394 	 * Check if del_cpus may contain a CPU that can become an active
1395 	 * housekeeping CPU again.
1396 	 */
1397 	if (del_cpus) {
1398 		cpumask_and(full_hk_cpus, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE),
1399 			    housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT));
1400 		if (cpumask_first_and_and(del_cpus, full_hk_cpus,
1401 					  cpu_active_mask) < nr_cpu_ids) {
1402 			res = false;
1403 			goto out;
1404 		}
1405 	}
1406 
1407 	cpumask_and(full_hk_cpus, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE),
1408 		    housekeeping_cpumask(HK_TYPE_DOMAIN));
1409 	cpumask_andnot(full_hk_cpus, full_hk_cpus, isolated_cpus);
1410 	cpumask_and(full_hk_cpus, full_hk_cpus, cpu_active_mask);
1411 	res = !cpumask_weight_andnot(full_hk_cpus, new_cpus);
1412 out:
1413 	free_cpumask_var(full_hk_cpus);
1414 	return res;
1415 }
1416 
1417 /**
1418  * validate_partition - Validate a cpuset partition configuration
1419  * @cs: The cpuset to validate
1420  * @parent: Parent partition of @cs, NULL for remote partition
1421  * @new_prs: The proposed new partition root state
1422  * @new_excpus: The new effective exclusive CPUs mask to validate
1423  * @del: exclusive CPUs to be deleted from @cs, can be NULL
1424  *
1425  * A remote partition gets its CPUs from top_cpuset, so top_cpuset is used
1426  * as the parent when @parent is NULL.
1427  *
1428  * The housekeeping check is skipped if @parent is not a valid partition
1429  * root as @cs can't be a valid local partition in that case.
1430  *
1431  * Return: PRS error code (0 if valid, non-zero error code if invalid)
1432  */
1433 static enum prs_errcode validate_partition(struct cpuset *cs,
1434 					   struct cpuset *parent, int new_prs,
1435 					   struct cpumask *new_excpus,
1436 					   struct cpumask *del)
1437 {
1438 	int parent_prs;
1439 
1440 	if (new_prs == PRS_MEMBER)
1441 		return PERR_NONE;
1442 
1443 	if (cpumask_empty(new_excpus))
1444 		return PERR_INVCPUS;
1445 
1446 	if (!parent)
1447 		parent = &top_cpuset;
1448 	parent_prs = parent->partition_root_state;
1449 	if (parent_prs > 0 &&
1450 	    prstate_housekeeping_conflict(new_prs, parent_prs, new_excpus, del))
1451 		return PERR_HKEEPING;
1452 
1453 	if (tasks_nocpu_error(parent, cs, new_excpus))
1454 		return PERR_NOCPUS;
1455 
1456 	return PERR_NONE;
1457 }
1458 
1459 /*
1460  * cpuset_update_sd_hk_unlock - Rebuild sched domains, update HK & unlock
1461  *
1462  * Update housekeeping cpumasks and rebuild sched domains if necessary and
1463  * then do a cpuset_full_unlock().
1464  * This should be called at the end of cpuset operation.
1465  */
1466 static void cpuset_update_sd_hk_unlock(void)
1467 	__releases(&cpuset_mutex)
1468 	__releases(&cpuset_top_mutex)
1469 {
1470 	/* force_sd_rebuild will be cleared in rebuild_sched_domains_locked() */
1471 	if (force_sd_rebuild)
1472 		rebuild_sched_domains_locked();
1473 
1474 	if (update_housekeeping) {
1475 		update_housekeeping = false;
1476 		cpumask_copy(isolated_hk_cpus, isolated_cpus);
1477 
1478 		/*
1479 		 * housekeeping_update() is now called without holding
1480 		 * cpus_read_lock and cpuset_mutex. Only cpuset_top_mutex
1481 		 * is still being held for mutual exclusion.
1482 		 */
1483 		mutex_unlock(&cpuset_mutex);
1484 		cpus_read_unlock();
1485 		WARN_ON_ONCE(housekeeping_update(isolated_hk_cpus));
1486 		mutex_unlock(&cpuset_top_mutex);
1487 	} else {
1488 		cpuset_full_unlock();
1489 	}
1490 }
1491 
1492 /*
1493  * Work function to invoke cpuset_update_sd_hk_unlock()
1494  */
1495 static void hk_sd_workfn(struct work_struct *work)
1496 {
1497 	cpuset_full_lock();
1498 	cpuset_update_sd_hk_unlock();
1499 }
1500 
1501 /**
1502  * rm_siblings_excl_cpus - Remove exclusive CPUs that are used by sibling cpusets
1503  * @parent: Parent cpuset containing all siblings
1504  * @cs: Current cpuset (will be skipped)
1505  * @excpus:  exclusive effective CPU mask to modify
1506  *
1507  * This function ensures the given @excpus mask doesn't include any CPUs that
1508  * are exclusively allocated to sibling cpusets. It walks through all siblings
1509  * of @cs under @parent and removes their exclusive CPUs from @excpus.
1510  */
1511 static int rm_siblings_excl_cpus(struct cpuset *parent, struct cpuset *cs,
1512 					struct cpumask *excpus)
1513 {
1514 	struct cgroup_subsys_state *css;
1515 	struct cpuset *sibling;
1516 	int retval = 0;
1517 
1518 	if (cpumask_empty(excpus))
1519 		return 0;
1520 
1521 	/*
1522 	 * Remove exclusive CPUs from siblings
1523 	 */
1524 	rcu_read_lock();
1525 	cpuset_for_each_child(sibling, css, parent) {
1526 		struct cpumask *sibling_xcpus;
1527 
1528 		if (sibling == cs)
1529 			continue;
1530 
1531 		/*
1532 		 * If exclusive_cpus is defined, effective_xcpus will always
1533 		 * be a subset. Otherwise, effective_xcpus will only be set
1534 		 * in a valid partition root.
1535 		 */
1536 		sibling_xcpus = cpumask_empty(sibling->exclusive_cpus)
1537 			      ? sibling->effective_xcpus
1538 			      : sibling->exclusive_cpus;
1539 
1540 		if (cpumask_intersects(excpus, sibling_xcpus)) {
1541 			cpumask_andnot(excpus, excpus, sibling_xcpus);
1542 			retval++;
1543 		}
1544 	}
1545 	rcu_read_unlock();
1546 
1547 	return retval;
1548 }
1549 
1550 /*
1551  * compute_excpus - compute effective exclusive CPUs
1552  * @cs: cpuset
1553  * @xcpus: effective exclusive CPUs value to be set
1554  * Return: 0 if there is no sibling conflict, > 0 otherwise
1555  *
1556  * If exclusive_cpus isn't explicitly set , we have to scan the sibling cpusets
1557  * and exclude their exclusive_cpus or effective_xcpus as well.
1558  */
1559 static int compute_excpus(struct cpuset *cs, struct cpumask *excpus)
1560 {
1561 	struct cpuset *parent = parent_cs(cs);
1562 
1563 	cpumask_and(excpus, user_xcpus(cs), parent->effective_xcpus);
1564 
1565 	if (!cpumask_empty(cs->exclusive_cpus))
1566 		return 0;
1567 
1568 	return rm_siblings_excl_cpus(parent, cs, excpus);
1569 }
1570 
1571 /*
1572  * compute_trialcs_excpus - Compute effective exclusive CPUs for a trial cpuset
1573  * @trialcs: The trial cpuset containing the proposed new configuration
1574  * @cs: The original cpuset that the trial configuration is based on
1575  * Return: 0 if successful with no sibling conflict, >0 if a conflict is found
1576  *
1577  * Computes the effective_xcpus for a trial configuration. @cs is provided to represent
1578  * the real cs.
1579  */
1580 static int compute_trialcs_excpus(struct cpuset *trialcs, struct cpuset *cs)
1581 {
1582 	struct cpuset *parent = parent_cs(trialcs);
1583 	struct cpumask *excpus = trialcs->effective_xcpus;
1584 
1585 	/* trialcs is member, cpuset.cpus has no impact to excpus */
1586 	if (cs_is_member(cs))
1587 		cpumask_and(excpus, trialcs->exclusive_cpus,
1588 				parent->effective_xcpus);
1589 	else
1590 		cpumask_and(excpus, user_xcpus(trialcs), parent->effective_xcpus);
1591 
1592 	return rm_siblings_excl_cpus(parent, cs, excpus);
1593 }
1594 
1595 /**
1596  * compute_partition_effective_cpumask - compute effective_cpus for partition
1597  * @cs: partition root cpuset
1598  * @new_ecpus: previously computed effective_cpus to be updated
1599  *
1600  * Compute the effective_cpus of a partition root by scanning effective_xcpus
1601  * of child partition roots and excluding their effective_xcpus.
1602  *
1603  * This has the side effect of invalidating valid child partition roots,
1604  * if necessary. Since it is called from either cpuset_hotplug_update_tasks()
1605  * or update_cpumasks_hier() where parent and children are modified
1606  * successively, we don't need to call update_parent_effective_cpumask()
1607  * and the child's effective_cpus will be updated in later iterations.
1608  */
1609 static void compute_partition_effective_cpumask(struct cpuset *cs,
1610 						struct cpumask *new_ecpus)
1611 {
1612 	struct cgroup_subsys_state *css;
1613 	struct cpuset *child;
1614 	bool populated = partition_is_populated(cs, NULL);
1615 	static cpumask_t xcpus;		/* Protected by cpuset_mutex */
1616 
1617 	/*
1618 	 * Check child partition roots to see if they should be invalidated
1619 	 * when all the effective_cpus will be used up and cs is populated.
1620 	 * It is better to sacrifice a child partition root than invalidating
1621 	 * the whole subtree.
1622 	 *
1623 	 * Return early if the effective_cpus is already empty.
1624 	 */
1625 	compute_excpus(cs, new_ecpus);
1626 	if (!cpumask_and(new_ecpus, new_ecpus, cpu_active_mask))
1627 		return;
1628 
1629 	rcu_read_lock();
1630 	cpuset_for_each_child(child, css, cs) {
1631 		if (!is_partition_valid(child))
1632 			continue;
1633 
1634 		/*
1635 		 * There shouldn't be a remote partition underneath another
1636 		 * partition root.
1637 		 */
1638 		WARN_ON_ONCE(is_remote_partition(child));
1639 		if (populated && cpumask_subset(new_ecpus, child->effective_xcpus)) {
1640 			int old_prs = child->partition_root_state;
1641 			int parent_prs = cs->partition_root_state;
1642 
1643 			/*
1644 			 * Invalidate child partition. Its CPUs still owned by
1645 			 * cs take the isolation state of cs. effective_xcpus
1646 			 * of cs may not be updated yet, so recompute it.
1647 			 */
1648 			spin_lock_irq(&callback_lock);
1649 			WRITE_ONCE(child->prs_err, PERR_NOCPUS);
1650 			compute_excpus(cs, &xcpus);
1651 			cpumask_and(&xcpus, &xcpus, child->effective_xcpus);
1652 			isolated_cpus_update(parent_prs, &xcpus);
1653 			child->partition_root_state = -old_prs;
1654 			spin_unlock_irq(&callback_lock);
1655 			notify_partition_change(child, old_prs);
1656 			continue;
1657 		}
1658 		cpumask_andnot(new_ecpus, new_ecpus,
1659 			       child->effective_xcpus);
1660 	}
1661 	rcu_read_unlock();
1662 }
1663 
1664 /**
1665  * compute_effective_cpumask - Compute the effective cpumask of the cpuset
1666  * @new_cpus: the temp variable for the new effective_cpus mask
1667  * @cs: the cpuset the need to recompute the new effective_cpus mask
1668  * @parent: the parent cpuset
1669  *
1670  * For v2, the parent's effective_cpus is inherited if cpumask is empty.
1671  * The result is valid only if the given cpuset isn't a partition root.
1672  */
1673 static void compute_effective_cpumask(struct cpumask *new_cpus,
1674 				      struct cpuset *cs, struct cpuset *parent)
1675 {
1676 	bool has_cpus;
1677 
1678 	if (is_remote_partition(cs) ||
1679 	    (is_partition_valid(cs) && is_partition_valid(parent)))
1680 		return compute_partition_effective_cpumask(cs, new_cpus);
1681 
1682 	has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus);
1683 	if (!has_cpus && is_in_v2_mode())
1684 		cpumask_copy(new_cpus, parent->effective_cpus);
1685 }
1686 
1687 /**
1688  * compute_effective_nodemask - Compute the effective nodemask of the cpuset
1689  * @new_mems: the temp variable for the new effective_mems mask
1690  * @cs: the cpuset the need to recompute the new effective_mems mask
1691  * @parent: the parent cpuset
1692  *
1693  * For v2, the parent's effective_mems is inherited if nodemask is empty.
1694  */
1695 static void compute_effective_nodemask(nodemask_t *new_mems,
1696 				       struct cpuset *cs, struct cpuset *parent)
1697 {
1698 	bool has_mems;
1699 
1700 	has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems);
1701 	if (!has_mems && is_in_v2_mode())
1702 		nodes_copy(*new_mems, parent->effective_mems);
1703 }
1704 
1705 /*
1706  * remote_partition_enable - Enable current cpuset as a remote partition root
1707  * @cs: the cpuset to update
1708  * @new_prs: new partition_root_state
1709  * @tmp: temporary masks
1710  * Return: 0 if successful, errcode if error
1711  *
1712  * Enable the current cpuset to become a remote partition root taking CPUs
1713  * directly from the top cpuset. cpuset_mutex must be held by the caller.
1714  */
1715 static int remote_partition_enable(struct cpuset *cs, int new_prs,
1716 				   struct tmpmasks *tmp)
1717 {
1718 	/*
1719 	 * The user must have sysadmin privilege.
1720 	 */
1721 	if (!capable(CAP_SYS_ADMIN))
1722 		return PERR_ACCESS;
1723 
1724 	/*
1725 	 * The requested exclusive_cpus must not be allocated to other
1726 	 * partitions and it can't use up all the root's effective_cpus.
1727 	 *
1728 	 * The effective_xcpus mask can contain offline CPUs, but there must
1729 	 * be at least one or more online CPUs present before it can be enabled.
1730 	 *
1731 	 * Note that creating a remote partition with any local partition root
1732 	 * above it or remote partition root underneath it is not allowed.
1733 	 */
1734 	compute_excpus(cs, tmp->new_cpus);
1735 	if (!cpumask_intersects(tmp->new_cpus, cpu_active_mask) ||
1736 	    cpumask_subset(top_cpuset.effective_cpus, tmp->new_cpus))
1737 		return PERR_INVCPUS;
1738 	if (cpumask_intersects(tmp->new_cpus, subpartitions_cpus))
1739 		return PERR_NOCPUS;
1740 	if (prstate_housekeeping_conflict(new_prs, PRS_ROOT, tmp->new_cpus, NULL))
1741 		return PERR_HKEEPING;
1742 
1743 	spin_lock_irq(&callback_lock);
1744 	partition_xcpus_add(new_prs, NULL, tmp->new_cpus);
1745 	cs->remote_partition = true;
1746 	cpumask_copy(cs->effective_xcpus, tmp->new_cpus);
1747 	spin_unlock_irq(&callback_lock);
1748 	cpuset_force_rebuild();
1749 	WRITE_ONCE(cs->prs_err, 0);
1750 
1751 	/*
1752 	 * Propagate changes in top_cpuset's effective_cpus down the hierarchy.
1753 	 */
1754 	cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus);
1755 	update_sibling_cpumasks(&top_cpuset, NULL, tmp);
1756 	return 0;
1757 }
1758 
1759 /*
1760  * remote_partition_disable - Remove current cpuset from remote partition list
1761  * @cs: the cpuset to update
1762  * @tmp: temporary masks
1763  *
1764  * The effective_cpus is also updated.
1765  *
1766  * cpuset_mutex must be held by the caller.
1767  */
1768 static void remote_partition_disable(struct cpuset *cs, struct tmpmasks *tmp)
1769 {
1770 	WARN_ON_ONCE(!is_remote_partition(cs));
1771 	/*
1772 	 * When a CPU is offlined, top_cpuset may end up with no available CPUs,
1773 	 * which should clear subpartitions_cpus. We should not emit a warning for this
1774 	 * scenario: the hierarchy is updated from top to bottom, so subpartitions_cpus
1775 	 * may already be cleared when disabling the partition.
1776 	 */
1777 	WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus) &&
1778 		     !cpumask_empty(subpartitions_cpus));
1779 
1780 	spin_lock_irq(&callback_lock);
1781 	cs->remote_partition = false;
1782 	partition_xcpus_del(cs->partition_root_state, NULL, cs->effective_xcpus);
1783 	if (cs->prs_err)
1784 		cs->partition_root_state = -cs->partition_root_state;
1785 	else
1786 		cs->partition_root_state = PRS_MEMBER;
1787 
1788 	/* effective_xcpus may need to be changed */
1789 	compute_excpus(cs, cs->effective_xcpus);
1790 	reset_partition_data(cs);
1791 	spin_unlock_irq(&callback_lock);
1792 	cpuset_force_rebuild();
1793 
1794 	/*
1795 	 * Propagate changes in top_cpuset's effective_cpus down the hierarchy.
1796 	 */
1797 	cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus);
1798 	update_sibling_cpumasks(&top_cpuset, NULL, tmp);
1799 }
1800 
1801 /*
1802  * remote_cpus_update - cpus_exclusive change of remote partition
1803  * @cs: the cpuset to be updated
1804  * @xcpus: the new exclusive_cpus mask, if non-NULL
1805  * @excpus: the new effective_xcpus mask
1806  * @tmp: temporary masks
1807  *
1808  * top_cpuset and subpartitions_cpus will be updated or partition can be
1809  * invalidated.
1810  */
1811 static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus,
1812 			       struct cpumask *excpus, struct tmpmasks *tmp)
1813 {
1814 	bool adding, deleting;
1815 	int prs = cs->partition_root_state;
1816 
1817 	if (WARN_ON_ONCE(!is_remote_partition(cs)))
1818 		return;
1819 
1820 	WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus));
1821 
1822 	if (cpumask_empty(excpus)) {
1823 		WRITE_ONCE(cs->prs_err, PERR_CPUSEMPTY);
1824 		goto invalidate;
1825 	}
1826 
1827 	adding   = cpumask_andnot(tmp->addmask, excpus, cs->effective_xcpus);
1828 	deleting = cpumask_andnot(tmp->delmask, cs->effective_xcpus, excpus);
1829 
1830 	/*
1831 	 * Additions of remote CPUs is only allowed if those CPUs are
1832 	 * not allocated to other partitions and there are effective_cpus
1833 	 * left in the top cpuset.
1834 	 */
1835 	if (adding) {
1836 		if (!capable(CAP_SYS_ADMIN))
1837 			WRITE_ONCE(cs->prs_err, PERR_ACCESS);
1838 		else if (cpumask_intersects(tmp->addmask, subpartitions_cpus) ||
1839 			 cpumask_subset(top_cpuset.effective_cpus, tmp->addmask))
1840 			WRITE_ONCE(cs->prs_err, PERR_NOCPUS);
1841 		else if (prstate_housekeeping_conflict(prs, PRS_ROOT,
1842 						       excpus, tmp->delmask))
1843 			WRITE_ONCE(cs->prs_err, PERR_HKEEPING);
1844 		if (cs->prs_err)
1845 			goto invalidate;
1846 	}
1847 
1848 	cs->remote_partition_updating = true;
1849 	spin_lock_irq(&callback_lock);
1850 	if (adding)
1851 		partition_xcpus_add(prs, NULL, tmp->addmask);
1852 	if (deleting)
1853 		partition_xcpus_del(prs, NULL, tmp->delmask);
1854 	/*
1855 	 * Need to update effective_xcpus and exclusive_cpus now as
1856 	 * update_sibling_cpumasks() below may iterate back to the same cs.
1857 	 */
1858 	cpumask_copy(cs->effective_xcpus, excpus);
1859 	if (xcpus)
1860 		cpumask_copy(cs->exclusive_cpus, xcpus);
1861 	spin_unlock_irq(&callback_lock);
1862 	if (adding || deleting)
1863 		cpuset_force_rebuild();
1864 
1865 	/*
1866 	 * Propagate changes in top_cpuset's effective_cpus down the hierarchy.
1867 	 */
1868 	cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus);
1869 	update_sibling_cpumasks(&top_cpuset, NULL, tmp);
1870 	cs->remote_partition_updating = false;
1871 	return;
1872 
1873 invalidate:
1874 	remote_partition_disable(cs, tmp);
1875 }
1876 
1877 /**
1878  * update_parent_effective_cpumask - update effective_cpus mask of parent cpuset
1879  * @cs:      The cpuset that requests change in partition root state
1880  * @cmd:     Partition root state change command
1881  * @newmask: Optional new effective xcpus for partcmd_update
1882  * @tmp:     Temporary addmask and delmask
1883  * Return:   0 or a partition root state error code
1884  *
1885  * For partcmd_enable*, the cpuset is being transformed from a non-partition
1886  * root to a partition root. The effective_xcpus (cpus_allowed if
1887  * effective_xcpus not set) mask of the given cpuset will be taken away from
1888  * parent's effective_cpus. The function will return 0 if all the CPUs listed
1889  * in effective_xcpus can be granted or an error code will be returned.
1890  *
1891  * For partcmd_disable, the cpuset is being transformed from a partition
1892  * root back to a non-partition root. Any CPUs in effective_xcpus will be
1893  * given back to parent's effective_cpus. 0 will always be returned.
1894  *
1895  * For partcmd_update, if the optional newmask is specified, the cpu list is
1896  * to be changed from effective_xcpus to newmask. Otherwise, effective_xcpus
1897  * will be computed and a valid partition will be invalidated if it is the new
1898  * cpumask is empty. The cpuset should either be a valid or invalid partition
1899  * root. The partition root state may change from valid to invalid or vice
1900  * versa.
1901  *
1902  * For partcmd_invalidate, the current partition will be made invalid.
1903  *
1904  * The partcmd_enable* and partcmd_disable commands are used by
1905  * update_prstate(). An error code may be returned and the caller will check
1906  * for error.
1907  *
1908  * The partcmd_update command is used by update_cpumasks_hier() with newmask
1909  * NULL and update_cpumask() with newmask set. The partcmd_invalidate is used
1910  * by update_cpumask() with NULL newmask. In both cases, the callers won't
1911  * check for error and so partition_root_state and prs_err will be updated
1912  * directly.
1913  */
1914 static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
1915 					   struct cpumask *newmask,
1916 					   struct tmpmasks *tmp)
1917 {
1918 	struct cpuset *parent = parent_cs(cs);
1919 	int adding;	/* Adding cpus to parent's effective_cpus	*/
1920 	int deleting;	/* Deleting cpus from parent's effective_cpus	*/
1921 	int old_prs, new_prs;
1922 	int part_error = PERR_NONE;	/* Partition error? */
1923 	int parent_prs = parent->partition_root_state;
1924 	bool nocpu;
1925 
1926 	lockdep_assert_cpuset_lock_held();
1927 	WARN_ON_ONCE(is_remote_partition(cs));	/* For local partition only */
1928 
1929 	/*
1930 	 * new_prs will only be changed for the partcmd_update and
1931 	 * partcmd_invalidate commands.
1932 	 */
1933 	adding = deleting = false;
1934 	old_prs = new_prs = cs->partition_root_state;
1935 
1936 	if (cmd == partcmd_invalidate) {
1937 		if (is_partition_invalid(cs))
1938 			return 0;
1939 
1940 		/*
1941 		 * Make the current partition invalid.
1942 		 */
1943 		if (is_partition_valid(parent))
1944 			adding = cpumask_and(tmp->addmask,
1945 					     cs->effective_xcpus,
1946 					     parent->effective_xcpus);
1947 		if (old_prs > 0)
1948 			new_prs = -old_prs;
1949 
1950 		goto write_error;
1951 	}
1952 
1953 	/*
1954 	 * The parent must be a partition root.
1955 	 * The new cpumask, if present, or the current cpus_allowed must
1956 	 * not be empty.
1957 	 */
1958 	if (!is_partition_valid(parent)) {
1959 		return is_partition_invalid(parent)
1960 		       ? PERR_INVPARENT : PERR_NOTPART;
1961 	}
1962 	if (!newmask && xcpus_empty(cs))
1963 		return PERR_CPUSEMPTY;
1964 
1965 	/*
1966 	 * Except for partcmd_disable and non-NULL newmask, use delmask
1967 	 * to compute the current effective xcpus value and use it for
1968 	 * tasks_nocpu_error check.
1969 	 */
1970 	if ((cmd != partcmd_disable) && !newmask) {
1971 		compute_excpus(cs, tmp->delmask);
1972 		nocpu = tasks_nocpu_error(parent, cs, tmp->delmask);
1973 	}
1974 
1975 	if ((cmd == partcmd_enable) || (cmd == partcmd_enablei)) {
1976 		new_prs = (cmd == partcmd_enable) ? PRS_ROOT : PRS_ISOLATED;
1977 
1978 		part_error = validate_partition(cs, parent, new_prs,
1979 						tmp->delmask, NULL);
1980 		if (part_error)
1981 			return part_error;
1982 
1983 		/*
1984 		 * This function will only be called when all the preliminary
1985 		 * checks have passed. At this point, the following condition
1986 		 * should hold.
1987 		 *
1988 		 * (cs->effective_xcpus & cpu_active_mask) ⊆ parent->effective_cpus
1989 		 *
1990 		 * Warn if it is not the case.
1991 		 */
1992 		cpumask_and(tmp->new_cpus, tmp->delmask, cpu_active_mask);
1993 		WARN_ON_ONCE(!cpumask_subset(tmp->new_cpus, parent->effective_cpus));
1994 		deleting = true;
1995 	} else if (cmd == partcmd_disable) {
1996 		/*
1997 		 * May need to add cpus back to parent's effective_cpus
1998 		 * (and maybe removed from subpartitions_cpus/isolated_cpus)
1999 		 * for valid partition root.
2000 		 */
2001 		if (is_partition_valid(cs)) {
2002 			cpumask_copy(tmp->addmask, cs->effective_xcpus);
2003 			adding = true;
2004 		}
2005 		new_prs = PRS_MEMBER;
2006 	} else if (newmask) {
2007 		/*
2008 		 * Empty cpumask is not allowed
2009 		 */
2010 		if (cpumask_empty(newmask)) {
2011 			part_error = PERR_CPUSEMPTY;
2012 			goto write_error;
2013 		}
2014 		if ((parent_prs == PRS_ISOLATED) && (old_prs == -PRS_ROOT)) {
2015 			part_error = PERR_ISOPARENT;
2016 			goto write_error;
2017 		}
2018 
2019 		/* Check newmask whether cpus are available for parent/cs */
2020 		nocpu = tasks_nocpu_error(parent, cs, newmask);
2021 
2022 		/*
2023 		 * partcmd_update with newmask:
2024 		 *
2025 		 * Compute add/delete mask to/from effective_cpus
2026 		 *
2027 		 * For valid partition:
2028 		 *   addmask = effective_xcpus & ~newmask
2029 		 *			      & parent->effective_xcpus
2030 		 *   delmask = newmask & ~effective_xcpus
2031 		 *		       & parent->effective_xcpus
2032 		 *
2033 		 * For invalid partition:
2034 		 *   delmask = newmask & parent->effective_xcpus
2035 		 *   The partition may become valid soon.
2036 		 */
2037 		if (is_partition_invalid(cs)) {
2038 			adding = false;
2039 			deleting = cpumask_and(tmp->delmask,
2040 					newmask, parent->effective_xcpus);
2041 		} else {
2042 			cpumask_andnot(tmp->addmask, cs->effective_xcpus, newmask);
2043 			adding = cpumask_and(tmp->addmask, tmp->addmask,
2044 					     parent->effective_xcpus);
2045 
2046 			cpumask_andnot(tmp->delmask, newmask, cs->effective_xcpus);
2047 			deleting = cpumask_and(tmp->delmask, tmp->delmask,
2048 					       parent->effective_xcpus);
2049 		}
2050 
2051 		/*
2052 		 * The new CPUs to be removed from parent's effective CPUs
2053 		 * must be present.
2054 		 */
2055 		if (deleting) {
2056 			cpumask_and(tmp->new_cpus, tmp->delmask, cpu_active_mask);
2057 			WARN_ON_ONCE(!cpumask_subset(tmp->new_cpus, parent->effective_cpus));
2058 		}
2059 
2060 		/*
2061 		 * Make partition invalid if parent's effective_cpus could
2062 		 * become empty and there are tasks in the parent.
2063 		 */
2064 		if (nocpu && (!adding ||
2065 		    !cpumask_intersects(tmp->addmask, cpu_active_mask))) {
2066 			part_error = PERR_NOCPUS;
2067 			deleting = false;
2068 			adding = cpumask_and(tmp->addmask,
2069 					     cs->effective_xcpus, parent->effective_xcpus);
2070 		}
2071 	} else {
2072 		/*
2073 		 * partcmd_update w/o newmask
2074 		 *
2075 		 * delmask = newly computed effective xcpus value
2076 		 *
2077 		 * This can be called from:
2078 		 * 1) update_cpumasks_hier()
2079 		 * 2) cpuset_hotplug_update_tasks()
2080 		 *
2081 		 * Check to see if it can be transitioned from valid to
2082 		 * invalid partition or vice versa.
2083 		 *
2084 		 * A partition error happens when parent has tasks and all
2085 		 * its effective CPUs will have to be distributed out.
2086 		 */
2087 		nocpu |= cpumask_empty(tmp->delmask);
2088 		if (nocpu) {
2089 			part_error = PERR_NOCPUS;
2090 			if (is_partition_valid(cs))
2091 				adding = cpumask_and(tmp->addmask,
2092 						     cs->effective_xcpus,
2093 						     parent->effective_xcpus);
2094 		} else if ((parent_prs == PRS_ISOLATED) &&
2095 			   ((old_prs == PRS_ROOT) || (old_prs == -PRS_ROOT))) {
2096 			/* Parent partition state changed to PRS_ISOLATED */
2097 			part_error = PERR_ISOPARENT;
2098 			if (old_prs == PRS_ROOT) {
2099 				cpumask_copy(tmp->addmask, cs->effective_xcpus);
2100 				adding = true;
2101 			}
2102 		} else if (is_partition_valid(cs) &&
2103 			   !cpumask_equal(tmp->delmask, cs->effective_xcpus)) {
2104 			/*
2105 			 * If there is an expansion of parent's effective_xcpus,
2106 			 * we may take more xcpus from parent. The parent's
2107 			 * effective_cpus should still have active CPUs or nocpu
2108 			 * should have been set. We also need to do a
2109 			 * housekeeping check for an isolated child with root
2110 			 * parent to make sure that it won't exhaust all the
2111 			 * active housekeeping CPUs.
2112 			 */
2113 			cpumask_andnot(tmp->addmask, cs->effective_xcpus, tmp->delmask);
2114 			if ((parent_prs == PRS_ROOT) && (old_prs == PRS_ISOLATED) &&
2115 			    prstate_housekeeping_conflict(old_prs, parent_prs,
2116 							  tmp->delmask, tmp->addmask)) {
2117 				adding = cpumask_and(tmp->addmask, cs->effective_xcpus,
2118 						     tmp->delmask);
2119 				part_error = PERR_HKEEPING;
2120 			} else {
2121 				deleting = cpumask_andnot(tmp->delmask, tmp->delmask,
2122 							  cs->effective_xcpus);
2123 			}
2124 		} else if (is_partition_invalid(cs)) {
2125 			struct cgroup_subsys_state *css;
2126 			struct cpuset *child;
2127 			bool exclusive = true;
2128 
2129 			/*
2130 			 * Convert invalid partition to valid has to pass the
2131 			 * cpu exclusivity test when cpuset.cpus.exclusive is
2132 			 * empty as well as the housekeeping check.
2133 			 */
2134 			if (prstate_housekeeping_conflict(cs->partition_root_state,
2135 							  parent_prs, tmp->delmask, NULL)) {
2136 				part_error = PERR_HKEEPING;
2137 				goto write_error;
2138 			}
2139 			if (cpumask_empty(cs->exclusive_cpus)) {
2140 				rcu_read_lock();
2141 				cpuset_for_each_child(child, css, parent) {
2142 					if (child == cs)
2143 						continue;
2144 					if (!cpusets_are_exclusive(cs, child)) {
2145 						exclusive = false;
2146 						break;
2147 					}
2148 				}
2149 				rcu_read_unlock();
2150 			}
2151 			if (exclusive)
2152 				deleting = !cpumask_empty(tmp->delmask);
2153 			else
2154 				part_error = PERR_NOTEXCL;
2155 		}
2156 	}
2157 
2158 write_error:
2159 	if (part_error)
2160 		WRITE_ONCE(cs->prs_err, part_error);
2161 
2162 	if (cmd == partcmd_update) {
2163 		/*
2164 		 * Check for possible transition between valid and invalid
2165 		 * partition root.
2166 		 */
2167 		switch (cs->partition_root_state) {
2168 		case PRS_ROOT:
2169 		case PRS_ISOLATED:
2170 			if (part_error)
2171 				new_prs = -old_prs;
2172 			break;
2173 		case PRS_INVALID_ROOT:
2174 		case PRS_INVALID_ISOLATED:
2175 			if (!part_error)
2176 				new_prs = -old_prs;
2177 			break;
2178 		}
2179 	}
2180 
2181 	if (!adding && !deleting && (new_prs == old_prs))
2182 		return 0;
2183 
2184 	/*
2185 	 * Change the parent's effective_cpus & effective_xcpus (top cpuset
2186 	 * only).
2187 	 *
2188 	 * Newly added CPUs will be removed from effective_cpus and
2189 	 * newly deleted ones will be added back to effective_cpus.
2190 	 */
2191 	spin_lock_irq(&callback_lock);
2192 	if (old_prs != new_prs)
2193 		cs->partition_root_state = new_prs;
2194 
2195 	/*
2196 	 * Adding to parent's effective_cpus means deletion CPUs from cs
2197 	 * and vice versa.
2198 	 */
2199 	if (adding)
2200 		partition_xcpus_del(old_prs, parent, tmp->addmask);
2201 	if (deleting)
2202 		partition_xcpus_add(new_prs, parent, tmp->delmask);
2203 
2204 	spin_unlock_irq(&callback_lock);
2205 
2206 	if (adding || deleting) {
2207 		cpuset_update_tasks_cpumask(parent, tmp->addmask);
2208 		update_sibling_cpumasks(parent, cs, tmp);
2209 	}
2210 
2211 	/*
2212 	 * For partcmd_update without newmask, it is being called from
2213 	 * cpuset_handle_hotplug(). Update the load balance flag and
2214 	 * scheduling domain accordingly.
2215 	 */
2216 	if ((cmd == partcmd_update) && !newmask)
2217 		update_partition_sd_lb(cs, old_prs);
2218 
2219 	notify_partition_change(cs, old_prs);
2220 	return 0;
2221 }
2222 
2223 /*
2224  * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree
2225  * @cs:  the cpuset to consider
2226  * @tmp: temp variables for calculating effective_cpus & partition setup
2227  * @force: don't skip any descendant cpusets if set
2228  *
2229  * When configured cpumask is changed, the effective cpumasks of this cpuset
2230  * and all its descendants need to be updated.
2231  *
2232  * On legacy hierarchy, effective_cpus will be the same with cpu_allowed.
2233  *
2234  * Called with cpuset_mutex held
2235  */
2236 static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp,
2237 				 bool force)
2238 {
2239 	struct cpuset *cp;
2240 	struct cgroup_subsys_state *pos_css;
2241 	int old_prs, new_prs;
2242 
2243 	rcu_read_lock();
2244 	cpuset_for_each_descendant_pre(cp, pos_css, cs) {
2245 		struct cpuset *parent = parent_cs(cp);
2246 		bool remote = is_remote_partition(cp);
2247 		bool update_parent = false;
2248 
2249 		old_prs = new_prs = cp->partition_root_state;
2250 
2251 		/*
2252 		 * For a child remote partition with remote_partition_updating
2253 		 * set, we can skip the whole subtree as the this function will
2254 		 * be called eventually. We can also skip the subtree if the
2255 		 * effective_xcpus isn't going to be changed. Otherwise, we
2256 		 * will have to call remote_cpus_update().
2257 		 *
2258 		 * remote_cpus_update() will reuse tmp->new_cpus only after
2259 		 * its value is being processed.
2260 		 */
2261 		if (remote && (cp != cs)) {
2262 			compute_excpus(cp, tmp->new_cpus);
2263 			if (cp->remote_partition_updating ||
2264 			    cpumask_equal(cp->effective_xcpus, tmp->new_cpus)) {
2265 				pos_css = css_rightmost_descendant(pos_css);
2266 				continue;
2267 			}
2268 			rcu_read_unlock();
2269 			remote_cpus_update(cp, NULL, tmp->new_cpus, tmp);
2270 			rcu_read_lock();
2271 
2272 			/* Remote partition may be invalidated */
2273 			new_prs = cp->partition_root_state;
2274 			remote = (new_prs == old_prs);
2275 		}
2276 
2277 		compute_effective_cpumask(tmp->new_cpus, cp, parent);
2278 
2279 		if (remote)
2280 			goto get_css;	/* Ready to update cpuset data */
2281 
2282 		/*
2283 		 * A partition with no effective_cpus is allowed as long as
2284 		 * there is no task associated with it. Call
2285 		 * update_parent_effective_cpumask() to check it.
2286 		 */
2287 		if (is_partition_valid(cp) && cpumask_empty(tmp->new_cpus)) {
2288 			update_parent = true;
2289 			goto update_parent_effective;
2290 		}
2291 
2292 		/*
2293 		 * Skip the whole subtree if
2294 		 * 1) the cpumask remains the same,
2295 		 * 2) has no partition root state,
2296 		 * 3) force flag not set, and
2297 		 * 4) for v2 load balance state same as its parent.
2298 		 */
2299 		if (!cp->partition_root_state && !force &&
2300 		    cpumask_equal(tmp->new_cpus, cp->effective_cpus) &&
2301 		    (!cpuset_v2() ||
2302 		    (is_sched_load_balance(parent) == is_sched_load_balance(cp)))) {
2303 			pos_css = css_rightmost_descendant(pos_css);
2304 			continue;
2305 		}
2306 
2307 update_parent_effective:
2308 		/*
2309 		 * update_parent_effective_cpumask() should have been called
2310 		 * for cs already in update_cpumask(). We should also call
2311 		 * cpuset_update_tasks_cpumask() again for tasks in the parent
2312 		 * cpuset if the parent's effective_cpus changes.
2313 		 */
2314 		if ((cp != cs) && old_prs) {
2315 			int parent_prs = parent->partition_root_state;
2316 
2317 			switch (parent_prs) {
2318 			case PRS_ROOT:
2319 			case PRS_ISOLATED:
2320 				update_parent = true;
2321 				break;
2322 
2323 			default:
2324 				/*
2325 				 * When parent is not a valid partition root,
2326 				 * child partition roots become invalid too.
2327 				 * The isolation state of their CPUs has been
2328 				 * settled when the parent gave them back.
2329 				 */
2330 				if (is_partition_valid(cp))
2331 					new_prs = -old_prs;
2332 
2333 				WRITE_ONCE(cp->prs_err,
2334 					   is_partition_invalid(parent)
2335 					   ? PERR_INVPARENT : PERR_NOTPART);
2336 				break;
2337 			}
2338 		}
2339 get_css:
2340 		if (!css_tryget_online(&cp->css))
2341 			continue;
2342 		rcu_read_unlock();
2343 
2344 		if (update_parent) {
2345 			update_parent_effective_cpumask(cp, partcmd_update, NULL, tmp);
2346 			/*
2347 			 * Capture current partition_root_state as it may have
2348 			 * changed. Always recompute effective cpumask as tmp->new_cpus
2349 			 * may be clobbered when update_sibling_cpumasks() is called
2350 			 * within update_parent_effective_cpumask().
2351 			 */
2352 			new_prs = cp->partition_root_state;
2353 			compute_effective_cpumask(tmp->new_cpus, cp, parent);
2354 		}
2355 
2356 		spin_lock_irq(&callback_lock);
2357 		cpumask_copy(cp->effective_cpus, tmp->new_cpus);
2358 		cp->partition_root_state = new_prs;
2359 		/*
2360 		 * Need to compute effective_xcpus if either exclusive_cpus
2361 		 * is non-empty or it is a valid partition root.
2362 		 */
2363 		if ((new_prs > 0) || !cpumask_empty(cp->exclusive_cpus))
2364 			compute_excpus(cp, cp->effective_xcpus);
2365 		if (new_prs <= 0)
2366 			reset_partition_data(cp);
2367 		spin_unlock_irq(&callback_lock);
2368 
2369 		notify_partition_change(cp, old_prs);
2370 
2371 		WARN_ON(!is_in_v2_mode() &&
2372 			!cpumask_equal(cp->cpus_allowed, cp->effective_cpus));
2373 
2374 		cpuset_update_tasks_cpumask(cp, tmp->new_cpus);
2375 
2376 		/*
2377 		 * On default hierarchy, inherit the CS_SCHED_LOAD_BALANCE
2378 		 * from parent if current cpuset isn't a valid partition root
2379 		 * and their load balance states differ.
2380 		 */
2381 		if (cpuset_v2() && !is_partition_valid(cp) &&
2382 		    (is_sched_load_balance(parent) != is_sched_load_balance(cp))) {
2383 			assign_bit(CS_SCHED_LOAD_BALANCE, &cp->flags,
2384 				   is_sched_load_balance(parent));
2385 		}
2386 
2387 		/*
2388 		 * On legacy hierarchy, if the effective cpumask of any non-
2389 		 * empty cpuset is changed, we need to rebuild sched domains.
2390 		 * On default hierarchy, the cpuset needs to be a partition
2391 		 * root as well.
2392 		 */
2393 		if (!cpumask_empty(cp->cpus_allowed) &&
2394 		    is_sched_load_balance(cp) &&
2395 		   (!cpuset_v2() || is_partition_valid(cp)))
2396 			cpuset_force_rebuild();
2397 
2398 		rcu_read_lock();
2399 		css_put(&cp->css);
2400 	}
2401 	rcu_read_unlock();
2402 }
2403 
2404 /**
2405  * update_sibling_cpumasks - Update siblings cpumasks
2406  * @parent:  Parent cpuset
2407  * @cs:      Current cpuset
2408  * @tmp:     Temp variables
2409  */
2410 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
2411 				    struct tmpmasks *tmp)
2412 {
2413 	struct cpuset *sibling;
2414 	struct cgroup_subsys_state *pos_css;
2415 
2416 	lockdep_assert_cpuset_lock_held();
2417 
2418 	/*
2419 	 * Check all its siblings and call update_cpumasks_hier()
2420 	 * if their effective_cpus will need to be changed.
2421 	 *
2422 	 * It is possible a change in parent's effective_cpus
2423 	 * due to a change in a child partition's effective_xcpus will impact
2424 	 * its siblings even if they do not inherit parent's effective_cpus
2425 	 * directly. It should not impact valid partition.
2426 	 *
2427 	 * The update_cpumasks_hier() function may sleep. So we have to
2428 	 * release the RCU read lock before calling it.
2429 	 */
2430 	rcu_read_lock();
2431 	cpuset_for_each_child(sibling, pos_css, parent) {
2432 		if (sibling == cs || is_partition_valid(sibling))
2433 			continue;
2434 
2435 		compute_effective_cpumask(tmp->new_cpus, sibling,
2436 					  parent);
2437 		if (cpumask_equal(tmp->new_cpus, sibling->effective_cpus))
2438 			continue;
2439 
2440 		if (!css_tryget_online(&sibling->css))
2441 			continue;
2442 
2443 		rcu_read_unlock();
2444 		update_cpumasks_hier(sibling, tmp, false);
2445 		rcu_read_lock();
2446 		css_put(&sibling->css);
2447 	}
2448 	rcu_read_unlock();
2449 }
2450 
2451 static int parse_cpuset_cpulist(const char *buf, struct cpumask *out_mask)
2452 {
2453 	int retval;
2454 
2455 	retval = cpulist_parse(buf, out_mask);
2456 	if (retval < 0)
2457 		return retval;
2458 	if (!cpumask_subset(out_mask, top_cpuset.cpus_allowed))
2459 		return -EINVAL;
2460 
2461 	return 0;
2462 }
2463 
2464 /**
2465  * partition_cpus_change - Handle partition state changes due to CPU mask updates
2466  * @cs: The target cpuset being modified
2467  * @trialcs: The trial cpuset containing proposed configuration changes
2468  * @tmp: Temporary masks for intermediate calculations
2469  *
2470  * This function handles partition state transitions triggered by CPU mask changes.
2471  * CPU modifications may cause a partition to be disabled or require state updates.
2472  */
2473 static void partition_cpus_change(struct cpuset *cs, struct cpuset *trialcs,
2474 				  struct tmpmasks *tmp)
2475 {
2476 	enum prs_errcode prs_err;
2477 	struct cpuset *parent = is_remote_partition(cs) ? NULL : parent_cs(cs);
2478 
2479 	if (cs_is_member(cs))
2480 		return;
2481 
2482 	/*
2483 	 * Check for housekeeping conflict by passing the whole
2484 	 * effective_xcpus as well as those CPUs that are to be
2485 	 * removed from cs as delmask.
2486 	 */
2487 	if (is_partition_valid(cs))
2488 		cpumask_andnot(tmp->delmask, cs->effective_xcpus,
2489 			       trialcs->effective_xcpus);
2490 	else
2491 		cpumask_clear(tmp->delmask);
2492 
2493 	prs_err = validate_partition(cs, parent, trialcs->partition_root_state,
2494 				     trialcs->effective_xcpus, tmp->delmask);
2495 	if (prs_err) {
2496 		WRITE_ONCE(cs->prs_err, prs_err);
2497 		trialcs->prs_err = prs_err;
2498 	}
2499 
2500 	if (is_remote_partition(cs)) {
2501 		if (trialcs->prs_err)
2502 			remote_partition_disable(cs, tmp);
2503 		else
2504 			remote_cpus_update(cs, trialcs->exclusive_cpus,
2505 					   trialcs->effective_xcpus, tmp);
2506 	} else {
2507 		if (trialcs->prs_err)
2508 			update_parent_effective_cpumask(cs, partcmd_invalidate,
2509 							NULL, tmp);
2510 		else
2511 			update_parent_effective_cpumask(cs, partcmd_update,
2512 							trialcs->effective_xcpus, tmp);
2513 	}
2514 }
2515 
2516 /**
2517  * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
2518  * @cs: the cpuset to consider
2519  * @trialcs: trial cpuset
2520  * @buf: buffer of cpu numbers written to this cpuset
2521  */
2522 static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
2523 			  const char *buf)
2524 {
2525 	int retval;
2526 	struct tmpmasks tmp;
2527 	bool force = false;
2528 	int old_prs = cs->partition_root_state;
2529 
2530 	retval = parse_cpuset_cpulist(buf, trialcs->cpus_allowed);
2531 	if (retval < 0)
2532 		return retval;
2533 
2534 	/* Nothing to do if the cpus didn't change */
2535 	if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed))
2536 		return 0;
2537 
2538 	compute_trialcs_excpus(trialcs, cs);
2539 	trialcs->prs_err = PERR_NONE;
2540 
2541 	retval = validate_change(cs, trialcs);
2542 	if (retval < 0)
2543 		return retval;
2544 
2545 	if (alloc_tmpmasks(&tmp))
2546 		return -ENOMEM;
2547 
2548 	/*
2549 	 * Check all the descendants in update_cpumasks_hier() if
2550 	 * effective_xcpus is to be changed.
2551 	 */
2552 	force = !cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus);
2553 
2554 	partition_cpus_change(cs, trialcs, &tmp);
2555 
2556 	spin_lock_irq(&callback_lock);
2557 	cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
2558 	cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus);
2559 	if (old_prs && !is_partition_valid(cs)) {
2560 		reset_partition_data(cs);
2561 		/* effective_xcpus may be cleared, update the subtree too */
2562 		force = true;
2563 	}
2564 	spin_unlock_irq(&callback_lock);
2565 
2566 	/* effective_cpus/effective_xcpus will be updated here */
2567 	update_cpumasks_hier(cs, &tmp, force);
2568 
2569 	/* Update CS_SCHED_LOAD_BALANCE and/or sched_domains, if necessary */
2570 	if (cs->partition_root_state)
2571 		update_partition_sd_lb(cs, old_prs);
2572 
2573 	free_tmpmasks(&tmp);
2574 	return retval;
2575 }
2576 
2577 /**
2578  * update_exclusive_cpumask - update the exclusive_cpus mask of a cpuset
2579  * @cs: the cpuset to consider
2580  * @trialcs: trial cpuset
2581  * @buf: buffer of cpu numbers written to this cpuset
2582  *
2583  * The tasks' cpumask will be updated if cs is a valid partition root.
2584  */
2585 static int update_exclusive_cpumask(struct cpuset *cs, struct cpuset *trialcs,
2586 				    const char *buf)
2587 {
2588 	int retval;
2589 	struct tmpmasks tmp;
2590 	bool force = false;
2591 	int old_prs = cs->partition_root_state;
2592 
2593 	retval = parse_cpuset_cpulist(buf, trialcs->exclusive_cpus);
2594 	if (retval < 0)
2595 		return retval;
2596 
2597 	/* Nothing to do if the CPUs didn't change */
2598 	if (cpumask_equal(cs->exclusive_cpus, trialcs->exclusive_cpus))
2599 		return 0;
2600 
2601 	/*
2602 	 * Reject the change if there is exclusive CPUs conflict with
2603 	 * the siblings.
2604 	 */
2605 	if (compute_trialcs_excpus(trialcs, cs))
2606 		return -EINVAL;
2607 
2608 	/*
2609 	 * Check all the descendants in update_cpumasks_hier() if
2610 	 * effective_xcpus is to be changed.
2611 	 */
2612 	force = !cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus);
2613 
2614 	retval = validate_change(cs, trialcs);
2615 	if (retval)
2616 		return retval;
2617 
2618 	if (alloc_tmpmasks(&tmp))
2619 		return -ENOMEM;
2620 
2621 	trialcs->prs_err = PERR_NONE;
2622 	partition_cpus_change(cs, trialcs, &tmp);
2623 
2624 	spin_lock_irq(&callback_lock);
2625 	cpumask_copy(cs->exclusive_cpus, trialcs->exclusive_cpus);
2626 	cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus);
2627 	if (old_prs && !is_partition_valid(cs)) {
2628 		reset_partition_data(cs);
2629 		/* effective_xcpus may be cleared, update the subtree too */
2630 		force = true;
2631 	}
2632 	spin_unlock_irq(&callback_lock);
2633 
2634 	/*
2635 	 * Call update_cpumasks_hier() to update effective_cpus/effective_xcpus
2636 	 * of the subtree when it is a valid partition root or effective_xcpus
2637 	 * is updated.
2638 	 */
2639 	if (is_partition_valid(cs) || force)
2640 		update_cpumasks_hier(cs, &tmp, force);
2641 
2642 	/* Update CS_SCHED_LOAD_BALANCE and/or sched_domains, if necessary */
2643 	if (cs->partition_root_state)
2644 		update_partition_sd_lb(cs, old_prs);
2645 
2646 	free_tmpmasks(&tmp);
2647 	return 0;
2648 }
2649 
2650 /*
2651  * Migrate memory region from one set of nodes to another.  This is
2652  * performed asynchronously as it can be called from process migration path
2653  * holding locks involved in process management.  All mm migrations are
2654  * performed in the queued order and can be waited for by flushing
2655  * cpuset_migrate_mm_wq.
2656  */
2657 
2658 struct cpuset_migrate_mm_work {
2659 	struct work_struct	work;
2660 	struct mm_struct	*mm;
2661 	nodemask_t		from;
2662 	nodemask_t		to;
2663 };
2664 
2665 static void cpuset_migrate_mm_workfn(struct work_struct *work)
2666 {
2667 	struct cpuset_migrate_mm_work *mwork =
2668 		container_of(work, struct cpuset_migrate_mm_work, work);
2669 
2670 	/* on a wq worker, no need to worry about %current's mems_allowed */
2671 	do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL);
2672 	mmput(mwork->mm);
2673 	kfree(mwork);
2674 }
2675 
2676 static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
2677 							const nodemask_t *to)
2678 {
2679 	struct cpuset_migrate_mm_work *mwork;
2680 
2681 	if (nodes_equal(*from, *to)) {
2682 		mmput(mm);
2683 		return;
2684 	}
2685 
2686 	mwork = kzalloc_obj(*mwork);
2687 	if (mwork) {
2688 		mwork->mm = mm;
2689 		mwork->from = *from;
2690 		mwork->to = *to;
2691 		INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn);
2692 		queue_work(cpuset_migrate_mm_wq, &mwork->work);
2693 	} else {
2694 		mmput(mm);
2695 	}
2696 }
2697 
2698 static void flush_migrate_mm_task_workfn(struct callback_head *head)
2699 {
2700 	flush_workqueue(cpuset_migrate_mm_wq);
2701 	kfree(head);
2702 }
2703 
2704 static void schedule_flush_migrate_mm(void)
2705 {
2706 	struct callback_head *flush_cb;
2707 
2708 	flush_cb = kzalloc_obj(struct callback_head);
2709 	if (!flush_cb)
2710 		return;
2711 
2712 	init_task_work(flush_cb, flush_migrate_mm_task_workfn);
2713 
2714 	if (task_work_add(current, flush_cb, TWA_RESUME))
2715 		kfree(flush_cb);
2716 }
2717 
2718 /*
2719  * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
2720  * @tsk: the task to change
2721  * @newmems: new nodes that the task will be set
2722  *
2723  * We use the mems_allowed_seq seqlock to safely update both tsk->mems_allowed
2724  * and rebind an eventual tasks' mempolicy. If the task is allocating in
2725  * parallel, it might temporarily see an empty intersection, which results in
2726  * a seqlock check and retry before OOM or allocation failure.
2727  */
2728 static void cpuset_change_task_nodemask(struct task_struct *tsk,
2729 					nodemask_t *newmems)
2730 {
2731 	task_lock(tsk);
2732 
2733 	local_irq_disable();
2734 	write_seqcount_begin(&tsk->mems_allowed_seq);
2735 
2736 	nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
2737 	mpol_rebind_task(tsk, newmems);
2738 	tsk->mems_allowed = *newmems;
2739 
2740 	write_seqcount_end(&tsk->mems_allowed_seq);
2741 	local_irq_enable();
2742 
2743 	task_unlock(tsk);
2744 }
2745 
2746 static void *cpuset_being_rebound;
2747 
2748 /**
2749  * cpuset_update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
2750  * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
2751  *
2752  * Iterate through each task of @cs updating its mems_allowed to the
2753  * effective cpuset's.  As this function is called with cpuset_mutex held,
2754  * cpuset membership stays stable.
2755  *
2756  * - cpuset_change_task_nodemask(): guarantee_online_mems()
2757  * - mpol_rebind_mm(): effective_mems
2758  * - cpuset_migrate_mm(): guarantee_online_mems()
2759  * - old_mems_allowed: guarantee_online_mems()
2760  *
2761  * For v2, guarantee_online_mems() should return a node mask that is the same
2762  * as the effective_mems of current cpuset.
2763  */
2764 void cpuset_update_tasks_nodemask(struct cpuset *cs)
2765 {
2766 	static nodemask_t newmems;	/* protected by cpuset_mutex */
2767 	struct css_task_iter it;
2768 	struct task_struct *task;
2769 
2770 	cpuset_being_rebound = cs;		/* causes mpol_dup() rebind */
2771 	guarantee_online_mems(cs, &newmems);
2772 
2773 	/*
2774 	 * The mpol_rebind_mm() call takes mmap_lock, which we couldn't
2775 	 * take while holding tasklist_lock.  Forks can happen - the
2776 	 * mpol_dup() cpuset_being_rebound check will catch such forks,
2777 	 * and rebind their vma mempolicies too.  Because we still hold
2778 	 * the global cpuset_mutex, we know that no other rebind effort
2779 	 * will be contending for the global variable cpuset_being_rebound.
2780 	 * It's ok if we rebind the same mm twice; mpol_rebind_mm()
2781 	 * is idempotent.  Also migrate pages in each mm to new nodes.
2782 	 */
2783 	css_task_iter_start(&cs->css, 0, &it);
2784 	while ((task = css_task_iter_next(&it))) {
2785 		struct mm_struct *mm;
2786 		bool migrate;
2787 
2788 		cpuset_change_task_nodemask(task, &newmems);
2789 
2790 		/* Rebind and migrate mm only for thread group leader */
2791 		if (!thread_group_leader(task))
2792 			continue;
2793 
2794 		mm = get_task_mm(task);
2795 		if (!mm)
2796 			continue;
2797 
2798 		migrate = is_memory_migrate(cs);
2799 
2800 		/*
2801 		 * For v1 we can have empty effective_mems, but we cannot
2802 		 * attach any tasks (see cpuset_can_attach_check()). For v2,
2803 		 * effective_mems is guaranteed to not be empty.
2804 		 */
2805 		mpol_rebind_mm(mm, &cs->effective_mems);
2806 		if (migrate)
2807 			cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems);
2808 		else
2809 			mmput(mm);
2810 	}
2811 	css_task_iter_end(&it);
2812 
2813 	/*
2814 	 * All the tasks' nodemasks have been updated, update
2815 	 * cs->old_mems_allowed.
2816 	 */
2817 	cs->old_mems_allowed = newmems;
2818 
2819 	/* We're done rebinding vmas to this cpuset's new mems_allowed. */
2820 	cpuset_being_rebound = NULL;
2821 }
2822 
2823 /*
2824  * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree
2825  * @cs: the cpuset to consider
2826  * @new_mems: a temp variable for calculating new effective_mems
2827  *
2828  * When configured nodemask is changed, the effective nodemasks of this cpuset
2829  * and all its descendants need to be updated.
2830  *
2831  * On legacy hierarchy, effective_mems will be the same with mems_allowed.
2832  *
2833  * Called with cpuset_mutex held
2834  */
2835 static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems)
2836 {
2837 	struct cpuset *cp;
2838 	struct cgroup_subsys_state *pos_css;
2839 
2840 	rcu_read_lock();
2841 	cpuset_for_each_descendant_pre(cp, pos_css, cs) {
2842 		struct cpuset *parent = parent_cs(cp);
2843 
2844 		compute_effective_nodemask(new_mems, cp, parent);
2845 
2846 		/* Skip the whole subtree if the nodemask remains the same. */
2847 		if (nodes_equal(*new_mems, cp->effective_mems)) {
2848 			pos_css = css_rightmost_descendant(pos_css);
2849 			continue;
2850 		}
2851 
2852 		if (!css_tryget_online(&cp->css))
2853 			continue;
2854 		rcu_read_unlock();
2855 
2856 		check_insane_mems_config(new_mems);
2857 
2858 		spin_lock_irq(&callback_lock);
2859 		cp->effective_mems = *new_mems;
2860 		spin_unlock_irq(&callback_lock);
2861 
2862 		WARN_ON(!is_in_v2_mode() &&
2863 			!nodes_equal(cp->mems_allowed, cp->effective_mems));
2864 
2865 		cpuset_update_tasks_nodemask(cp);
2866 
2867 		rcu_read_lock();
2868 		css_put(&cp->css);
2869 	}
2870 	rcu_read_unlock();
2871 }
2872 
2873 /*
2874  * Handle user request to change the 'mems' memory placement
2875  * of a cpuset.  Needs to validate the request, update the
2876  * cpusets mems_allowed, and for each task in the cpuset,
2877  * update mems_allowed and rebind task's mempolicy and any vma
2878  * mempolicies and if the cpuset is marked 'memory_migrate',
2879  * migrate the tasks pages to the new memory.
2880  *
2881  * Call with cpuset_mutex held. May take callback_lock during call.
2882  * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
2883  * lock each such tasks mm->mmap_lock, scan its vma's and rebind
2884  * their mempolicies to the cpusets new mems_allowed.
2885  */
2886 static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
2887 			   const char *buf)
2888 {
2889 	int retval;
2890 
2891 	/*
2892 	 * An empty mems_allowed is ok iff there are no tasks in the cpuset.
2893 	 * The validate_change() call ensures that cpusets with tasks have memory.
2894 	 */
2895 	retval = nodelist_parse(buf, trialcs->mems_allowed);
2896 	if (retval < 0)
2897 		return retval;
2898 
2899 	if (!nodes_subset(trialcs->mems_allowed,
2900 			  top_cpuset.mems_allowed))
2901 		return -EINVAL;
2902 
2903 	/* No change? nothing to do */
2904 	if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed))
2905 		return 0;
2906 
2907 	retval = validate_change(cs, trialcs);
2908 	if (retval < 0)
2909 		return retval;
2910 
2911 	spin_lock_irq(&callback_lock);
2912 	cs->mems_allowed = trialcs->mems_allowed;
2913 	spin_unlock_irq(&callback_lock);
2914 
2915 	/* use trialcs->mems_allowed as a temp variable */
2916 	update_nodemasks_hier(cs, &trialcs->mems_allowed);
2917 	return 0;
2918 }
2919 
2920 bool current_cpuset_is_being_rebound(void)
2921 {
2922 	bool ret;
2923 
2924 	rcu_read_lock();
2925 	ret = task_cs(current) == cpuset_being_rebound;
2926 	rcu_read_unlock();
2927 
2928 	return ret;
2929 }
2930 
2931 /**
2932  * update_prstate - update partition_root_state
2933  * @cs: the cpuset to update
2934  * @new_prs: new partition root state
2935  * Return: 0 if successful, != 0 if error
2936  *
2937  * Call with cpuset_mutex held.
2938  */
2939 static int update_prstate(struct cpuset *cs, int new_prs)
2940 {
2941 	int err = PERR_NONE, old_prs = cs->partition_root_state;
2942 	struct cpuset *parent = parent_cs(cs);
2943 	struct tmpmasks tmpmask;
2944 	int parent_prs = parent->partition_root_state;
2945 	bool isolcpus_updated = false;
2946 	bool disable_partition = false;
2947 
2948 	if (old_prs == new_prs)
2949 		return 0;
2950 
2951 	/*
2952 	 * Treat a previously invalid partition root as if it is a "member".
2953 	 */
2954 	if (new_prs && is_partition_invalid(cs))
2955 		old_prs = PRS_MEMBER;
2956 
2957 	if (alloc_tmpmasks(&tmpmask))
2958 		return -ENOMEM;
2959 
2960 	if (!old_prs) {
2961 		/*
2962 		 * cpus_allowed and exclusive_cpus cannot be both empty.
2963 		 */
2964 		if (xcpus_empty(cs)) {
2965 			err = PERR_CPUSEMPTY;
2966 			goto out;
2967 		}
2968 
2969 		/*
2970 		 * We don't support the creation of a new local partition with
2971 		 * a remote partition underneath it. This unsupported
2972 		 * setting can happen only if parent is the top_cpuset because
2973 		 * a remote partition cannot be created underneath an existing
2974 		 * local or remote partition.
2975 		 */
2976 		if ((parent == &top_cpuset) &&
2977 		    cpumask_intersects(cs->exclusive_cpus, subpartitions_cpus)) {
2978 			err = PERR_REMOTE;
2979 			goto out;
2980 		}
2981 
2982 		/*
2983 		 * If parent is valid partition, enable local partiion.
2984 		 * Otherwise, enable a remote partition.
2985 		 */
2986 		if (is_partition_valid(parent)) {
2987 			enum partition_cmd cmd = (new_prs == PRS_ROOT)
2988 					       ? partcmd_enable : partcmd_enablei;
2989 
2990 			/*
2991 			 * It is not allowed to create a root partition under
2992 			 * an isolated parent partition.
2993 			 */
2994 			if ((new_prs == PRS_ROOT) && (parent_prs == PRS_ISOLATED)) {
2995 				err = PERR_ISOPARENT;
2996 				goto out;
2997 			}
2998 			err = update_parent_effective_cpumask(cs, cmd, NULL, &tmpmask);
2999 		} else {
3000 			err = remote_partition_enable(cs, new_prs, &tmpmask);
3001 		}
3002 	} else if (old_prs && new_prs) {
3003 		int parent_prs = is_remote_partition(cs)
3004 				 ? PRS_ROOT : parent->partition_root_state;
3005 
3006 		/*
3007 		 * A change in load balance state only, no change in cpumasks.
3008 		 * Need to update isolated_cpus.
3009 		 */
3010 		if (prstate_housekeeping_conflict(new_prs, parent_prs,
3011 						  cs->effective_xcpus, NULL))
3012 			err = PERR_HKEEPING;
3013 		else if (parent_prs == PRS_ISOLATED)
3014 			err = PERR_ISOPARENT;	/* old_prs should be isolated */
3015 
3016 		if (err) {
3017 			disable_partition = true;
3018 		} else {
3019 			struct cgroup_subsys_state *css;
3020 			struct cpuset *child;
3021 
3022 			/*
3023 			 * Need to scan child partitions, if present, and
3024 			 * exclude their effective_xcpus from cs. In the case
3025 			 * of switching to isolated partition with child root
3026 			 * partitions, those child partitions will be
3027 			 * invalidated in update_cpumasks_hier().
3028 			 */
3029 			isolcpus_updated = true;
3030 			cpumask_copy(tmpmask.new_cpus, cs->effective_xcpus);
3031 			rcu_read_lock();
3032 			cpuset_for_each_child(child, css, cs) {
3033 				if (!is_partition_valid(child))
3034 					continue;
3035 				cpumask_andnot(tmpmask.new_cpus, tmpmask.new_cpus,
3036 					       child->effective_xcpus);
3037 			}
3038 			rcu_read_unlock();
3039 		}
3040 	} else {
3041 		/*
3042 		 * Switching back to member is always allowed even if it
3043 		 * disables child partitions.
3044 		 */
3045 		disable_partition = true;
3046 	}
3047 
3048 	if (disable_partition) {
3049 		if (is_remote_partition(cs))
3050 			remote_partition_disable(cs, &tmpmask);
3051 		else
3052 			update_parent_effective_cpumask(cs, partcmd_disable,
3053 							NULL, &tmpmask);
3054 
3055 		/*
3056 		 * Invalidation of child partitions will be done in
3057 		 * update_cpumasks_hier() below.
3058 		 */
3059 	}
3060 out:
3061 	/*
3062 	 * Make partition invalid if an error happens.
3063 	 */
3064 	if (err)
3065 		new_prs = -new_prs;
3066 
3067 	spin_lock_irq(&callback_lock);
3068 	cs->partition_root_state = new_prs;
3069 	WRITE_ONCE(cs->prs_err, err);
3070 	if (!is_partition_valid(cs))
3071 		reset_partition_data(cs);
3072 	else if (isolcpus_updated)
3073 		isolated_cpus_update(new_prs, tmpmask.new_cpus);
3074 	spin_unlock_irq(&callback_lock);
3075 
3076 	/* Force update if partition is disabled & update effective_xcpus */
3077 	update_cpumasks_hier(cs, &tmpmask, disable_partition);
3078 
3079 	/* A newly created partition must have effective_xcpus set */
3080 	WARN_ON_ONCE(!old_prs && (new_prs > 0)
3081 			      && cpumask_empty(cs->effective_xcpus));
3082 
3083 	/* Update sched domains and load balance flag */
3084 	update_partition_sd_lb(cs, old_prs);
3085 
3086 	notify_partition_change(cs, old_prs);
3087 	free_tmpmasks(&tmpmask);
3088 	return 0;
3089 }
3090 
3091 /*
3092  * Check to see if a cpuset can accept a new task
3093  * For v1, cpus_allowed and mems_allowed can't be empty.
3094  * For v2, effective_cpus can't be empty.
3095  * Note that in v1, effective_cpus = cpus_allowed.
3096  *
3097  * Also set the boolean flag passed in by @psetsched depending on if
3098  * security_task_setscheduler() call is needed and @oldcs is not NULL.
3099  */
3100 static int cpuset_can_attach_check(struct cpuset *cs, struct cpuset *oldcs,
3101 				   bool *psetsched)
3102 {
3103 	bool cpus_updated, mems_updated;
3104 
3105 	if (cpumask_empty(cs->effective_cpus) ||
3106 	   (!is_in_v2_mode() && nodes_empty(cs->mems_allowed)))
3107 		return -ENOSPC;
3108 
3109 	if (!oldcs)
3110 		return 0;
3111 
3112 	if (!llist_on_list(&oldcs->attach_node))
3113 		llist_add(&oldcs->attach_node, &src_cs_head);
3114 
3115 	if (!llist_on_list(&cs->attach_node))
3116 		llist_add(&cs->attach_node, &dst_cs_head);
3117 
3118 	cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus);
3119 	mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems);
3120 
3121 	if (cpus_updated)
3122 		attach_ctx.cpus_updated = true;
3123 	if (mems_updated)
3124 		attach_ctx.mems_updated = true;
3125 
3126 	/*
3127 	 * Skip rights over task setsched check in v2 when nothing changes for
3128 	 * the current oldcs/cs pair, migration permission derives from
3129 	 * hierarchy ownership in cgroup_procs_write_permission()).
3130 	 */
3131 	*psetsched = !cpuset_v2() || cpus_updated || mems_updated;
3132 
3133 	/*
3134 	 * A v1 cpuset with tasks will have no CPU left only when CPU hotplug
3135 	 * brings the last online CPU offline as users are not allowed to empty
3136 	 * cpuset.cpus when there are active tasks inside. When that happens,
3137 	 * we should allow tasks to migrate out without security check to make
3138 	 * sure they will be able to run after migration.
3139 	 */
3140 	if (!is_in_v2_mode() && cpumask_empty(oldcs->effective_cpus))
3141 		*psetsched = false;
3142 
3143 	return 0;
3144 }
3145 
3146 static int cpuset_reserve_dl_bw(void)
3147 {
3148 	struct cpuset *cs;
3149 	int cpu, ret;
3150 
3151 	llist_for_each_entry(cs, dst_cs_head.first, attach_node) {
3152 		if (!cs->sum_migrate_dl_bw)
3153 			continue;
3154 
3155 		cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
3156 		if (unlikely(cpu >= nr_cpu_ids))
3157 			return -EINVAL;
3158 
3159 		ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw);
3160 		if (ret)
3161 			return ret;
3162 
3163 		cs->dl_bw_cpu = cpu;
3164 	}
3165 	return 0;
3166 }
3167 
3168 /*
3169  * Clear and optionally apply (@cancel is false) the attach related data in the
3170  * source or destination cpuset.
3171  */
3172 static void clear_attach_data(struct llist_head *head, bool cancel)
3173 {
3174 	struct cpuset *cs, *next;
3175 	struct llist_node *lnode = __llist_del_all(head);
3176 
3177 	llist_for_each_entry_safe(cs, next, lnode, attach_node) {
3178 		init_llist_node(&cs->attach_node);
3179 		if (cs->nr_migrate_dl_tasks) {
3180 			if (!cancel)
3181 				atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks);
3182 			else if (cs->dl_bw_cpu >= 0) /* && cancel */
3183 				dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw);
3184 			cs->nr_migrate_dl_tasks = 0;
3185 			cs->sum_migrate_dl_bw = 0;
3186 			cs->dl_bw_cpu = -1;
3187 		}
3188 	}
3189 }
3190 
3191 /* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
3192 static int cpuset_can_attach(struct cgroup_taskset *tset)
3193 {
3194 	struct cgroup_subsys_state *css;
3195 	struct cpuset *cs, *oldcs;
3196 	struct task_struct *task;
3197 	bool setsched_check;
3198 	int ret;
3199 
3200 	cs = oldcs = NULL;
3201 	mutex_lock(&cpuset_mutex);
3202 	attach_ctx.old_cs = NULL;	/* Used later in cpuset_attach_task() */
3203 	attach_ctx.cpus_updated = false;
3204 	attach_ctx.mems_updated = false;
3205 	attach_ctx.many_dest_cs = false;
3206 
3207 	/*
3208 	 * The attach_ctx.old_cs is used mainly by cpuset_migrate_mm() to get
3209 	 * the old_mems_allowed value. There are two ways that many-to-one
3210 	 * cpuset migration can happen:
3211 	 * 1) A multithread application with threads in different cpusets is
3212 	 *    wholely migrated to a new cpuset.
3213 	 * 2) Disabling v2 cpuset controller will move all the tasks in child
3214 	 *    cpusets to the parent cpuset.
3215 	 *
3216 	 * In the former case, it is the mm setting of the group leader that
3217 	 * really matters. So attach_ctx.old_cs should track the oldcs of the
3218 	 * group leader. It falls back to the oldcs of the first task if there
3219 	 * is no group leader in the taskset. In the latter case, effective_mems
3220 	 * of child cpusets must always be a subset of the parent. So no real
3221 	 * page migration will be necessary no matter which child cpuset is
3222 	 * selected as attach_ctx.old_cs.
3223 	 *
3224 	 * For a v2 threaded subtree where cpuset isn't enabled in some of the
3225 	 * cgroups, it is possible that oldcs == cs for some of the tasks.
3226 	 * In this case, we can skip checking on those tasks as there is no
3227 	 * actual migration wrt cpuset.
3228 	 */
3229 	cgroup_taskset_for_each(task, css, tset) {
3230 		struct cpuset *new_cs = css_cs(css);
3231 		struct cpuset *new_oldcs = task_cs(task);
3232 
3233 		if ((new_oldcs != oldcs) || (new_cs != cs)) {
3234 			if (cs && (new_cs != cs))
3235 				attach_ctx.many_dest_cs = true;
3236 			cs = new_cs;
3237 			oldcs = new_oldcs;
3238 			if (oldcs == cs)
3239 				continue;
3240 			if (!attach_ctx.old_cs)
3241 				attach_ctx.old_cs = oldcs;
3242 			ret = cpuset_can_attach_check(cs, oldcs, &setsched_check);
3243 			if (ret)
3244 				goto out_unlock;
3245 		}
3246 
3247 		if (oldcs == cs)
3248 			continue;
3249 
3250 		ret = task_can_attach(task);
3251 		if (ret)
3252 			goto out_unlock;
3253 
3254 		/* Update attach_ctx.old_cs to the latest group leader */
3255 		if (task == task->group_leader)
3256 			attach_ctx.old_cs = task_cs(task);
3257 
3258 		if (setsched_check) {
3259 			ret = security_task_setscheduler(task);
3260 			if (ret)
3261 				goto out_unlock;
3262 		}
3263 
3264 		if (dl_task(task)) {
3265 			/*
3266 			 * Count all migrating DL tasks for cpuset task accounting.
3267 			 * Only tasks that need a root-domain bandwidth move
3268 			 * contribute to sum_migrate_dl_bw.
3269 			 */
3270 			cs->nr_migrate_dl_tasks++;
3271 			oldcs->nr_migrate_dl_tasks--;
3272 			if (dl_task_needs_bw_move(task, cs->effective_cpus))
3273 				cs->sum_migrate_dl_bw += task->dl.dl_bw;
3274 		}
3275 	}
3276 
3277 	/*
3278 	 * The only case where there are multiple destination cpusets for
3279 	 * task migration is when enabling a v2 cpuset controllers where
3280 	 * tasks will be migrated to multiple child cpusets from a parent
3281 	 * cpuset with the same effective CPUs and memory nodes. IOW,
3282 	 * both attach_cpus_updated and attach_mems_updated should be false.
3283 	 * If not, it is a condition that the current code cannot handle.
3284 	 * Print a warning and abort the attach operation as further code
3285 	 * change may be needed.
3286 	 */
3287 	if (WARN_ON_ONCE(attach_ctx.many_dest_cs && (!cpuset_v2() ||
3288 			 attach_ctx.cpus_updated || attach_ctx.mems_updated))) {
3289 		ret = -EINVAL;
3290 		goto out_unlock;
3291 	}
3292 
3293 	ret = cpuset_reserve_dl_bw();
3294 
3295 out_unlock:
3296 	if (ret) {
3297 		clear_attach_data(&src_cs_head, true);
3298 		clear_attach_data(&dst_cs_head, true);
3299 	} else {
3300 		attach_ctx.in_progress++;
3301 	}
3302 
3303 	mutex_unlock(&cpuset_mutex);
3304 	return ret;
3305 }
3306 
3307 static void cpuset_cancel_attach(struct cgroup_taskset *tset)
3308 {
3309 	mutex_lock(&cpuset_mutex);
3310 	dec_attach_in_progress_locked();
3311 	clear_attach_data(&src_cs_head, true);
3312 	clear_attach_data(&dst_cs_head, true);
3313 	mutex_unlock(&cpuset_mutex);
3314 }
3315 
3316 /*
3317  * Protected by cpuset_mutex. cpus_attach is used only by cpuset_attach_task()
3318  * but we can't allocate it dynamically there.  Define it global and
3319  * allocate from cpuset_init().
3320  */
3321 static cpumask_var_t cpus_attach;
3322 
3323 static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task)
3324 {
3325 	struct mm_struct *mm;
3326 
3327 	lockdep_assert_cpuset_lock_held();
3328 
3329 	if (cs != &top_cpuset)
3330 		guarantee_active_cpus(task, cpus_attach);
3331 	else
3332 		cpumask_andnot(cpus_attach, task_cpu_possible_mask(task),
3333 			       subpartitions_cpus);
3334 	/*
3335 	 * can_attach beforehand should guarantee that this doesn't
3336 	 * fail.  TODO: have a better way to handle failure here
3337 	 */
3338 	WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach));
3339 
3340 	if (cpuset_v2() && !attach_ctx.mems_updated)
3341 		return;
3342 
3343 	cpuset_change_task_nodemask(task, &attach_ctx.nodemask_to);
3344 	cpuset1_update_task_spread_flags(cs, task);
3345 
3346 	if ((task != task->group_leader) || !attach_ctx.mems_updated)
3347 		return;
3348 
3349 	/*
3350 	 * Change mm for threadgroup leader. This is expensive and may
3351 	 * sleep and should be moved outside migration path proper.
3352 	 */
3353 	mm = get_task_mm(task);
3354 	if (mm) {
3355 		struct cpuset *oldcs = attach_ctx.old_cs;
3356 
3357 		mpol_rebind_mm(mm, &cs->effective_mems);
3358 
3359 		/*
3360 		 * old_mems_allowed is the same with mems_allowed
3361 		 * here, except if this task is being moved
3362 		 * automatically due to hotplug.  In that case
3363 		 * @mems_allowed has been updated and is empty, so
3364 		 * @old_mems_allowed is the right nodesets that we
3365 		 * migrate mm from.
3366 		 */
3367 		if (is_memory_migrate(cs)) {
3368 			cpuset_migrate_mm(mm, &oldcs->old_mems_allowed,
3369 					  &attach_ctx.nodemask_to);
3370 			attach_ctx.task_work_queued = true;
3371 		} else {
3372 			mmput(mm);
3373 		}
3374 	}
3375 }
3376 
3377 static void cpuset_attach(struct cgroup_taskset *tset)
3378 {
3379 	struct task_struct *task;
3380 	struct cgroup_subsys_state *css;
3381 	struct cpuset *cs;
3382 
3383 	cgroup_taskset_first(tset, &css);
3384 	cs = css_cs(css);
3385 
3386 	lockdep_assert_cpus_held();	/* see cgroup_attach_lock() */
3387 	mutex_lock(&cpuset_mutex);
3388 	attach_ctx.task_work_queued = false;
3389 	guarantee_online_mems(cs, &attach_ctx.nodemask_to);
3390 
3391 	/*
3392 	 * attach_ctx.old_cs can only be NULL if no task is actually migrating.
3393 	 * This is highly unlikely. If it happens at all, we can skip task
3394 	 * iteration and setting old_mems_allowed.
3395 	 */
3396 	if (unlikely(!attach_ctx.old_cs))
3397 		goto out;
3398 
3399 	/*
3400 	 * In the default hierarchy, enabling cpuset in the child cgroups
3401 	 * will trigger a cpuset_attach() call with no change in effective cpus
3402 	 * and mems. In that case, we can optimize out by skipping the task
3403 	 * iteration and the destination cpuset list is iterated to set
3404 	 * old_mems_allowed.
3405 	 */
3406 	if (cpuset_v2() && !attach_ctx.cpus_updated && !attach_ctx.mems_updated) {
3407 		llist_for_each_entry(cs, dst_cs_head.first, attach_node)
3408 			cs->old_mems_allowed = attach_ctx.nodemask_to;
3409 		goto out;
3410 	}
3411 
3412 	cgroup_taskset_for_each(task, css, tset)
3413 		cpuset_attach_task(cs, task);
3414 
3415 	if (attach_ctx.task_work_queued)
3416 		schedule_flush_migrate_mm();
3417 	cs->old_mems_allowed = attach_ctx.nodemask_to;
3418 out:
3419 	clear_attach_data(&src_cs_head, false);
3420 	clear_attach_data(&dst_cs_head, false);
3421 	dec_attach_in_progress_locked();
3422 
3423 	mutex_unlock(&cpuset_mutex);
3424 }
3425 
3426 /*
3427  * Common handling for a write to a "cpus" or "mems" file.
3428  */
3429 ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
3430 				    char *buf, size_t nbytes, loff_t off)
3431 {
3432 	struct cpuset *cs = css_cs(of_css(of));
3433 	struct cpuset *trialcs;
3434 	int retval = -ENODEV;
3435 
3436 	/* root is read-only */
3437 	if (cs == &top_cpuset)
3438 		return -EACCES;
3439 
3440 	buf = strstrip(buf);
3441 
3442 	/* cpuset_mutex acquired in wait_attach_done_lock() */
3443 	mutex_lock(&cpuset_top_mutex);
3444 	cpus_read_lock();
3445 	wait_attach_done_lock();
3446 
3447 	if (!is_cpuset_online(cs))
3448 		goto out_unlock;
3449 
3450 	trialcs = dup_or_alloc_cpuset(cs);
3451 	if (!trialcs) {
3452 		retval = -ENOMEM;
3453 		goto out_unlock;
3454 	}
3455 
3456 	switch (of_cft(of)->private) {
3457 	case FILE_CPULIST:
3458 		retval = update_cpumask(cs, trialcs, buf);
3459 		break;
3460 	case FILE_EXCLUSIVE_CPULIST:
3461 		retval = update_exclusive_cpumask(cs, trialcs, buf);
3462 		break;
3463 	case FILE_MEMLIST:
3464 		retval = update_nodemask(cs, trialcs, buf);
3465 		break;
3466 	default:
3467 		retval = -EINVAL;
3468 		break;
3469 	}
3470 
3471 	free_cpuset(trialcs);
3472 out_unlock:
3473 	cpuset_update_sd_hk_unlock();
3474 	if (of_cft(of)->private == FILE_MEMLIST)
3475 		schedule_flush_migrate_mm();
3476 	return retval ?: nbytes;
3477 }
3478 
3479 /*
3480  * These ascii lists should be read in a single call, by using a user
3481  * buffer large enough to hold the entire map.  If read in smaller
3482  * chunks, there is no guarantee of atomicity.  Since the display format
3483  * used, list of ranges of sequential numbers, is variable length,
3484  * and since these maps can change value dynamically, one could read
3485  * gibberish by doing partial reads while a list was changing.
3486  */
3487 int cpuset_common_seq_show(struct seq_file *sf, void *v)
3488 {
3489 	struct cpuset *cs = css_cs(seq_css(sf));
3490 	cpuset_filetype_t type = seq_cft(sf)->private;
3491 	int ret = 0;
3492 
3493 	spin_lock_irq(&callback_lock);
3494 
3495 	switch (type) {
3496 	case FILE_CPULIST:
3497 		seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_allowed));
3498 		break;
3499 	case FILE_MEMLIST:
3500 		seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed));
3501 		break;
3502 	case FILE_EFFECTIVE_CPULIST:
3503 		seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus));
3504 		break;
3505 	case FILE_EFFECTIVE_MEMLIST:
3506 		seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems));
3507 		break;
3508 	case FILE_EXCLUSIVE_CPULIST:
3509 		seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->exclusive_cpus));
3510 		break;
3511 	case FILE_EFFECTIVE_XCPULIST:
3512 		seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_xcpus));
3513 		break;
3514 	case FILE_SUBPARTS_CPULIST:
3515 		seq_printf(sf, "%*pbl\n", cpumask_pr_args(subpartitions_cpus));
3516 		break;
3517 	case FILE_ISOLATED_CPULIST:
3518 		seq_printf(sf, "%*pbl\n", cpumask_pr_args(isolated_cpus));
3519 		break;
3520 	default:
3521 		ret = -EINVAL;
3522 	}
3523 
3524 	spin_unlock_irq(&callback_lock);
3525 	return ret;
3526 }
3527 
3528 static int cpuset_partition_show(struct seq_file *seq, void *v)
3529 {
3530 	struct cpuset *cs = css_cs(seq_css(seq));
3531 	const char *err, *type = NULL;
3532 
3533 	switch (cs->partition_root_state) {
3534 	case PRS_ROOT:
3535 		seq_puts(seq, "root\n");
3536 		break;
3537 	case PRS_ISOLATED:
3538 		seq_puts(seq, "isolated\n");
3539 		break;
3540 	case PRS_MEMBER:
3541 		seq_puts(seq, "member\n");
3542 		break;
3543 	case PRS_INVALID_ROOT:
3544 		type = "root";
3545 		fallthrough;
3546 	case PRS_INVALID_ISOLATED:
3547 		if (!type)
3548 			type = "isolated";
3549 		err = perr_strings[READ_ONCE(cs->prs_err)];
3550 		if (err)
3551 			seq_printf(seq, "%s invalid (%s)\n", type, err);
3552 		else
3553 			seq_printf(seq, "%s invalid\n", type);
3554 		break;
3555 	}
3556 	return 0;
3557 }
3558 
3559 static ssize_t cpuset_partition_write(struct kernfs_open_file *of, char *buf,
3560 				     size_t nbytes, loff_t off)
3561 {
3562 	struct cpuset *cs = css_cs(of_css(of));
3563 	int val;
3564 	int retval = -ENODEV;
3565 
3566 	buf = strstrip(buf);
3567 
3568 	if (!strcmp(buf, "root"))
3569 		val = PRS_ROOT;
3570 	else if (!strcmp(buf, "member"))
3571 		val = PRS_MEMBER;
3572 	else if (!strcmp(buf, "isolated"))
3573 		val = PRS_ISOLATED;
3574 	else
3575 		return -EINVAL;
3576 
3577 	mutex_lock(&cpuset_top_mutex);
3578 	cpus_read_lock();
3579 	wait_attach_done_lock();
3580 
3581 	if (is_cpuset_online(cs))
3582 		retval = update_prstate(cs, val);
3583 	cpuset_update_sd_hk_unlock();
3584 	return retval ?: nbytes;
3585 }
3586 
3587 /*
3588  * This is currently a minimal set for the default hierarchy. It can be
3589  * expanded later on by migrating more features and control files from v1.
3590  */
3591 static struct cftype dfl_files[] = {
3592 	{
3593 		.name = "cpus",
3594 		.seq_show = cpuset_common_seq_show,
3595 		.write = cpuset_write_resmask,
3596 		.max_write_len = (100U + 6 * NR_CPUS),
3597 		.private = FILE_CPULIST,
3598 		.flags = CFTYPE_NOT_ON_ROOT,
3599 	},
3600 
3601 	{
3602 		.name = "mems",
3603 		.seq_show = cpuset_common_seq_show,
3604 		.write = cpuset_write_resmask,
3605 		.max_write_len = (100U + 6 * MAX_NUMNODES),
3606 		.private = FILE_MEMLIST,
3607 		.flags = CFTYPE_NOT_ON_ROOT,
3608 	},
3609 
3610 	{
3611 		.name = "cpus.effective",
3612 		.seq_show = cpuset_common_seq_show,
3613 		.private = FILE_EFFECTIVE_CPULIST,
3614 	},
3615 
3616 	{
3617 		.name = "mems.effective",
3618 		.seq_show = cpuset_common_seq_show,
3619 		.private = FILE_EFFECTIVE_MEMLIST,
3620 	},
3621 
3622 	{
3623 		.name = "cpus.partition",
3624 		.seq_show = cpuset_partition_show,
3625 		.write = cpuset_partition_write,
3626 		.private = FILE_PARTITION_ROOT,
3627 		.flags = CFTYPE_NOT_ON_ROOT,
3628 		.file_offset = offsetof(struct cpuset, partition_file),
3629 	},
3630 
3631 	{
3632 		.name = "cpus.exclusive",
3633 		.seq_show = cpuset_common_seq_show,
3634 		.write = cpuset_write_resmask,
3635 		.max_write_len = (100U + 6 * NR_CPUS),
3636 		.private = FILE_EXCLUSIVE_CPULIST,
3637 		.flags = CFTYPE_NOT_ON_ROOT,
3638 	},
3639 
3640 	{
3641 		.name = "cpus.exclusive.effective",
3642 		.seq_show = cpuset_common_seq_show,
3643 		.private = FILE_EFFECTIVE_XCPULIST,
3644 		.flags = CFTYPE_NOT_ON_ROOT,
3645 	},
3646 
3647 	{
3648 		.name = "cpus.subpartitions",
3649 		.seq_show = cpuset_common_seq_show,
3650 		.private = FILE_SUBPARTS_CPULIST,
3651 		.flags = CFTYPE_ONLY_ON_ROOT | CFTYPE_DEBUG,
3652 	},
3653 
3654 	{
3655 		.name = "cpus.isolated",
3656 		.seq_show = cpuset_common_seq_show,
3657 		.private = FILE_ISOLATED_CPULIST,
3658 		.flags = CFTYPE_ONLY_ON_ROOT,
3659 	},
3660 
3661 	{ }	/* terminate */
3662 };
3663 
3664 
3665 /**
3666  * cpuset_css_alloc - Allocate a cpuset css
3667  * @parent_css: Parent css of the control group that the new cpuset will be
3668  *              part of
3669  * Return: cpuset css on success, -ENOMEM on failure.
3670  *
3671  * Allocate and initialize a new cpuset css, for non-NULL @parent_css, return
3672  * top cpuset css otherwise.
3673  */
3674 static struct cgroup_subsys_state *
3675 cpuset_css_alloc(struct cgroup_subsys_state *parent_css)
3676 {
3677 	struct cpuset *cs;
3678 
3679 	if (!parent_css)
3680 		return &top_cpuset.css;
3681 
3682 	cs = dup_or_alloc_cpuset(NULL);
3683 	if (!cs)
3684 		return ERR_PTR(-ENOMEM);
3685 
3686 	__set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
3687 	cpuset1_init(cs);
3688 
3689 	/* Set CS_MEMORY_MIGRATE for default hierarchy */
3690 	if (cpuset_v2())
3691 		__set_bit(CS_MEMORY_MIGRATE, &cs->flags);
3692 
3693 	return &cs->css;
3694 }
3695 
3696 static int cpuset_css_online(struct cgroup_subsys_state *css)
3697 {
3698 	struct cpuset *cs = css_cs(css);
3699 	struct cpuset *parent = parent_cs(cs);
3700 
3701 	if (!parent)
3702 		return 0;
3703 
3704 	cpuset_full_lock();
3705 	/*
3706 	 * For v2, clear CS_SCHED_LOAD_BALANCE if parent is isolated
3707 	 */
3708 	if (cpuset_v2() && !is_sched_load_balance(parent))
3709 		clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
3710 
3711 	cpuset_inc();
3712 
3713 	spin_lock_irq(&callback_lock);
3714 	if (is_in_v2_mode()) {
3715 		cpumask_copy(cs->effective_cpus, parent->effective_cpus);
3716 		cs->effective_mems = parent->effective_mems;
3717 	}
3718 	spin_unlock_irq(&callback_lock);
3719 	cpuset1_online_css(css);
3720 
3721 	cpuset_full_unlock();
3722 	return 0;
3723 }
3724 
3725 static void cpuset_css_offline(struct cgroup_subsys_state *css)
3726 {
3727 	struct cpuset *cs = css_cs(css);
3728 
3729 	cpuset_full_lock();
3730 	cpuset1_offline_css(cs);
3731 	cpuset_dec();
3732 	cpuset_full_unlock();
3733 }
3734 
3735 /*
3736  * If a dying cpuset has the 'cpus.partition' enabled, turn it off by
3737  * changing it back to member to free its exclusive CPUs back to the pool to
3738  * be used by other online cpusets.
3739  */
3740 static void cpuset_css_killed(struct cgroup_subsys_state *css)
3741 {
3742 	struct cpuset *cs = css_cs(css);
3743 
3744 	cpuset_full_lock();
3745 	/* Reset valid partition back to member */
3746 	if (is_partition_valid(cs))
3747 		update_prstate(cs, PRS_MEMBER);
3748 	cpuset_update_sd_hk_unlock();
3749 }
3750 
3751 static void cpuset_css_free(struct cgroup_subsys_state *css)
3752 {
3753 	struct cpuset *cs = css_cs(css);
3754 
3755 	free_cpuset(cs);
3756 }
3757 
3758 static void cpuset_bind(struct cgroup_subsys_state *root_css)
3759 {
3760 	mutex_lock(&cpuset_mutex);
3761 	spin_lock_irq(&callback_lock);
3762 
3763 	WRITE_ONCE(cpuset_v2_mode,
3764 		   !!(cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE));
3765 	if (is_in_v2_mode()) {
3766 		cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask);
3767 		cpumask_copy(top_cpuset.effective_xcpus, cpu_possible_mask);
3768 		top_cpuset.mems_allowed = node_possible_map;
3769 	} else {
3770 		cpumask_copy(top_cpuset.cpus_allowed,
3771 			     top_cpuset.effective_cpus);
3772 		top_cpuset.mems_allowed = top_cpuset.effective_mems;
3773 	}
3774 
3775 	spin_unlock_irq(&callback_lock);
3776 	mutex_unlock(&cpuset_mutex);
3777 }
3778 
3779 /*
3780  * In case the child is cloned into a cpuset different from its parent,
3781  * additional checks are done to see if the move is allowed.
3782  */
3783 static int cpuset_can_fork(struct task_struct *task, struct css_set *cset)
3784 {
3785 	struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]);
3786 	bool same_cs;
3787 	int ret;
3788 
3789 	rcu_read_lock();
3790 	same_cs = (cs == task_cs(current));
3791 	rcu_read_unlock();
3792 
3793 	if (same_cs)
3794 		return 0;
3795 
3796 	lockdep_assert_held(&cgroup_mutex);
3797 	mutex_lock(&cpuset_mutex);
3798 
3799 	/* Check to see if task is allowed in the cpuset */
3800 	ret = cpuset_can_attach_check(cs, NULL, NULL);
3801 	if (ret)
3802 		goto out_unlock;
3803 
3804 	ret = task_can_attach(task);
3805 	if (ret)
3806 		goto out_unlock;
3807 
3808 	ret = security_task_setscheduler(task);
3809 	if (ret)
3810 		goto out_unlock;
3811 
3812 	attach_ctx.in_progress++;
3813 out_unlock:
3814 	mutex_unlock(&cpuset_mutex);
3815 	return ret;
3816 }
3817 
3818 static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset)
3819 {
3820 	struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]);
3821 	bool same_cs;
3822 
3823 	rcu_read_lock();
3824 	same_cs = (cs == task_cs(current));
3825 	rcu_read_unlock();
3826 
3827 	if (same_cs)
3828 		return;
3829 
3830 	dec_attach_in_progress();
3831 }
3832 
3833 /*
3834  * Make sure the new task conform to the current state of its parent,
3835  * which could have been changed by cpuset just after it inherits the
3836  * state from the parent and before it sits on the cgroup's task list.
3837  */
3838 static void cpuset_fork(struct task_struct *task)
3839 {
3840 	struct cpuset *cs, *oldcs;
3841 
3842 	rcu_read_lock();
3843 	cs = task_cs(task);
3844 	oldcs = task_cs(current);
3845 	rcu_read_unlock();
3846 
3847 	if (cs == oldcs) {
3848 		unsigned int seq;
3849 
3850 		if (cs == &top_cpuset)
3851 			return;
3852 
3853 		set_cpus_allowed_ptr(task, current->cpus_ptr);
3854 		/*
3855 		 * A cpuset.mems update can update current->mems_allowed
3856 		 * concurrently, so retry it again if it really happens.
3857 		 */
3858 		do {
3859 			seq = read_mems_allowed_begin();
3860 			task->mems_allowed = current->mems_allowed;
3861 		} while (read_mems_allowed_retry(seq));
3862 		return;
3863 	}
3864 
3865 	/* CLONE_INTO_CGROUP */
3866 	mutex_lock(&cpuset_mutex);
3867 	guarantee_online_mems(cs, &attach_ctx.nodemask_to);
3868 	cs->old_mems_allowed = attach_ctx.nodemask_to;
3869 
3870 	/*
3871 	 * Assume CPUs and memory nodes are updated
3872 	 * A CLONE_INTO_CGROUP operation should have taken the cgroup mutex
3873 	 * and so there shouldn't be a competing cpuset_attach() operation.
3874 	 */
3875 	attach_ctx.cpus_updated = attach_ctx.mems_updated = true;
3876 	attach_ctx.task_work_queued = false;
3877 	attach_ctx.old_cs = oldcs;
3878 	cpuset_attach_task(cs, task);
3879 	if (attach_ctx.task_work_queued)
3880 		schedule_flush_migrate_mm();
3881 
3882 	dec_attach_in_progress_locked();
3883 	mutex_unlock(&cpuset_mutex);
3884 }
3885 
3886 struct cgroup_subsys cpuset_cgrp_subsys = {
3887 	.css_alloc	= cpuset_css_alloc,
3888 	.css_online	= cpuset_css_online,
3889 	.css_offline	= cpuset_css_offline,
3890 	.css_killed	= cpuset_css_killed,
3891 	.css_free	= cpuset_css_free,
3892 	.can_attach	= cpuset_can_attach,
3893 	.cancel_attach	= cpuset_cancel_attach,
3894 	.attach		= cpuset_attach,
3895 	.bind		= cpuset_bind,
3896 	.can_fork	= cpuset_can_fork,
3897 	.cancel_fork	= cpuset_cancel_fork,
3898 	.fork		= cpuset_fork,
3899 #ifdef CONFIG_CPUSETS_V1
3900 	.legacy_cftypes	= cpuset1_files,
3901 #endif
3902 	.dfl_cftypes	= dfl_files,
3903 	.early_init	= true,
3904 	.threaded	= true,
3905 };
3906 
3907 /**
3908  * cpuset_init - initialize cpusets at system boot
3909  *
3910  * Description: Initialize top_cpuset
3911  **/
3912 
3913 int __init cpuset_init(void)
3914 {
3915 	BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL));
3916 	BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL));
3917 	BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_xcpus, GFP_KERNEL));
3918 	BUG_ON(!alloc_cpumask_var(&top_cpuset.exclusive_cpus, GFP_KERNEL));
3919 	BUG_ON(!zalloc_cpumask_var(&subpartitions_cpus, GFP_KERNEL));
3920 	BUG_ON(!zalloc_cpumask_var(&isolated_cpus, GFP_KERNEL));
3921 	BUG_ON(!zalloc_cpumask_var(&isolated_hk_cpus, GFP_KERNEL));
3922 
3923 	cpumask_setall(top_cpuset.cpus_allowed);
3924 	nodes_setall(top_cpuset.mems_allowed);
3925 	cpumask_setall(top_cpuset.effective_cpus);
3926 	cpumask_setall(top_cpuset.effective_xcpus);
3927 	cpumask_setall(top_cpuset.exclusive_cpus);
3928 	nodes_setall(top_cpuset.effective_mems);
3929 	init_llist_node(&top_cpuset.attach_node);
3930 
3931 	cpuset1_init(&top_cpuset);
3932 
3933 	BUG_ON(!alloc_cpumask_var(&cpus_attach, GFP_KERNEL));
3934 
3935 	if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT))
3936 		cpumask_andnot(isolated_cpus, cpu_possible_mask,
3937 			       housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT));
3938 
3939 	return 0;
3940 }
3941 
3942 static void
3943 hotplug_update_tasks(struct cpuset *cs,
3944 		     struct cpumask *new_cpus, nodemask_t *new_mems,
3945 		     bool cpus_updated, bool mems_updated)
3946 {
3947 	/* A partition root is allowed to have empty effective cpus */
3948 	if (cpumask_empty(new_cpus) && !is_partition_valid(cs))
3949 		cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus);
3950 	if (nodes_empty(*new_mems))
3951 		*new_mems = parent_cs(cs)->effective_mems;
3952 
3953 	spin_lock_irq(&callback_lock);
3954 	cpumask_copy(cs->effective_cpus, new_cpus);
3955 	cs->effective_mems = *new_mems;
3956 	spin_unlock_irq(&callback_lock);
3957 
3958 	if (cpus_updated)
3959 		cpuset_update_tasks_cpumask(cs, new_cpus);
3960 	if (mems_updated)
3961 		cpuset_update_tasks_nodemask(cs);
3962 }
3963 
3964 void cpuset_force_rebuild(void)
3965 {
3966 	force_sd_rebuild = true;
3967 }
3968 
3969 /**
3970  * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
3971  * @cs: cpuset in interest
3972  * @tmp: the tmpmasks structure pointer
3973  *
3974  * Compare @cs's cpu and mem masks against top_cpuset and if some have gone
3975  * offline, update @cs accordingly.  If @cs ends up with no CPU or memory,
3976  * all its tasks are moved to the nearest ancestor with both resources.
3977  */
3978 static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
3979 {
3980 	static cpumask_t new_cpus;
3981 	static nodemask_t new_mems;
3982 	bool cpus_updated;
3983 	bool mems_updated;
3984 	int partcmd = -1;
3985 	struct cpuset *parent;
3986 
3987 	wait_attach_done_lock();
3988 	parent = parent_cs(cs);
3989 	compute_effective_cpumask(&new_cpus, cs, parent);
3990 	compute_effective_nodemask(&new_mems, cs, parent);
3991 
3992 	if (!tmp || !cs->partition_root_state)
3993 		goto update_tasks;
3994 
3995 	if (is_remote_partition(cs) && (cpumask_empty(subpartitions_cpus) ||
3996 				       (cpumask_empty(&new_cpus) &&
3997 					partition_is_populated(cs, NULL)))) {
3998 		WRITE_ONCE(cs->prs_err, PERR_HOTPLUG);
3999 		remote_partition_disable(cs, tmp);
4000 		compute_effective_cpumask(&new_cpus, cs, parent);
4001 	}
4002 
4003 	/*
4004 	 * Force the partition to become invalid if either one of
4005 	 * the following conditions hold:
4006 	 * 1) empty effective cpus but not valid empty partition.
4007 	 * 2) parent is invalid or doesn't grant any cpus to child
4008 	 *    partitions.
4009 	 * 3) subpartitions_cpus is empty.
4010 	 */
4011 	if (is_local_partition(cs) &&
4012 	    (!is_partition_valid(parent) ||
4013 	     tasks_nocpu_error(parent, cs, &new_cpus) ||
4014 	     cpumask_empty(subpartitions_cpus)))
4015 		partcmd = partcmd_invalidate;
4016 	/*
4017 	 * On the other hand, an invalid partition root may be transitioned
4018 	 * back to a regular one if its effective xcpus has an active CPU.
4019 	 */
4020 	else if (is_partition_valid(parent) && is_partition_invalid(cs) &&
4021 		 cpumask_intersects(cs->effective_xcpus, cpu_active_mask))
4022 		partcmd = partcmd_update;
4023 
4024 	if (partcmd >= 0) {
4025 		update_parent_effective_cpumask(cs, partcmd, NULL, tmp);
4026 		if (is_partition_valid(cs) || (partcmd == partcmd_invalidate)) {
4027 			compute_effective_cpumask(&new_cpus, cs, parent);
4028 			cpuset_force_rebuild();
4029 		}
4030 	}
4031 
4032 update_tasks:
4033 	cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus);
4034 	mems_updated = !nodes_equal(new_mems, cs->effective_mems);
4035 	if (!cpus_updated && !mems_updated)
4036 		goto unlock;	/* Hotplug doesn't affect this cpuset */
4037 
4038 	if (mems_updated)
4039 		check_insane_mems_config(&new_mems);
4040 
4041 	if (is_in_v2_mode())
4042 		hotplug_update_tasks(cs, &new_cpus, &new_mems,
4043 				     cpus_updated, mems_updated);
4044 	else
4045 		cpuset1_hotplug_update_tasks(cs, &new_cpus, &new_mems,
4046 					    cpus_updated, mems_updated);
4047 
4048 unlock:
4049 	mutex_unlock(&cpuset_mutex);
4050 }
4051 
4052 /**
4053  * cpuset_handle_hotplug - handle CPU/memory hot{,un}plug for a cpuset
4054  *
4055  * This function is called after either CPU or memory configuration has
4056  * changed and updates cpuset accordingly.  The top_cpuset is always
4057  * synchronized to cpu_active_mask and N_MEMORY, which is necessary in
4058  * order to make cpusets transparent (of no affect) on systems that are
4059  * actively using CPU hotplug but making no active use of cpusets.
4060  *
4061  * Non-root cpusets are only affected by offlining.  If any CPUs or memory
4062  * nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on
4063  * all descendants.
4064  *
4065  * Note that CPU offlining during suspend is ignored.  We don't modify
4066  * cpusets across suspend/resume cycles at all.
4067  *
4068  * CPU / memory hotplug is handled synchronously.
4069  */
4070 static void cpuset_handle_hotplug(void)
4071 {
4072 	static DECLARE_WORK(hk_sd_work, hk_sd_workfn);
4073 	static cpumask_t new_cpus;
4074 	static nodemask_t new_mems;
4075 	bool cpus_updated, mems_updated;
4076 	bool on_dfl;
4077 	struct tmpmasks tmp, *ptmp = NULL;
4078 
4079 	lockdep_assert_cpus_held();
4080 	mutex_lock(&cpuset_mutex);
4081 
4082 	on_dfl = is_in_v2_mode();
4083 	if (on_dfl && !alloc_tmpmasks(&tmp))
4084 		ptmp = &tmp;
4085 
4086 	/* fetch the available cpus/mems and find out which changed how */
4087 	cpumask_copy(&new_cpus, cpu_active_mask);
4088 	new_mems = node_states[N_MEMORY];
4089 
4090 	/*
4091 	 * If subpartitions_cpus is populated, it is likely that the check
4092 	 * below will produce a false positive on cpus_updated when the cpu
4093 	 * list isn't changed. It is extra work, but it is better to be safe.
4094 	 */
4095 	cpus_updated = !cpumask_equal(top_cpuset.effective_cpus, &new_cpus) ||
4096 		       !cpumask_empty(subpartitions_cpus);
4097 	mems_updated = !nodes_equal(top_cpuset.effective_mems, new_mems);
4098 
4099 	/* For v1, synchronize cpus_allowed to cpu_active_mask */
4100 	if (cpus_updated) {
4101 		cpuset_force_rebuild();
4102 		spin_lock_irq(&callback_lock);
4103 		if (!on_dfl)
4104 			cpumask_copy(top_cpuset.cpus_allowed, &new_cpus);
4105 		/*
4106 		 * Make sure that CPUs allocated to child partitions
4107 		 * do not show up in effective_cpus. If no CPU is left,
4108 		 * we clear the subpartitions_cpus & let the child partitions
4109 		 * fight for the CPUs again.
4110 		 */
4111 		if (!cpumask_empty(subpartitions_cpus)) {
4112 			if (cpumask_subset(&new_cpus, subpartitions_cpus)) {
4113 				cpumask_clear(subpartitions_cpus);
4114 			} else {
4115 				cpumask_andnot(&new_cpus, &new_cpus,
4116 					       subpartitions_cpus);
4117 			}
4118 		}
4119 		cpumask_copy(top_cpuset.effective_cpus, &new_cpus);
4120 		spin_unlock_irq(&callback_lock);
4121 		/* we don't mess with cpumasks of tasks in top_cpuset */
4122 	}
4123 
4124 	/* synchronize mems_allowed to N_MEMORY */
4125 	if (mems_updated) {
4126 		spin_lock_irq(&callback_lock);
4127 		if (!on_dfl)
4128 			top_cpuset.mems_allowed = new_mems;
4129 		top_cpuset.effective_mems = new_mems;
4130 		spin_unlock_irq(&callback_lock);
4131 		cpuset_update_tasks_nodemask(&top_cpuset);
4132 	}
4133 
4134 	mutex_unlock(&cpuset_mutex);
4135 
4136 	/* if cpus or mems changed, we need to propagate to descendants */
4137 	if (cpus_updated || mems_updated) {
4138 		struct cpuset *cs;
4139 		struct cgroup_subsys_state *pos_css;
4140 
4141 		rcu_read_lock();
4142 		cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
4143 			if (cs == &top_cpuset || !css_tryget_online(&cs->css))
4144 				continue;
4145 			rcu_read_unlock();
4146 
4147 			cpuset_hotplug_update_tasks(cs, ptmp);
4148 
4149 			rcu_read_lock();
4150 			css_put(&cs->css);
4151 		}
4152 		rcu_read_unlock();
4153 	}
4154 
4155 	/*
4156 	 * rebuild_sched_domains() will always be called directly if needed
4157 	 * to make sure that newly added or removed CPU will be reflected in
4158 	 * the sched domains. However, if isolated partition invalidation
4159 	 * or recreation is being done (update_housekeeping set), a work item
4160 	 * will be queued to call housekeeping_update() to update the
4161 	 * corresponding housekeeping cpumasks after some slight delay.
4162 	 *
4163 	 * We rely on WORK_STRUCT_PENDING_BIT to not requeue a work item that
4164 	 * is still pending. Before the pending bit is cleared, the work data
4165 	 * is copied out and work item dequeued. So it is possible to queue
4166 	 * the work again before the hk_sd_workfn() is invoked to process the
4167 	 * previously queued work. Since hk_sd_workfn() doesn't use the work
4168 	 * item at all, this is not a problem.
4169 	 */
4170 	if (force_sd_rebuild)
4171 		rebuild_sched_domains_cpuslocked();
4172 	if (update_housekeeping)
4173 		queue_work(system_dfl_wq, &hk_sd_work);
4174 
4175 	free_tmpmasks(ptmp);
4176 }
4177 
4178 void cpuset_update_active_cpus(void)
4179 {
4180 	/*
4181 	 * We're inside cpu hotplug critical region which usually nests
4182 	 * inside cgroup synchronization.  Bounce actual hotplug processing
4183 	 * to a work item to avoid reverse locking order.
4184 	 */
4185 	cpuset_handle_hotplug();
4186 }
4187 
4188 /*
4189  * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY].
4190  * Call this routine anytime after node_states[N_MEMORY] changes.
4191  * See cpuset_update_active_cpus() for CPU hotplug handling.
4192  */
4193 static int cpuset_track_online_nodes(struct notifier_block *self,
4194 				unsigned long action, void *arg)
4195 {
4196 	cpuset_handle_hotplug();
4197 	return NOTIFY_OK;
4198 }
4199 
4200 /**
4201  * cpuset_init_smp - initialize cpus_allowed
4202  *
4203  * Description: Finish top cpuset after cpu, node maps are initialized
4204  */
4205 void __init cpuset_init_smp(void)
4206 {
4207 	/*
4208 	 * cpus_allowd/mems_allowed set to v2 values in the initial
4209 	 * cpuset_bind() call will be reset to v1 values in another
4210 	 * cpuset_bind() call when v1 cpuset is mounted.
4211 	 */
4212 	top_cpuset.old_mems_allowed = top_cpuset.mems_allowed;
4213 
4214 	cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask);
4215 	top_cpuset.effective_mems = node_states[N_MEMORY];
4216 
4217 	hotplug_node_notifier(cpuset_track_online_nodes, CPUSET_CALLBACK_PRI);
4218 
4219 	cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0);
4220 	BUG_ON(!cpuset_migrate_mm_wq);
4221 }
4222 
4223 /*
4224  * Return cpus_allowed mask from a task's cpuset.
4225  */
4226 static void __cpuset_cpus_allowed_locked(struct task_struct *tsk, struct cpumask *pmask)
4227 {
4228 	struct cpuset *cs;
4229 
4230 	cs = task_cs(tsk);
4231 	if (cs != &top_cpuset)
4232 		guarantee_active_cpus(tsk, pmask);
4233 	/*
4234 	 * Tasks in the top cpuset won't get update to their cpumasks
4235 	 * when a hotplug online/offline event happens. So we include all
4236 	 * offline cpus in the allowed cpu list.
4237 	 */
4238 	if ((cs == &top_cpuset) || cpumask_empty(pmask)) {
4239 		const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
4240 
4241 		/*
4242 		 * We first exclude cpus allocated to partitions. If there is no
4243 		 * allowable online cpu left, we fall back to all possible cpus.
4244 		 */
4245 		cpumask_andnot(pmask, possible_mask, subpartitions_cpus);
4246 		if (!cpumask_intersects(pmask, cpu_active_mask))
4247 			cpumask_copy(pmask, possible_mask);
4248 	}
4249 }
4250 
4251 /**
4252  * cpuset_cpus_allowed_locked - return cpus_allowed mask from a task's cpuset.
4253  * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
4254  * @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
4255  *
4256  * Similir to cpuset_cpus_allowed() except that the caller must have acquired
4257  * cpuset_mutex.
4258  */
4259 void cpuset_cpus_allowed_locked(struct task_struct *tsk, struct cpumask *pmask)
4260 {
4261 	lockdep_assert_cpuset_lock_held();
4262 	__cpuset_cpus_allowed_locked(tsk, pmask);
4263 }
4264 
4265 /**
4266  * cpuset_cpus_allowed - return cpus_allowed mask from a task's cpuset.
4267  * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
4268  * @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
4269  *
4270  * Description: Returns the cpumask_var_t cpus_allowed of the cpuset
4271  * attached to the specified @tsk.  Guaranteed to return some non-empty
4272  * subset of cpu_active_mask, even if this means going outside the
4273  * tasks cpuset, except when the task is in the top cpuset.
4274  **/
4275 
4276 void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
4277 {
4278 	unsigned long flags;
4279 
4280 	spin_lock_irqsave(&callback_lock, flags);
4281 	__cpuset_cpus_allowed_locked(tsk, pmask);
4282 	spin_unlock_irqrestore(&callback_lock, flags);
4283 }
4284 
4285 /**
4286  * cpuset_cpus_allowed_fallback - final fallback before complete catastrophe.
4287  * @tsk: pointer to task_struct with which the scheduler is struggling
4288  *
4289  * Description: In the case that the scheduler cannot find an allowed cpu in
4290  * tsk->cpus_allowed, we fall back to task_cs(tsk)->cpus_allowed. In legacy
4291  * mode however, this value is the same as task_cs(tsk)->effective_cpus,
4292  * which will not contain a sane cpumask during cases such as cpu hotplugging.
4293  * This is the absolute last resort for the scheduler and it is only used if
4294  * _every_ other avenue has been traveled.
4295  *
4296  * Returns true if the affinity of @tsk was changed, false otherwise.
4297  **/
4298 
4299 bool cpuset_cpus_allowed_fallback(struct task_struct *tsk)
4300 {
4301 	const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
4302 	const struct cpumask *cs_mask;
4303 	bool changed = false;
4304 
4305 	rcu_read_lock();
4306 	cs_mask = task_cs(tsk)->cpus_allowed;
4307 	if (is_in_v2_mode() && cpumask_subset(cs_mask, possible_mask)) {
4308 		set_cpus_allowed_force(tsk, cs_mask);
4309 		changed = true;
4310 	}
4311 	rcu_read_unlock();
4312 
4313 	/*
4314 	 * We own tsk->cpus_allowed, nobody can change it under us.
4315 	 *
4316 	 * But we used cs && cs->cpus_allowed lockless and thus can
4317 	 * race with cgroup_attach_task() or update_cpumask() and get
4318 	 * the wrong tsk->cpus_allowed. However, both cases imply the
4319 	 * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr()
4320 	 * which takes task_rq_lock().
4321 	 *
4322 	 * If we are called after it dropped the lock we must see all
4323 	 * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary
4324 	 * set any mask even if it is not right from task_cs() pov,
4325 	 * the pending set_cpus_allowed_ptr() will fix things.
4326 	 *
4327 	 * select_fallback_rq() will fix things ups and set cpu_possible_mask
4328 	 * if required.
4329 	 */
4330 	return changed;
4331 }
4332 
4333 /*
4334  * Returns the number of CPUs available for this cgroup.
4335  *
4336  * This only really works for cgroup-v2 where all the controllers are mounted
4337  * in the same hierarchy. If not cgroup-v2 or no cpuset controller is
4338  * configured it reverts to num_online_cpus().
4339  */
4340 int cpuset_num_cpus(struct cgroup *cgrp)
4341 {
4342 	int nr = num_online_cpus();
4343 	struct cpuset *cs;
4344 
4345 	if (cpuset_v2()) {
4346 		guard(rcu)();
4347 		cs = css_cs(cgroup_e_css(cgrp, &cpuset_cgrp_subsys));
4348 		if (cs)
4349 			nr = cpumask_weight(cs->effective_cpus);
4350 	}
4351 
4352 	return nr;
4353 }
4354 
4355 void __init cpuset_init_current_mems_allowed(void)
4356 {
4357 	nodes_setall(current->mems_allowed);
4358 }
4359 
4360 /**
4361  * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
4362  * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
4363  *
4364  * Description: Returns the nodemask_t mems_allowed of the cpuset
4365  * attached to the specified @tsk.  Guaranteed to return some non-empty
4366  * subset of node_states[N_MEMORY], even if this means going outside the
4367  * tasks cpuset.
4368  **/
4369 
4370 nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
4371 {
4372 	nodemask_t mask;
4373 	unsigned long flags;
4374 
4375 	spin_lock_irqsave(&callback_lock, flags);
4376 	guarantee_online_mems(task_cs(tsk), &mask);
4377 	spin_unlock_irqrestore(&callback_lock, flags);
4378 
4379 	return mask;
4380 }
4381 
4382 /**
4383  * cpuset_nodemask_valid_mems_allowed - check nodemask vs. current mems_allowed
4384  * @nodemask: the nodemask to be checked
4385  *
4386  * Are any of the nodes in the nodemask allowed in current->mems_allowed?
4387  *
4388  * A cpuset update rebinds the task's mempolicy and changes mems_allowed in
4389  * separate steps, so retry if an update raced with the check.
4390  */
4391 int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask)
4392 {
4393 	unsigned int seq;
4394 	int ret;
4395 
4396 	do {
4397 		seq = read_mems_allowed_begin();
4398 		ret = nodes_intersects(*nodemask, current->mems_allowed);
4399 	} while (read_mems_allowed_retry(seq));
4400 
4401 	return ret;
4402 }
4403 
4404 /*
4405  * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
4406  * mem_hardwall ancestor to the specified cpuset.  Call holding
4407  * callback_lock.  If no ancestor is mem_exclusive or mem_hardwall
4408  * (an unusual configuration), then returns the root cpuset.
4409  */
4410 static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
4411 {
4412 	while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs))
4413 		cs = parent_cs(cs);
4414 	return cs;
4415 }
4416 
4417 /*
4418  * cpuset_current_node_allowed - Can current task allocate on a memory node?
4419  * @node: is this an allowed node?
4420  * @gfp_mask: memory allocation flags
4421  *
4422  * If we're in interrupt, yes, we can always allocate.  If @node is set in
4423  * current's mems_allowed, yes.  If it's not a __GFP_HARDWALL request and this
4424  * node is set in the nearest hardwalled cpuset ancestor to current's cpuset,
4425  * yes.  If current has access to memory reserves as an oom victim, yes.
4426  * If the current task is PF_EXITING, yes. Otherwise, no.
4427  *
4428  * GFP_USER allocations are marked with the __GFP_HARDWALL bit,
4429  * and do not allow allocations outside the current tasks cpuset
4430  * unless the task has been OOM killed or is exiting.
4431  * GFP_KERNEL allocations are not so marked, so can escape to the
4432  * nearest enclosing hardwalled ancestor cpuset.
4433  *
4434  * Scanning up parent cpusets requires callback_lock.  The
4435  * page allocator only calls here with __GFP_HARDWALL bit
4436  * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the
4437  * current tasks mems_allowed came up empty on the first pass over
4438  * the zonelist.  So only GFP_KERNEL allocations, if all nodes in the
4439  * cpuset are short of memory, might require taking the callback_lock.
4440  *
4441  * The first call here from mm/page_alloc:get_page_from_freelist()
4442  * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
4443  * so no allocation on a node outside the cpuset is allowed (unless
4444  * in interrupt, of course).  The PF_EXITING check must therefore
4445  * come before the __GFP_HARDWALL check, otherwise a dying task
4446  * would be blocked on the fast path.
4447  *
4448  * The second pass through get_page_from_freelist() doesn't even call here for
4449  * GFP_ATOMIC calls.  That, and the checks below have the combined affect that:
4450  *	in_interrupt - any node ok (current task context irrelevant)
4451  *	GFP_ATOMIC   - any node ok
4452  *	tsk_is_oom_victim   - any node ok
4453  *	PF_EXITING   - any node ok (let dying task exit quickly)
4454  *	GFP_KERNEL   - any node in enclosing hardwalled cpuset ok
4455  *	GFP_USER     - only nodes in current tasks mems allowed ok.
4456  */
4457 bool cpuset_current_node_allowed(int node, gfp_t gfp_mask)
4458 {
4459 	struct cpuset *cs;		/* current cpuset ancestors */
4460 	bool allowed;			/* is allocation in zone z allowed? */
4461 	unsigned long flags;
4462 
4463 	if (in_interrupt())
4464 		return true;
4465 	if (node_isset(node, current->mems_allowed))
4466 		return true;
4467 	/*
4468 	 * Allow tasks that have access to memory reserves because they have
4469 	 * been OOM killed to get memory anywhere.
4470 	 */
4471 	if (unlikely(tsk_is_oom_victim(current)))
4472 		return true;
4473 	if (current->flags & PF_EXITING) /* Let dying task have memory */
4474 		return true;
4475 	if (gfp_mask & __GFP_HARDWALL)	/* If hardwall request, stop here */
4476 		return false;
4477 
4478 	if (cpuset_v2())
4479 		return true;
4480 
4481 	/* Not hardwall and node outside mems_allowed: scan up cpusets */
4482 	spin_lock_irqsave(&callback_lock, flags);
4483 
4484 	cs = nearest_hardwall_ancestor(task_cs(current));
4485 	allowed = node_isset(node, cs->mems_allowed);
4486 
4487 	spin_unlock_irqrestore(&callback_lock, flags);
4488 	return allowed;
4489 }
4490 
4491 /**
4492  * cpuset_nodes_allowed - return effective_mems mask from a cgroup cpuset.
4493  * @cgroup: pointer to struct cgroup.
4494  * @mask: pointer to struct nodemask_t to be returned.
4495  *
4496  * Returns effective_mems mask from a cgroup cpuset if it is cgroup v2 and
4497  * has cpuset subsys. Otherwise, returns node_states[N_MEMORY].
4498  *
4499  * This function intentionally avoids taking the cpuset_mutex or callback_lock
4500  * when accessing effective_mems. This is because the obtained effective_mems
4501  * is stale immediately after the query anyway (e.g., effective_mems is updated
4502  * immediately after releasing the lock but before returning).
4503  *
4504  * As a result, returned @mask may be empty because cs->effective_mems can be
4505  * rebound during this call. Besides, nodes in @mask are not guaranteed to be
4506  * online due to hot plugins. Callers should check the mask for validity on
4507  * return based on its subsequent use.
4508  **/
4509 void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask)
4510 {
4511 	struct cgroup_subsys_state *css;
4512 	struct cpuset *cs;
4513 
4514 	/*
4515 	 * In v1, mem_cgroup and cpuset are unlikely in the same hierarchy
4516 	 * and mems_allowed is likely to be empty even if we could get to it,
4517 	 * so return directly to avoid taking a global lock on the empty check.
4518 	 */
4519 	if (!cgroup || !cpuset_v2()) {
4520 		nodes_copy(*mask, node_states[N_MEMORY]);
4521 		return;
4522 	}
4523 
4524 	css = cgroup_get_e_css(cgroup, &cpuset_cgrp_subsys);
4525 	if (!css) {
4526 		nodes_copy(*mask, node_states[N_MEMORY]);
4527 		return;
4528 	}
4529 
4530 	/*
4531 	 * The reference taken via cgroup_get_e_css is sufficient to
4532 	 * protect css, but it does not imply safe accesses to effective_mems.
4533 	 *
4534 	 * Normally, accessing effective_mems would require the cpuset_mutex
4535 	 * or callback_lock - but the correctness of this information is stale
4536 	 * immediately after the query anyway. We do not acquire the lock
4537 	 * during this process to save lock contention in exchange for racing
4538 	 * against mems_allowed rebinds.
4539 	 */
4540 	cs = container_of(css, struct cpuset, css);
4541 	nodes_copy(*mask, cs->effective_mems);
4542 	css_put(css);
4543 }
4544 
4545 /**
4546  * cpuset_spread_node() - On which node to begin search for a page
4547  * @rotor: round robin rotor
4548  *
4549  * If a task is marked PFA_SPREAD_PAGE and a page cache allocation uses
4550  * cpuset_mem_spread_node() to determine where to start looking, spread the
4551  * starting node around the task's mems_allowed nodes instead of starting on
4552  * the local node.
4553  *
4554  * We don't have to worry about the returned node being offline
4555  * because "it can't happen", and even if it did, it would be ok.
4556  *
4557  * The routines calling guarantee_online_mems() are careful to
4558  * only set nodes in task->mems_allowed that are online.  So it
4559  * should not be possible for the following code to return an
4560  * offline node.  But if it did, that would be ok, as this routine
4561  * is not returning the node where the allocation must be, only
4562  * the node where the search should start.  The zonelist used by
4563  * the allocator will include all nodes.  If the slab allocator
4564  * is passed an offline node, it will fall back to the local node.
4565  * See kmem_cache_alloc_node().
4566  */
4567 static int cpuset_spread_node(int *rotor)
4568 {
4569 	return *rotor = next_node_in(*rotor, current->mems_allowed);
4570 }
4571 
4572 /**
4573  * cpuset_mem_spread_node() - On which node to begin search for a file page
4574  */
4575 int cpuset_mem_spread_node(void)
4576 {
4577 	if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE)
4578 		current->cpuset_mem_spread_rotor =
4579 			node_random(&current->mems_allowed);
4580 
4581 	return cpuset_spread_node(&current->cpuset_mem_spread_rotor);
4582 }
4583 
4584 /**
4585  * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
4586  * @tsk1: pointer to task_struct of some task.
4587  * @tsk2: pointer to task_struct of some other task.
4588  *
4589  * Description: Return true if @tsk1's mems_allowed intersects the
4590  * mems_allowed of @tsk2.  Used by the OOM killer to determine if
4591  * one of the task's memory usage might impact the memory available
4592  * to the other.
4593  **/
4594 
4595 int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
4596 				   const struct task_struct *tsk2)
4597 {
4598 	return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);
4599 }
4600 
4601 /**
4602  * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed
4603  *
4604  * Description: Prints current's name, cpuset name, and cached copy of its
4605  * mems_allowed to the kernel log.
4606  */
4607 void cpuset_print_current_mems_allowed(void)
4608 {
4609 	struct cgroup *cgrp;
4610 
4611 	rcu_read_lock();
4612 
4613 	cgrp = task_cs(current)->css.cgroup;
4614 	pr_cont(",cpuset=");
4615 	pr_cont_cgroup_name(cgrp);
4616 	pr_cont(",mems_allowed=%*pbl",
4617 		nodemask_pr_args(&current->mems_allowed));
4618 
4619 	rcu_read_unlock();
4620 }
4621 
4622 /* Display task mems_allowed in /proc/<pid>/status file. */
4623 void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task)
4624 {
4625 	seq_printf(m, "Mems_allowed:\t%*pb\n",
4626 		   nodemask_pr_args(&task->mems_allowed));
4627 	seq_printf(m, "Mems_allowed_list:\t%*pbl\n",
4628 		   nodemask_pr_args(&task->mems_allowed));
4629 }
4630