Lines Matching +full:cs +full:- +full:out
7 * Copyright (C) 2004-2007 Silicon Graphics, Inc.
11 * sysfs is Copyright (c) 2001-3 Patrick Mochel
13 * 2003-10-10 Written by Simon Derr.
14 * 2003-10-22 Updates by Stephen Hemminger.
15 * 2004 May-July Rework by Paul Jackson.
24 #include "cpuset-internal.h"
50 * node binding, add this key to provide a quick low-cost judgment
74 * Exclusive CPUs distributed out to local or remote sub-partitions of
96 * - update_partition_sd_lb()
97 * - update_cpumasks_hier()
98 * - cpuset_update_flag()
99 * - cpuset_hotplug_update_tasks()
100 * - cpuset_handle_hotplug()
104 * Note that update_relax_domain_level() in cpuset-v1.c can still call
112 * 0 - member (not a partition root)
113 * 1 - partition root
114 * 2 - partition root without load balancing (isolated)
115 * -1 - invalid partition root
116 * -2 - invalid isolated partition root
118 * There are 2 types of partitions - local or remote. Local partitions are
132 #define PRS_INVALID_ROOT -1
133 #define PRS_INVALID_ISOLATED -2
146 struct cpuset *cs = task_cs(p); in inc_dl_tasks_cs() local
148 cs->nr_deadline_tasks++; in inc_dl_tasks_cs()
153 struct cpuset *cs = task_cs(p); in dec_dl_tasks_cs() local
155 cs->nr_deadline_tasks--; in dec_dl_tasks_cs()
158 static inline bool is_partition_valid(const struct cpuset *cs) in is_partition_valid() argument
160 return cs->partition_root_state > 0; in is_partition_valid()
163 static inline bool is_partition_invalid(const struct cpuset *cs) in is_partition_invalid() argument
165 return cs->partition_root_state < 0; in is_partition_invalid()
168 static inline bool cs_is_member(const struct cpuset *cs) in cs_is_member() argument
170 return cs->partition_root_state == PRS_MEMBER; in cs_is_member()
176 static inline void make_partition_invalid(struct cpuset *cs) in make_partition_invalid() argument
178 if (cs->partition_root_state > 0) in make_partition_invalid()
179 cs->partition_root_state = -cs->partition_root_state; in make_partition_invalid()
185 static inline void notify_partition_change(struct cpuset *cs, int old_prs) in notify_partition_change() argument
187 if (old_prs == cs->partition_root_state) in notify_partition_change()
189 cgroup_file_notify(&cs->partition_file); in notify_partition_change()
192 if (is_partition_valid(cs)) in notify_partition_change()
193 WRITE_ONCE(cs->prs_err, PERR_NONE); in notify_partition_change()
214 .relax_domain_level = -1,
219 * There are two global locks guarding cpuset structures - cpuset_mutex and
223 * paths that rely on priority inheritance (e.g. scheduler - on RT) for
240 * If a task is only holding callback_lock, then it has read-only
248 * small pieces of code, such as when reading out possibly multi-word
255 * cpuset_lock - Acquire the global cpuset mutex
271 * cpuset_full_lock - Acquire full protection for cpuset modification
316 * decrease cs->attach_in_progress.
317 * wake_up cpuset_attach_wq if cs->attach_in_progress==0.
319 static inline void dec_attach_in_progress_locked(struct cpuset *cs) in dec_attach_in_progress_locked() argument
323 cs->attach_in_progress--; in dec_attach_in_progress_locked()
324 if (!cs->attach_in_progress) in dec_attach_in_progress_locked()
328 static inline void dec_attach_in_progress(struct cpuset *cs) in dec_attach_in_progress() argument
331 dec_attach_in_progress_locked(cs); in dec_attach_in_progress()
352 (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE); in is_in_v2_mode()
356 * partition_is_populated - check if partition has tasks
357 * @cs: partition root to be checked
361 * It is assumed that @cs is a valid partition root. @excluded_child should
362 * be non-NULL when this cpuset is going to become a partition itself.
364 static inline bool partition_is_populated(struct cpuset *cs, in partition_is_populated() argument
370 if (cs->css.cgroup->nr_populated_csets) in partition_is_populated()
372 if (!excluded_child && !cs->nr_subparts) in partition_is_populated()
373 return cgroup_is_populated(cs->css.cgroup); in partition_is_populated()
376 cpuset_for_each_child(child, css, cs) { in partition_is_populated()
381 if (cgroup_is_populated(child->css.cgroup)) { in partition_is_populated()
396 * One way or another, we guarantee to return some non-empty subset
405 struct cpuset *cs; in guarantee_active_cpus() local
411 cs = task_cs(tsk); in guarantee_active_cpus()
413 while (!cpumask_intersects(cs->effective_cpus, pmask)) in guarantee_active_cpus()
414 cs = parent_cs(cs); in guarantee_active_cpus()
416 cpumask_and(pmask, pmask, cs->effective_cpus); in guarantee_active_cpus()
426 * One way or another, we guarantee to return some non-empty subset
431 static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask) in guarantee_online_mems() argument
433 while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY])) in guarantee_online_mems()
434 cs = parent_cs(cs); in guarantee_online_mems()
435 nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]); in guarantee_online_mems()
439 * alloc_cpumasks - Allocate an array of cpumask variables
442 * Return: 0 if successful, -ENOMEM otherwise.
445 * success, -ENOMEM on allocation failure. On failure, any previously
454 while (--i >= 0) in alloc_cpumasks()
456 return -ENOMEM; in alloc_cpumasks()
463 * alloc_tmpmasks - Allocate temporary cpumasks for cpuset operations.
465 * Return: 0 on success, -ENOMEM on allocation failure
474 &tmp->new_cpus, in alloc_tmpmasks()
475 &tmp->addmask, in alloc_tmpmasks()
476 &tmp->delmask in alloc_tmpmasks()
483 * free_tmpmasks - free cpumasks in a tmpmasks structure
491 free_cpumask_var(tmp->new_cpus); in free_tmpmasks()
492 free_cpumask_var(tmp->addmask); in free_tmpmasks()
493 free_cpumask_var(tmp->delmask); in free_tmpmasks()
497 * dup_or_alloc_cpuset - Duplicate or allocate a new cpuset
498 * @cs: Source cpuset to duplicate (NULL for a fresh allocation)
501 * 1. Duplicating an existing cpuset (if @cs is non-NULL), or
502 * 2. Allocating a fresh cpuset with zero-initialized masks (if @cs is NULL)
506 static struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs) in dup_or_alloc_cpuset() argument
511 trial = cs ? kmemdup(cs, sizeof(*cs), GFP_KERNEL) : in dup_or_alloc_cpuset()
512 kzalloc(sizeof(*cs), GFP_KERNEL); in dup_or_alloc_cpuset()
518 &trial->cpus_allowed, in dup_or_alloc_cpuset()
519 &trial->effective_cpus, in dup_or_alloc_cpuset()
520 &trial->effective_xcpus, in dup_or_alloc_cpuset()
521 &trial->exclusive_cpus in dup_or_alloc_cpuset()
530 if (cs) { in dup_or_alloc_cpuset()
531 cpumask_copy(trial->cpus_allowed, cs->cpus_allowed); in dup_or_alloc_cpuset()
532 cpumask_copy(trial->effective_cpus, cs->effective_cpus); in dup_or_alloc_cpuset()
533 cpumask_copy(trial->effective_xcpus, cs->effective_xcpus); in dup_or_alloc_cpuset()
534 cpumask_copy(trial->exclusive_cpus, cs->exclusive_cpus); in dup_or_alloc_cpuset()
541 * free_cpuset - free the cpuset
542 * @cs: the cpuset to be freed
544 static inline void free_cpuset(struct cpuset *cs) in free_cpuset() argument
546 free_cpumask_var(cs->cpus_allowed); in free_cpuset()
547 free_cpumask_var(cs->effective_cpus); in free_cpuset()
548 free_cpumask_var(cs->effective_xcpus); in free_cpuset()
549 free_cpumask_var(cs->exclusive_cpus); in free_cpuset()
550 kfree(cs); in free_cpuset()
554 static inline struct cpumask *user_xcpus(struct cpuset *cs) in user_xcpus() argument
556 return cpumask_empty(cs->exclusive_cpus) ? cs->cpus_allowed in user_xcpus()
557 : cs->exclusive_cpus; in user_xcpus()
560 static inline bool xcpus_empty(struct cpuset *cs) in xcpus_empty() argument
562 return cpumask_empty(cs->cpus_allowed) && in xcpus_empty()
563 cpumask_empty(cs->exclusive_cpus); in xcpus_empty()
567 * cpusets_are_exclusive() - check if two cpusets are exclusive
582 * cpus_excl_conflict - Check if two cpusets have exclusive CPU conflicts
600 if (cpumask_intersects(cs1->exclusive_cpus, cs2->exclusive_cpus)) in cpus_excl_conflict()
604 if (!cpumask_empty(cs1->cpus_allowed) && in cpus_excl_conflict()
605 cpumask_subset(cs1->cpus_allowed, cs2->exclusive_cpus)) in cpus_excl_conflict()
608 if (!cpumask_empty(cs2->cpus_allowed) && in cpus_excl_conflict()
609 cpumask_subset(cs2->cpus_allowed, cs1->exclusive_cpus)) in cpus_excl_conflict()
618 return nodes_intersects(cs1->mems_allowed, cs2->mems_allowed); in mems_excl_conflict()
623 * validate_change() - Used to validate that any proposed cpuset change
631 * 'cur' is the address of an actual, in-use cpuset. Operations
639 * Return 0 if valid, -errno if not.
653 goto out; in validate_change()
657 goto out; in validate_change()
662 * Cpusets with tasks - existing or newly being attached - can't in validate_change()
665 ret = -ENOSPC; in validate_change()
666 if ((cgroup_is_populated(cur->css.cgroup) || cur->attach_in_progress)) { in validate_change()
667 if (!cpumask_empty(cur->cpus_allowed) && in validate_change()
668 cpumask_empty(trial->cpus_allowed)) in validate_change()
669 goto out; in validate_change()
670 if (!nodes_empty(cur->mems_allowed) && in validate_change()
671 nodes_empty(trial->mems_allowed)) in validate_change()
672 goto out; in validate_change()
684 * for non-isolated partition root. At this point, the target in validate_change()
692 ret = -EBUSY; in validate_change()
694 !cpuset_cpumask_can_shrink(cur->effective_cpus, user_xcpus(trial))) in validate_change()
695 goto out; in validate_change()
701 ret = -EINVAL; in validate_change()
706 goto out; in validate_change()
708 goto out; in validate_change()
712 out: in validate_change()
724 return cpumask_intersects(a->effective_cpus, b->effective_cpus); in cpusets_overlap()
730 if (dattr->relax_domain_level < c->relax_domain_level) in update_domain_attr()
731 dattr->relax_domain_level = c->relax_domain_level; in update_domain_attr()
744 if (cpumask_empty(cp->cpus_allowed)) { in update_domain_attr_tree()
758 /* jump label reference count + the top-level cpuset */ in nr_cpusets()
766 * A 'partial partition' is a set of non-overlapping subsets whose
773 * See "What is sched_load_balance" in Documentation/admin-guide/cgroup-v1/cpusets.rst
784 * cp - cpuset pointer, used (together with pos_css) to perform a
785 * top-down scan of all cpusets. For our purposes, rebuilding
788 * csa - (for CpuSet Array) Array of pointers to all the cpusets
795 * doms - Conversion of 'csa' to an array of cpumasks, for passing to
805 * and merging them using a union-find algorithm.
815 struct cpuset *cp; /* top-down scan of cpusets */ in generate_sched_domains()
875 if (!cpumask_empty(cp->cpus_allowed) && in generate_sched_domains()
877 cpumask_intersects(cp->cpus_allowed, in generate_sched_domains()
882 !cpumask_empty(cp->effective_cpus)) in generate_sched_domains()
892 * non-empty effective_cpus will be saved into csn[]. in generate_sched_domains()
894 if ((cp->partition_root_state == PRS_ROOT) && in generate_sched_domains()
895 !cpumask_empty(cp->effective_cpus)) in generate_sched_domains()
902 if (!is_partition_valid(cp) && cpumask_empty(cp->exclusive_cpus)) in generate_sched_domains()
909 * we can optimize out unneeded sched domains scanning. in generate_sched_domains()
915 uf_node_init(&csa[i]->node); in generate_sched_domains()
926 uf_union(&csa[i]->node, &csa[j]->node); in generate_sched_domains()
933 if (uf_find(&csa[i]->node) == &csa[i]->node) in generate_sched_domains()
954 * to SD_ATTR_INIT. Also non-isolating partition root CPUs are a in generate_sched_domains()
964 cpumask_and(doms[i], csa[i]->effective_cpus, in generate_sched_domains()
967 cpumask_copy(doms[i], csa[i]->effective_cpus); in generate_sched_domains()
977 if (uf_find(&csa[j]->node) == &csa[i]->node) { in generate_sched_domains()
986 cpumask_or(dp, dp, csa[j]->effective_cpus); in generate_sched_domains()
1012 static void dl_update_tasks_root_domain(struct cpuset *cs) in dl_update_tasks_root_domain() argument
1017 if (cs->nr_deadline_tasks == 0) in dl_update_tasks_root_domain()
1020 css_task_iter_start(&cs->css, 0, &it); in dl_update_tasks_root_domain()
1030 struct cpuset *cs = NULL; in dl_rebuild_rd_accounting() local
1048 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { in dl_rebuild_rd_accounting()
1050 if (cpumask_empty(cs->effective_cpus)) { in dl_rebuild_rd_accounting()
1055 css_get(&cs->css); in dl_rebuild_rd_accounting()
1059 dl_update_tasks_root_domain(cs); in dl_rebuild_rd_accounting()
1062 css_put(&cs->css); in dl_rebuild_rd_accounting()
1070 * If the flag 'sched_load_balance' of any cpuset with non-empty
1072 * which has that flag enabled, or if any cpuset with a non-empty
1083 struct cpuset *cs; in rebuild_sched_domains_locked() local
1110 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { in rebuild_sched_domains_locked()
1111 if (!is_partition_valid(cs)) { in rebuild_sched_domains_locked()
1115 if (!cpumask_subset(cs->effective_cpus, in rebuild_sched_domains_locked()
1158 * cpuset_update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
1159 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
1162 * Iterate through each task of @cs updating its cpus_allowed to the
1173 void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus) in cpuset_update_tasks_cpumask() argument
1177 bool top_cs = cs == &top_cpuset; in cpuset_update_tasks_cpumask()
1179 css_task_iter_start(&cs->css, 0, &it); in cpuset_update_tasks_cpumask()
1189 if (task->flags & PF_NO_SETAFFINITY) in cpuset_update_tasks_cpumask()
1193 cpumask_and(new_cpus, possible_mask, cs->effective_cpus); in cpuset_update_tasks_cpumask()
1201 * compute_effective_cpumask - Compute the effective cpumask of the cpuset
1203 * @cs: the cpuset the need to recompute the new effective_cpus mask
1209 struct cpuset *cs, struct cpuset *parent) in compute_effective_cpumask() argument
1211 cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); in compute_effective_cpumask()
1225 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
1233 static int update_partition_exclusive_flag(struct cpuset *cs, int new_prs) in update_partition_exclusive_flag() argument
1237 if (exclusive && !is_cpu_exclusive(cs)) { in update_partition_exclusive_flag()
1238 if (cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 1)) in update_partition_exclusive_flag()
1240 } else if (!exclusive && is_cpu_exclusive(cs)) { in update_partition_exclusive_flag()
1242 cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 0); in update_partition_exclusive_flag()
1254 static void update_partition_sd_lb(struct cpuset *cs, int old_prs) in update_partition_sd_lb() argument
1256 int new_prs = cs->partition_root_state; in update_partition_sd_lb()
1261 * If cs is not a valid partition root, the load balance state in update_partition_sd_lb()
1267 new_lb = is_sched_load_balance(parent_cs(cs)); in update_partition_sd_lb()
1269 if (new_lb != !!is_sched_load_balance(cs)) { in update_partition_sd_lb()
1272 set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in update_partition_sd_lb()
1274 clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in update_partition_sd_lb()
1282 * tasks_nocpu_error - Return true if tasks will have no effective_cpus
1284 static bool tasks_nocpu_error(struct cpuset *parent, struct cpuset *cs, in tasks_nocpu_error() argument
1288 * A populated partition (cs or parent) can't have empty effective_cpus in tasks_nocpu_error()
1290 return (cpumask_subset(parent->effective_cpus, xcpus) && in tasks_nocpu_error()
1291 partition_is_populated(parent, cs)) || in tasks_nocpu_error()
1293 partition_is_populated(cs, NULL)); in tasks_nocpu_error()
1296 static void reset_partition_data(struct cpuset *cs) in reset_partition_data() argument
1298 struct cpuset *parent = parent_cs(cs); in reset_partition_data()
1305 cs->nr_subparts = 0; in reset_partition_data()
1306 if (cpumask_empty(cs->exclusive_cpus)) { in reset_partition_data()
1307 cpumask_clear(cs->effective_xcpus); in reset_partition_data()
1308 if (is_cpu_exclusive(cs)) in reset_partition_data()
1309 clear_bit(CS_CPU_EXCLUSIVE, &cs->flags); in reset_partition_data()
1311 if (!cpumask_and(cs->effective_cpus, parent->effective_cpus, cs->cpus_allowed)) in reset_partition_data()
1312 cpumask_copy(cs->effective_cpus, parent->effective_cpus); in reset_partition_data()
1316 * isolated_cpus_update - Update the isolated_cpus mask
1331 * partition_xcpus_add - Add new exclusive CPUs to partition
1353 isolcpus_updated = (new_prs != parent->partition_root_state); in partition_xcpus_add()
1355 isolated_cpus_update(parent->partition_root_state, new_prs, in partition_xcpus_add()
1358 cpumask_andnot(parent->effective_cpus, parent->effective_cpus, xcpus); in partition_xcpus_add()
1363 * partition_xcpus_del - Remove exclusive CPUs from partition
1384 isolcpus_updated = (old_prs != parent->partition_root_state); in partition_xcpus_del()
1386 isolated_cpus_update(old_prs, parent->partition_root_state, in partition_xcpus_del()
1390 cpumask_or(parent->effective_cpus, parent->effective_cpus, xcpus); in partition_xcpus_del()
1408 * cpuset_cpu_is_isolated - Check if the given CPU is isolated
1419 * rm_siblings_excl_cpus - Remove exclusive CPUs that are used by sibling cpusets
1421 * @cs: Current cpuset (will be skipped)
1426 * of @cs under @parent and removes their exclusive CPUs from @excpus.
1428 static int rm_siblings_excl_cpus(struct cpuset *parent, struct cpuset *cs, in rm_siblings_excl_cpus() argument
1443 if (sibling == cs) in rm_siblings_excl_cpus()
1446 if (cpumask_intersects(excpus, sibling->exclusive_cpus)) { in rm_siblings_excl_cpus()
1447 cpumask_andnot(excpus, excpus, sibling->exclusive_cpus); in rm_siblings_excl_cpus()
1451 if (cpumask_intersects(excpus, sibling->effective_xcpus)) { in rm_siblings_excl_cpus()
1452 cpumask_andnot(excpus, excpus, sibling->effective_xcpus); in rm_siblings_excl_cpus()
1462 * compute_excpus - compute effective exclusive CPUs
1463 * @cs: cpuset
1470 static int compute_excpus(struct cpuset *cs, struct cpumask *excpus) in compute_excpus() argument
1472 struct cpuset *parent = parent_cs(cs); in compute_excpus()
1474 cpumask_and(excpus, user_xcpus(cs), parent->effective_xcpus); in compute_excpus()
1476 if (!cpumask_empty(cs->exclusive_cpus)) in compute_excpus()
1479 return rm_siblings_excl_cpus(parent, cs, excpus); in compute_excpus()
1483 * compute_trialcs_excpus - Compute effective exclusive CPUs for a trial cpuset
1485 * @cs: The original cpuset that the trial configuration is based on
1488 * Computes the effective_xcpus for a trial configuration. @cs is provided to represent
1489 * the real cs.
1491 static int compute_trialcs_excpus(struct cpuset *trialcs, struct cpuset *cs) in compute_trialcs_excpus() argument
1494 struct cpumask *excpus = trialcs->effective_xcpus; in compute_trialcs_excpus()
1497 if (cs_is_member(cs)) in compute_trialcs_excpus()
1498 cpumask_and(excpus, trialcs->exclusive_cpus, in compute_trialcs_excpus()
1499 parent->effective_xcpus); in compute_trialcs_excpus()
1501 cpumask_and(excpus, user_xcpus(trialcs), parent->effective_xcpus); in compute_trialcs_excpus()
1503 return rm_siblings_excl_cpus(parent, cs, excpus); in compute_trialcs_excpus()
1506 static inline bool is_remote_partition(struct cpuset *cs) in is_remote_partition() argument
1508 return !list_empty(&cs->remote_sibling); in is_remote_partition()
1511 static inline bool is_local_partition(struct cpuset *cs) in is_local_partition() argument
1513 return is_partition_valid(cs) && !is_remote_partition(cs); in is_local_partition()
1517 * remote_partition_enable - Enable current cpuset as a remote partition root
1518 * @cs: the cpuset to update
1526 static int remote_partition_enable(struct cpuset *cs, int new_prs, in remote_partition_enable() argument
1547 compute_excpus(cs, tmp->new_cpus); in remote_partition_enable()
1548 WARN_ON_ONCE(cpumask_intersects(tmp->new_cpus, subpartitions_cpus)); in remote_partition_enable()
1549 if (!cpumask_intersects(tmp->new_cpus, cpu_active_mask) || in remote_partition_enable()
1550 cpumask_subset(top_cpuset.effective_cpus, tmp->new_cpus)) in remote_partition_enable()
1554 isolcpus_updated = partition_xcpus_add(new_prs, NULL, tmp->new_cpus); in remote_partition_enable()
1555 list_add(&cs->remote_sibling, &remote_children); in remote_partition_enable()
1556 cpumask_copy(cs->effective_xcpus, tmp->new_cpus); in remote_partition_enable()
1560 cs->prs_err = 0; in remote_partition_enable()
1565 cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus); in remote_partition_enable()
1571 * remote_partition_disable - Remove current cpuset from remote partition list
1572 * @cs: the cpuset to update
1579 static void remote_partition_disable(struct cpuset *cs, struct tmpmasks *tmp) in remote_partition_disable() argument
1583 WARN_ON_ONCE(!is_remote_partition(cs)); in remote_partition_disable()
1584 WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus)); in remote_partition_disable()
1587 list_del_init(&cs->remote_sibling); in remote_partition_disable()
1588 isolcpus_updated = partition_xcpus_del(cs->partition_root_state, in remote_partition_disable()
1589 NULL, cs->effective_xcpus); in remote_partition_disable()
1590 if (cs->prs_err) in remote_partition_disable()
1591 cs->partition_root_state = -cs->partition_root_state; in remote_partition_disable()
1593 cs->partition_root_state = PRS_MEMBER; in remote_partition_disable()
1596 compute_excpus(cs, cs->effective_xcpus); in remote_partition_disable()
1597 reset_partition_data(cs); in remote_partition_disable()
1605 cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus); in remote_partition_disable()
1610 * remote_cpus_update - cpus_exclusive change of remote partition
1611 * @cs: the cpuset to be updated
1612 * @xcpus: the new exclusive_cpus mask, if non-NULL
1619 static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, in remote_cpus_update() argument
1623 int prs = cs->partition_root_state; in remote_cpus_update()
1626 if (WARN_ON_ONCE(!is_remote_partition(cs))) in remote_cpus_update()
1629 WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus)); in remote_cpus_update()
1632 cs->prs_err = PERR_CPUSEMPTY; in remote_cpus_update()
1636 adding = cpumask_andnot(tmp->addmask, excpus, cs->effective_xcpus); in remote_cpus_update()
1637 deleting = cpumask_andnot(tmp->delmask, cs->effective_xcpus, excpus); in remote_cpus_update()
1645 WARN_ON_ONCE(cpumask_intersects(tmp->addmask, subpartitions_cpus)); in remote_cpus_update()
1647 cs->prs_err = PERR_ACCESS; in remote_cpus_update()
1648 else if (cpumask_intersects(tmp->addmask, subpartitions_cpus) || in remote_cpus_update()
1649 cpumask_subset(top_cpuset.effective_cpus, tmp->addmask)) in remote_cpus_update()
1650 cs->prs_err = PERR_NOCPUS; in remote_cpus_update()
1651 if (cs->prs_err) in remote_cpus_update()
1657 isolcpus_updated += partition_xcpus_add(prs, NULL, tmp->addmask); in remote_cpus_update()
1659 isolcpus_updated += partition_xcpus_del(prs, NULL, tmp->delmask); in remote_cpus_update()
1662 * update_sibling_cpumasks() below may iterate back to the same cs. in remote_cpus_update()
1664 cpumask_copy(cs->effective_xcpus, excpus); in remote_cpus_update()
1666 cpumask_copy(cs->exclusive_cpus, xcpus); in remote_cpus_update()
1675 cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus); in remote_cpus_update()
1680 remote_partition_disable(cs, tmp); in remote_cpus_update()
1684 * prstate_housekeeping_conflict - check for partition & housekeeping conflicts
1704 * update_parent_effective_cpumask - update effective_cpus mask of parent cpuset
1705 * @cs: The cpuset that requests change in partition root state
1711 * For partcmd_enable*, the cpuset is being transformed from a non-partition
1718 * root back to a non-partition root. Any CPUs in effective_xcpus will be
1740 static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, in update_parent_effective_cpumask() argument
1744 struct cpuset *parent = parent_cs(cs); in update_parent_effective_cpumask()
1751 struct cpumask *xcpus = user_xcpus(cs); in update_parent_effective_cpumask()
1755 WARN_ON_ONCE(is_remote_partition(cs)); /* For local partition only */ in update_parent_effective_cpumask()
1762 old_prs = new_prs = cs->partition_root_state; in update_parent_effective_cpumask()
1765 if (is_partition_invalid(cs)) in update_parent_effective_cpumask()
1772 adding = cpumask_and(tmp->addmask, in update_parent_effective_cpumask()
1773 xcpus, parent->effective_xcpus); in update_parent_effective_cpumask()
1775 new_prs = -old_prs; in update_parent_effective_cpumask()
1776 subparts_delta--; in update_parent_effective_cpumask()
1790 if (!newmask && xcpus_empty(cs)) in update_parent_effective_cpumask()
1793 nocpu = tasks_nocpu_error(parent, cs, xcpus); in update_parent_effective_cpumask()
1801 xcpus = tmp->delmask; in update_parent_effective_cpumask()
1802 if (compute_excpus(cs, xcpus)) in update_parent_effective_cpumask()
1803 WARN_ON_ONCE(!cpumask_empty(cs->exclusive_cpus)); in update_parent_effective_cpumask()
1816 if (tasks_nocpu_error(parent, cs, xcpus)) in update_parent_effective_cpumask()
1824 * (cs->effective_xcpus & cpu_active_mask) ⊆ parent->effective_cpus in update_parent_effective_cpumask()
1828 cpumask_and(tmp->new_cpus, xcpus, cpu_active_mask); in update_parent_effective_cpumask()
1829 WARN_ON_ONCE(!cpumask_subset(tmp->new_cpus, parent->effective_cpus)); in update_parent_effective_cpumask()
1840 if (is_partition_valid(cs)) { in update_parent_effective_cpumask()
1841 cpumask_copy(tmp->addmask, cs->effective_xcpus); in update_parent_effective_cpumask()
1843 subparts_delta--; in update_parent_effective_cpumask()
1855 /* Check newmask again, whether cpus are available for parent/cs */ in update_parent_effective_cpumask()
1856 nocpu |= tasks_nocpu_error(parent, cs, newmask); in update_parent_effective_cpumask()
1865 * & parent->effective_xcpus in update_parent_effective_cpumask()
1867 * & parent->effective_xcpus in update_parent_effective_cpumask()
1870 * delmask = newmask & parent->effective_xcpus in update_parent_effective_cpumask()
1872 if (is_partition_invalid(cs)) { in update_parent_effective_cpumask()
1874 deleting = cpumask_and(tmp->delmask, in update_parent_effective_cpumask()
1875 newmask, parent->effective_xcpus); in update_parent_effective_cpumask()
1877 cpumask_andnot(tmp->addmask, xcpus, newmask); in update_parent_effective_cpumask()
1878 adding = cpumask_and(tmp->addmask, tmp->addmask, in update_parent_effective_cpumask()
1879 parent->effective_xcpus); in update_parent_effective_cpumask()
1881 cpumask_andnot(tmp->delmask, newmask, xcpus); in update_parent_effective_cpumask()
1882 deleting = cpumask_and(tmp->delmask, tmp->delmask, in update_parent_effective_cpumask()
1883 parent->effective_xcpus); in update_parent_effective_cpumask()
1890 cpumask_and(tmp->new_cpus, tmp->delmask, cpu_active_mask); in update_parent_effective_cpumask()
1891 WARN_ON_ONCE(!cpumask_subset(tmp->new_cpus, parent->effective_cpus)); in update_parent_effective_cpumask()
1899 !cpumask_intersects(tmp->addmask, cpu_active_mask))) { in update_parent_effective_cpumask()
1902 adding = cpumask_and(tmp->addmask, in update_parent_effective_cpumask()
1903 xcpus, parent->effective_xcpus); in update_parent_effective_cpumask()
1909 * delmask = effective_xcpus & parent->effective_cpus in update_parent_effective_cpumask()
1919 * its effective CPUs will have to be distributed out. in update_parent_effective_cpumask()
1923 if (is_partition_valid(cs)) in update_parent_effective_cpumask()
1924 adding = cpumask_and(tmp->addmask, in update_parent_effective_cpumask()
1925 xcpus, parent->effective_xcpus); in update_parent_effective_cpumask()
1926 } else if (is_partition_invalid(cs) && !cpumask_empty(xcpus) && in update_parent_effective_cpumask()
1927 cpumask_subset(xcpus, parent->effective_xcpus)) { in update_parent_effective_cpumask()
1938 if (child == cs) in update_parent_effective_cpumask()
1940 if (!cpusets_are_exclusive(cs, child)) { in update_parent_effective_cpumask()
1947 deleting = cpumask_and(tmp->delmask, in update_parent_effective_cpumask()
1948 xcpus, parent->effective_cpus); in update_parent_effective_cpumask()
1956 WRITE_ONCE(cs->prs_err, part_error); in update_parent_effective_cpumask()
1963 switch (cs->partition_root_state) { in update_parent_effective_cpumask()
1967 new_prs = -old_prs; in update_parent_effective_cpumask()
1968 subparts_delta--; in update_parent_effective_cpumask()
1974 new_prs = -old_prs; in update_parent_effective_cpumask()
1988 * CPU lists in cs haven't been updated yet. So defer it to later. in update_parent_effective_cpumask()
1991 int err = update_partition_exclusive_flag(cs, new_prs); in update_parent_effective_cpumask()
2006 cs->partition_root_state = new_prs; in update_parent_effective_cpumask()
2008 cs->nr_subparts = 0; in update_parent_effective_cpumask()
2011 * Adding to parent's effective_cpus means deletion CPUs from cs in update_parent_effective_cpumask()
2016 tmp->addmask); in update_parent_effective_cpumask()
2019 tmp->delmask); in update_parent_effective_cpumask()
2022 parent->nr_subparts += subparts_delta; in update_parent_effective_cpumask()
2023 WARN_ON_ONCE(parent->nr_subparts < 0); in update_parent_effective_cpumask()
2029 update_partition_exclusive_flag(cs, new_prs); in update_parent_effective_cpumask()
2032 cpuset_update_tasks_cpumask(parent, tmp->addmask); in update_parent_effective_cpumask()
2033 update_sibling_cpumasks(parent, cs, tmp); in update_parent_effective_cpumask()
2042 update_partition_sd_lb(cs, old_prs); in update_parent_effective_cpumask()
2044 notify_partition_change(cs, old_prs); in update_parent_effective_cpumask()
2049 * compute_partition_effective_cpumask - compute effective_cpus for partition
2050 * @cs: partition root cpuset
2064 static void compute_partition_effective_cpumask(struct cpuset *cs, in compute_partition_effective_cpumask() argument
2069 bool populated = partition_is_populated(cs, NULL); in compute_partition_effective_cpumask()
2079 compute_excpus(cs, new_ecpus); in compute_partition_effective_cpumask()
2083 cpuset_for_each_child(child, css, cs) { in compute_partition_effective_cpumask()
2092 child->prs_err = 0; in compute_partition_effective_cpumask()
2093 if (!cpumask_subset(child->effective_xcpus, in compute_partition_effective_cpumask()
2094 cs->effective_xcpus)) in compute_partition_effective_cpumask()
2095 child->prs_err = PERR_INVCPUS; in compute_partition_effective_cpumask()
2097 cpumask_subset(new_ecpus, child->effective_xcpus)) in compute_partition_effective_cpumask()
2098 child->prs_err = PERR_NOCPUS; in compute_partition_effective_cpumask()
2100 if (child->prs_err) { in compute_partition_effective_cpumask()
2101 int old_prs = child->partition_root_state; in compute_partition_effective_cpumask()
2108 cs->nr_subparts--; in compute_partition_effective_cpumask()
2109 child->nr_subparts = 0; in compute_partition_effective_cpumask()
2115 child->effective_xcpus); in compute_partition_effective_cpumask()
2121 * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree
2122 * @cs: the cpuset to consider
2133 static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, in update_cpumasks_hier() argument
2142 cpuset_for_each_descendant_pre(cp, pos_css, cs) { in update_cpumasks_hier()
2147 old_prs = new_prs = cp->partition_root_state; in update_cpumasks_hier()
2150 * For child remote partition root (!= cs), we need to call in update_cpumasks_hier()
2154 * remote_cpus_update() will reuse tmp->new_cpus only after in update_cpumasks_hier()
2157 if (remote && (cp != cs)) { in update_cpumasks_hier()
2158 compute_excpus(cp, tmp->new_cpus); in update_cpumasks_hier()
2159 if (cpumask_equal(cp->effective_xcpus, tmp->new_cpus)) { in update_cpumasks_hier()
2164 remote_cpus_update(cp, NULL, tmp->new_cpus, tmp); in update_cpumasks_hier()
2168 new_prs = cp->partition_root_state; in update_cpumasks_hier()
2173 compute_partition_effective_cpumask(cp, tmp->new_cpus); in update_cpumasks_hier()
2175 compute_effective_cpumask(tmp->new_cpus, cp, parent); in update_cpumasks_hier()
2185 if (is_partition_valid(cp) && cpumask_empty(tmp->new_cpus)) { in update_cpumasks_hier()
2194 * out all its CPUs. in update_cpumasks_hier()
2196 if (is_in_v2_mode() && !remote && cpumask_empty(tmp->new_cpus)) in update_cpumasks_hier()
2197 cpumask_copy(tmp->new_cpus, parent->effective_cpus); in update_cpumasks_hier()
2206 if (!cp->partition_root_state && !force && in update_cpumasks_hier()
2207 cpumask_equal(tmp->new_cpus, cp->effective_cpus) && in update_cpumasks_hier()
2217 * for cs already in update_cpumask(). We should also call in update_cpumasks_hier()
2221 if ((cp != cs) && old_prs) { in update_cpumasks_hier()
2222 switch (parent->partition_root_state) { in update_cpumasks_hier()
2235 new_prs = -cp->partition_root_state; in update_cpumasks_hier()
2236 WRITE_ONCE(cp->prs_err, in update_cpumasks_hier()
2243 if (!css_tryget_online(&cp->css)) in update_cpumasks_hier()
2253 new_prs = cp->partition_root_state; in update_cpumasks_hier()
2257 cpumask_copy(cp->effective_cpus, tmp->new_cpus); in update_cpumasks_hier()
2258 cp->partition_root_state = new_prs; in update_cpumasks_hier()
2259 if (!cpumask_empty(cp->exclusive_cpus) && (cp != cs)) in update_cpumasks_hier()
2260 compute_excpus(cp, cp->effective_xcpus); in update_cpumasks_hier()
2266 if ((new_prs > 0) && cpumask_empty(cp->exclusive_cpus)) in update_cpumasks_hier()
2267 cpumask_and(cp->effective_xcpus, in update_cpumasks_hier()
2268 cp->cpus_allowed, parent->effective_xcpus); in update_cpumasks_hier()
2276 !cpumask_equal(cp->cpus_allowed, cp->effective_cpus)); in update_cpumasks_hier()
2278 cpuset_update_tasks_cpumask(cp, cp->effective_cpus); in update_cpumasks_hier()
2288 set_bit(CS_SCHED_LOAD_BALANCE, &cp->flags); in update_cpumasks_hier()
2290 clear_bit(CS_SCHED_LOAD_BALANCE, &cp->flags); in update_cpumasks_hier()
2294 * On legacy hierarchy, if the effective cpumask of any non- in update_cpumasks_hier()
2299 if (!cpumask_empty(cp->cpus_allowed) && in update_cpumasks_hier()
2305 css_put(&cp->css); in update_cpumasks_hier()
2314 * update_sibling_cpumasks - Update siblings cpumasks
2316 * @cs: Current cpuset
2319 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs, in update_sibling_cpumasks() argument
2341 if (sibling == cs) in update_sibling_cpumasks()
2344 compute_effective_cpumask(tmp->new_cpus, sibling, in update_sibling_cpumasks()
2346 if (cpumask_equal(tmp->new_cpus, sibling->effective_cpus)) in update_sibling_cpumasks()
2356 if (!css_tryget_online(&sibling->css)) in update_sibling_cpumasks()
2362 css_put(&sibling->css); in update_sibling_cpumasks()
2375 return -EINVAL; in parse_cpuset_cpulist()
2381 * validate_partition - Validate a cpuset partition configuration
2382 * @cs: The cpuset to validate
2388 * Return: PRS error code (0 if valid, non-zero error code if invalid)
2390 static enum prs_errcode validate_partition(struct cpuset *cs, struct cpuset *trialcs) in validate_partition() argument
2392 struct cpuset *parent = parent_cs(cs); in validate_partition()
2397 if (cpumask_empty(trialcs->effective_xcpus)) in validate_partition()
2400 if (prstate_housekeeping_conflict(trialcs->partition_root_state, in validate_partition()
2401 trialcs->effective_xcpus)) in validate_partition()
2404 if (tasks_nocpu_error(parent, cs, trialcs->effective_xcpus)) in validate_partition()
2410 static int cpus_allowed_validate_change(struct cpuset *cs, struct cpuset *trialcs, in cpus_allowed_validate_change() argument
2414 struct cpuset *parent = parent_cs(cs); in cpus_allowed_validate_change()
2416 retval = validate_change(cs, trialcs); in cpus_allowed_validate_change()
2418 if ((retval == -EINVAL) && cpuset_v2()) { in cpus_allowed_validate_change()
2423 * The -EINVAL error code indicates that partition sibling in cpus_allowed_validate_change()
2429 trialcs->prs_err = PERR_NOTEXCL; in cpus_allowed_validate_change()
2435 cpumask_intersects(xcpus, cp->effective_xcpus)) { in cpus_allowed_validate_change()
2448 * partition_cpus_change - Handle partition state changes due to CPU mask updates
2449 * @cs: The target cpuset being modified
2456 static void partition_cpus_change(struct cpuset *cs, struct cpuset *trialcs, in partition_cpus_change() argument
2461 if (cs_is_member(cs)) in partition_cpus_change()
2464 prs_err = validate_partition(cs, trialcs); in partition_cpus_change()
2466 trialcs->prs_err = cs->prs_err = prs_err; in partition_cpus_change()
2468 if (is_remote_partition(cs)) { in partition_cpus_change()
2469 if (trialcs->prs_err) in partition_cpus_change()
2470 remote_partition_disable(cs, tmp); in partition_cpus_change()
2472 remote_cpus_update(cs, trialcs->exclusive_cpus, in partition_cpus_change()
2473 trialcs->effective_xcpus, tmp); in partition_cpus_change()
2475 if (trialcs->prs_err) in partition_cpus_change()
2476 update_parent_effective_cpumask(cs, partcmd_invalidate, in partition_cpus_change()
2479 update_parent_effective_cpumask(cs, partcmd_update, in partition_cpus_change()
2480 trialcs->effective_xcpus, tmp); in partition_cpus_change()
2485 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
2486 * @cs: the cpuset to consider
2490 static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, in update_cpumask() argument
2496 int old_prs = cs->partition_root_state; in update_cpumask()
2498 retval = parse_cpuset_cpulist(buf, trialcs->cpus_allowed); in update_cpumask()
2503 if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed)) in update_cpumask()
2507 return -ENOMEM; in update_cpumask()
2509 compute_trialcs_excpus(trialcs, cs); in update_cpumask()
2510 trialcs->prs_err = PERR_NONE; in update_cpumask()
2512 retval = cpus_allowed_validate_change(cs, trialcs, &tmp); in update_cpumask()
2520 force = !cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus); in update_cpumask()
2522 partition_cpus_change(cs, trialcs, &tmp); in update_cpumask()
2525 cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed); in update_cpumask()
2526 cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus); in update_cpumask()
2527 if ((old_prs > 0) && !is_partition_valid(cs)) in update_cpumask()
2528 reset_partition_data(cs); in update_cpumask()
2532 update_cpumasks_hier(cs, &tmp, force); in update_cpumask()
2535 if (cs->partition_root_state) in update_cpumask()
2536 update_partition_sd_lb(cs, old_prs); in update_cpumask()
2543 * update_exclusive_cpumask - update the exclusive_cpus mask of a cpuset
2544 * @cs: the cpuset to consider
2548 * The tasks' cpumask will be updated if cs is a valid partition root.
2550 static int update_exclusive_cpumask(struct cpuset *cs, struct cpuset *trialcs, in update_exclusive_cpumask() argument
2556 int old_prs = cs->partition_root_state; in update_exclusive_cpumask()
2558 retval = parse_cpuset_cpulist(buf, trialcs->exclusive_cpus); in update_exclusive_cpumask()
2563 if (cpumask_equal(cs->exclusive_cpus, trialcs->exclusive_cpus)) in update_exclusive_cpumask()
2570 if (compute_trialcs_excpus(trialcs, cs)) in update_exclusive_cpumask()
2571 return -EINVAL; in update_exclusive_cpumask()
2577 force = !cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus); in update_exclusive_cpumask()
2579 retval = validate_change(cs, trialcs); in update_exclusive_cpumask()
2584 return -ENOMEM; in update_exclusive_cpumask()
2586 trialcs->prs_err = PERR_NONE; in update_exclusive_cpumask()
2587 partition_cpus_change(cs, trialcs, &tmp); in update_exclusive_cpumask()
2590 cpumask_copy(cs->exclusive_cpus, trialcs->exclusive_cpus); in update_exclusive_cpumask()
2591 cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus); in update_exclusive_cpumask()
2592 if ((old_prs > 0) && !is_partition_valid(cs)) in update_exclusive_cpumask()
2593 reset_partition_data(cs); in update_exclusive_cpumask()
2601 if (is_partition_valid(cs) || force) in update_exclusive_cpumask()
2602 update_cpumasks_hier(cs, &tmp, force); in update_exclusive_cpumask()
2605 if (cs->partition_root_state) in update_exclusive_cpumask()
2606 update_partition_sd_lb(cs, old_prs); in update_exclusive_cpumask()
2633 do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL); in cpuset_migrate_mm_workfn()
2634 mmput(mwork->mm); in cpuset_migrate_mm_workfn()
2650 mwork->mm = mm; in cpuset_migrate_mm()
2651 mwork->from = *from; in cpuset_migrate_mm()
2652 mwork->to = *to; in cpuset_migrate_mm()
2653 INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn); in cpuset_migrate_mm()
2654 queue_work(cpuset_migrate_mm_wq, &mwork->work); in cpuset_migrate_mm()
2681 * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
2685 * We use the mems_allowed_seq seqlock to safely update both tsk->mems_allowed
2696 write_seqcount_begin(&tsk->mems_allowed_seq); in cpuset_change_task_nodemask()
2698 nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems); in cpuset_change_task_nodemask()
2700 tsk->mems_allowed = *newmems; in cpuset_change_task_nodemask()
2702 write_seqcount_end(&tsk->mems_allowed_seq); in cpuset_change_task_nodemask()
2711 * cpuset_update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
2712 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
2714 * Iterate through each task of @cs updating its mems_allowed to the
2718 void cpuset_update_tasks_nodemask(struct cpuset *cs) in cpuset_update_tasks_nodemask() argument
2724 cpuset_being_rebound = cs; /* causes mpol_dup() rebind */ in cpuset_update_tasks_nodemask()
2726 guarantee_online_mems(cs, &newmems); in cpuset_update_tasks_nodemask()
2730 * take while holding tasklist_lock. Forks can happen - the in cpuset_update_tasks_nodemask()
2738 css_task_iter_start(&cs->css, 0, &it); in cpuset_update_tasks_nodemask()
2749 migrate = is_memory_migrate(cs); in cpuset_update_tasks_nodemask()
2751 mpol_rebind_mm(mm, &cs->mems_allowed); in cpuset_update_tasks_nodemask()
2753 cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems); in cpuset_update_tasks_nodemask()
2761 * cs->old_mems_allowed. in cpuset_update_tasks_nodemask()
2763 cs->old_mems_allowed = newmems; in cpuset_update_tasks_nodemask()
2770 * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree
2771 * @cs: the cpuset to consider
2781 static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) in update_nodemasks_hier() argument
2787 cpuset_for_each_descendant_pre(cp, pos_css, cs) { in update_nodemasks_hier()
2790 nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems); in update_nodemasks_hier()
2797 *new_mems = parent->effective_mems; in update_nodemasks_hier()
2800 if (nodes_equal(*new_mems, cp->effective_mems)) { in update_nodemasks_hier()
2805 if (!css_tryget_online(&cp->css)) in update_nodemasks_hier()
2810 cp->effective_mems = *new_mems; in update_nodemasks_hier()
2814 !nodes_equal(cp->mems_allowed, cp->effective_mems)); in update_nodemasks_hier()
2819 css_put(&cp->css); in update_nodemasks_hier()
2833 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
2834 * lock each such tasks mm->mmap_lock, scan its vma's and rebind
2837 static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs, in update_nodemask() argument
2846 retval = nodelist_parse(buf, trialcs->mems_allowed); in update_nodemask()
2850 if (!nodes_subset(trialcs->mems_allowed, in update_nodemask()
2852 retval = -EINVAL; in update_nodemask()
2856 if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) { in update_nodemask()
2857 retval = 0; /* Too easy - nothing to do */ in update_nodemask()
2860 retval = validate_change(cs, trialcs); in update_nodemask()
2864 check_insane_mems_config(&trialcs->mems_allowed); in update_nodemask()
2867 cs->mems_allowed = trialcs->mems_allowed; in update_nodemask()
2870 /* use trialcs->mems_allowed as a temp variable */ in update_nodemask()
2871 update_nodemasks_hier(cs, &trialcs->mems_allowed); in update_nodemask()
2888 * cpuset_update_flag - read a 0 or a 1 in a file and update associated flag
2890 * cs: the cpuset to update
2896 int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, in cpuset_update_flag() argument
2904 trialcs = dup_or_alloc_cpuset(cs); in cpuset_update_flag()
2906 return -ENOMEM; in cpuset_update_flag()
2909 set_bit(bit, &trialcs->flags); in cpuset_update_flag()
2911 clear_bit(bit, &trialcs->flags); in cpuset_update_flag()
2913 err = validate_change(cs, trialcs); in cpuset_update_flag()
2915 goto out; in cpuset_update_flag()
2917 balance_flag_changed = (is_sched_load_balance(cs) != in cpuset_update_flag()
2920 spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs)) in cpuset_update_flag()
2921 || (is_spread_page(cs) != is_spread_page(trialcs))); in cpuset_update_flag()
2924 cs->flags = trialcs->flags; in cpuset_update_flag()
2927 if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) { in cpuset_update_flag()
2935 cpuset1_update_tasks_flags(cs); in cpuset_update_flag()
2936 out: in cpuset_update_flag()
2942 * update_prstate - update partition_root_state
2943 * @cs: the cpuset to update
2949 static int update_prstate(struct cpuset *cs, int new_prs) in update_prstate() argument
2951 int err = PERR_NONE, old_prs = cs->partition_root_state; in update_prstate()
2952 struct cpuset *parent = parent_cs(cs); in update_prstate()
2962 if (new_prs && is_partition_invalid(cs)) in update_prstate()
2966 return -ENOMEM; in update_prstate()
2968 err = update_partition_exclusive_flag(cs, new_prs); in update_prstate()
2970 goto out; in update_prstate()
2976 if (xcpus_empty(cs)) { in update_prstate()
2978 goto out; in update_prstate()
2989 cpumask_intersects(cs->exclusive_cpus, subpartitions_cpus)) { in update_prstate()
2991 goto out; in update_prstate()
3002 err = update_parent_effective_cpumask(cs, cmd, NULL, &tmpmask); in update_prstate()
3004 err = remote_partition_enable(cs, new_prs, &tmpmask); in update_prstate()
3017 if (is_remote_partition(cs)) in update_prstate()
3018 remote_partition_disable(cs, &tmpmask); in update_prstate()
3020 update_parent_effective_cpumask(cs, partcmd_disable, in update_prstate()
3028 out: in update_prstate()
3034 new_prs = -new_prs; in update_prstate()
3035 update_partition_exclusive_flag(cs, new_prs); in update_prstate()
3039 cs->partition_root_state = new_prs; in update_prstate()
3040 WRITE_ONCE(cs->prs_err, err); in update_prstate()
3041 if (!is_partition_valid(cs)) in update_prstate()
3042 reset_partition_data(cs); in update_prstate()
3044 isolated_cpus_update(old_prs, new_prs, cs->effective_xcpus); in update_prstate()
3049 update_cpumasks_hier(cs, &tmpmask, !new_prs); in update_prstate()
3053 && cpumask_empty(cs->effective_xcpus)); in update_prstate()
3056 update_partition_sd_lb(cs, old_prs); in update_prstate()
3058 notify_partition_change(cs, old_prs); in update_prstate()
3073 static int cpuset_can_attach_check(struct cpuset *cs) in cpuset_can_attach_check() argument
3075 if (cpumask_empty(cs->effective_cpus) || in cpuset_can_attach_check()
3076 (!is_in_v2_mode() && nodes_empty(cs->mems_allowed))) in cpuset_can_attach_check()
3077 return -ENOSPC; in cpuset_can_attach_check()
3081 static void reset_migrate_dl_data(struct cpuset *cs) in reset_migrate_dl_data() argument
3083 cs->nr_migrate_dl_tasks = 0; in reset_migrate_dl_data()
3084 cs->sum_migrate_dl_bw = 0; in reset_migrate_dl_data()
3091 struct cpuset *cs, *oldcs; in cpuset_can_attach() local
3099 cs = css_cs(css); in cpuset_can_attach()
3104 ret = cpuset_can_attach_check(cs); in cpuset_can_attach()
3108 cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus); in cpuset_can_attach()
3109 mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); in cpuset_can_attach()
3128 cs->nr_migrate_dl_tasks++; in cpuset_can_attach()
3129 cs->sum_migrate_dl_bw += task->dl.dl_bw; in cpuset_can_attach()
3133 if (!cs->nr_migrate_dl_tasks) in cpuset_can_attach()
3136 if (!cpumask_intersects(oldcs->effective_cpus, cs->effective_cpus)) { in cpuset_can_attach()
3137 int cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); in cpuset_can_attach()
3140 reset_migrate_dl_data(cs); in cpuset_can_attach()
3141 ret = -EINVAL; in cpuset_can_attach()
3145 ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); in cpuset_can_attach()
3147 reset_migrate_dl_data(cs); in cpuset_can_attach()
3157 cs->attach_in_progress++; in cpuset_can_attach()
3166 struct cpuset *cs; in cpuset_cancel_attach() local
3169 cs = css_cs(css); in cpuset_cancel_attach()
3172 dec_attach_in_progress_locked(cs); in cpuset_cancel_attach()
3174 if (cs->nr_migrate_dl_tasks) { in cpuset_cancel_attach()
3175 int cpu = cpumask_any(cs->effective_cpus); in cpuset_cancel_attach()
3177 dl_bw_free(cpu, cs->sum_migrate_dl_bw); in cpuset_cancel_attach()
3178 reset_migrate_dl_data(cs); in cpuset_cancel_attach()
3192 static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task) in cpuset_attach_task() argument
3196 if (cs != &top_cpuset) in cpuset_attach_task()
3208 cpuset1_update_task_spread_flags(cs, task); in cpuset_attach_task()
3216 struct cpuset *cs; in cpuset_attach() local
3222 cs = css_cs(css); in cpuset_attach()
3226 cpus_updated = !cpumask_equal(cs->effective_cpus, in cpuset_attach()
3227 oldcs->effective_cpus); in cpuset_attach()
3228 mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); in cpuset_attach()
3233 * in effective cpus and mems. In that case, we can optimize out in cpuset_attach()
3237 cpuset_attach_nodemask_to = cs->effective_mems; in cpuset_attach()
3238 goto out; in cpuset_attach()
3241 guarantee_online_mems(cs, &cpuset_attach_nodemask_to); in cpuset_attach()
3244 cpuset_attach_task(cs, task); in cpuset_attach()
3252 cpuset_attach_nodemask_to = cs->effective_mems; in cpuset_attach()
3253 if (!is_memory_migrate(cs) && !mems_updated) in cpuset_attach()
3254 goto out; in cpuset_attach()
3270 if (is_memory_migrate(cs)) { in cpuset_attach()
3271 cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, in cpuset_attach()
3279 out: in cpuset_attach()
3282 cs->old_mems_allowed = cpuset_attach_nodemask_to; in cpuset_attach()
3284 if (cs->nr_migrate_dl_tasks) { in cpuset_attach()
3285 cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks; in cpuset_attach()
3286 oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks; in cpuset_attach()
3287 reset_migrate_dl_data(cs); in cpuset_attach()
3290 dec_attach_in_progress_locked(cs); in cpuset_attach()
3301 struct cpuset *cs = css_cs(of_css(of)); in cpuset_write_resmask() local
3303 int retval = -ENODEV; in cpuset_write_resmask()
3305 /* root is read-only */ in cpuset_write_resmask()
3306 if (cs == &top_cpuset) in cpuset_write_resmask()
3307 return -EACCES; in cpuset_write_resmask()
3311 if (!is_cpuset_online(cs)) in cpuset_write_resmask()
3314 trialcs = dup_or_alloc_cpuset(cs); in cpuset_write_resmask()
3316 retval = -ENOMEM; in cpuset_write_resmask()
3320 switch (of_cft(of)->private) { in cpuset_write_resmask()
3322 retval = update_cpumask(cs, trialcs, buf); in cpuset_write_resmask()
3325 retval = update_exclusive_cpumask(cs, trialcs, buf); in cpuset_write_resmask()
3328 retval = update_nodemask(cs, trialcs, buf); in cpuset_write_resmask()
3331 retval = -EINVAL; in cpuset_write_resmask()
3340 if (of_cft(of)->private == FILE_MEMLIST) in cpuset_write_resmask()
3355 struct cpuset *cs = css_cs(seq_css(sf)); in cpuset_common_seq_show() local
3356 cpuset_filetype_t type = seq_cft(sf)->private; in cpuset_common_seq_show()
3363 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_allowed)); in cpuset_common_seq_show()
3366 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed)); in cpuset_common_seq_show()
3369 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus)); in cpuset_common_seq_show()
3372 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems)); in cpuset_common_seq_show()
3375 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->exclusive_cpus)); in cpuset_common_seq_show()
3378 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_xcpus)); in cpuset_common_seq_show()
3387 ret = -EINVAL; in cpuset_common_seq_show()
3396 struct cpuset *cs = css_cs(seq_css(seq)); in cpuset_partition_show() local
3399 switch (cs->partition_root_state) { in cpuset_partition_show()
3415 err = perr_strings[READ_ONCE(cs->prs_err)]; in cpuset_partition_show()
3428 struct cpuset *cs = css_cs(of_css(of)); in cpuset_partition_write() local
3430 int retval = -ENODEV; in cpuset_partition_write()
3441 return -EINVAL; in cpuset_partition_write()
3444 if (is_cpuset_online(cs)) in cpuset_partition_write()
3445 retval = update_prstate(cs, val); in cpuset_partition_write()
3529 * cpuset_css_alloc - Allocate a cpuset css
3532 * Return: cpuset css on success, -ENOMEM on failure.
3534 * Allocate and initialize a new cpuset css, for non-NULL @parent_css, return
3540 struct cpuset *cs; in cpuset_css_alloc() local
3545 cs = dup_or_alloc_cpuset(NULL); in cpuset_css_alloc()
3546 if (!cs) in cpuset_css_alloc()
3547 return ERR_PTR(-ENOMEM); in cpuset_css_alloc()
3549 __set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in cpuset_css_alloc()
3550 fmeter_init(&cs->fmeter); in cpuset_css_alloc()
3551 cs->relax_domain_level = -1; in cpuset_css_alloc()
3552 INIT_LIST_HEAD(&cs->remote_sibling); in cpuset_css_alloc()
3556 __set_bit(CS_MEMORY_MIGRATE, &cs->flags); in cpuset_css_alloc()
3558 return &cs->css; in cpuset_css_alloc()
3563 struct cpuset *cs = css_cs(css); in cpuset_css_online() local
3564 struct cpuset *parent = parent_cs(cs); in cpuset_css_online()
3573 set_bit(CS_SPREAD_PAGE, &cs->flags); in cpuset_css_online()
3575 set_bit(CS_SPREAD_SLAB, &cs->flags); in cpuset_css_online()
3580 clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in cpuset_css_online()
3586 cpumask_copy(cs->effective_cpus, parent->effective_cpus); in cpuset_css_online()
3587 cs->effective_mems = parent->effective_mems; in cpuset_css_online()
3591 if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags)) in cpuset_css_online()
3597 * historical reasons - the flag may be specified during mount. in cpuset_css_online()
3600 * refuse to clone the configuration - thereby refusing the task to in cpuset_css_online()
3604 * changed to grant parent->cpus_allowed-sibling_cpus_exclusive in cpuset_css_online()
3617 cs->mems_allowed = parent->mems_allowed; in cpuset_css_online()
3618 cs->effective_mems = parent->mems_allowed; in cpuset_css_online()
3619 cpumask_copy(cs->cpus_allowed, parent->cpus_allowed); in cpuset_css_online()
3620 cpumask_copy(cs->effective_cpus, parent->cpus_allowed); in cpuset_css_online()
3636 struct cpuset *cs = css_cs(css); in cpuset_css_offline() local
3639 if (!cpuset_v2() && is_sched_load_balance(cs)) in cpuset_css_offline()
3640 cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); in cpuset_css_offline()
3653 struct cpuset *cs = css_cs(css); in cpuset_css_killed() local
3657 if (is_partition_valid(cs)) in cpuset_css_killed()
3658 update_prstate(cs, PRS_MEMBER); in cpuset_css_killed()
3664 struct cpuset *cs = css_cs(css); in cpuset_css_free() local
3666 free_cpuset(cs); in cpuset_css_free()
3694 struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]); in cpuset_can_fork() local
3699 same_cs = (cs == task_cs(current)); in cpuset_can_fork()
3709 ret = cpuset_can_attach_check(cs); in cpuset_can_fork()
3725 cs->attach_in_progress++; in cpuset_can_fork()
3733 struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]); in cpuset_cancel_fork() local
3737 same_cs = (cs == task_cs(current)); in cpuset_cancel_fork()
3743 dec_attach_in_progress(cs); in cpuset_cancel_fork()
3753 struct cpuset *cs; in cpuset_fork() local
3757 cs = task_cs(task); in cpuset_fork()
3758 same_cs = (cs == task_cs(current)); in cpuset_fork()
3762 if (cs == &top_cpuset) in cpuset_fork()
3765 set_cpus_allowed_ptr(task, current->cpus_ptr); in cpuset_fork()
3766 task->mems_allowed = current->mems_allowed; in cpuset_fork()
3772 guarantee_online_mems(cs, &cpuset_attach_nodemask_to); in cpuset_fork()
3773 cpuset_attach_task(cs, task); in cpuset_fork()
3775 dec_attach_in_progress_locked(cs); in cpuset_fork()
3801 * cpuset_init - initialize cpusets at system boot
3838 hotplug_update_tasks(struct cpuset *cs, in hotplug_update_tasks() argument
3843 if (cpumask_empty(new_cpus) && !is_partition_valid(cs)) in hotplug_update_tasks()
3844 cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus); in hotplug_update_tasks()
3846 *new_mems = parent_cs(cs)->effective_mems; in hotplug_update_tasks()
3849 cpumask_copy(cs->effective_cpus, new_cpus); in hotplug_update_tasks()
3850 cs->effective_mems = *new_mems; in hotplug_update_tasks()
3854 cpuset_update_tasks_cpumask(cs, new_cpus); in hotplug_update_tasks()
3856 cpuset_update_tasks_nodemask(cs); in hotplug_update_tasks()
3865 * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
3866 * @cs: cpuset in interest
3869 * Compare @cs's cpu and mem masks against top_cpuset and if some have gone
3870 * offline, update @cs accordingly. If @cs ends up with no CPU or memory,
3873 static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) in cpuset_hotplug_update_tasks() argument
3880 int partcmd = -1; in cpuset_hotplug_update_tasks()
3883 wait_event(cpuset_attach_wq, cs->attach_in_progress == 0); in cpuset_hotplug_update_tasks()
3891 if (cs->attach_in_progress) { in cpuset_hotplug_update_tasks()
3896 parent = parent_cs(cs); in cpuset_hotplug_update_tasks()
3897 compute_effective_cpumask(&new_cpus, cs, parent); in cpuset_hotplug_update_tasks()
3898 nodes_and(new_mems, cs->mems_allowed, parent->effective_mems); in cpuset_hotplug_update_tasks()
3900 if (!tmp || !cs->partition_root_state) in cpuset_hotplug_update_tasks()
3907 remote = is_remote_partition(cs); in cpuset_hotplug_update_tasks()
3908 if (remote || (is_partition_valid(cs) && is_partition_valid(parent))) in cpuset_hotplug_update_tasks()
3909 compute_partition_effective_cpumask(cs, &new_cpus); in cpuset_hotplug_update_tasks()
3912 partition_is_populated(cs, NULL)) { in cpuset_hotplug_update_tasks()
3913 cs->prs_err = PERR_HOTPLUG; in cpuset_hotplug_update_tasks()
3914 remote_partition_disable(cs, tmp); in cpuset_hotplug_update_tasks()
3915 compute_effective_cpumask(&new_cpus, cs, parent); in cpuset_hotplug_update_tasks()
3926 if (is_local_partition(cs) && (!is_partition_valid(parent) || in cpuset_hotplug_update_tasks()
3927 tasks_nocpu_error(parent, cs, &new_cpus))) in cpuset_hotplug_update_tasks()
3931 * back to a regular one with a non-empty effective xcpus. in cpuset_hotplug_update_tasks()
3933 else if (is_partition_valid(parent) && is_partition_invalid(cs) && in cpuset_hotplug_update_tasks()
3934 !cpumask_empty(cs->effective_xcpus)) in cpuset_hotplug_update_tasks()
3938 update_parent_effective_cpumask(cs, partcmd, NULL, tmp); in cpuset_hotplug_update_tasks()
3939 if ((partcmd == partcmd_invalidate) || is_partition_valid(cs)) { in cpuset_hotplug_update_tasks()
3940 compute_partition_effective_cpumask(cs, &new_cpus); in cpuset_hotplug_update_tasks()
3946 cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus); in cpuset_hotplug_update_tasks()
3947 mems_updated = !nodes_equal(new_mems, cs->effective_mems); in cpuset_hotplug_update_tasks()
3955 hotplug_update_tasks(cs, &new_cpus, &new_mems, in cpuset_hotplug_update_tasks()
3958 cpuset1_hotplug_update_tasks(cs, &new_cpus, &new_mems, in cpuset_hotplug_update_tasks()
3966 * cpuset_handle_hotplug - handle CPU/memory hot{,un}plug for a cpuset
3974 * Non-root cpusets are only affected by offlining. If any CPUs or memory
3997 /* fetch the available cpus/mems and find out which changed how */ in cpuset_handle_hotplug()
4050 struct cpuset *cs; in cpuset_handle_hotplug() local
4054 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { in cpuset_handle_hotplug()
4055 if (cs == &top_cpuset || !css_tryget_online(&cs->css)) in cpuset_handle_hotplug()
4059 cpuset_hotplug_update_tasks(cs, ptmp); in cpuset_handle_hotplug()
4062 css_put(&cs->css); in cpuset_handle_hotplug()
4097 * cpuset_init_smp - initialize cpus_allowed
4120 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
4121 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
4125 * attached to the specified @tsk. Guaranteed to return some non-empty
4133 struct cpuset *cs; in cpuset_cpus_allowed() local
4137 cs = task_cs(tsk); in cpuset_cpus_allowed()
4138 if (cs != &top_cpuset) in cpuset_cpus_allowed()
4145 if ((cs == &top_cpuset) || cpumask_empty(pmask)) { in cpuset_cpus_allowed()
4161 * cpuset_cpus_allowed_fallback - final fallback before complete catastrophe.
4165 * tsk->cpus_allowed, we fall back to task_cs(tsk)->cpus_allowed. In legacy
4166 * mode however, this value is the same as task_cs(tsk)->effective_cpus,
4181 cs_mask = task_cs(tsk)->cpus_allowed; in cpuset_cpus_allowed_fallback()
4189 * We own tsk->cpus_allowed, nobody can change it under us. in cpuset_cpus_allowed_fallback()
4191 * But we used cs && cs->cpus_allowed lockless and thus can in cpuset_cpus_allowed_fallback()
4193 * the wrong tsk->cpus_allowed. However, both cases imply the in cpuset_cpus_allowed_fallback()
4194 * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr() in cpuset_cpus_allowed_fallback()
4198 * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary in cpuset_cpus_allowed_fallback()
4210 nodes_setall(current->mems_allowed); in cpuset_init_current_mems_allowed()
4214 * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
4215 * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
4218 * attached to the specified @tsk. Guaranteed to return some non-empty
4236 * cpuset_nodemask_valid_mems_allowed - check nodemask vs. current mems_allowed
4239 * Are any of the nodes in the nodemask allowed in current->mems_allowed?
4243 return nodes_intersects(*nodemask, current->mems_allowed); in cpuset_nodemask_valid_mems_allowed()
4247 * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
4252 static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) in nearest_hardwall_ancestor() argument
4254 while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs)) in nearest_hardwall_ancestor()
4255 cs = parent_cs(cs); in nearest_hardwall_ancestor()
4256 return cs; in nearest_hardwall_ancestor()
4260 * cpuset_current_node_allowed - Can current task allocate on a memory node?
4293 * in_interrupt - any node ok (current task context irrelevant)
4294 * GFP_ATOMIC - any node ok
4295 * tsk_is_oom_victim - any node ok
4296 * GFP_KERNEL - any node in enclosing hardwalled cpuset ok
4297 * GFP_USER - only nodes in current tasks mems allowed ok.
4301 struct cpuset *cs; /* current cpuset ancestors */ in cpuset_current_node_allowed() local
4307 if (node_isset(node, current->mems_allowed)) in cpuset_current_node_allowed()
4318 if (current->flags & PF_EXITING) /* Let dying task have memory */ in cpuset_current_node_allowed()
4324 cs = nearest_hardwall_ancestor(task_cs(current)); in cpuset_current_node_allowed()
4325 allowed = node_isset(node, cs->mems_allowed); in cpuset_current_node_allowed()
4334 struct cpuset *cs; in cpuset_node_allowed() local
4351 * or callback_lock - but node_isset is atomic and the reference in cpuset_node_allowed()
4355 * relax locking here to avoid taking global locks - while accepting in cpuset_node_allowed()
4361 cs = container_of(css, struct cpuset, css); in cpuset_node_allowed()
4362 allowed = node_isset(nid, cs->effective_mems); in cpuset_node_allowed()
4368 * cpuset_spread_node() - On which node to begin search for a page
4384 * only set nodes in task->mems_allowed that are online. So it
4395 return *rotor = next_node_in(*rotor, current->mems_allowed); in cpuset_spread_node()
4399 * cpuset_mem_spread_node() - On which node to begin search for a file page
4403 if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE) in cpuset_mem_spread_node()
4404 current->cpuset_mem_spread_rotor = in cpuset_mem_spread_node()
4405 node_random(¤t->mems_allowed); in cpuset_mem_spread_node()
4407 return cpuset_spread_node(¤t->cpuset_mem_spread_rotor); in cpuset_mem_spread_node()
4411 * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
4424 return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed); in cpuset_mems_allowed_intersects()
4428 * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed
4439 cgrp = task_cs(current)->css.cgroup; in cpuset_print_current_mems_allowed()
4443 nodemask_pr_args(¤t->mems_allowed)); in cpuset_print_current_mems_allowed()
4452 nodemask_pr_args(&task->mems_allowed)); in cpuset_task_status_allowed()
4454 nodemask_pr_args(&task->mems_allowed)); in cpuset_task_status_allowed()