1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * kernel/cpuset.c 4 * 5 * Processor and Memory placement constraints for sets of tasks. 6 * 7 * Copyright (C) 2003 BULL SA. 8 * Copyright (C) 2004-2007 Silicon Graphics, Inc. 9 * Copyright (C) 2006 Google, Inc 10 * 11 * Portions derived from Patrick Mochel's sysfs code. 12 * sysfs is Copyright (c) 2001-3 Patrick Mochel 13 * 14 * 2003-10-10 Written by Simon Derr. 15 * 2003-10-22 Updates by Stephen Hemminger. 16 * 2004 May-July Rework by Paul Jackson. 17 * 2006 Rework by Paul Menage to use generic cgroups 18 * 2008 Rework of the scheduler domains and CPU hotplug handling 19 * by Max Krasnyansky 20 */ 21 #include "cpuset-internal.h" 22 23 #include <linux/init.h> 24 #include <linux/interrupt.h> 25 #include <linux/kernel.h> 26 #include <linux/mempolicy.h> 27 #include <linux/mm.h> 28 #include <linux/memory.h> 29 #include <linux/rcupdate.h> 30 #include <linux/sched.h> 31 #include <linux/sched/deadline.h> 32 #include <linux/sched/mm.h> 33 #include <linux/sched/task.h> 34 #include <linux/security.h> 35 #include <linux/oom.h> 36 #include <linux/sched/isolation.h> 37 #include <linux/wait.h> 38 #include <linux/workqueue.h> 39 #include <linux/task_work.h> 40 #include <linux/llist.h> 41 42 DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key); 43 DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key); 44 45 /* 46 * There could be abnormal cpuset configurations for cpu or memory 47 * node binding, add this key to provide a quick low-cost judgment 48 * of the situation. 49 */ 50 DEFINE_STATIC_KEY_FALSE(cpusets_insane_config_key); 51 52 static const char * const perr_strings[] = { 53 [PERR_INVCPUS] = "Invalid cpu list in cpuset.cpus.exclusive", 54 [PERR_INVPARENT] = "Parent is an invalid partition root", 55 [PERR_NOTPART] = "Parent is not a partition root", 56 [PERR_NOTEXCL] = "Cpu list in cpuset.cpus not exclusive", 57 [PERR_NOCPUS] = "Parent unable to distribute cpu downstream", 58 [PERR_HOTPLUG] = "No cpu available due to hotplug", 59 [PERR_CPUSEMPTY] = "cpuset.cpus and cpuset.cpus.exclusive are empty", 60 [PERR_HKEEPING] = "partition config conflicts with housekeeping setup", 61 [PERR_ACCESS] = "Enable partition not permitted", 62 [PERR_REMOTE] = "Have remote partition underneath", 63 [PERR_ISOPARENT] = "Root partition is not allowed with isolated parent", 64 }; 65 66 /* 67 * CPUSET Locking Convention 68 * ------------------------- 69 * 70 * Below are the four global/local locks guarding cpuset structures in lock 71 * acquisition order: 72 * - cpuset_top_mutex 73 * - cpu_hotplug_lock (cpus_read_lock/cpus_write_lock) 74 * - cpuset_mutex 75 * - callback_lock (raw spinlock) 76 * 77 * As cpuset will now indirectly flush a number of different workqueues in 78 * housekeeping_update() to update housekeeping cpumasks when the set of 79 * isolated CPUs is going to be changed, it may be vulnerable to deadlock 80 * if we hold cpus_read_lock while calling into housekeeping_update(). 81 * 82 * The first cpuset_top_mutex will be held except when calling into 83 * cpuset_handle_hotplug() from the CPU hotplug code where cpus_write_lock 84 * and cpuset_mutex will be held instead. The main purpose of this mutex 85 * is to prevent regular cpuset control file write actions from interfering 86 * with the call to housekeeping_update(), though CPU hotplug operation can 87 * still happen in parallel. This mutex also provides protection for some 88 * internal variables. 89 * 90 * A task must hold all the remaining three locks to modify externally visible 91 * or used fields of cpusets, though some of the internally used cpuset fields 92 * and internal variables can be modified without holding callback_lock. If only 93 * reliable read access of the externally used fields are needed, a task can 94 * hold either cpuset_mutex or callback_lock which are exposed to other 95 * external subsystems. 96 * 97 * If a task holds cpu_hotplug_lock and cpuset_mutex, it blocks others, 98 * ensuring that it is the only task able to also acquire callback_lock and 99 * be able to modify cpusets. It can perform various checks on the cpuset 100 * structure first, knowing nothing will change. It can also allocate memory 101 * without holding callback_lock. While it is performing these checks, various 102 * callback routines can briefly acquire callback_lock to query cpusets. Once 103 * it is ready to make the changes, it takes callback_lock, blocking everyone 104 * else. 105 * 106 * Calls to the kernel memory allocator cannot be made while holding 107 * callback_lock which is a spinlock, as the memory allocator may sleep or 108 * call back into cpuset code and acquire callback_lock. 109 * 110 * Now, the task_struct fields mems_allowed and mempolicy may be changed 111 * by other task, we use alloc_lock in the task_struct fields to protect 112 * them. 113 * 114 * The cpuset_common_seq_show() handlers only hold callback_lock across 115 * small pieces of code, such as when reading out possibly multi-word 116 * cpumasks and nodemasks. 117 */ 118 119 static DEFINE_MUTEX(cpuset_top_mutex); 120 static DEFINE_MUTEX(cpuset_mutex); 121 122 /* 123 * File level internal variables below follow one of the following exclusion 124 * rules. 125 * 126 * RWCS: Read/write-able by holding either cpus_write_lock (and optionally 127 * cpuset_mutex) or both cpus_read_lock and cpuset_mutex. 128 * 129 * CSCB: Readable by holding either cpuset_mutex or callback_lock. Writable 130 * by holding both cpuset_mutex and callback_lock. 131 * 132 * T: Read/write-able by holding the cpuset_top_mutex. 133 */ 134 135 /* 136 * For local partitions, update to subpartitions_cpus & isolated_cpus is done 137 * in update_parent_effective_cpumask(). For remote partitions, it is done in 138 * the remote_partition_*() and remote_cpus_update() helpers. 139 */ 140 /* 141 * Exclusive CPUs distributed out to local or remote sub-partitions of 142 * top_cpuset 143 */ 144 static cpumask_var_t subpartitions_cpus; /* RWCS */ 145 146 /* 147 * Exclusive CPUs in isolated partitions (shown in cpuset.cpus.isolated) 148 */ 149 static cpumask_var_t isolated_cpus; /* CSCB */ 150 151 /* 152 * Set if housekeeping cpumasks are to be updated. 153 */ 154 static bool update_housekeeping; /* RWCS */ 155 156 /* 157 * Set if "cpuset_v2_mode" mount option is used 158 * Cached at bind time and not lock protected; accessed via {READ,WRITE}_ONCE 159 */ 160 static bool cpuset_v2_mode; 161 162 /* 163 * Copy of isolated_cpus to be passed to housekeeping_update() 164 */ 165 static cpumask_var_t isolated_hk_cpus; /* T */ 166 167 /* 168 * A flag to force sched domain rebuild at the end of an operation. 169 * It can be set in 170 * - update_partition_sd_lb() 171 * - update_cpumasks_hier() 172 * - cpuset_hotplug_update_tasks() 173 * - cpuset_handle_hotplug() 174 * 175 * Protected by cpuset_mutex (with cpus_read_lock held) or cpus_write_lock. 176 * 177 * Note that update_relax_domain_level() in cpuset-v1.c can still call 178 * rebuild_sched_domains_locked() directly without using this flag. 179 */ 180 static bool force_sd_rebuild; /* RWCS */ 181 182 /* 183 * Partition root states: 184 * 185 * 0 - member (not a partition root) 186 * 1 - partition root 187 * 2 - partition root without load balancing (isolated) 188 * -1 - invalid partition root 189 * -2 - invalid isolated partition root 190 * 191 * There are 2 types of partitions - local or remote. Local partitions are 192 * those whose parents are partition root themselves. Setting of 193 * cpuset.cpus.exclusive are optional in setting up local partitions. 194 * Remote partitions are those whose parents are not partition roots. Passing 195 * down exclusive CPUs by setting cpuset.cpus.exclusive along its ancestor 196 * nodes are mandatory in creating a remote partition. 197 * 198 * For simplicity, a local partition can be created under a local or remote 199 * partition but a remote partition cannot have any partition root in its 200 * ancestor chain except the cgroup root. 201 * 202 * A valid partition can be formed by setting exclusive_cpus or cpus_allowed 203 * if exclusive_cpus is not set. In the case of partition with empty 204 * exclusive_cpus, all the conflicting exclusive CPUs specified in the 205 * following cpumasks of sibling cpusets will be removed from its 206 * cpus_allowed in determining its effective_xcpus. 207 * - effective_xcpus 208 * - exclusive_cpus 209 * 210 * The "cpuset.cpus.exclusive" control file should be used for setting up 211 * partition if the users want to get as many CPUs as possible. 212 */ 213 #define PRS_MEMBER 0 214 #define PRS_ROOT 1 215 #define PRS_ISOLATED 2 216 #define PRS_INVALID_ROOT -1 217 #define PRS_INVALID_ISOLATED -2 218 219 /* 220 * Temporary cpumasks for working with partitions that are passed among 221 * functions to avoid memory allocation in inner functions. 222 */ 223 struct tmpmasks { 224 cpumask_var_t addmask, delmask; /* For partition root */ 225 cpumask_var_t new_cpus; /* For update_cpumasks_hier() */ 226 }; 227 228 void inc_dl_tasks_cs(struct task_struct *p) 229 { 230 struct cpuset *cs = task_cs(p); 231 232 atomic_inc(&cs->nr_deadline_tasks); 233 } 234 235 void dec_dl_tasks_cs(struct task_struct *p) 236 { 237 struct cpuset *cs = task_cs(p); 238 239 atomic_dec(&cs->nr_deadline_tasks); 240 } 241 242 static inline bool is_partition_valid(const struct cpuset *cs) 243 { 244 return cs->partition_root_state > 0; 245 } 246 247 static inline bool is_partition_invalid(const struct cpuset *cs) 248 { 249 return cs->partition_root_state < 0; 250 } 251 252 static inline bool cs_is_member(const struct cpuset *cs) 253 { 254 return cs->partition_root_state == PRS_MEMBER; 255 } 256 257 /* 258 * Send notification event of whenever partition_root_state changes. 259 */ 260 static inline void notify_partition_change(struct cpuset *cs, int old_prs) 261 { 262 if (old_prs == cs->partition_root_state) 263 return; 264 cgroup_file_notify(&cs->partition_file); 265 266 /* Reset prs_err if not invalid */ 267 if (is_partition_valid(cs)) 268 WRITE_ONCE(cs->prs_err, PERR_NONE); 269 } 270 271 /* 272 * The top_cpuset is always synchronized to cpu_active_mask and we should avoid 273 * using cpu_online_mask as much as possible. An active CPU is always an online 274 * CPU, but not vice versa. cpu_active_mask and cpu_online_mask can differ 275 * during hotplug operations. A CPU is marked active at the last stage of CPU 276 * bringup (CPUHP_AP_ACTIVE). It is also the stage where cpuset hotplug code 277 * will be called to update the sched domains so that the scheduler can move 278 * a normal task to a newly active CPU or remove tasks away from a newly 279 * inactivated CPU. The online bit is set much earlier in the CPU bringup 280 * process and cleared much later in CPU teardown. 281 * 282 * If cpu_online_mask is used while a hotunplug operation is happening in 283 * parallel, we may leave an offline CPU in cpu_allowed or some other masks. 284 */ 285 struct cpuset top_cpuset = { 286 .flags = BIT(CS_CPU_EXCLUSIVE) | 287 BIT(CS_MEM_EXCLUSIVE) | BIT(CS_SCHED_LOAD_BALANCE), 288 .partition_root_state = PRS_ROOT, 289 .dl_bw_cpu = -1, 290 }; 291 292 /** 293 * cpuset_lock - Acquire the global cpuset mutex 294 * 295 * This locks the global cpuset mutex to prevent modifications to cpuset 296 * hierarchy and configurations. This helper is not enough to make modification. 297 */ 298 void cpuset_lock(void) 299 { 300 mutex_lock(&cpuset_mutex); 301 } 302 303 void cpuset_unlock(void) 304 { 305 mutex_unlock(&cpuset_mutex); 306 } 307 308 void lockdep_assert_cpuset_lock_held(void) 309 { 310 lockdep_assert_held(&cpuset_mutex); 311 } 312 313 /** 314 * cpuset_full_lock - Acquire full protection for cpuset modification 315 * 316 * Takes both CPU hotplug read lock (cpus_read_lock()) and cpuset mutex 317 * to safely modify cpuset data. 318 */ 319 void cpuset_full_lock(void) 320 { 321 mutex_lock(&cpuset_top_mutex); 322 cpus_read_lock(); 323 mutex_lock(&cpuset_mutex); 324 } 325 326 void cpuset_full_unlock(void) 327 { 328 mutex_unlock(&cpuset_mutex); 329 cpus_read_unlock(); 330 mutex_unlock(&cpuset_top_mutex); 331 } 332 333 #ifdef CONFIG_LOCKDEP 334 bool lockdep_is_cpuset_held(void) 335 { 336 return lockdep_is_held(&cpuset_mutex) || 337 lockdep_is_held(&cpuset_top_mutex); 338 } 339 #endif 340 341 static DEFINE_SPINLOCK(callback_lock); 342 343 void cpuset_callback_lock_irq(void) 344 { 345 spin_lock_irq(&callback_lock); 346 } 347 348 void cpuset_callback_unlock_irq(void) 349 { 350 spin_unlock_irq(&callback_lock); 351 } 352 353 static struct workqueue_struct *cpuset_migrate_mm_wq; 354 355 static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq); 356 357 /* 358 * Cpuset task attach context 359 * Protected by cpuset_mutex 360 */ 361 static struct { 362 int in_progress; 363 bool cpus_updated; 364 bool mems_updated; 365 bool task_work_queued; 366 bool many_dest_cs; /* Have many destination cpusets */ 367 struct cpuset *old_cs; /* Source cpuset */ 368 nodemask_t nodemask_to; 369 } attach_ctx; 370 static LLIST_HEAD(src_cs_head); 371 static LLIST_HEAD(dst_cs_head); 372 373 /* 374 * Wait if task attach is in progress until it is done and then acquire 375 * cpuset_mutex before returning. 376 */ 377 static void wait_attach_done_lock(void) 378 __acquires(&cpuset_mutex) 379 { 380 for (;;) { 381 mutex_lock(&cpuset_mutex); 382 if (!attach_ctx.in_progress) 383 return; 384 385 mutex_unlock(&cpuset_mutex); 386 387 /* Wait until attach operation is done to prevent racing */ 388 wait_event(cpuset_attach_wq, attach_ctx.in_progress == 0); 389 } 390 } 391 392 static inline void check_insane_mems_config(nodemask_t *nodes) 393 { 394 if (!cpusets_insane_config() && 395 movable_only_nodes(nodes)) { 396 static_branch_enable_cpuslocked(&cpusets_insane_config_key); 397 pr_info("Unsupported (movable nodes only) cpuset configuration detected (nmask=%*pbl)!\n" 398 "Cpuset allocations might fail even with a lot of memory available.\n", 399 nodemask_pr_args(nodes)); 400 } 401 } 402 403 /* 404 * decrease attach_ctx.in_progress. 405 * wake_up cpuset_attach_wq if attach_ctx.in_progress==0. 406 */ 407 static inline void dec_attach_in_progress_locked(void) 408 { 409 lockdep_assert_cpuset_lock_held(); 410 411 attach_ctx.in_progress--; 412 if (!attach_ctx.in_progress) 413 wake_up(&cpuset_attach_wq); 414 } 415 416 static inline void dec_attach_in_progress(void) 417 { 418 mutex_lock(&cpuset_mutex); 419 dec_attach_in_progress_locked(); 420 mutex_unlock(&cpuset_mutex); 421 } 422 423 static inline bool cpuset_v2(void) 424 { 425 return !IS_ENABLED(CONFIG_CPUSETS_V1) || 426 cgroup_subsys_on_dfl(cpuset_cgrp_subsys); 427 } 428 429 /* 430 * Cgroup v2 behavior is used on the "cpus" and "mems" control files when 431 * on default hierarchy or when the cpuset_v2_mode flag is set by mounting 432 * the v1 cpuset cgroup filesystem with the "cpuset_v2_mode" mount option. 433 * With v2 behavior, "cpus" and "mems" are always what the users have 434 * requested and won't be changed by hotplug events. Only the effective 435 * cpus or mems will be affected. 436 */ 437 static inline bool is_in_v2_mode(void) 438 { 439 return cpuset_v2() || READ_ONCE(cpuset_v2_mode); 440 } 441 442 /** 443 * partition_is_populated - check if partition has tasks 444 * @cs: partition root to be checked 445 * @excluded_child: a child cpuset to be excluded in task checking 446 * Return: true if there are tasks, false otherwise 447 * 448 * @cs should be a valid partition root or going to become a partition root. 449 * @excluded_child should be non-NULL when this cpuset is going to become a 450 * partition itself. 451 * 452 * Note that a remote partition is not allowed underneath a valid local 453 * or remote partition. So if a non-partition root child is populated, 454 * the whole partition is considered populated. 455 */ 456 static inline bool partition_is_populated(struct cpuset *cs, 457 struct cpuset *excluded_child) 458 { 459 struct cpuset *cp; 460 struct cgroup_subsys_state *pos_css; 461 462 /* 463 * We cannot call cs_is_populated(cs) directly, as 464 * nr_populated_domain_children may include populated 465 * csets from descendants that are partitions. 466 */ 467 if (cgroup_has_tasks(cs->css.cgroup)) 468 return true; 469 470 rcu_read_lock(); 471 cpuset_for_each_descendant_pre(cp, pos_css, cs) { 472 if (cp == cs) 473 continue; 474 475 if (cp == excluded_child || is_partition_valid(cp)) { 476 pos_css = css_rightmost_descendant(pos_css); 477 continue; 478 } 479 480 if (cpuset_is_populated(cp)) { 481 rcu_read_unlock(); 482 return true; 483 } 484 } 485 rcu_read_unlock(); 486 return false; 487 } 488 489 /* 490 * Return in pmask the portion of a task's cpusets's cpus_allowed that 491 * are online and are capable of running the task. If none are found, 492 * walk up the cpuset hierarchy until we find one that does have some 493 * appropriate cpus. 494 * 495 * One way or another, we guarantee to return some non-empty subset 496 * of cpu_active_mask. 497 * 498 * Call with callback_lock or cpuset_mutex held. 499 */ 500 static void guarantee_active_cpus(struct task_struct *tsk, 501 struct cpumask *pmask) 502 { 503 const struct cpumask *possible_mask = task_cpu_possible_mask(tsk); 504 struct cpuset *cs; 505 506 if (WARN_ON(!cpumask_and(pmask, possible_mask, cpu_active_mask))) 507 cpumask_copy(pmask, cpu_active_mask); 508 509 rcu_read_lock(); 510 cs = task_cs(tsk); 511 512 while (!cpumask_intersects(cs->effective_cpus, pmask)) { 513 cs = parent_cs(cs); 514 if (unlikely(!cs)) { 515 /* 516 * The top cpuset doesn't have any active cpu as a 517 * consequence of a race between its caller and the cpu 518 * hotplug operation where cpu_active_mask is updated 519 * asynchronously before cpuset_handle_hotplug() is 520 * being called to adjust the effective_cpus of the 521 * affected cpusets. But we know the top cpuset's 522 * effective_cpus is on its way to be identical to 523 * cpu_active_mask minus the exclusive CPUs dedicated 524 * to other valid cpuset partitions. Just pass back 525 * the filtered cpu_active_mask in this case. 526 */ 527 goto out_unlock; 528 } 529 } 530 cpumask_and(pmask, pmask, cs->effective_cpus); 531 out_unlock: 532 rcu_read_unlock(); 533 } 534 535 /* 536 * Return in *pmask the portion of a cpusets's mems_allowed that 537 * are online, with memory. If none are online with memory, walk 538 * up the cpuset hierarchy until we find one that does have some 539 * online mems. The top cpuset always has some mems online. With v2, 540 * effective_mems should always contain online memory nodes except 541 * during the transition period where a memory node hotunplug operation 542 * is in progress. 543 * 544 * One way or another, we guarantee to return some non-empty subset 545 * of node_states[N_MEMORY]. 546 * 547 * Call with callback_lock or cpuset_mutex held. 548 */ 549 static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask) 550 { 551 while (!nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY])) 552 cs = parent_cs(cs); 553 } 554 555 /** 556 * alloc_cpumasks - Allocate an array of cpumask variables 557 * @pmasks: Pointer to array of cpumask_var_t pointers 558 * @size: Number of cpumasks to allocate 559 * Return: 0 if successful, -ENOMEM otherwise. 560 * 561 * Allocates @size cpumasks and initializes them to empty. Returns 0 on 562 * success, -ENOMEM on allocation failure. On failure, any previously 563 * allocated cpumasks are freed. 564 */ 565 static inline int alloc_cpumasks(cpumask_var_t *pmasks[], u32 size) 566 { 567 int i; 568 569 for (i = 0; i < size; i++) { 570 if (!zalloc_cpumask_var(pmasks[i], GFP_KERNEL)) { 571 while (--i >= 0) 572 free_cpumask_var(*pmasks[i]); 573 return -ENOMEM; 574 } 575 } 576 return 0; 577 } 578 579 /** 580 * alloc_tmpmasks - Allocate temporary cpumasks for cpuset operations. 581 * @tmp: Pointer to tmpmasks structure to populate 582 * Return: 0 on success, -ENOMEM on allocation failure 583 */ 584 static inline int alloc_tmpmasks(struct tmpmasks *tmp) 585 { 586 /* 587 * Array of pointers to the three cpumask_var_t fields in tmpmasks. 588 * Note: Array size must match actual number of masks (3) 589 */ 590 cpumask_var_t *pmask[3] = { 591 &tmp->new_cpus, 592 &tmp->addmask, 593 &tmp->delmask 594 }; 595 596 return alloc_cpumasks(pmask, ARRAY_SIZE(pmask)); 597 } 598 599 /** 600 * free_tmpmasks - free cpumasks in a tmpmasks structure 601 * @tmp: the tmpmasks structure pointer 602 */ 603 static inline void free_tmpmasks(struct tmpmasks *tmp) 604 { 605 if (!tmp) 606 return; 607 608 free_cpumask_var(tmp->new_cpus); 609 free_cpumask_var(tmp->addmask); 610 free_cpumask_var(tmp->delmask); 611 } 612 613 /** 614 * dup_or_alloc_cpuset - Duplicate or allocate a new cpuset 615 * @cs: Source cpuset to duplicate (NULL for a fresh allocation) 616 * 617 * Creates a new cpuset by either: 618 * 1. Duplicating an existing cpuset (if @cs is non-NULL), or 619 * 2. Allocating a fresh cpuset with zero-initialized masks (if @cs is NULL) 620 * 621 * Return: Pointer to newly allocated cpuset on success, NULL on failure 622 */ 623 struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs) 624 { 625 struct cpuset *trial; 626 627 /* Allocate base structure */ 628 trial = cs ? kmemdup(cs, sizeof(*cs), GFP_KERNEL) : 629 kzalloc_obj(*cs); 630 if (!trial) 631 return NULL; 632 633 trial->dl_bw_cpu = -1; 634 init_llist_node(&trial->attach_node); 635 636 /* Setup cpumask pointer array */ 637 cpumask_var_t *pmask[4] = { 638 &trial->cpus_allowed, 639 &trial->effective_cpus, 640 &trial->effective_xcpus, 641 &trial->exclusive_cpus 642 }; 643 644 if (alloc_cpumasks(pmask, ARRAY_SIZE(pmask))) { 645 kfree(trial); 646 return NULL; 647 } 648 649 /* Copy masks if duplicating */ 650 if (cs) { 651 cpumask_copy(trial->cpus_allowed, cs->cpus_allowed); 652 cpumask_copy(trial->effective_cpus, cs->effective_cpus); 653 cpumask_copy(trial->effective_xcpus, cs->effective_xcpus); 654 cpumask_copy(trial->exclusive_cpus, cs->exclusive_cpus); 655 } 656 657 return trial; 658 } 659 660 /** 661 * free_cpuset - free the cpuset 662 * @cs: the cpuset to be freed 663 */ 664 void free_cpuset(struct cpuset *cs) 665 { 666 free_cpumask_var(cs->cpus_allowed); 667 free_cpumask_var(cs->effective_cpus); 668 free_cpumask_var(cs->effective_xcpus); 669 free_cpumask_var(cs->exclusive_cpus); 670 kfree(cs); 671 } 672 673 /* Return user specified exclusive CPUs */ 674 static inline struct cpumask *user_xcpus(struct cpuset *cs) 675 { 676 return cpumask_empty(cs->exclusive_cpus) ? cs->cpus_allowed 677 : cs->exclusive_cpus; 678 } 679 680 static inline bool xcpus_empty(struct cpuset *cs) 681 { 682 return cpumask_empty(cs->cpus_allowed) && 683 cpumask_empty(cs->exclusive_cpus); 684 } 685 686 /* 687 * cpusets_are_exclusive() - check if two cpusets are exclusive 688 * 689 * Return true if exclusive, false if not 690 */ 691 static inline bool cpusets_are_exclusive(struct cpuset *cs1, struct cpuset *cs2) 692 { 693 struct cpumask *xcpus1 = user_xcpus(cs1); 694 struct cpumask *xcpus2 = user_xcpus(cs2); 695 696 if (cpumask_intersects(xcpus1, xcpus2)) 697 return false; 698 return true; 699 } 700 701 /** 702 * cpus_excl_conflict - Check if two cpusets have exclusive CPU conflicts 703 * @trial: the trial cpuset to be checked 704 * @sibling: a sibling cpuset to be checked against 705 * @xcpus_changed: set if exclusive_cpus has been set 706 * 707 * Returns: true if CPU exclusivity conflict exists, false otherwise 708 * 709 * Conflict detection rules: 710 * o cgroup v1 711 * See cpuset1_cpus_excl_conflict() 712 * o cgroup v2 713 * - The exclusive_cpus values cannot overlap. 714 * - New exclusive_cpus cannot be a superset of a sibling's cpus_allowed. 715 */ 716 static inline bool cpus_excl_conflict(struct cpuset *trial, struct cpuset *sibling, 717 bool xcpus_changed) 718 { 719 if (!cpuset_v2()) 720 return cpuset1_cpus_excl_conflict(trial, sibling); 721 722 /* The cpus_allowed of a sibling cpuset cannot be a subset of the new exclusive_cpus */ 723 if (xcpus_changed && !cpumask_empty(sibling->cpus_allowed) && 724 cpumask_subset(sibling->cpus_allowed, trial->exclusive_cpus)) 725 return true; 726 727 /* Exclusive_cpus cannot intersect */ 728 return cpumask_intersects(trial->exclusive_cpus, sibling->exclusive_cpus); 729 } 730 731 static inline bool mems_excl_conflict(struct cpuset *cs1, struct cpuset *cs2) 732 { 733 if ((is_mem_exclusive(cs1) || is_mem_exclusive(cs2))) 734 return nodes_intersects(cs1->mems_allowed, cs2->mems_allowed); 735 return false; 736 } 737 738 /* 739 * validate_change() - Used to validate that any proposed cpuset change 740 * follows the structural rules for cpusets. 741 * 742 * If we replaced the flag and mask values of the current cpuset 743 * (cur) with those values in the trial cpuset (trial), would 744 * our various subset and exclusive rules still be valid? Presumes 745 * cpuset_mutex held. 746 * 747 * 'cur' is the address of an actual, in-use cpuset. Operations 748 * such as list traversal that depend on the actual address of the 749 * cpuset in the list must use cur below, not trial. 750 * 751 * 'trial' is the address of bulk structure copy of cur, with 752 * perhaps one or more of the fields cpus_allowed, mems_allowed, 753 * or flags changed to new, trial values. 754 * 755 * Return 0 if valid, -errno if not. 756 */ 757 758 int validate_change(struct cpuset *cur, struct cpuset *trial) 759 { 760 struct cgroup_subsys_state *css; 761 struct cpuset *c, *par; 762 bool xcpus_changed; 763 int ret = 0; 764 765 rcu_read_lock(); 766 767 if (!is_in_v2_mode()) 768 ret = cpuset1_validate_change(cur, trial); 769 if (ret) 770 goto out; 771 772 /* Remaining checks don't apply to root cpuset */ 773 if (cur == &top_cpuset) 774 goto out; 775 776 par = parent_cs(cur); 777 778 /* 779 * We can't shrink if we won't have enough room for SCHED_DEADLINE 780 * tasks. This check is not done when scheduling is disabled as the 781 * users should know what they are doing. 782 * 783 * For v1, effective_cpus == cpus_allowed & user_xcpus() returns 784 * cpus_allowed. 785 * 786 * For v2, is_partition_valid(cur) & is_sched_load_balance() are true 787 * only for non-isolated partition root. At this point, the target 788 * effective_cpus isn't computed yet. user_xcpus() is the best 789 * approximation. 790 * 791 * TBD: May need to precompute the real effective_cpus here in case 792 * incorrect scheduling of SCHED_DEADLINE tasks in a partition 793 * becomes an issue. 794 */ 795 ret = -EBUSY; 796 if ((is_partition_valid(cur) || (!cpuset_v2() && is_cpu_exclusive(cur))) && 797 is_sched_load_balance(cur) && 798 !cpuset_cpumask_can_shrink(cur->effective_cpus, user_xcpus(trial))) 799 goto out; 800 801 /* 802 * If either I or some sibling (!= me) is exclusive, we can't 803 * overlap. exclusive_cpus cannot overlap with each other if set. 804 */ 805 ret = -EINVAL; 806 xcpus_changed = !cpumask_equal(cur->exclusive_cpus, trial->exclusive_cpus); 807 cpuset_for_each_child(c, css, par) { 808 if (c == cur) 809 continue; 810 if (cpus_excl_conflict(trial, c, xcpus_changed)) 811 goto out; 812 if (mems_excl_conflict(trial, c)) 813 goto out; 814 } 815 816 ret = 0; 817 out: 818 rcu_read_unlock(); 819 return ret; 820 } 821 822 #ifdef CONFIG_SMP 823 824 /* 825 * generate_sched_domains() 826 * 827 * This function builds a partial partition of the systems CPUs 828 * A 'partial partition' is a set of non-overlapping subsets whose 829 * union is a subset of that set. 830 * The output of this function needs to be passed to kernel/sched/core.c 831 * partition_sched_domains() routine, which will rebuild the scheduler's 832 * load balancing domains (sched domains) as specified by that partial 833 * partition. 834 * 835 * See "What is sched_load_balance" in Documentation/admin-guide/cgroup-v1/cpusets.rst 836 * for a background explanation of this. 837 * 838 * Does not return errors, on the theory that the callers of this 839 * routine would rather not worry about failures to rebuild sched 840 * domains when operating in the severe memory shortage situations 841 * that could cause allocation failures below. 842 * 843 * Must be called with cpuset_mutex held. 844 * 845 * The three key local variables below are: 846 * cp - cpuset pointer, used (together with pos_css) to perform a 847 * top-down scan of all cpusets. For our purposes, rebuilding 848 * the schedulers sched domains, we can ignore !is_sched_load_ 849 * balance cpusets. 850 * csa - (for CpuSet Array) Array of pointers to all the cpusets 851 * that need to be load balanced, for convenient iterative 852 * access by the subsequent code that finds the best partition, 853 * i.e the set of domains (subsets) of CPUs such that the 854 * cpus_allowed of every cpuset marked is_sched_load_balance 855 * is a subset of one of these domains, while there are as 856 * many such domains as possible, each as small as possible. 857 * doms - Conversion of 'csa' to an array of cpumasks, for passing to 858 * the kernel/sched/core.c routine partition_sched_domains() in a 859 * convenient format, that can be easily compared to the prior 860 * value to determine what partition elements (sched domains) 861 * were changed (added or removed.) 862 */ 863 static int generate_sched_domains(cpumask_var_t **domains, 864 struct sched_domain_attr **attributes) 865 { 866 struct cpuset *cp; /* top-down scan of cpusets */ 867 struct cpuset **csa; /* array of all cpuset ptrs */ 868 int i, j; /* indices for partition finding loops */ 869 cpumask_var_t *doms; /* resulting partition; i.e. sched domains */ 870 struct sched_domain_attr *dattr; /* attributes for custom domains */ 871 int ndoms = 0; /* number of sched domains in result */ 872 struct cgroup_subsys_state *pos_css; 873 874 if (!cpuset_v2()) 875 return cpuset1_generate_sched_domains(domains, attributes); 876 877 doms = NULL; 878 dattr = NULL; 879 csa = NULL; 880 881 /* Special case for the 99% of systems with one, full, sched domain */ 882 if (cpumask_empty(subpartitions_cpus)) { 883 ndoms = 1; 884 /* !csa will be checked and can be correctly handled */ 885 goto generate_doms; 886 } 887 888 csa = kmalloc_objs(cp, nr_cpusets()); 889 if (!csa) 890 goto done; 891 892 /* Find how many partitions and cache them to csa[] */ 893 rcu_read_lock(); 894 cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) { 895 /* 896 * Only valid partition roots that are not isolated and with 897 * non-empty effective_cpus will be saved into csa[]. 898 */ 899 if ((cp->partition_root_state == PRS_ROOT) && 900 !cpumask_empty(cp->effective_cpus)) 901 csa[ndoms++] = cp; 902 903 /* 904 * Skip @cp's subtree if not a partition root and has no 905 * exclusive CPUs to be granted to child cpusets. 906 */ 907 if (!is_partition_valid(cp) && cpumask_empty(cp->exclusive_cpus)) 908 pos_css = css_rightmost_descendant(pos_css); 909 } 910 rcu_read_unlock(); 911 912 for (i = 0; i < ndoms; i++) { 913 for (j = i + 1; j < ndoms; j++) { 914 if (cpusets_overlap(csa[i], csa[j])) 915 /* 916 * Cgroup v2 shouldn't pass down overlapping 917 * partition root cpusets. 918 */ 919 WARN_ON_ONCE(1); 920 } 921 } 922 923 generate_doms: 924 doms = alloc_sched_domains(ndoms); 925 if (!doms) 926 goto done; 927 928 /* 929 * The rest of the code, including the scheduler, can deal with 930 * dattr==NULL case. No need to abort if alloc fails. 931 */ 932 dattr = kmalloc_objs(struct sched_domain_attr, ndoms); 933 934 /* 935 * Cgroup v2 doesn't support domain attributes, just set all of them 936 * to SD_ATTR_INIT. Also non-isolating partition root CPUs are a 937 * subset of HK_TYPE_DOMAIN_BOOT housekeeping CPUs. 938 */ 939 for (i = 0; i < ndoms; i++) { 940 /* 941 * The top cpuset may contain some boot time isolated 942 * CPUs that need to be excluded from the sched domain. 943 */ 944 if (!csa || csa[i] == &top_cpuset) 945 cpumask_and(doms[i], top_cpuset.effective_cpus, 946 housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)); 947 else 948 cpumask_copy(doms[i], csa[i]->effective_cpus); 949 if (dattr) 950 dattr[i] = SD_ATTR_INIT; 951 } 952 953 done: 954 kfree(csa); 955 956 /* 957 * Fallback to the default domain if kmalloc() failed. 958 * See comments in partition_sched_domains(). 959 */ 960 if (doms == NULL) 961 ndoms = 1; 962 963 *domains = doms; 964 *attributes = dattr; 965 return ndoms; 966 } 967 968 static void dl_update_tasks_root_domain(struct cpuset *cs) 969 { 970 struct css_task_iter it; 971 struct task_struct *task; 972 973 if (atomic_read(&cs->nr_deadline_tasks) == 0) 974 return; 975 976 css_task_iter_start(&cs->css, 0, &it); 977 978 while ((task = css_task_iter_next(&it))) 979 dl_add_task_root_domain(task); 980 981 css_task_iter_end(&it); 982 } 983 984 void dl_rebuild_rd_accounting(void) 985 { 986 struct cpuset *cs = NULL; 987 struct cgroup_subsys_state *pos_css; 988 int cpu; 989 u64 cookie = ++dl_cookie; 990 991 lockdep_assert_cpuset_lock_held(); 992 lockdep_assert_cpus_held(); 993 lockdep_assert_held(&sched_domains_mutex); 994 995 rcu_read_lock(); 996 997 for_each_possible_cpu(cpu) { 998 if (dl_bw_visited(cpu, cookie)) 999 continue; 1000 1001 dl_clear_root_domain_cpu(cpu); 1002 } 1003 1004 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { 1005 1006 if (cpumask_empty(cs->effective_cpus)) { 1007 pos_css = css_rightmost_descendant(pos_css); 1008 continue; 1009 } 1010 1011 css_get(&cs->css); 1012 1013 rcu_read_unlock(); 1014 1015 dl_update_tasks_root_domain(cs); 1016 1017 rcu_read_lock(); 1018 css_put(&cs->css); 1019 } 1020 rcu_read_unlock(); 1021 } 1022 1023 /* 1024 * Rebuild scheduler domains. 1025 * 1026 * If the flag 'sched_load_balance' of any cpuset with non-empty 1027 * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset 1028 * which has that flag enabled, or if any cpuset with a non-empty 1029 * 'cpus' is removed, then call this routine to rebuild the 1030 * scheduler's dynamic sched domains. 1031 * 1032 * Call with cpuset_mutex held. Takes cpus_read_lock(). 1033 */ 1034 void rebuild_sched_domains_locked(void) 1035 { 1036 struct sched_domain_attr *attr; 1037 cpumask_var_t *doms; 1038 int ndoms; 1039 int i; 1040 1041 lockdep_assert_cpus_held(); 1042 lockdep_assert_cpuset_lock_held(); 1043 force_sd_rebuild = false; 1044 1045 /* Generate domain masks and attrs */ 1046 ndoms = generate_sched_domains(&doms, &attr); 1047 1048 /* 1049 * cpuset_hotplug_workfn is invoked synchronously now, thus this 1050 * function should not race with CPU hotplug. And the effective CPUs 1051 * must not include any offline CPUs. Passing an offline CPU in the 1052 * doms to partition_sched_domains() will trigger a kernel panic. 1053 * 1054 * We perform a final check here: if the doms contains any 1055 * offline CPUs, a warning is emitted and we return directly to 1056 * prevent the panic. 1057 */ 1058 for (i = 0; doms && i < ndoms; i++) { 1059 if (WARN_ON_ONCE(!cpumask_subset(doms[i], cpu_active_mask))) { 1060 free_sched_domains(doms, ndoms); 1061 kfree(attr); 1062 return; 1063 } 1064 } 1065 1066 /* Have scheduler rebuild the domains */ 1067 partition_sched_domains(ndoms, doms, attr); 1068 } 1069 #else /* !CONFIG_SMP */ 1070 void rebuild_sched_domains_locked(void) 1071 { 1072 } 1073 #endif /* CONFIG_SMP */ 1074 1075 static void rebuild_sched_domains_cpuslocked(void) 1076 { 1077 mutex_lock(&cpuset_mutex); 1078 rebuild_sched_domains_locked(); 1079 mutex_unlock(&cpuset_mutex); 1080 } 1081 1082 void rebuild_sched_domains(void) 1083 { 1084 cpus_read_lock(); 1085 rebuild_sched_domains_cpuslocked(); 1086 cpus_read_unlock(); 1087 } 1088 1089 void cpuset_reset_sched_domains(void) 1090 { 1091 mutex_lock(&cpuset_mutex); 1092 partition_sched_domains(1, NULL, NULL); 1093 mutex_unlock(&cpuset_mutex); 1094 } 1095 1096 /** 1097 * cpuset_update_tasks_cpumask - Update the cpumasks of tasks in the cpuset. 1098 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed 1099 * @new_cpus: the temp variable for the new effective_cpus mask 1100 * 1101 * Iterate through each task of @cs updating its cpus_allowed to the 1102 * effective cpuset's. As this function is called with cpuset_mutex held, 1103 * cpuset membership stays stable. 1104 * 1105 * For top_cpuset, task_cpu_possible_mask() is used instead of effective_cpus 1106 * to make sure all offline CPUs are also included as hotplug code won't 1107 * update cpumasks for tasks in top_cpuset. 1108 * 1109 * As task_cpu_possible_mask() can be task dependent in arm64, we have to 1110 * do cpu masking per task instead of doing it once for all. 1111 */ 1112 void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus) 1113 { 1114 struct css_task_iter it; 1115 struct task_struct *task; 1116 bool top_cs = cs == &top_cpuset; 1117 1118 css_task_iter_start(&cs->css, 0, &it); 1119 while ((task = css_task_iter_next(&it))) { 1120 const struct cpumask *possible_mask = task_cpu_possible_mask(task); 1121 1122 if (top_cs) { 1123 /* 1124 * PF_KTHREAD tasks are handled by housekeeping. 1125 * PF_NO_SETAFFINITY tasks are ignored. 1126 */ 1127 if (task->flags & (PF_KTHREAD | PF_NO_SETAFFINITY)) 1128 continue; 1129 cpumask_andnot(new_cpus, possible_mask, subpartitions_cpus); 1130 } else { 1131 cpumask_and(new_cpus, possible_mask, cs->effective_cpus); 1132 } 1133 set_cpus_allowed_ptr(task, new_cpus); 1134 } 1135 css_task_iter_end(&it); 1136 } 1137 1138 /* 1139 * Commands for update_parent_effective_cpumask 1140 */ 1141 enum partition_cmd { 1142 partcmd_enable, /* Enable partition root */ 1143 partcmd_enablei, /* Enable isolated partition root */ 1144 partcmd_disable, /* Disable partition root */ 1145 partcmd_update, /* Update parent's effective_cpus */ 1146 partcmd_invalidate, /* Make partition invalid */ 1147 }; 1148 1149 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs, 1150 struct tmpmasks *tmp); 1151 1152 /* 1153 * Update partition load balance flag and/or rebuild sched domain 1154 * 1155 * Changing load balance flag will automatically call 1156 * rebuild_sched_domains_locked(). 1157 * This function is for cgroup v2 only. 1158 */ 1159 static void update_partition_sd_lb(struct cpuset *cs, int old_prs) 1160 { 1161 int new_prs = cs->partition_root_state; 1162 bool rebuild_domains = (new_prs > 0) || (old_prs > 0); 1163 bool new_lb; 1164 1165 /* 1166 * If cs is not a valid partition root, the load balance state 1167 * will follow its parent. 1168 */ 1169 if (new_prs > 0) { 1170 new_lb = (new_prs != PRS_ISOLATED); 1171 } else { 1172 new_lb = is_sched_load_balance(parent_cs(cs)); 1173 } 1174 if (new_lb != !!is_sched_load_balance(cs)) { 1175 rebuild_domains = true; 1176 assign_bit(CS_SCHED_LOAD_BALANCE, &cs->flags, new_lb); 1177 } 1178 1179 if (rebuild_domains) 1180 cpuset_force_rebuild(); 1181 } 1182 1183 /* 1184 * tasks_nocpu_error - Return true if tasks will have no effective_cpus 1185 */ 1186 static bool tasks_nocpu_error(struct cpuset *parent, struct cpuset *cs, 1187 struct cpumask *xcpus) 1188 { 1189 /* 1190 * A populated partition (cs or parent) can't have empty effective_cpus 1191 */ 1192 return (cpumask_subset(parent->effective_cpus, xcpus) && 1193 partition_is_populated(parent, cs)) || 1194 (!cpumask_intersects(xcpus, cpu_active_mask) && 1195 partition_is_populated(cs, NULL)); 1196 } 1197 1198 static void reset_partition_data(struct cpuset *cs) 1199 { 1200 struct cpuset *parent = parent_cs(cs); 1201 1202 if (!cpuset_v2()) 1203 return; 1204 1205 lockdep_assert_held(&callback_lock); 1206 1207 if (cpumask_empty(cs->exclusive_cpus)) 1208 cpumask_clear(cs->effective_xcpus); 1209 1210 if (!cpumask_and(cs->effective_cpus, parent->effective_cpus, cs->cpus_allowed)) 1211 cpumask_copy(cs->effective_cpus, parent->effective_cpus); 1212 } 1213 1214 static inline bool is_remote_partition(struct cpuset *cs) 1215 { 1216 return cs->remote_partition; 1217 } 1218 1219 static inline bool is_local_partition(struct cpuset *cs) 1220 { 1221 return is_partition_valid(cs) && !is_remote_partition(cs); 1222 } 1223 1224 /* Return true if isolated_cpus changes. */ 1225 static bool isolated_cpu_update(int new_prs, int cpu) 1226 { 1227 lockdep_assert_held(&callback_lock); 1228 lockdep_assert_held(&cpuset_mutex); 1229 1230 if (new_prs == PRS_ISOLATED) { 1231 if (cpumask_test_cpu(cpu, isolated_cpus)) 1232 return false; 1233 cpumask_set_cpu(cpu, isolated_cpus); 1234 return true; 1235 } 1236 1237 /* CPUs isolated at boot must remain isolated. */ 1238 if (!cpumask_test_cpu(cpu, 1239 housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)) || 1240 !cpumask_test_cpu(cpu, isolated_cpus)) 1241 return false; 1242 cpumask_clear_cpu(cpu, isolated_cpus); 1243 return true; 1244 } 1245 1246 /* 1247 * isolated_cpus_update - Set the isolation state of @xcpus 1248 * @new_prs: partition root state that @xcpus will belong to 1249 * @xcpus: exclusive CPUs whose isolation state may change 1250 * 1251 * CPUs isolated at boot time are kept isolated. 1252 */ 1253 static void isolated_cpus_update(int new_prs, struct cpumask *xcpus) 1254 { 1255 bool updated = false; 1256 int cpu; 1257 1258 lockdep_assert_held(&callback_lock); 1259 lockdep_assert_held(&cpuset_mutex); 1260 for_each_cpu(cpu, xcpus) 1261 updated |= isolated_cpu_update(new_prs, cpu); 1262 if (updated) 1263 update_housekeeping = true; 1264 } 1265 1266 /* 1267 * partition_xcpus_add - Add new exclusive CPUs to partition 1268 * @new_prs: new partition_root_state 1269 * @parent: parent cpuset 1270 * @xcpus: exclusive CPUs to be added 1271 * 1272 * Remote partition if parent == NULL 1273 */ 1274 static void partition_xcpus_add(int new_prs, struct cpuset *parent, 1275 struct cpumask *xcpus) 1276 { 1277 WARN_ON_ONCE(new_prs < 0); 1278 lockdep_assert_held(&callback_lock); 1279 1280 if (cpumask_empty(xcpus)) 1281 return; 1282 1283 if (!parent) 1284 parent = &top_cpuset; 1285 1286 if (parent == &top_cpuset) 1287 cpumask_or(subpartitions_cpus, subpartitions_cpus, xcpus); 1288 1289 isolated_cpus_update(new_prs, xcpus); 1290 1291 cpumask_andnot(parent->effective_cpus, parent->effective_cpus, xcpus); 1292 } 1293 1294 /* 1295 * partition_xcpus_del - Remove exclusive CPUs from partition 1296 * @old_prs: old partition_root_state 1297 * @parent: parent cpuset 1298 * @xcpus: exclusive CPUs to be removed 1299 * 1300 * Remote partition if parent == NULL 1301 */ 1302 static void partition_xcpus_del(int old_prs, struct cpuset *parent, 1303 struct cpumask *xcpus) 1304 { 1305 WARN_ON_ONCE(old_prs < 0); 1306 lockdep_assert_held(&callback_lock); 1307 1308 if (cpumask_empty(xcpus)) 1309 return; 1310 1311 if (!parent) 1312 parent = &top_cpuset; 1313 1314 if (parent == &top_cpuset) 1315 cpumask_andnot(subpartitions_cpus, subpartitions_cpus, xcpus); 1316 1317 /* CPUs given back, including those of descendants, take parent's state */ 1318 isolated_cpus_update(parent->partition_root_state, xcpus); 1319 1320 cpumask_or(parent->effective_cpus, parent->effective_cpus, xcpus); 1321 cpumask_and(parent->effective_cpus, parent->effective_cpus, cpu_active_mask); 1322 } 1323 1324 /* 1325 * prstate_housekeeping_conflict - check for partition & housekeeping conflicts 1326 * @new_prs: new partition root state to be checked 1327 * @parent_prs: Parent partition root state 1328 * @new_cpus: the set of new effective xcpus in the current cpuset 1329 * @del_cpus: old CPUs going to be removed from current cpuset, can be NULL 1330 * Return: true if there is conflict, false otherwise 1331 * 1332 * There are two different housekeeping conflicts to be checked: 1333 * 1) If new_prs is PRS_ROOT, none of the @new_cpus can be a boot-time isolated 1334 * CPU. IOW, the whole @new_cpus must be a subset of HK_TYPE_DOMAIN_BOOT. 1335 * 2) If nohz_full is enabled and we have isolated CPUs, their combination must 1336 * still leave housekeeping CPUs. This check is only needed if new_prs 1337 * differs from parent_prs and one of them is PRS_ISOLATED. 1338 */ 1339 static bool prstate_housekeeping_conflict(int new_prs, int parent_prs, 1340 struct cpumask *new_cpus, 1341 struct cpumask *del_cpus) 1342 { 1343 cpumask_var_t full_hk_cpus; 1344 int res; 1345 1346 WARN_ON_ONCE(parent_prs <= 0); 1347 if (!new_prs) 1348 return false; 1349 1350 /* 1351 * For a currently invalid partition root state, do the check assuming 1352 * that it may be converted back to valid. 1353 * 1354 * There are code paths where an invalid partition may be reverted back 1355 * to a valid partition but we still need to run the housekeeping check 1356 * before the conversion happens to make sure the switch is valid. We 1357 * can either do the negative to positive PRS switch individually in all 1358 * these code paths or do it here which I think will make the code a bit 1359 * easier to read. 1360 */ 1361 if (new_prs < 0) 1362 new_prs = -new_prs; 1363 1364 if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT) && (new_prs == PRS_ROOT) && 1365 !cpumask_subset(new_cpus, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT))) 1366 return true; 1367 1368 if (del_cpus && cpumask_empty(del_cpus)) 1369 del_cpus = NULL; 1370 1371 /* 1372 * Exit 2nd check early and return false if 1373 * 1) there is no boot time isolated nohz_full CPU 1374 * 2) parent and child has the same partition state 1375 * 3) new_prs is PRS_ROOT, parent_prs is PRS_ISOLATED and no CPU will 1376 * be returned back to the parent. 1377 */ 1378 if (!housekeeping_enabled(HK_TYPE_KERNEL_NOISE) || 1379 (new_prs == parent_prs) || (!del_cpus && (new_prs == PRS_ROOT))) 1380 return false; 1381 1382 /* 1383 * Now make sure that @new_cpus contains new CPUs to be isolated and 1384 * @del_cpus contains isolated CPUs to be un-isolated. 1385 * With the guards above, @new_cpus can't be NULL after the swap. 1386 */ 1387 if (parent_prs == PRS_ISOLATED) 1388 swap(new_cpus, del_cpus); 1389 1390 if (!alloc_cpumask_var(&full_hk_cpus, GFP_KERNEL)) 1391 return true; 1392 1393 /* 1394 * Check if del_cpus may contain a CPU that can become an active 1395 * housekeeping CPU again. 1396 */ 1397 if (del_cpus) { 1398 cpumask_and(full_hk_cpus, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE), 1399 housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)); 1400 if (cpumask_first_and_and(del_cpus, full_hk_cpus, 1401 cpu_active_mask) < nr_cpu_ids) { 1402 res = false; 1403 goto out; 1404 } 1405 } 1406 1407 cpumask_and(full_hk_cpus, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE), 1408 housekeeping_cpumask(HK_TYPE_DOMAIN)); 1409 cpumask_andnot(full_hk_cpus, full_hk_cpus, isolated_cpus); 1410 cpumask_and(full_hk_cpus, full_hk_cpus, cpu_active_mask); 1411 res = !cpumask_weight_andnot(full_hk_cpus, new_cpus); 1412 out: 1413 free_cpumask_var(full_hk_cpus); 1414 return res; 1415 } 1416 1417 /** 1418 * validate_partition - Validate a cpuset partition configuration 1419 * @cs: The cpuset to validate 1420 * @parent: Parent partition of @cs, NULL for remote partition 1421 * @new_prs: The proposed new partition root state 1422 * @new_excpus: The new effective exclusive CPUs mask to validate 1423 * @del: exclusive CPUs to be deleted from @cs, can be NULL 1424 * 1425 * A remote partition gets its CPUs from top_cpuset, so top_cpuset is used 1426 * as the parent when @parent is NULL. 1427 * 1428 * The housekeeping check is skipped if @parent is not a valid partition 1429 * root as @cs can't be a valid local partition in that case. 1430 * 1431 * Return: PRS error code (0 if valid, non-zero error code if invalid) 1432 */ 1433 static enum prs_errcode validate_partition(struct cpuset *cs, 1434 struct cpuset *parent, int new_prs, 1435 struct cpumask *new_excpus, 1436 struct cpumask *del) 1437 { 1438 int parent_prs; 1439 1440 if (new_prs == PRS_MEMBER) 1441 return PERR_NONE; 1442 1443 if (cpumask_empty(new_excpus)) 1444 return PERR_INVCPUS; 1445 1446 if (!parent) 1447 parent = &top_cpuset; 1448 parent_prs = parent->partition_root_state; 1449 if (parent_prs > 0 && 1450 prstate_housekeeping_conflict(new_prs, parent_prs, new_excpus, del)) 1451 return PERR_HKEEPING; 1452 1453 if (tasks_nocpu_error(parent, cs, new_excpus)) 1454 return PERR_NOCPUS; 1455 1456 return PERR_NONE; 1457 } 1458 1459 /* 1460 * cpuset_update_sd_hk_unlock - Rebuild sched domains, update HK & unlock 1461 * 1462 * Update housekeeping cpumasks and rebuild sched domains if necessary and 1463 * then do a cpuset_full_unlock(). 1464 * This should be called at the end of cpuset operation. 1465 */ 1466 static void cpuset_update_sd_hk_unlock(void) 1467 __releases(&cpuset_mutex) 1468 __releases(&cpuset_top_mutex) 1469 { 1470 /* force_sd_rebuild will be cleared in rebuild_sched_domains_locked() */ 1471 if (force_sd_rebuild) 1472 rebuild_sched_domains_locked(); 1473 1474 if (update_housekeeping) { 1475 update_housekeeping = false; 1476 cpumask_copy(isolated_hk_cpus, isolated_cpus); 1477 1478 /* 1479 * housekeeping_update() is now called without holding 1480 * cpus_read_lock and cpuset_mutex. Only cpuset_top_mutex 1481 * is still being held for mutual exclusion. 1482 */ 1483 mutex_unlock(&cpuset_mutex); 1484 cpus_read_unlock(); 1485 WARN_ON_ONCE(housekeeping_update(isolated_hk_cpus)); 1486 mutex_unlock(&cpuset_top_mutex); 1487 } else { 1488 cpuset_full_unlock(); 1489 } 1490 } 1491 1492 /* 1493 * Work function to invoke cpuset_update_sd_hk_unlock() 1494 */ 1495 static void hk_sd_workfn(struct work_struct *work) 1496 { 1497 cpuset_full_lock(); 1498 cpuset_update_sd_hk_unlock(); 1499 } 1500 1501 /** 1502 * rm_siblings_excl_cpus - Remove exclusive CPUs that are used by sibling cpusets 1503 * @parent: Parent cpuset containing all siblings 1504 * @cs: Current cpuset (will be skipped) 1505 * @excpus: exclusive effective CPU mask to modify 1506 * 1507 * This function ensures the given @excpus mask doesn't include any CPUs that 1508 * are exclusively allocated to sibling cpusets. It walks through all siblings 1509 * of @cs under @parent and removes their exclusive CPUs from @excpus. 1510 */ 1511 static int rm_siblings_excl_cpus(struct cpuset *parent, struct cpuset *cs, 1512 struct cpumask *excpus) 1513 { 1514 struct cgroup_subsys_state *css; 1515 struct cpuset *sibling; 1516 int retval = 0; 1517 1518 if (cpumask_empty(excpus)) 1519 return 0; 1520 1521 /* 1522 * Remove exclusive CPUs from siblings 1523 */ 1524 rcu_read_lock(); 1525 cpuset_for_each_child(sibling, css, parent) { 1526 struct cpumask *sibling_xcpus; 1527 1528 if (sibling == cs) 1529 continue; 1530 1531 /* 1532 * If exclusive_cpus is defined, effective_xcpus will always 1533 * be a subset. Otherwise, effective_xcpus will only be set 1534 * in a valid partition root. 1535 */ 1536 sibling_xcpus = cpumask_empty(sibling->exclusive_cpus) 1537 ? sibling->effective_xcpus 1538 : sibling->exclusive_cpus; 1539 1540 if (cpumask_intersects(excpus, sibling_xcpus)) { 1541 cpumask_andnot(excpus, excpus, sibling_xcpus); 1542 retval++; 1543 } 1544 } 1545 rcu_read_unlock(); 1546 1547 return retval; 1548 } 1549 1550 /* 1551 * compute_excpus - compute effective exclusive CPUs 1552 * @cs: cpuset 1553 * @xcpus: effective exclusive CPUs value to be set 1554 * Return: 0 if there is no sibling conflict, > 0 otherwise 1555 * 1556 * If exclusive_cpus isn't explicitly set , we have to scan the sibling cpusets 1557 * and exclude their exclusive_cpus or effective_xcpus as well. 1558 */ 1559 static int compute_excpus(struct cpuset *cs, struct cpumask *excpus) 1560 { 1561 struct cpuset *parent = parent_cs(cs); 1562 1563 cpumask_and(excpus, user_xcpus(cs), parent->effective_xcpus); 1564 1565 if (!cpumask_empty(cs->exclusive_cpus)) 1566 return 0; 1567 1568 return rm_siblings_excl_cpus(parent, cs, excpus); 1569 } 1570 1571 /* 1572 * compute_trialcs_excpus - Compute effective exclusive CPUs for a trial cpuset 1573 * @trialcs: The trial cpuset containing the proposed new configuration 1574 * @cs: The original cpuset that the trial configuration is based on 1575 * Return: 0 if successful with no sibling conflict, >0 if a conflict is found 1576 * 1577 * Computes the effective_xcpus for a trial configuration. @cs is provided to represent 1578 * the real cs. 1579 */ 1580 static int compute_trialcs_excpus(struct cpuset *trialcs, struct cpuset *cs) 1581 { 1582 struct cpuset *parent = parent_cs(trialcs); 1583 struct cpumask *excpus = trialcs->effective_xcpus; 1584 1585 /* trialcs is member, cpuset.cpus has no impact to excpus */ 1586 if (cs_is_member(cs)) 1587 cpumask_and(excpus, trialcs->exclusive_cpus, 1588 parent->effective_xcpus); 1589 else 1590 cpumask_and(excpus, user_xcpus(trialcs), parent->effective_xcpus); 1591 1592 return rm_siblings_excl_cpus(parent, cs, excpus); 1593 } 1594 1595 /** 1596 * compute_partition_effective_cpumask - compute effective_cpus for partition 1597 * @cs: partition root cpuset 1598 * @new_ecpus: previously computed effective_cpus to be updated 1599 * 1600 * Compute the effective_cpus of a partition root by scanning effective_xcpus 1601 * of child partition roots and excluding their effective_xcpus. 1602 * 1603 * This has the side effect of invalidating valid child partition roots, 1604 * if necessary. Since it is called from either cpuset_hotplug_update_tasks() 1605 * or update_cpumasks_hier() where parent and children are modified 1606 * successively, we don't need to call update_parent_effective_cpumask() 1607 * and the child's effective_cpus will be updated in later iterations. 1608 */ 1609 static void compute_partition_effective_cpumask(struct cpuset *cs, 1610 struct cpumask *new_ecpus) 1611 { 1612 struct cgroup_subsys_state *css; 1613 struct cpuset *child; 1614 bool populated = partition_is_populated(cs, NULL); 1615 static cpumask_t xcpus; /* Protected by cpuset_mutex */ 1616 1617 /* 1618 * Check child partition roots to see if they should be invalidated 1619 * when all the effective_cpus will be used up and cs is populated. 1620 * It is better to sacrifice a child partition root than invalidating 1621 * the whole subtree. 1622 * 1623 * Return early if the effective_cpus is already empty. 1624 */ 1625 compute_excpus(cs, new_ecpus); 1626 if (!cpumask_and(new_ecpus, new_ecpus, cpu_active_mask)) 1627 return; 1628 1629 rcu_read_lock(); 1630 cpuset_for_each_child(child, css, cs) { 1631 if (!is_partition_valid(child)) 1632 continue; 1633 1634 /* 1635 * There shouldn't be a remote partition underneath another 1636 * partition root. 1637 */ 1638 WARN_ON_ONCE(is_remote_partition(child)); 1639 if (populated && cpumask_subset(new_ecpus, child->effective_xcpus)) { 1640 int old_prs = child->partition_root_state; 1641 int parent_prs = cs->partition_root_state; 1642 1643 /* 1644 * Invalidate child partition. Its CPUs still owned by 1645 * cs take the isolation state of cs. effective_xcpus 1646 * of cs may not be updated yet, so recompute it. 1647 */ 1648 spin_lock_irq(&callback_lock); 1649 WRITE_ONCE(child->prs_err, PERR_NOCPUS); 1650 compute_excpus(cs, &xcpus); 1651 cpumask_and(&xcpus, &xcpus, child->effective_xcpus); 1652 isolated_cpus_update(parent_prs, &xcpus); 1653 child->partition_root_state = -old_prs; 1654 spin_unlock_irq(&callback_lock); 1655 notify_partition_change(child, old_prs); 1656 continue; 1657 } 1658 cpumask_andnot(new_ecpus, new_ecpus, 1659 child->effective_xcpus); 1660 } 1661 rcu_read_unlock(); 1662 } 1663 1664 /** 1665 * compute_effective_cpumask - Compute the effective cpumask of the cpuset 1666 * @new_cpus: the temp variable for the new effective_cpus mask 1667 * @cs: the cpuset the need to recompute the new effective_cpus mask 1668 * @parent: the parent cpuset 1669 * 1670 * For v2, the parent's effective_cpus is inherited if cpumask is empty. 1671 * The result is valid only if the given cpuset isn't a partition root. 1672 */ 1673 static void compute_effective_cpumask(struct cpumask *new_cpus, 1674 struct cpuset *cs, struct cpuset *parent) 1675 { 1676 bool has_cpus; 1677 1678 if (is_remote_partition(cs) || 1679 (is_partition_valid(cs) && is_partition_valid(parent))) 1680 return compute_partition_effective_cpumask(cs, new_cpus); 1681 1682 has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); 1683 if (!has_cpus && is_in_v2_mode()) 1684 cpumask_copy(new_cpus, parent->effective_cpus); 1685 } 1686 1687 /** 1688 * compute_effective_nodemask - Compute the effective nodemask of the cpuset 1689 * @new_mems: the temp variable for the new effective_mems mask 1690 * @cs: the cpuset the need to recompute the new effective_mems mask 1691 * @parent: the parent cpuset 1692 * 1693 * For v2, the parent's effective_mems is inherited if nodemask is empty. 1694 */ 1695 static void compute_effective_nodemask(nodemask_t *new_mems, 1696 struct cpuset *cs, struct cpuset *parent) 1697 { 1698 bool has_mems; 1699 1700 has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems); 1701 if (!has_mems && is_in_v2_mode()) 1702 nodes_copy(*new_mems, parent->effective_mems); 1703 } 1704 1705 /* 1706 * remote_partition_enable - Enable current cpuset as a remote partition root 1707 * @cs: the cpuset to update 1708 * @new_prs: new partition_root_state 1709 * @tmp: temporary masks 1710 * Return: 0 if successful, errcode if error 1711 * 1712 * Enable the current cpuset to become a remote partition root taking CPUs 1713 * directly from the top cpuset. cpuset_mutex must be held by the caller. 1714 */ 1715 static int remote_partition_enable(struct cpuset *cs, int new_prs, 1716 struct tmpmasks *tmp) 1717 { 1718 /* 1719 * The user must have sysadmin privilege. 1720 */ 1721 if (!capable(CAP_SYS_ADMIN)) 1722 return PERR_ACCESS; 1723 1724 /* 1725 * The requested exclusive_cpus must not be allocated to other 1726 * partitions and it can't use up all the root's effective_cpus. 1727 * 1728 * The effective_xcpus mask can contain offline CPUs, but there must 1729 * be at least one or more online CPUs present before it can be enabled. 1730 * 1731 * Note that creating a remote partition with any local partition root 1732 * above it or remote partition root underneath it is not allowed. 1733 */ 1734 compute_excpus(cs, tmp->new_cpus); 1735 if (!cpumask_intersects(tmp->new_cpus, cpu_active_mask) || 1736 cpumask_subset(top_cpuset.effective_cpus, tmp->new_cpus)) 1737 return PERR_INVCPUS; 1738 if (cpumask_intersects(tmp->new_cpus, subpartitions_cpus)) 1739 return PERR_NOCPUS; 1740 if (prstate_housekeeping_conflict(new_prs, PRS_ROOT, tmp->new_cpus, NULL)) 1741 return PERR_HKEEPING; 1742 1743 spin_lock_irq(&callback_lock); 1744 partition_xcpus_add(new_prs, NULL, tmp->new_cpus); 1745 cs->remote_partition = true; 1746 cpumask_copy(cs->effective_xcpus, tmp->new_cpus); 1747 spin_unlock_irq(&callback_lock); 1748 cpuset_force_rebuild(); 1749 WRITE_ONCE(cs->prs_err, 0); 1750 1751 /* 1752 * Propagate changes in top_cpuset's effective_cpus down the hierarchy. 1753 */ 1754 cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus); 1755 update_sibling_cpumasks(&top_cpuset, NULL, tmp); 1756 return 0; 1757 } 1758 1759 /* 1760 * remote_partition_disable - Remove current cpuset from remote partition list 1761 * @cs: the cpuset to update 1762 * @tmp: temporary masks 1763 * 1764 * The effective_cpus is also updated. 1765 * 1766 * cpuset_mutex must be held by the caller. 1767 */ 1768 static void remote_partition_disable(struct cpuset *cs, struct tmpmasks *tmp) 1769 { 1770 WARN_ON_ONCE(!is_remote_partition(cs)); 1771 /* 1772 * When a CPU is offlined, top_cpuset may end up with no available CPUs, 1773 * which should clear subpartitions_cpus. We should not emit a warning for this 1774 * scenario: the hierarchy is updated from top to bottom, so subpartitions_cpus 1775 * may already be cleared when disabling the partition. 1776 */ 1777 WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus) && 1778 !cpumask_empty(subpartitions_cpus)); 1779 1780 spin_lock_irq(&callback_lock); 1781 cs->remote_partition = false; 1782 partition_xcpus_del(cs->partition_root_state, NULL, cs->effective_xcpus); 1783 if (cs->prs_err) 1784 cs->partition_root_state = -cs->partition_root_state; 1785 else 1786 cs->partition_root_state = PRS_MEMBER; 1787 1788 /* effective_xcpus may need to be changed */ 1789 compute_excpus(cs, cs->effective_xcpus); 1790 reset_partition_data(cs); 1791 spin_unlock_irq(&callback_lock); 1792 cpuset_force_rebuild(); 1793 1794 /* 1795 * Propagate changes in top_cpuset's effective_cpus down the hierarchy. 1796 */ 1797 cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus); 1798 update_sibling_cpumasks(&top_cpuset, NULL, tmp); 1799 } 1800 1801 /* 1802 * remote_cpus_update - cpus_exclusive change of remote partition 1803 * @cs: the cpuset to be updated 1804 * @xcpus: the new exclusive_cpus mask, if non-NULL 1805 * @excpus: the new effective_xcpus mask 1806 * @tmp: temporary masks 1807 * 1808 * top_cpuset and subpartitions_cpus will be updated or partition can be 1809 * invalidated. 1810 */ 1811 static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, 1812 struct cpumask *excpus, struct tmpmasks *tmp) 1813 { 1814 bool adding, deleting; 1815 int prs = cs->partition_root_state; 1816 1817 if (WARN_ON_ONCE(!is_remote_partition(cs))) 1818 return; 1819 1820 WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus)); 1821 1822 if (cpumask_empty(excpus)) { 1823 WRITE_ONCE(cs->prs_err, PERR_CPUSEMPTY); 1824 goto invalidate; 1825 } 1826 1827 adding = cpumask_andnot(tmp->addmask, excpus, cs->effective_xcpus); 1828 deleting = cpumask_andnot(tmp->delmask, cs->effective_xcpus, excpus); 1829 1830 /* 1831 * Additions of remote CPUs is only allowed if those CPUs are 1832 * not allocated to other partitions and there are effective_cpus 1833 * left in the top cpuset. 1834 */ 1835 if (adding) { 1836 if (!capable(CAP_SYS_ADMIN)) 1837 WRITE_ONCE(cs->prs_err, PERR_ACCESS); 1838 else if (cpumask_intersects(tmp->addmask, subpartitions_cpus) || 1839 cpumask_subset(top_cpuset.effective_cpus, tmp->addmask)) 1840 WRITE_ONCE(cs->prs_err, PERR_NOCPUS); 1841 else if (prstate_housekeeping_conflict(prs, PRS_ROOT, 1842 excpus, tmp->delmask)) 1843 WRITE_ONCE(cs->prs_err, PERR_HKEEPING); 1844 if (cs->prs_err) 1845 goto invalidate; 1846 } 1847 1848 cs->remote_partition_updating = true; 1849 spin_lock_irq(&callback_lock); 1850 if (adding) 1851 partition_xcpus_add(prs, NULL, tmp->addmask); 1852 if (deleting) 1853 partition_xcpus_del(prs, NULL, tmp->delmask); 1854 /* 1855 * Need to update effective_xcpus and exclusive_cpus now as 1856 * update_sibling_cpumasks() below may iterate back to the same cs. 1857 */ 1858 cpumask_copy(cs->effective_xcpus, excpus); 1859 if (xcpus) 1860 cpumask_copy(cs->exclusive_cpus, xcpus); 1861 spin_unlock_irq(&callback_lock); 1862 if (adding || deleting) 1863 cpuset_force_rebuild(); 1864 1865 /* 1866 * Propagate changes in top_cpuset's effective_cpus down the hierarchy. 1867 */ 1868 cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus); 1869 update_sibling_cpumasks(&top_cpuset, NULL, tmp); 1870 cs->remote_partition_updating = false; 1871 return; 1872 1873 invalidate: 1874 remote_partition_disable(cs, tmp); 1875 } 1876 1877 /** 1878 * update_parent_effective_cpumask - update effective_cpus mask of parent cpuset 1879 * @cs: The cpuset that requests change in partition root state 1880 * @cmd: Partition root state change command 1881 * @newmask: Optional new effective xcpus for partcmd_update 1882 * @tmp: Temporary addmask and delmask 1883 * Return: 0 or a partition root state error code 1884 * 1885 * For partcmd_enable*, the cpuset is being transformed from a non-partition 1886 * root to a partition root. The effective_xcpus (cpus_allowed if 1887 * effective_xcpus not set) mask of the given cpuset will be taken away from 1888 * parent's effective_cpus. The function will return 0 if all the CPUs listed 1889 * in effective_xcpus can be granted or an error code will be returned. 1890 * 1891 * For partcmd_disable, the cpuset is being transformed from a partition 1892 * root back to a non-partition root. Any CPUs in effective_xcpus will be 1893 * given back to parent's effective_cpus. 0 will always be returned. 1894 * 1895 * For partcmd_update, if the optional newmask is specified, the cpu list is 1896 * to be changed from effective_xcpus to newmask. Otherwise, effective_xcpus 1897 * will be computed and a valid partition will be invalidated if it is the new 1898 * cpumask is empty. The cpuset should either be a valid or invalid partition 1899 * root. The partition root state may change from valid to invalid or vice 1900 * versa. 1901 * 1902 * For partcmd_invalidate, the current partition will be made invalid. 1903 * 1904 * The partcmd_enable* and partcmd_disable commands are used by 1905 * update_prstate(). An error code may be returned and the caller will check 1906 * for error. 1907 * 1908 * The partcmd_update command is used by update_cpumasks_hier() with newmask 1909 * NULL and update_cpumask() with newmask set. The partcmd_invalidate is used 1910 * by update_cpumask() with NULL newmask. In both cases, the callers won't 1911 * check for error and so partition_root_state and prs_err will be updated 1912 * directly. 1913 */ 1914 static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, 1915 struct cpumask *newmask, 1916 struct tmpmasks *tmp) 1917 { 1918 struct cpuset *parent = parent_cs(cs); 1919 int adding; /* Adding cpus to parent's effective_cpus */ 1920 int deleting; /* Deleting cpus from parent's effective_cpus */ 1921 int old_prs, new_prs; 1922 int part_error = PERR_NONE; /* Partition error? */ 1923 int parent_prs = parent->partition_root_state; 1924 bool nocpu; 1925 1926 lockdep_assert_cpuset_lock_held(); 1927 WARN_ON_ONCE(is_remote_partition(cs)); /* For local partition only */ 1928 1929 /* 1930 * new_prs will only be changed for the partcmd_update and 1931 * partcmd_invalidate commands. 1932 */ 1933 adding = deleting = false; 1934 old_prs = new_prs = cs->partition_root_state; 1935 1936 if (cmd == partcmd_invalidate) { 1937 if (is_partition_invalid(cs)) 1938 return 0; 1939 1940 /* 1941 * Make the current partition invalid. 1942 */ 1943 if (is_partition_valid(parent)) 1944 adding = cpumask_and(tmp->addmask, 1945 cs->effective_xcpus, 1946 parent->effective_xcpus); 1947 if (old_prs > 0) 1948 new_prs = -old_prs; 1949 1950 goto write_error; 1951 } 1952 1953 /* 1954 * The parent must be a partition root. 1955 * The new cpumask, if present, or the current cpus_allowed must 1956 * not be empty. 1957 */ 1958 if (!is_partition_valid(parent)) { 1959 return is_partition_invalid(parent) 1960 ? PERR_INVPARENT : PERR_NOTPART; 1961 } 1962 if (!newmask && xcpus_empty(cs)) 1963 return PERR_CPUSEMPTY; 1964 1965 /* 1966 * Except for partcmd_disable and non-NULL newmask, use delmask 1967 * to compute the current effective xcpus value and use it for 1968 * tasks_nocpu_error check. 1969 */ 1970 if ((cmd != partcmd_disable) && !newmask) { 1971 compute_excpus(cs, tmp->delmask); 1972 nocpu = tasks_nocpu_error(parent, cs, tmp->delmask); 1973 } 1974 1975 if ((cmd == partcmd_enable) || (cmd == partcmd_enablei)) { 1976 new_prs = (cmd == partcmd_enable) ? PRS_ROOT : PRS_ISOLATED; 1977 1978 part_error = validate_partition(cs, parent, new_prs, 1979 tmp->delmask, NULL); 1980 if (part_error) 1981 return part_error; 1982 1983 /* 1984 * This function will only be called when all the preliminary 1985 * checks have passed. At this point, the following condition 1986 * should hold. 1987 * 1988 * (cs->effective_xcpus & cpu_active_mask) ⊆ parent->effective_cpus 1989 * 1990 * Warn if it is not the case. 1991 */ 1992 cpumask_and(tmp->new_cpus, tmp->delmask, cpu_active_mask); 1993 WARN_ON_ONCE(!cpumask_subset(tmp->new_cpus, parent->effective_cpus)); 1994 deleting = true; 1995 } else if (cmd == partcmd_disable) { 1996 /* 1997 * May need to add cpus back to parent's effective_cpus 1998 * (and maybe removed from subpartitions_cpus/isolated_cpus) 1999 * for valid partition root. 2000 */ 2001 if (is_partition_valid(cs)) { 2002 cpumask_copy(tmp->addmask, cs->effective_xcpus); 2003 adding = true; 2004 } 2005 new_prs = PRS_MEMBER; 2006 } else if (newmask) { 2007 /* 2008 * Empty cpumask is not allowed 2009 */ 2010 if (cpumask_empty(newmask)) { 2011 part_error = PERR_CPUSEMPTY; 2012 goto write_error; 2013 } 2014 if ((parent_prs == PRS_ISOLATED) && (old_prs == -PRS_ROOT)) { 2015 part_error = PERR_ISOPARENT; 2016 goto write_error; 2017 } 2018 2019 /* Check newmask whether cpus are available for parent/cs */ 2020 nocpu = tasks_nocpu_error(parent, cs, newmask); 2021 2022 /* 2023 * partcmd_update with newmask: 2024 * 2025 * Compute add/delete mask to/from effective_cpus 2026 * 2027 * For valid partition: 2028 * addmask = effective_xcpus & ~newmask 2029 * & parent->effective_xcpus 2030 * delmask = newmask & ~effective_xcpus 2031 * & parent->effective_xcpus 2032 * 2033 * For invalid partition: 2034 * delmask = newmask & parent->effective_xcpus 2035 * The partition may become valid soon. 2036 */ 2037 if (is_partition_invalid(cs)) { 2038 adding = false; 2039 deleting = cpumask_and(tmp->delmask, 2040 newmask, parent->effective_xcpus); 2041 } else { 2042 cpumask_andnot(tmp->addmask, cs->effective_xcpus, newmask); 2043 adding = cpumask_and(tmp->addmask, tmp->addmask, 2044 parent->effective_xcpus); 2045 2046 cpumask_andnot(tmp->delmask, newmask, cs->effective_xcpus); 2047 deleting = cpumask_and(tmp->delmask, tmp->delmask, 2048 parent->effective_xcpus); 2049 } 2050 2051 /* 2052 * The new CPUs to be removed from parent's effective CPUs 2053 * must be present. 2054 */ 2055 if (deleting) { 2056 cpumask_and(tmp->new_cpus, tmp->delmask, cpu_active_mask); 2057 WARN_ON_ONCE(!cpumask_subset(tmp->new_cpus, parent->effective_cpus)); 2058 } 2059 2060 /* 2061 * Make partition invalid if parent's effective_cpus could 2062 * become empty and there are tasks in the parent. 2063 */ 2064 if (nocpu && (!adding || 2065 !cpumask_intersects(tmp->addmask, cpu_active_mask))) { 2066 part_error = PERR_NOCPUS; 2067 deleting = false; 2068 adding = cpumask_and(tmp->addmask, 2069 cs->effective_xcpus, parent->effective_xcpus); 2070 } 2071 } else { 2072 /* 2073 * partcmd_update w/o newmask 2074 * 2075 * delmask = newly computed effective xcpus value 2076 * 2077 * This can be called from: 2078 * 1) update_cpumasks_hier() 2079 * 2) cpuset_hotplug_update_tasks() 2080 * 2081 * Check to see if it can be transitioned from valid to 2082 * invalid partition or vice versa. 2083 * 2084 * A partition error happens when parent has tasks and all 2085 * its effective CPUs will have to be distributed out. 2086 */ 2087 nocpu |= cpumask_empty(tmp->delmask); 2088 if (nocpu) { 2089 part_error = PERR_NOCPUS; 2090 if (is_partition_valid(cs)) 2091 adding = cpumask_and(tmp->addmask, 2092 cs->effective_xcpus, 2093 parent->effective_xcpus); 2094 } else if ((parent_prs == PRS_ISOLATED) && 2095 ((old_prs == PRS_ROOT) || (old_prs == -PRS_ROOT))) { 2096 /* Parent partition state changed to PRS_ISOLATED */ 2097 part_error = PERR_ISOPARENT; 2098 if (old_prs == PRS_ROOT) { 2099 cpumask_copy(tmp->addmask, cs->effective_xcpus); 2100 adding = true; 2101 } 2102 } else if (is_partition_valid(cs) && 2103 !cpumask_equal(tmp->delmask, cs->effective_xcpus)) { 2104 /* 2105 * If there is an expansion of parent's effective_xcpus, 2106 * we may take more xcpus from parent. The parent's 2107 * effective_cpus should still have active CPUs or nocpu 2108 * should have been set. We also need to do a 2109 * housekeeping check for an isolated child with root 2110 * parent to make sure that it won't exhaust all the 2111 * active housekeeping CPUs. 2112 */ 2113 cpumask_andnot(tmp->addmask, cs->effective_xcpus, tmp->delmask); 2114 if ((parent_prs == PRS_ROOT) && (old_prs == PRS_ISOLATED) && 2115 prstate_housekeeping_conflict(old_prs, parent_prs, 2116 tmp->delmask, tmp->addmask)) { 2117 adding = cpumask_and(tmp->addmask, cs->effective_xcpus, 2118 tmp->delmask); 2119 part_error = PERR_HKEEPING; 2120 } else { 2121 deleting = cpumask_andnot(tmp->delmask, tmp->delmask, 2122 cs->effective_xcpus); 2123 } 2124 } else if (is_partition_invalid(cs)) { 2125 struct cgroup_subsys_state *css; 2126 struct cpuset *child; 2127 bool exclusive = true; 2128 2129 /* 2130 * Convert invalid partition to valid has to pass the 2131 * cpu exclusivity test when cpuset.cpus.exclusive is 2132 * empty as well as the housekeeping check. 2133 */ 2134 if (prstate_housekeeping_conflict(cs->partition_root_state, 2135 parent_prs, tmp->delmask, NULL)) { 2136 part_error = PERR_HKEEPING; 2137 goto write_error; 2138 } 2139 if (cpumask_empty(cs->exclusive_cpus)) { 2140 rcu_read_lock(); 2141 cpuset_for_each_child(child, css, parent) { 2142 if (child == cs) 2143 continue; 2144 if (!cpusets_are_exclusive(cs, child)) { 2145 exclusive = false; 2146 break; 2147 } 2148 } 2149 rcu_read_unlock(); 2150 } 2151 if (exclusive) 2152 deleting = !cpumask_empty(tmp->delmask); 2153 else 2154 part_error = PERR_NOTEXCL; 2155 } 2156 } 2157 2158 write_error: 2159 if (part_error) 2160 WRITE_ONCE(cs->prs_err, part_error); 2161 2162 if (cmd == partcmd_update) { 2163 /* 2164 * Check for possible transition between valid and invalid 2165 * partition root. 2166 */ 2167 switch (cs->partition_root_state) { 2168 case PRS_ROOT: 2169 case PRS_ISOLATED: 2170 if (part_error) 2171 new_prs = -old_prs; 2172 break; 2173 case PRS_INVALID_ROOT: 2174 case PRS_INVALID_ISOLATED: 2175 if (!part_error) 2176 new_prs = -old_prs; 2177 break; 2178 } 2179 } 2180 2181 if (!adding && !deleting && (new_prs == old_prs)) 2182 return 0; 2183 2184 /* 2185 * Change the parent's effective_cpus & effective_xcpus (top cpuset 2186 * only). 2187 * 2188 * Newly added CPUs will be removed from effective_cpus and 2189 * newly deleted ones will be added back to effective_cpus. 2190 */ 2191 spin_lock_irq(&callback_lock); 2192 if (old_prs != new_prs) 2193 cs->partition_root_state = new_prs; 2194 2195 /* 2196 * Adding to parent's effective_cpus means deletion CPUs from cs 2197 * and vice versa. 2198 */ 2199 if (adding) 2200 partition_xcpus_del(old_prs, parent, tmp->addmask); 2201 if (deleting) 2202 partition_xcpus_add(new_prs, parent, tmp->delmask); 2203 2204 spin_unlock_irq(&callback_lock); 2205 2206 if (adding || deleting) { 2207 cpuset_update_tasks_cpumask(parent, tmp->addmask); 2208 update_sibling_cpumasks(parent, cs, tmp); 2209 } 2210 2211 /* 2212 * For partcmd_update without newmask, it is being called from 2213 * cpuset_handle_hotplug(). Update the load balance flag and 2214 * scheduling domain accordingly. 2215 */ 2216 if ((cmd == partcmd_update) && !newmask) 2217 update_partition_sd_lb(cs, old_prs); 2218 2219 notify_partition_change(cs, old_prs); 2220 return 0; 2221 } 2222 2223 /* 2224 * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree 2225 * @cs: the cpuset to consider 2226 * @tmp: temp variables for calculating effective_cpus & partition setup 2227 * @force: don't skip any descendant cpusets if set 2228 * 2229 * When configured cpumask is changed, the effective cpumasks of this cpuset 2230 * and all its descendants need to be updated. 2231 * 2232 * On legacy hierarchy, effective_cpus will be the same with cpu_allowed. 2233 * 2234 * Called with cpuset_mutex held 2235 */ 2236 static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, 2237 bool force) 2238 { 2239 struct cpuset *cp; 2240 struct cgroup_subsys_state *pos_css; 2241 int old_prs, new_prs; 2242 2243 rcu_read_lock(); 2244 cpuset_for_each_descendant_pre(cp, pos_css, cs) { 2245 struct cpuset *parent = parent_cs(cp); 2246 bool remote = is_remote_partition(cp); 2247 bool update_parent = false; 2248 2249 old_prs = new_prs = cp->partition_root_state; 2250 2251 /* 2252 * For a child remote partition with remote_partition_updating 2253 * set, we can skip the whole subtree as the this function will 2254 * be called eventually. We can also skip the subtree if the 2255 * effective_xcpus isn't going to be changed. Otherwise, we 2256 * will have to call remote_cpus_update(). 2257 * 2258 * remote_cpus_update() will reuse tmp->new_cpus only after 2259 * its value is being processed. 2260 */ 2261 if (remote && (cp != cs)) { 2262 compute_excpus(cp, tmp->new_cpus); 2263 if (cp->remote_partition_updating || 2264 cpumask_equal(cp->effective_xcpus, tmp->new_cpus)) { 2265 pos_css = css_rightmost_descendant(pos_css); 2266 continue; 2267 } 2268 rcu_read_unlock(); 2269 remote_cpus_update(cp, NULL, tmp->new_cpus, tmp); 2270 rcu_read_lock(); 2271 2272 /* Remote partition may be invalidated */ 2273 new_prs = cp->partition_root_state; 2274 remote = (new_prs == old_prs); 2275 } 2276 2277 compute_effective_cpumask(tmp->new_cpus, cp, parent); 2278 2279 if (remote) 2280 goto get_css; /* Ready to update cpuset data */ 2281 2282 /* 2283 * A partition with no effective_cpus is allowed as long as 2284 * there is no task associated with it. Call 2285 * update_parent_effective_cpumask() to check it. 2286 */ 2287 if (is_partition_valid(cp) && cpumask_empty(tmp->new_cpus)) { 2288 update_parent = true; 2289 goto update_parent_effective; 2290 } 2291 2292 /* 2293 * Skip the whole subtree if 2294 * 1) the cpumask remains the same, 2295 * 2) has no partition root state, 2296 * 3) force flag not set, and 2297 * 4) for v2 load balance state same as its parent. 2298 */ 2299 if (!cp->partition_root_state && !force && 2300 cpumask_equal(tmp->new_cpus, cp->effective_cpus) && 2301 (!cpuset_v2() || 2302 (is_sched_load_balance(parent) == is_sched_load_balance(cp)))) { 2303 pos_css = css_rightmost_descendant(pos_css); 2304 continue; 2305 } 2306 2307 update_parent_effective: 2308 /* 2309 * update_parent_effective_cpumask() should have been called 2310 * for cs already in update_cpumask(). We should also call 2311 * cpuset_update_tasks_cpumask() again for tasks in the parent 2312 * cpuset if the parent's effective_cpus changes. 2313 */ 2314 if ((cp != cs) && old_prs) { 2315 int parent_prs = parent->partition_root_state; 2316 2317 switch (parent_prs) { 2318 case PRS_ROOT: 2319 case PRS_ISOLATED: 2320 update_parent = true; 2321 break; 2322 2323 default: 2324 /* 2325 * When parent is not a valid partition root, 2326 * child partition roots become invalid too. 2327 * The isolation state of their CPUs has been 2328 * settled when the parent gave them back. 2329 */ 2330 if (is_partition_valid(cp)) 2331 new_prs = -old_prs; 2332 2333 WRITE_ONCE(cp->prs_err, 2334 is_partition_invalid(parent) 2335 ? PERR_INVPARENT : PERR_NOTPART); 2336 break; 2337 } 2338 } 2339 get_css: 2340 if (!css_tryget_online(&cp->css)) 2341 continue; 2342 rcu_read_unlock(); 2343 2344 if (update_parent) { 2345 update_parent_effective_cpumask(cp, partcmd_update, NULL, tmp); 2346 /* 2347 * Capture current partition_root_state as it may have 2348 * changed. Always recompute effective cpumask as tmp->new_cpus 2349 * may be clobbered when update_sibling_cpumasks() is called 2350 * within update_parent_effective_cpumask(). 2351 */ 2352 new_prs = cp->partition_root_state; 2353 compute_effective_cpumask(tmp->new_cpus, cp, parent); 2354 } 2355 2356 spin_lock_irq(&callback_lock); 2357 cpumask_copy(cp->effective_cpus, tmp->new_cpus); 2358 cp->partition_root_state = new_prs; 2359 /* 2360 * Need to compute effective_xcpus if either exclusive_cpus 2361 * is non-empty or it is a valid partition root. 2362 */ 2363 if ((new_prs > 0) || !cpumask_empty(cp->exclusive_cpus)) 2364 compute_excpus(cp, cp->effective_xcpus); 2365 if (new_prs <= 0) 2366 reset_partition_data(cp); 2367 spin_unlock_irq(&callback_lock); 2368 2369 notify_partition_change(cp, old_prs); 2370 2371 WARN_ON(!is_in_v2_mode() && 2372 !cpumask_equal(cp->cpus_allowed, cp->effective_cpus)); 2373 2374 cpuset_update_tasks_cpumask(cp, tmp->new_cpus); 2375 2376 /* 2377 * On default hierarchy, inherit the CS_SCHED_LOAD_BALANCE 2378 * from parent if current cpuset isn't a valid partition root 2379 * and their load balance states differ. 2380 */ 2381 if (cpuset_v2() && !is_partition_valid(cp) && 2382 (is_sched_load_balance(parent) != is_sched_load_balance(cp))) { 2383 assign_bit(CS_SCHED_LOAD_BALANCE, &cp->flags, 2384 is_sched_load_balance(parent)); 2385 } 2386 2387 /* 2388 * On legacy hierarchy, if the effective cpumask of any non- 2389 * empty cpuset is changed, we need to rebuild sched domains. 2390 * On default hierarchy, the cpuset needs to be a partition 2391 * root as well. 2392 */ 2393 if (!cpumask_empty(cp->cpus_allowed) && 2394 is_sched_load_balance(cp) && 2395 (!cpuset_v2() || is_partition_valid(cp))) 2396 cpuset_force_rebuild(); 2397 2398 rcu_read_lock(); 2399 css_put(&cp->css); 2400 } 2401 rcu_read_unlock(); 2402 } 2403 2404 /** 2405 * update_sibling_cpumasks - Update siblings cpumasks 2406 * @parent: Parent cpuset 2407 * @cs: Current cpuset 2408 * @tmp: Temp variables 2409 */ 2410 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs, 2411 struct tmpmasks *tmp) 2412 { 2413 struct cpuset *sibling; 2414 struct cgroup_subsys_state *pos_css; 2415 2416 lockdep_assert_cpuset_lock_held(); 2417 2418 /* 2419 * Check all its siblings and call update_cpumasks_hier() 2420 * if their effective_cpus will need to be changed. 2421 * 2422 * It is possible a change in parent's effective_cpus 2423 * due to a change in a child partition's effective_xcpus will impact 2424 * its siblings even if they do not inherit parent's effective_cpus 2425 * directly. It should not impact valid partition. 2426 * 2427 * The update_cpumasks_hier() function may sleep. So we have to 2428 * release the RCU read lock before calling it. 2429 */ 2430 rcu_read_lock(); 2431 cpuset_for_each_child(sibling, pos_css, parent) { 2432 if (sibling == cs || is_partition_valid(sibling)) 2433 continue; 2434 2435 compute_effective_cpumask(tmp->new_cpus, sibling, 2436 parent); 2437 if (cpumask_equal(tmp->new_cpus, sibling->effective_cpus)) 2438 continue; 2439 2440 if (!css_tryget_online(&sibling->css)) 2441 continue; 2442 2443 rcu_read_unlock(); 2444 update_cpumasks_hier(sibling, tmp, false); 2445 rcu_read_lock(); 2446 css_put(&sibling->css); 2447 } 2448 rcu_read_unlock(); 2449 } 2450 2451 static int parse_cpuset_cpulist(const char *buf, struct cpumask *out_mask) 2452 { 2453 int retval; 2454 2455 retval = cpulist_parse(buf, out_mask); 2456 if (retval < 0) 2457 return retval; 2458 if (!cpumask_subset(out_mask, top_cpuset.cpus_allowed)) 2459 return -EINVAL; 2460 2461 return 0; 2462 } 2463 2464 /** 2465 * partition_cpus_change - Handle partition state changes due to CPU mask updates 2466 * @cs: The target cpuset being modified 2467 * @trialcs: The trial cpuset containing proposed configuration changes 2468 * @tmp: Temporary masks for intermediate calculations 2469 * 2470 * This function handles partition state transitions triggered by CPU mask changes. 2471 * CPU modifications may cause a partition to be disabled or require state updates. 2472 */ 2473 static void partition_cpus_change(struct cpuset *cs, struct cpuset *trialcs, 2474 struct tmpmasks *tmp) 2475 { 2476 enum prs_errcode prs_err; 2477 struct cpuset *parent = is_remote_partition(cs) ? NULL : parent_cs(cs); 2478 2479 if (cs_is_member(cs)) 2480 return; 2481 2482 /* 2483 * Check for housekeeping conflict by passing the whole 2484 * effective_xcpus as well as those CPUs that are to be 2485 * removed from cs as delmask. 2486 */ 2487 if (is_partition_valid(cs)) 2488 cpumask_andnot(tmp->delmask, cs->effective_xcpus, 2489 trialcs->effective_xcpus); 2490 else 2491 cpumask_clear(tmp->delmask); 2492 2493 prs_err = validate_partition(cs, parent, trialcs->partition_root_state, 2494 trialcs->effective_xcpus, tmp->delmask); 2495 if (prs_err) { 2496 WRITE_ONCE(cs->prs_err, prs_err); 2497 trialcs->prs_err = prs_err; 2498 } 2499 2500 if (is_remote_partition(cs)) { 2501 if (trialcs->prs_err) 2502 remote_partition_disable(cs, tmp); 2503 else 2504 remote_cpus_update(cs, trialcs->exclusive_cpus, 2505 trialcs->effective_xcpus, tmp); 2506 } else { 2507 if (trialcs->prs_err) 2508 update_parent_effective_cpumask(cs, partcmd_invalidate, 2509 NULL, tmp); 2510 else 2511 update_parent_effective_cpumask(cs, partcmd_update, 2512 trialcs->effective_xcpus, tmp); 2513 } 2514 } 2515 2516 /** 2517 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it 2518 * @cs: the cpuset to consider 2519 * @trialcs: trial cpuset 2520 * @buf: buffer of cpu numbers written to this cpuset 2521 */ 2522 static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, 2523 const char *buf) 2524 { 2525 int retval; 2526 struct tmpmasks tmp; 2527 bool force = false; 2528 int old_prs = cs->partition_root_state; 2529 2530 retval = parse_cpuset_cpulist(buf, trialcs->cpus_allowed); 2531 if (retval < 0) 2532 return retval; 2533 2534 /* Nothing to do if the cpus didn't change */ 2535 if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed)) 2536 return 0; 2537 2538 compute_trialcs_excpus(trialcs, cs); 2539 trialcs->prs_err = PERR_NONE; 2540 2541 retval = validate_change(cs, trialcs); 2542 if (retval < 0) 2543 return retval; 2544 2545 if (alloc_tmpmasks(&tmp)) 2546 return -ENOMEM; 2547 2548 /* 2549 * Check all the descendants in update_cpumasks_hier() if 2550 * effective_xcpus is to be changed. 2551 */ 2552 force = !cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus); 2553 2554 partition_cpus_change(cs, trialcs, &tmp); 2555 2556 spin_lock_irq(&callback_lock); 2557 cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed); 2558 cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus); 2559 if (old_prs && !is_partition_valid(cs)) { 2560 reset_partition_data(cs); 2561 /* effective_xcpus may be cleared, update the subtree too */ 2562 force = true; 2563 } 2564 spin_unlock_irq(&callback_lock); 2565 2566 /* effective_cpus/effective_xcpus will be updated here */ 2567 update_cpumasks_hier(cs, &tmp, force); 2568 2569 /* Update CS_SCHED_LOAD_BALANCE and/or sched_domains, if necessary */ 2570 if (cs->partition_root_state) 2571 update_partition_sd_lb(cs, old_prs); 2572 2573 free_tmpmasks(&tmp); 2574 return retval; 2575 } 2576 2577 /** 2578 * update_exclusive_cpumask - update the exclusive_cpus mask of a cpuset 2579 * @cs: the cpuset to consider 2580 * @trialcs: trial cpuset 2581 * @buf: buffer of cpu numbers written to this cpuset 2582 * 2583 * The tasks' cpumask will be updated if cs is a valid partition root. 2584 */ 2585 static int update_exclusive_cpumask(struct cpuset *cs, struct cpuset *trialcs, 2586 const char *buf) 2587 { 2588 int retval; 2589 struct tmpmasks tmp; 2590 bool force = false; 2591 int old_prs = cs->partition_root_state; 2592 2593 retval = parse_cpuset_cpulist(buf, trialcs->exclusive_cpus); 2594 if (retval < 0) 2595 return retval; 2596 2597 /* Nothing to do if the CPUs didn't change */ 2598 if (cpumask_equal(cs->exclusive_cpus, trialcs->exclusive_cpus)) 2599 return 0; 2600 2601 /* 2602 * Reject the change if there is exclusive CPUs conflict with 2603 * the siblings. 2604 */ 2605 if (compute_trialcs_excpus(trialcs, cs)) 2606 return -EINVAL; 2607 2608 /* 2609 * Check all the descendants in update_cpumasks_hier() if 2610 * effective_xcpus is to be changed. 2611 */ 2612 force = !cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus); 2613 2614 retval = validate_change(cs, trialcs); 2615 if (retval) 2616 return retval; 2617 2618 if (alloc_tmpmasks(&tmp)) 2619 return -ENOMEM; 2620 2621 trialcs->prs_err = PERR_NONE; 2622 partition_cpus_change(cs, trialcs, &tmp); 2623 2624 spin_lock_irq(&callback_lock); 2625 cpumask_copy(cs->exclusive_cpus, trialcs->exclusive_cpus); 2626 cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus); 2627 if (old_prs && !is_partition_valid(cs)) { 2628 reset_partition_data(cs); 2629 /* effective_xcpus may be cleared, update the subtree too */ 2630 force = true; 2631 } 2632 spin_unlock_irq(&callback_lock); 2633 2634 /* 2635 * Call update_cpumasks_hier() to update effective_cpus/effective_xcpus 2636 * of the subtree when it is a valid partition root or effective_xcpus 2637 * is updated. 2638 */ 2639 if (is_partition_valid(cs) || force) 2640 update_cpumasks_hier(cs, &tmp, force); 2641 2642 /* Update CS_SCHED_LOAD_BALANCE and/or sched_domains, if necessary */ 2643 if (cs->partition_root_state) 2644 update_partition_sd_lb(cs, old_prs); 2645 2646 free_tmpmasks(&tmp); 2647 return 0; 2648 } 2649 2650 /* 2651 * Migrate memory region from one set of nodes to another. This is 2652 * performed asynchronously as it can be called from process migration path 2653 * holding locks involved in process management. All mm migrations are 2654 * performed in the queued order and can be waited for by flushing 2655 * cpuset_migrate_mm_wq. 2656 */ 2657 2658 struct cpuset_migrate_mm_work { 2659 struct work_struct work; 2660 struct mm_struct *mm; 2661 nodemask_t from; 2662 nodemask_t to; 2663 }; 2664 2665 static void cpuset_migrate_mm_workfn(struct work_struct *work) 2666 { 2667 struct cpuset_migrate_mm_work *mwork = 2668 container_of(work, struct cpuset_migrate_mm_work, work); 2669 2670 /* on a wq worker, no need to worry about %current's mems_allowed */ 2671 do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL); 2672 mmput(mwork->mm); 2673 kfree(mwork); 2674 } 2675 2676 static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from, 2677 const nodemask_t *to) 2678 { 2679 struct cpuset_migrate_mm_work *mwork; 2680 2681 if (nodes_equal(*from, *to)) { 2682 mmput(mm); 2683 return; 2684 } 2685 2686 mwork = kzalloc_obj(*mwork); 2687 if (mwork) { 2688 mwork->mm = mm; 2689 mwork->from = *from; 2690 mwork->to = *to; 2691 INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn); 2692 queue_work(cpuset_migrate_mm_wq, &mwork->work); 2693 } else { 2694 mmput(mm); 2695 } 2696 } 2697 2698 static void flush_migrate_mm_task_workfn(struct callback_head *head) 2699 { 2700 flush_workqueue(cpuset_migrate_mm_wq); 2701 kfree(head); 2702 } 2703 2704 static void schedule_flush_migrate_mm(void) 2705 { 2706 struct callback_head *flush_cb; 2707 2708 flush_cb = kzalloc_obj(struct callback_head); 2709 if (!flush_cb) 2710 return; 2711 2712 init_task_work(flush_cb, flush_migrate_mm_task_workfn); 2713 2714 if (task_work_add(current, flush_cb, TWA_RESUME)) 2715 kfree(flush_cb); 2716 } 2717 2718 /* 2719 * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy 2720 * @tsk: the task to change 2721 * @newmems: new nodes that the task will be set 2722 * 2723 * We use the mems_allowed_seq seqlock to safely update both tsk->mems_allowed 2724 * and rebind an eventual tasks' mempolicy. If the task is allocating in 2725 * parallel, it might temporarily see an empty intersection, which results in 2726 * a seqlock check and retry before OOM or allocation failure. 2727 */ 2728 static void cpuset_change_task_nodemask(struct task_struct *tsk, 2729 nodemask_t *newmems) 2730 { 2731 task_lock(tsk); 2732 2733 local_irq_disable(); 2734 write_seqcount_begin(&tsk->mems_allowed_seq); 2735 2736 nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems); 2737 mpol_rebind_task(tsk, newmems); 2738 tsk->mems_allowed = *newmems; 2739 2740 write_seqcount_end(&tsk->mems_allowed_seq); 2741 local_irq_enable(); 2742 2743 task_unlock(tsk); 2744 } 2745 2746 static void *cpuset_being_rebound; 2747 2748 /** 2749 * cpuset_update_tasks_nodemask - Update the nodemasks of tasks in the cpuset. 2750 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed 2751 * 2752 * Iterate through each task of @cs updating its mems_allowed to the 2753 * effective cpuset's. As this function is called with cpuset_mutex held, 2754 * cpuset membership stays stable. 2755 * 2756 * - cpuset_change_task_nodemask(): guarantee_online_mems() 2757 * - mpol_rebind_mm(): effective_mems 2758 * - cpuset_migrate_mm(): guarantee_online_mems() 2759 * - old_mems_allowed: guarantee_online_mems() 2760 * 2761 * For v2, guarantee_online_mems() should return a node mask that is the same 2762 * as the effective_mems of current cpuset. 2763 */ 2764 void cpuset_update_tasks_nodemask(struct cpuset *cs) 2765 { 2766 static nodemask_t newmems; /* protected by cpuset_mutex */ 2767 struct css_task_iter it; 2768 struct task_struct *task; 2769 2770 cpuset_being_rebound = cs; /* causes mpol_dup() rebind */ 2771 guarantee_online_mems(cs, &newmems); 2772 2773 /* 2774 * The mpol_rebind_mm() call takes mmap_lock, which we couldn't 2775 * take while holding tasklist_lock. Forks can happen - the 2776 * mpol_dup() cpuset_being_rebound check will catch such forks, 2777 * and rebind their vma mempolicies too. Because we still hold 2778 * the global cpuset_mutex, we know that no other rebind effort 2779 * will be contending for the global variable cpuset_being_rebound. 2780 * It's ok if we rebind the same mm twice; mpol_rebind_mm() 2781 * is idempotent. Also migrate pages in each mm to new nodes. 2782 */ 2783 css_task_iter_start(&cs->css, 0, &it); 2784 while ((task = css_task_iter_next(&it))) { 2785 struct mm_struct *mm; 2786 bool migrate; 2787 2788 cpuset_change_task_nodemask(task, &newmems); 2789 2790 /* Rebind and migrate mm only for thread group leader */ 2791 if (!thread_group_leader(task)) 2792 continue; 2793 2794 mm = get_task_mm(task); 2795 if (!mm) 2796 continue; 2797 2798 migrate = is_memory_migrate(cs); 2799 2800 /* 2801 * For v1 we can have empty effective_mems, but we cannot 2802 * attach any tasks (see cpuset_can_attach_check()). For v2, 2803 * effective_mems is guaranteed to not be empty. 2804 */ 2805 mpol_rebind_mm(mm, &cs->effective_mems); 2806 if (migrate) 2807 cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems); 2808 else 2809 mmput(mm); 2810 } 2811 css_task_iter_end(&it); 2812 2813 /* 2814 * All the tasks' nodemasks have been updated, update 2815 * cs->old_mems_allowed. 2816 */ 2817 cs->old_mems_allowed = newmems; 2818 2819 /* We're done rebinding vmas to this cpuset's new mems_allowed. */ 2820 cpuset_being_rebound = NULL; 2821 } 2822 2823 /* 2824 * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree 2825 * @cs: the cpuset to consider 2826 * @new_mems: a temp variable for calculating new effective_mems 2827 * 2828 * When configured nodemask is changed, the effective nodemasks of this cpuset 2829 * and all its descendants need to be updated. 2830 * 2831 * On legacy hierarchy, effective_mems will be the same with mems_allowed. 2832 * 2833 * Called with cpuset_mutex held 2834 */ 2835 static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) 2836 { 2837 struct cpuset *cp; 2838 struct cgroup_subsys_state *pos_css; 2839 2840 rcu_read_lock(); 2841 cpuset_for_each_descendant_pre(cp, pos_css, cs) { 2842 struct cpuset *parent = parent_cs(cp); 2843 2844 compute_effective_nodemask(new_mems, cp, parent); 2845 2846 /* Skip the whole subtree if the nodemask remains the same. */ 2847 if (nodes_equal(*new_mems, cp->effective_mems)) { 2848 pos_css = css_rightmost_descendant(pos_css); 2849 continue; 2850 } 2851 2852 if (!css_tryget_online(&cp->css)) 2853 continue; 2854 rcu_read_unlock(); 2855 2856 check_insane_mems_config(new_mems); 2857 2858 spin_lock_irq(&callback_lock); 2859 cp->effective_mems = *new_mems; 2860 spin_unlock_irq(&callback_lock); 2861 2862 WARN_ON(!is_in_v2_mode() && 2863 !nodes_equal(cp->mems_allowed, cp->effective_mems)); 2864 2865 cpuset_update_tasks_nodemask(cp); 2866 2867 rcu_read_lock(); 2868 css_put(&cp->css); 2869 } 2870 rcu_read_unlock(); 2871 } 2872 2873 /* 2874 * Handle user request to change the 'mems' memory placement 2875 * of a cpuset. Needs to validate the request, update the 2876 * cpusets mems_allowed, and for each task in the cpuset, 2877 * update mems_allowed and rebind task's mempolicy and any vma 2878 * mempolicies and if the cpuset is marked 'memory_migrate', 2879 * migrate the tasks pages to the new memory. 2880 * 2881 * Call with cpuset_mutex held. May take callback_lock during call. 2882 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs, 2883 * lock each such tasks mm->mmap_lock, scan its vma's and rebind 2884 * their mempolicies to the cpusets new mems_allowed. 2885 */ 2886 static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs, 2887 const char *buf) 2888 { 2889 int retval; 2890 2891 /* 2892 * An empty mems_allowed is ok iff there are no tasks in the cpuset. 2893 * The validate_change() call ensures that cpusets with tasks have memory. 2894 */ 2895 retval = nodelist_parse(buf, trialcs->mems_allowed); 2896 if (retval < 0) 2897 return retval; 2898 2899 if (!nodes_subset(trialcs->mems_allowed, 2900 top_cpuset.mems_allowed)) 2901 return -EINVAL; 2902 2903 /* No change? nothing to do */ 2904 if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) 2905 return 0; 2906 2907 retval = validate_change(cs, trialcs); 2908 if (retval < 0) 2909 return retval; 2910 2911 spin_lock_irq(&callback_lock); 2912 cs->mems_allowed = trialcs->mems_allowed; 2913 spin_unlock_irq(&callback_lock); 2914 2915 /* use trialcs->mems_allowed as a temp variable */ 2916 update_nodemasks_hier(cs, &trialcs->mems_allowed); 2917 return 0; 2918 } 2919 2920 bool current_cpuset_is_being_rebound(void) 2921 { 2922 bool ret; 2923 2924 rcu_read_lock(); 2925 ret = task_cs(current) == cpuset_being_rebound; 2926 rcu_read_unlock(); 2927 2928 return ret; 2929 } 2930 2931 /** 2932 * update_prstate - update partition_root_state 2933 * @cs: the cpuset to update 2934 * @new_prs: new partition root state 2935 * Return: 0 if successful, != 0 if error 2936 * 2937 * Call with cpuset_mutex held. 2938 */ 2939 static int update_prstate(struct cpuset *cs, int new_prs) 2940 { 2941 int err = PERR_NONE, old_prs = cs->partition_root_state; 2942 struct cpuset *parent = parent_cs(cs); 2943 struct tmpmasks tmpmask; 2944 int parent_prs = parent->partition_root_state; 2945 bool isolcpus_updated = false; 2946 bool disable_partition = false; 2947 2948 if (old_prs == new_prs) 2949 return 0; 2950 2951 /* 2952 * Treat a previously invalid partition root as if it is a "member". 2953 */ 2954 if (new_prs && is_partition_invalid(cs)) 2955 old_prs = PRS_MEMBER; 2956 2957 if (alloc_tmpmasks(&tmpmask)) 2958 return -ENOMEM; 2959 2960 if (!old_prs) { 2961 /* 2962 * cpus_allowed and exclusive_cpus cannot be both empty. 2963 */ 2964 if (xcpus_empty(cs)) { 2965 err = PERR_CPUSEMPTY; 2966 goto out; 2967 } 2968 2969 /* 2970 * We don't support the creation of a new local partition with 2971 * a remote partition underneath it. This unsupported 2972 * setting can happen only if parent is the top_cpuset because 2973 * a remote partition cannot be created underneath an existing 2974 * local or remote partition. 2975 */ 2976 if ((parent == &top_cpuset) && 2977 cpumask_intersects(cs->exclusive_cpus, subpartitions_cpus)) { 2978 err = PERR_REMOTE; 2979 goto out; 2980 } 2981 2982 /* 2983 * If parent is valid partition, enable local partiion. 2984 * Otherwise, enable a remote partition. 2985 */ 2986 if (is_partition_valid(parent)) { 2987 enum partition_cmd cmd = (new_prs == PRS_ROOT) 2988 ? partcmd_enable : partcmd_enablei; 2989 2990 /* 2991 * It is not allowed to create a root partition under 2992 * an isolated parent partition. 2993 */ 2994 if ((new_prs == PRS_ROOT) && (parent_prs == PRS_ISOLATED)) { 2995 err = PERR_ISOPARENT; 2996 goto out; 2997 } 2998 err = update_parent_effective_cpumask(cs, cmd, NULL, &tmpmask); 2999 } else { 3000 err = remote_partition_enable(cs, new_prs, &tmpmask); 3001 } 3002 } else if (old_prs && new_prs) { 3003 int parent_prs = is_remote_partition(cs) 3004 ? PRS_ROOT : parent->partition_root_state; 3005 3006 /* 3007 * A change in load balance state only, no change in cpumasks. 3008 * Need to update isolated_cpus. 3009 */ 3010 if (prstate_housekeeping_conflict(new_prs, parent_prs, 3011 cs->effective_xcpus, NULL)) 3012 err = PERR_HKEEPING; 3013 else if (parent_prs == PRS_ISOLATED) 3014 err = PERR_ISOPARENT; /* old_prs should be isolated */ 3015 3016 if (err) { 3017 disable_partition = true; 3018 } else { 3019 struct cgroup_subsys_state *css; 3020 struct cpuset *child; 3021 3022 /* 3023 * Need to scan child partitions, if present, and 3024 * exclude their effective_xcpus from cs. In the case 3025 * of switching to isolated partition with child root 3026 * partitions, those child partitions will be 3027 * invalidated in update_cpumasks_hier(). 3028 */ 3029 isolcpus_updated = true; 3030 cpumask_copy(tmpmask.new_cpus, cs->effective_xcpus); 3031 rcu_read_lock(); 3032 cpuset_for_each_child(child, css, cs) { 3033 if (!is_partition_valid(child)) 3034 continue; 3035 cpumask_andnot(tmpmask.new_cpus, tmpmask.new_cpus, 3036 child->effective_xcpus); 3037 } 3038 rcu_read_unlock(); 3039 } 3040 } else { 3041 /* 3042 * Switching back to member is always allowed even if it 3043 * disables child partitions. 3044 */ 3045 disable_partition = true; 3046 } 3047 3048 if (disable_partition) { 3049 if (is_remote_partition(cs)) 3050 remote_partition_disable(cs, &tmpmask); 3051 else 3052 update_parent_effective_cpumask(cs, partcmd_disable, 3053 NULL, &tmpmask); 3054 3055 /* 3056 * Invalidation of child partitions will be done in 3057 * update_cpumasks_hier() below. 3058 */ 3059 } 3060 out: 3061 /* 3062 * Make partition invalid if an error happens. 3063 */ 3064 if (err) 3065 new_prs = -new_prs; 3066 3067 spin_lock_irq(&callback_lock); 3068 cs->partition_root_state = new_prs; 3069 WRITE_ONCE(cs->prs_err, err); 3070 if (!is_partition_valid(cs)) 3071 reset_partition_data(cs); 3072 else if (isolcpus_updated) 3073 isolated_cpus_update(new_prs, tmpmask.new_cpus); 3074 spin_unlock_irq(&callback_lock); 3075 3076 /* Force update if partition is disabled & update effective_xcpus */ 3077 update_cpumasks_hier(cs, &tmpmask, disable_partition); 3078 3079 /* A newly created partition must have effective_xcpus set */ 3080 WARN_ON_ONCE(!old_prs && (new_prs > 0) 3081 && cpumask_empty(cs->effective_xcpus)); 3082 3083 /* Update sched domains and load balance flag */ 3084 update_partition_sd_lb(cs, old_prs); 3085 3086 notify_partition_change(cs, old_prs); 3087 free_tmpmasks(&tmpmask); 3088 return 0; 3089 } 3090 3091 /* 3092 * Check to see if a cpuset can accept a new task 3093 * For v1, cpus_allowed and mems_allowed can't be empty. 3094 * For v2, effective_cpus can't be empty. 3095 * Note that in v1, effective_cpus = cpus_allowed. 3096 * 3097 * Also set the boolean flag passed in by @psetsched depending on if 3098 * security_task_setscheduler() call is needed and @oldcs is not NULL. 3099 */ 3100 static int cpuset_can_attach_check(struct cpuset *cs, struct cpuset *oldcs, 3101 bool *psetsched) 3102 { 3103 bool cpus_updated, mems_updated; 3104 3105 if (cpumask_empty(cs->effective_cpus) || 3106 (!is_in_v2_mode() && nodes_empty(cs->mems_allowed))) 3107 return -ENOSPC; 3108 3109 if (!oldcs) 3110 return 0; 3111 3112 if (!llist_on_list(&oldcs->attach_node)) 3113 llist_add(&oldcs->attach_node, &src_cs_head); 3114 3115 if (!llist_on_list(&cs->attach_node)) 3116 llist_add(&cs->attach_node, &dst_cs_head); 3117 3118 cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus); 3119 mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); 3120 3121 if (cpus_updated) 3122 attach_ctx.cpus_updated = true; 3123 if (mems_updated) 3124 attach_ctx.mems_updated = true; 3125 3126 /* 3127 * Skip rights over task setsched check in v2 when nothing changes for 3128 * the current oldcs/cs pair, migration permission derives from 3129 * hierarchy ownership in cgroup_procs_write_permission()). 3130 */ 3131 *psetsched = !cpuset_v2() || cpus_updated || mems_updated; 3132 3133 /* 3134 * A v1 cpuset with tasks will have no CPU left only when CPU hotplug 3135 * brings the last online CPU offline as users are not allowed to empty 3136 * cpuset.cpus when there are active tasks inside. When that happens, 3137 * we should allow tasks to migrate out without security check to make 3138 * sure they will be able to run after migration. 3139 */ 3140 if (!is_in_v2_mode() && cpumask_empty(oldcs->effective_cpus)) 3141 *psetsched = false; 3142 3143 return 0; 3144 } 3145 3146 static int cpuset_reserve_dl_bw(void) 3147 { 3148 struct cpuset *cs; 3149 int cpu, ret; 3150 3151 llist_for_each_entry(cs, dst_cs_head.first, attach_node) { 3152 if (!cs->sum_migrate_dl_bw) 3153 continue; 3154 3155 cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); 3156 if (unlikely(cpu >= nr_cpu_ids)) 3157 return -EINVAL; 3158 3159 ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); 3160 if (ret) 3161 return ret; 3162 3163 cs->dl_bw_cpu = cpu; 3164 } 3165 return 0; 3166 } 3167 3168 /* 3169 * Clear and optionally apply (@cancel is false) the attach related data in the 3170 * source or destination cpuset. 3171 */ 3172 static void clear_attach_data(struct llist_head *head, bool cancel) 3173 { 3174 struct cpuset *cs, *next; 3175 struct llist_node *lnode = __llist_del_all(head); 3176 3177 llist_for_each_entry_safe(cs, next, lnode, attach_node) { 3178 init_llist_node(&cs->attach_node); 3179 if (cs->nr_migrate_dl_tasks) { 3180 if (!cancel) 3181 atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks); 3182 else if (cs->dl_bw_cpu >= 0) /* && cancel */ 3183 dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw); 3184 cs->nr_migrate_dl_tasks = 0; 3185 cs->sum_migrate_dl_bw = 0; 3186 cs->dl_bw_cpu = -1; 3187 } 3188 } 3189 } 3190 3191 /* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */ 3192 static int cpuset_can_attach(struct cgroup_taskset *tset) 3193 { 3194 struct cgroup_subsys_state *css; 3195 struct cpuset *cs, *oldcs; 3196 struct task_struct *task; 3197 bool setsched_check; 3198 int ret; 3199 3200 cs = oldcs = NULL; 3201 mutex_lock(&cpuset_mutex); 3202 attach_ctx.old_cs = NULL; /* Used later in cpuset_attach_task() */ 3203 attach_ctx.cpus_updated = false; 3204 attach_ctx.mems_updated = false; 3205 attach_ctx.many_dest_cs = false; 3206 3207 /* 3208 * The attach_ctx.old_cs is used mainly by cpuset_migrate_mm() to get 3209 * the old_mems_allowed value. There are two ways that many-to-one 3210 * cpuset migration can happen: 3211 * 1) A multithread application with threads in different cpusets is 3212 * wholely migrated to a new cpuset. 3213 * 2) Disabling v2 cpuset controller will move all the tasks in child 3214 * cpusets to the parent cpuset. 3215 * 3216 * In the former case, it is the mm setting of the group leader that 3217 * really matters. So attach_ctx.old_cs should track the oldcs of the 3218 * group leader. It falls back to the oldcs of the first task if there 3219 * is no group leader in the taskset. In the latter case, effective_mems 3220 * of child cpusets must always be a subset of the parent. So no real 3221 * page migration will be necessary no matter which child cpuset is 3222 * selected as attach_ctx.old_cs. 3223 * 3224 * For a v2 threaded subtree where cpuset isn't enabled in some of the 3225 * cgroups, it is possible that oldcs == cs for some of the tasks. 3226 * In this case, we can skip checking on those tasks as there is no 3227 * actual migration wrt cpuset. 3228 */ 3229 cgroup_taskset_for_each(task, css, tset) { 3230 struct cpuset *new_cs = css_cs(css); 3231 struct cpuset *new_oldcs = task_cs(task); 3232 3233 if ((new_oldcs != oldcs) || (new_cs != cs)) { 3234 if (cs && (new_cs != cs)) 3235 attach_ctx.many_dest_cs = true; 3236 cs = new_cs; 3237 oldcs = new_oldcs; 3238 if (oldcs == cs) 3239 continue; 3240 if (!attach_ctx.old_cs) 3241 attach_ctx.old_cs = oldcs; 3242 ret = cpuset_can_attach_check(cs, oldcs, &setsched_check); 3243 if (ret) 3244 goto out_unlock; 3245 } 3246 3247 if (oldcs == cs) 3248 continue; 3249 3250 ret = task_can_attach(task); 3251 if (ret) 3252 goto out_unlock; 3253 3254 /* Update attach_ctx.old_cs to the latest group leader */ 3255 if (task == task->group_leader) 3256 attach_ctx.old_cs = task_cs(task); 3257 3258 if (setsched_check) { 3259 ret = security_task_setscheduler(task); 3260 if (ret) 3261 goto out_unlock; 3262 } 3263 3264 if (dl_task(task)) { 3265 /* 3266 * Count all migrating DL tasks for cpuset task accounting. 3267 * Only tasks that need a root-domain bandwidth move 3268 * contribute to sum_migrate_dl_bw. 3269 */ 3270 cs->nr_migrate_dl_tasks++; 3271 oldcs->nr_migrate_dl_tasks--; 3272 if (dl_task_needs_bw_move(task, cs->effective_cpus)) 3273 cs->sum_migrate_dl_bw += task->dl.dl_bw; 3274 } 3275 } 3276 3277 /* 3278 * The only case where there are multiple destination cpusets for 3279 * task migration is when enabling a v2 cpuset controllers where 3280 * tasks will be migrated to multiple child cpusets from a parent 3281 * cpuset with the same effective CPUs and memory nodes. IOW, 3282 * both attach_cpus_updated and attach_mems_updated should be false. 3283 * If not, it is a condition that the current code cannot handle. 3284 * Print a warning and abort the attach operation as further code 3285 * change may be needed. 3286 */ 3287 if (WARN_ON_ONCE(attach_ctx.many_dest_cs && (!cpuset_v2() || 3288 attach_ctx.cpus_updated || attach_ctx.mems_updated))) { 3289 ret = -EINVAL; 3290 goto out_unlock; 3291 } 3292 3293 ret = cpuset_reserve_dl_bw(); 3294 3295 out_unlock: 3296 if (ret) { 3297 clear_attach_data(&src_cs_head, true); 3298 clear_attach_data(&dst_cs_head, true); 3299 } else { 3300 attach_ctx.in_progress++; 3301 } 3302 3303 mutex_unlock(&cpuset_mutex); 3304 return ret; 3305 } 3306 3307 static void cpuset_cancel_attach(struct cgroup_taskset *tset) 3308 { 3309 mutex_lock(&cpuset_mutex); 3310 dec_attach_in_progress_locked(); 3311 clear_attach_data(&src_cs_head, true); 3312 clear_attach_data(&dst_cs_head, true); 3313 mutex_unlock(&cpuset_mutex); 3314 } 3315 3316 /* 3317 * Protected by cpuset_mutex. cpus_attach is used only by cpuset_attach_task() 3318 * but we can't allocate it dynamically there. Define it global and 3319 * allocate from cpuset_init(). 3320 */ 3321 static cpumask_var_t cpus_attach; 3322 3323 static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task) 3324 { 3325 struct mm_struct *mm; 3326 3327 lockdep_assert_cpuset_lock_held(); 3328 3329 if (cs != &top_cpuset) 3330 guarantee_active_cpus(task, cpus_attach); 3331 else 3332 cpumask_andnot(cpus_attach, task_cpu_possible_mask(task), 3333 subpartitions_cpus); 3334 /* 3335 * can_attach beforehand should guarantee that this doesn't 3336 * fail. TODO: have a better way to handle failure here 3337 */ 3338 WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach)); 3339 3340 if (cpuset_v2() && !attach_ctx.mems_updated) 3341 return; 3342 3343 cpuset_change_task_nodemask(task, &attach_ctx.nodemask_to); 3344 cpuset1_update_task_spread_flags(cs, task); 3345 3346 if ((task != task->group_leader) || !attach_ctx.mems_updated) 3347 return; 3348 3349 /* 3350 * Change mm for threadgroup leader. This is expensive and may 3351 * sleep and should be moved outside migration path proper. 3352 */ 3353 mm = get_task_mm(task); 3354 if (mm) { 3355 struct cpuset *oldcs = attach_ctx.old_cs; 3356 3357 mpol_rebind_mm(mm, &cs->effective_mems); 3358 3359 /* 3360 * old_mems_allowed is the same with mems_allowed 3361 * here, except if this task is being moved 3362 * automatically due to hotplug. In that case 3363 * @mems_allowed has been updated and is empty, so 3364 * @old_mems_allowed is the right nodesets that we 3365 * migrate mm from. 3366 */ 3367 if (is_memory_migrate(cs)) { 3368 cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, 3369 &attach_ctx.nodemask_to); 3370 attach_ctx.task_work_queued = true; 3371 } else { 3372 mmput(mm); 3373 } 3374 } 3375 } 3376 3377 static void cpuset_attach(struct cgroup_taskset *tset) 3378 { 3379 struct task_struct *task; 3380 struct cgroup_subsys_state *css; 3381 struct cpuset *cs; 3382 3383 cgroup_taskset_first(tset, &css); 3384 cs = css_cs(css); 3385 3386 lockdep_assert_cpus_held(); /* see cgroup_attach_lock() */ 3387 mutex_lock(&cpuset_mutex); 3388 attach_ctx.task_work_queued = false; 3389 guarantee_online_mems(cs, &attach_ctx.nodemask_to); 3390 3391 /* 3392 * attach_ctx.old_cs can only be NULL if no task is actually migrating. 3393 * This is highly unlikely. If it happens at all, we can skip task 3394 * iteration and setting old_mems_allowed. 3395 */ 3396 if (unlikely(!attach_ctx.old_cs)) 3397 goto out; 3398 3399 /* 3400 * In the default hierarchy, enabling cpuset in the child cgroups 3401 * will trigger a cpuset_attach() call with no change in effective cpus 3402 * and mems. In that case, we can optimize out by skipping the task 3403 * iteration and the destination cpuset list is iterated to set 3404 * old_mems_allowed. 3405 */ 3406 if (cpuset_v2() && !attach_ctx.cpus_updated && !attach_ctx.mems_updated) { 3407 llist_for_each_entry(cs, dst_cs_head.first, attach_node) 3408 cs->old_mems_allowed = attach_ctx.nodemask_to; 3409 goto out; 3410 } 3411 3412 cgroup_taskset_for_each(task, css, tset) 3413 cpuset_attach_task(cs, task); 3414 3415 if (attach_ctx.task_work_queued) 3416 schedule_flush_migrate_mm(); 3417 cs->old_mems_allowed = attach_ctx.nodemask_to; 3418 out: 3419 clear_attach_data(&src_cs_head, false); 3420 clear_attach_data(&dst_cs_head, false); 3421 dec_attach_in_progress_locked(); 3422 3423 mutex_unlock(&cpuset_mutex); 3424 } 3425 3426 /* 3427 * Common handling for a write to a "cpus" or "mems" file. 3428 */ 3429 ssize_t cpuset_write_resmask(struct kernfs_open_file *of, 3430 char *buf, size_t nbytes, loff_t off) 3431 { 3432 struct cpuset *cs = css_cs(of_css(of)); 3433 struct cpuset *trialcs; 3434 int retval = -ENODEV; 3435 3436 /* root is read-only */ 3437 if (cs == &top_cpuset) 3438 return -EACCES; 3439 3440 buf = strstrip(buf); 3441 3442 /* cpuset_mutex acquired in wait_attach_done_lock() */ 3443 mutex_lock(&cpuset_top_mutex); 3444 cpus_read_lock(); 3445 wait_attach_done_lock(); 3446 3447 if (!is_cpuset_online(cs)) 3448 goto out_unlock; 3449 3450 trialcs = dup_or_alloc_cpuset(cs); 3451 if (!trialcs) { 3452 retval = -ENOMEM; 3453 goto out_unlock; 3454 } 3455 3456 switch (of_cft(of)->private) { 3457 case FILE_CPULIST: 3458 retval = update_cpumask(cs, trialcs, buf); 3459 break; 3460 case FILE_EXCLUSIVE_CPULIST: 3461 retval = update_exclusive_cpumask(cs, trialcs, buf); 3462 break; 3463 case FILE_MEMLIST: 3464 retval = update_nodemask(cs, trialcs, buf); 3465 break; 3466 default: 3467 retval = -EINVAL; 3468 break; 3469 } 3470 3471 free_cpuset(trialcs); 3472 out_unlock: 3473 cpuset_update_sd_hk_unlock(); 3474 if (of_cft(of)->private == FILE_MEMLIST) 3475 schedule_flush_migrate_mm(); 3476 return retval ?: nbytes; 3477 } 3478 3479 /* 3480 * These ascii lists should be read in a single call, by using a user 3481 * buffer large enough to hold the entire map. If read in smaller 3482 * chunks, there is no guarantee of atomicity. Since the display format 3483 * used, list of ranges of sequential numbers, is variable length, 3484 * and since these maps can change value dynamically, one could read 3485 * gibberish by doing partial reads while a list was changing. 3486 */ 3487 int cpuset_common_seq_show(struct seq_file *sf, void *v) 3488 { 3489 struct cpuset *cs = css_cs(seq_css(sf)); 3490 cpuset_filetype_t type = seq_cft(sf)->private; 3491 int ret = 0; 3492 3493 spin_lock_irq(&callback_lock); 3494 3495 switch (type) { 3496 case FILE_CPULIST: 3497 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_allowed)); 3498 break; 3499 case FILE_MEMLIST: 3500 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed)); 3501 break; 3502 case FILE_EFFECTIVE_CPULIST: 3503 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus)); 3504 break; 3505 case FILE_EFFECTIVE_MEMLIST: 3506 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems)); 3507 break; 3508 case FILE_EXCLUSIVE_CPULIST: 3509 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->exclusive_cpus)); 3510 break; 3511 case FILE_EFFECTIVE_XCPULIST: 3512 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_xcpus)); 3513 break; 3514 case FILE_SUBPARTS_CPULIST: 3515 seq_printf(sf, "%*pbl\n", cpumask_pr_args(subpartitions_cpus)); 3516 break; 3517 case FILE_ISOLATED_CPULIST: 3518 seq_printf(sf, "%*pbl\n", cpumask_pr_args(isolated_cpus)); 3519 break; 3520 default: 3521 ret = -EINVAL; 3522 } 3523 3524 spin_unlock_irq(&callback_lock); 3525 return ret; 3526 } 3527 3528 static int cpuset_partition_show(struct seq_file *seq, void *v) 3529 { 3530 struct cpuset *cs = css_cs(seq_css(seq)); 3531 const char *err, *type = NULL; 3532 3533 switch (cs->partition_root_state) { 3534 case PRS_ROOT: 3535 seq_puts(seq, "root\n"); 3536 break; 3537 case PRS_ISOLATED: 3538 seq_puts(seq, "isolated\n"); 3539 break; 3540 case PRS_MEMBER: 3541 seq_puts(seq, "member\n"); 3542 break; 3543 case PRS_INVALID_ROOT: 3544 type = "root"; 3545 fallthrough; 3546 case PRS_INVALID_ISOLATED: 3547 if (!type) 3548 type = "isolated"; 3549 err = perr_strings[READ_ONCE(cs->prs_err)]; 3550 if (err) 3551 seq_printf(seq, "%s invalid (%s)\n", type, err); 3552 else 3553 seq_printf(seq, "%s invalid\n", type); 3554 break; 3555 } 3556 return 0; 3557 } 3558 3559 static ssize_t cpuset_partition_write(struct kernfs_open_file *of, char *buf, 3560 size_t nbytes, loff_t off) 3561 { 3562 struct cpuset *cs = css_cs(of_css(of)); 3563 int val; 3564 int retval = -ENODEV; 3565 3566 buf = strstrip(buf); 3567 3568 if (!strcmp(buf, "root")) 3569 val = PRS_ROOT; 3570 else if (!strcmp(buf, "member")) 3571 val = PRS_MEMBER; 3572 else if (!strcmp(buf, "isolated")) 3573 val = PRS_ISOLATED; 3574 else 3575 return -EINVAL; 3576 3577 mutex_lock(&cpuset_top_mutex); 3578 cpus_read_lock(); 3579 wait_attach_done_lock(); 3580 3581 if (is_cpuset_online(cs)) 3582 retval = update_prstate(cs, val); 3583 cpuset_update_sd_hk_unlock(); 3584 return retval ?: nbytes; 3585 } 3586 3587 /* 3588 * This is currently a minimal set for the default hierarchy. It can be 3589 * expanded later on by migrating more features and control files from v1. 3590 */ 3591 static struct cftype dfl_files[] = { 3592 { 3593 .name = "cpus", 3594 .seq_show = cpuset_common_seq_show, 3595 .write = cpuset_write_resmask, 3596 .max_write_len = (100U + 6 * NR_CPUS), 3597 .private = FILE_CPULIST, 3598 .flags = CFTYPE_NOT_ON_ROOT, 3599 }, 3600 3601 { 3602 .name = "mems", 3603 .seq_show = cpuset_common_seq_show, 3604 .write = cpuset_write_resmask, 3605 .max_write_len = (100U + 6 * MAX_NUMNODES), 3606 .private = FILE_MEMLIST, 3607 .flags = CFTYPE_NOT_ON_ROOT, 3608 }, 3609 3610 { 3611 .name = "cpus.effective", 3612 .seq_show = cpuset_common_seq_show, 3613 .private = FILE_EFFECTIVE_CPULIST, 3614 }, 3615 3616 { 3617 .name = "mems.effective", 3618 .seq_show = cpuset_common_seq_show, 3619 .private = FILE_EFFECTIVE_MEMLIST, 3620 }, 3621 3622 { 3623 .name = "cpus.partition", 3624 .seq_show = cpuset_partition_show, 3625 .write = cpuset_partition_write, 3626 .private = FILE_PARTITION_ROOT, 3627 .flags = CFTYPE_NOT_ON_ROOT, 3628 .file_offset = offsetof(struct cpuset, partition_file), 3629 }, 3630 3631 { 3632 .name = "cpus.exclusive", 3633 .seq_show = cpuset_common_seq_show, 3634 .write = cpuset_write_resmask, 3635 .max_write_len = (100U + 6 * NR_CPUS), 3636 .private = FILE_EXCLUSIVE_CPULIST, 3637 .flags = CFTYPE_NOT_ON_ROOT, 3638 }, 3639 3640 { 3641 .name = "cpus.exclusive.effective", 3642 .seq_show = cpuset_common_seq_show, 3643 .private = FILE_EFFECTIVE_XCPULIST, 3644 .flags = CFTYPE_NOT_ON_ROOT, 3645 }, 3646 3647 { 3648 .name = "cpus.subpartitions", 3649 .seq_show = cpuset_common_seq_show, 3650 .private = FILE_SUBPARTS_CPULIST, 3651 .flags = CFTYPE_ONLY_ON_ROOT | CFTYPE_DEBUG, 3652 }, 3653 3654 { 3655 .name = "cpus.isolated", 3656 .seq_show = cpuset_common_seq_show, 3657 .private = FILE_ISOLATED_CPULIST, 3658 .flags = CFTYPE_ONLY_ON_ROOT, 3659 }, 3660 3661 { } /* terminate */ 3662 }; 3663 3664 3665 /** 3666 * cpuset_css_alloc - Allocate a cpuset css 3667 * @parent_css: Parent css of the control group that the new cpuset will be 3668 * part of 3669 * Return: cpuset css on success, -ENOMEM on failure. 3670 * 3671 * Allocate and initialize a new cpuset css, for non-NULL @parent_css, return 3672 * top cpuset css otherwise. 3673 */ 3674 static struct cgroup_subsys_state * 3675 cpuset_css_alloc(struct cgroup_subsys_state *parent_css) 3676 { 3677 struct cpuset *cs; 3678 3679 if (!parent_css) 3680 return &top_cpuset.css; 3681 3682 cs = dup_or_alloc_cpuset(NULL); 3683 if (!cs) 3684 return ERR_PTR(-ENOMEM); 3685 3686 __set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); 3687 cpuset1_init(cs); 3688 3689 /* Set CS_MEMORY_MIGRATE for default hierarchy */ 3690 if (cpuset_v2()) 3691 __set_bit(CS_MEMORY_MIGRATE, &cs->flags); 3692 3693 return &cs->css; 3694 } 3695 3696 static int cpuset_css_online(struct cgroup_subsys_state *css) 3697 { 3698 struct cpuset *cs = css_cs(css); 3699 struct cpuset *parent = parent_cs(cs); 3700 3701 if (!parent) 3702 return 0; 3703 3704 cpuset_full_lock(); 3705 /* 3706 * For v2, clear CS_SCHED_LOAD_BALANCE if parent is isolated 3707 */ 3708 if (cpuset_v2() && !is_sched_load_balance(parent)) 3709 clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); 3710 3711 cpuset_inc(); 3712 3713 spin_lock_irq(&callback_lock); 3714 if (is_in_v2_mode()) { 3715 cpumask_copy(cs->effective_cpus, parent->effective_cpus); 3716 cs->effective_mems = parent->effective_mems; 3717 } 3718 spin_unlock_irq(&callback_lock); 3719 cpuset1_online_css(css); 3720 3721 cpuset_full_unlock(); 3722 return 0; 3723 } 3724 3725 static void cpuset_css_offline(struct cgroup_subsys_state *css) 3726 { 3727 struct cpuset *cs = css_cs(css); 3728 3729 cpuset_full_lock(); 3730 cpuset1_offline_css(cs); 3731 cpuset_dec(); 3732 cpuset_full_unlock(); 3733 } 3734 3735 /* 3736 * If a dying cpuset has the 'cpus.partition' enabled, turn it off by 3737 * changing it back to member to free its exclusive CPUs back to the pool to 3738 * be used by other online cpusets. 3739 */ 3740 static void cpuset_css_killed(struct cgroup_subsys_state *css) 3741 { 3742 struct cpuset *cs = css_cs(css); 3743 3744 cpuset_full_lock(); 3745 /* Reset valid partition back to member */ 3746 if (is_partition_valid(cs)) 3747 update_prstate(cs, PRS_MEMBER); 3748 cpuset_update_sd_hk_unlock(); 3749 } 3750 3751 static void cpuset_css_free(struct cgroup_subsys_state *css) 3752 { 3753 struct cpuset *cs = css_cs(css); 3754 3755 free_cpuset(cs); 3756 } 3757 3758 static void cpuset_bind(struct cgroup_subsys_state *root_css) 3759 { 3760 mutex_lock(&cpuset_mutex); 3761 spin_lock_irq(&callback_lock); 3762 3763 WRITE_ONCE(cpuset_v2_mode, 3764 !!(cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE)); 3765 if (is_in_v2_mode()) { 3766 cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask); 3767 cpumask_copy(top_cpuset.effective_xcpus, cpu_possible_mask); 3768 top_cpuset.mems_allowed = node_possible_map; 3769 } else { 3770 cpumask_copy(top_cpuset.cpus_allowed, 3771 top_cpuset.effective_cpus); 3772 top_cpuset.mems_allowed = top_cpuset.effective_mems; 3773 } 3774 3775 spin_unlock_irq(&callback_lock); 3776 mutex_unlock(&cpuset_mutex); 3777 } 3778 3779 /* 3780 * In case the child is cloned into a cpuset different from its parent, 3781 * additional checks are done to see if the move is allowed. 3782 */ 3783 static int cpuset_can_fork(struct task_struct *task, struct css_set *cset) 3784 { 3785 struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]); 3786 bool same_cs; 3787 int ret; 3788 3789 rcu_read_lock(); 3790 same_cs = (cs == task_cs(current)); 3791 rcu_read_unlock(); 3792 3793 if (same_cs) 3794 return 0; 3795 3796 lockdep_assert_held(&cgroup_mutex); 3797 mutex_lock(&cpuset_mutex); 3798 3799 /* Check to see if task is allowed in the cpuset */ 3800 ret = cpuset_can_attach_check(cs, NULL, NULL); 3801 if (ret) 3802 goto out_unlock; 3803 3804 ret = task_can_attach(task); 3805 if (ret) 3806 goto out_unlock; 3807 3808 ret = security_task_setscheduler(task); 3809 if (ret) 3810 goto out_unlock; 3811 3812 attach_ctx.in_progress++; 3813 out_unlock: 3814 mutex_unlock(&cpuset_mutex); 3815 return ret; 3816 } 3817 3818 static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset) 3819 { 3820 struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]); 3821 bool same_cs; 3822 3823 rcu_read_lock(); 3824 same_cs = (cs == task_cs(current)); 3825 rcu_read_unlock(); 3826 3827 if (same_cs) 3828 return; 3829 3830 dec_attach_in_progress(); 3831 } 3832 3833 /* 3834 * Make sure the new task conform to the current state of its parent, 3835 * which could have been changed by cpuset just after it inherits the 3836 * state from the parent and before it sits on the cgroup's task list. 3837 */ 3838 static void cpuset_fork(struct task_struct *task) 3839 { 3840 struct cpuset *cs, *oldcs; 3841 3842 rcu_read_lock(); 3843 cs = task_cs(task); 3844 oldcs = task_cs(current); 3845 rcu_read_unlock(); 3846 3847 if (cs == oldcs) { 3848 unsigned int seq; 3849 3850 if (cs == &top_cpuset) 3851 return; 3852 3853 set_cpus_allowed_ptr(task, current->cpus_ptr); 3854 /* 3855 * A cpuset.mems update can update current->mems_allowed 3856 * concurrently, so retry it again if it really happens. 3857 */ 3858 do { 3859 seq = read_mems_allowed_begin(); 3860 task->mems_allowed = current->mems_allowed; 3861 } while (read_mems_allowed_retry(seq)); 3862 return; 3863 } 3864 3865 /* CLONE_INTO_CGROUP */ 3866 mutex_lock(&cpuset_mutex); 3867 guarantee_online_mems(cs, &attach_ctx.nodemask_to); 3868 cs->old_mems_allowed = attach_ctx.nodemask_to; 3869 3870 /* 3871 * Assume CPUs and memory nodes are updated 3872 * A CLONE_INTO_CGROUP operation should have taken the cgroup mutex 3873 * and so there shouldn't be a competing cpuset_attach() operation. 3874 */ 3875 attach_ctx.cpus_updated = attach_ctx.mems_updated = true; 3876 attach_ctx.task_work_queued = false; 3877 attach_ctx.old_cs = oldcs; 3878 cpuset_attach_task(cs, task); 3879 if (attach_ctx.task_work_queued) 3880 schedule_flush_migrate_mm(); 3881 3882 dec_attach_in_progress_locked(); 3883 mutex_unlock(&cpuset_mutex); 3884 } 3885 3886 struct cgroup_subsys cpuset_cgrp_subsys = { 3887 .css_alloc = cpuset_css_alloc, 3888 .css_online = cpuset_css_online, 3889 .css_offline = cpuset_css_offline, 3890 .css_killed = cpuset_css_killed, 3891 .css_free = cpuset_css_free, 3892 .can_attach = cpuset_can_attach, 3893 .cancel_attach = cpuset_cancel_attach, 3894 .attach = cpuset_attach, 3895 .bind = cpuset_bind, 3896 .can_fork = cpuset_can_fork, 3897 .cancel_fork = cpuset_cancel_fork, 3898 .fork = cpuset_fork, 3899 #ifdef CONFIG_CPUSETS_V1 3900 .legacy_cftypes = cpuset1_files, 3901 #endif 3902 .dfl_cftypes = dfl_files, 3903 .early_init = true, 3904 .threaded = true, 3905 }; 3906 3907 /** 3908 * cpuset_init - initialize cpusets at system boot 3909 * 3910 * Description: Initialize top_cpuset 3911 **/ 3912 3913 int __init cpuset_init(void) 3914 { 3915 BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL)); 3916 BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL)); 3917 BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_xcpus, GFP_KERNEL)); 3918 BUG_ON(!alloc_cpumask_var(&top_cpuset.exclusive_cpus, GFP_KERNEL)); 3919 BUG_ON(!zalloc_cpumask_var(&subpartitions_cpus, GFP_KERNEL)); 3920 BUG_ON(!zalloc_cpumask_var(&isolated_cpus, GFP_KERNEL)); 3921 BUG_ON(!zalloc_cpumask_var(&isolated_hk_cpus, GFP_KERNEL)); 3922 3923 cpumask_setall(top_cpuset.cpus_allowed); 3924 nodes_setall(top_cpuset.mems_allowed); 3925 cpumask_setall(top_cpuset.effective_cpus); 3926 cpumask_setall(top_cpuset.effective_xcpus); 3927 cpumask_setall(top_cpuset.exclusive_cpus); 3928 nodes_setall(top_cpuset.effective_mems); 3929 init_llist_node(&top_cpuset.attach_node); 3930 3931 cpuset1_init(&top_cpuset); 3932 3933 BUG_ON(!alloc_cpumask_var(&cpus_attach, GFP_KERNEL)); 3934 3935 if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) 3936 cpumask_andnot(isolated_cpus, cpu_possible_mask, 3937 housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)); 3938 3939 return 0; 3940 } 3941 3942 static void 3943 hotplug_update_tasks(struct cpuset *cs, 3944 struct cpumask *new_cpus, nodemask_t *new_mems, 3945 bool cpus_updated, bool mems_updated) 3946 { 3947 /* A partition root is allowed to have empty effective cpus */ 3948 if (cpumask_empty(new_cpus) && !is_partition_valid(cs)) 3949 cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus); 3950 if (nodes_empty(*new_mems)) 3951 *new_mems = parent_cs(cs)->effective_mems; 3952 3953 spin_lock_irq(&callback_lock); 3954 cpumask_copy(cs->effective_cpus, new_cpus); 3955 cs->effective_mems = *new_mems; 3956 spin_unlock_irq(&callback_lock); 3957 3958 if (cpus_updated) 3959 cpuset_update_tasks_cpumask(cs, new_cpus); 3960 if (mems_updated) 3961 cpuset_update_tasks_nodemask(cs); 3962 } 3963 3964 void cpuset_force_rebuild(void) 3965 { 3966 force_sd_rebuild = true; 3967 } 3968 3969 /** 3970 * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug 3971 * @cs: cpuset in interest 3972 * @tmp: the tmpmasks structure pointer 3973 * 3974 * Compare @cs's cpu and mem masks against top_cpuset and if some have gone 3975 * offline, update @cs accordingly. If @cs ends up with no CPU or memory, 3976 * all its tasks are moved to the nearest ancestor with both resources. 3977 */ 3978 static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) 3979 { 3980 static cpumask_t new_cpus; 3981 static nodemask_t new_mems; 3982 bool cpus_updated; 3983 bool mems_updated; 3984 int partcmd = -1; 3985 struct cpuset *parent; 3986 3987 wait_attach_done_lock(); 3988 parent = parent_cs(cs); 3989 compute_effective_cpumask(&new_cpus, cs, parent); 3990 compute_effective_nodemask(&new_mems, cs, parent); 3991 3992 if (!tmp || !cs->partition_root_state) 3993 goto update_tasks; 3994 3995 if (is_remote_partition(cs) && (cpumask_empty(subpartitions_cpus) || 3996 (cpumask_empty(&new_cpus) && 3997 partition_is_populated(cs, NULL)))) { 3998 WRITE_ONCE(cs->prs_err, PERR_HOTPLUG); 3999 remote_partition_disable(cs, tmp); 4000 compute_effective_cpumask(&new_cpus, cs, parent); 4001 } 4002 4003 /* 4004 * Force the partition to become invalid if either one of 4005 * the following conditions hold: 4006 * 1) empty effective cpus but not valid empty partition. 4007 * 2) parent is invalid or doesn't grant any cpus to child 4008 * partitions. 4009 * 3) subpartitions_cpus is empty. 4010 */ 4011 if (is_local_partition(cs) && 4012 (!is_partition_valid(parent) || 4013 tasks_nocpu_error(parent, cs, &new_cpus) || 4014 cpumask_empty(subpartitions_cpus))) 4015 partcmd = partcmd_invalidate; 4016 /* 4017 * On the other hand, an invalid partition root may be transitioned 4018 * back to a regular one if its effective xcpus has an active CPU. 4019 */ 4020 else if (is_partition_valid(parent) && is_partition_invalid(cs) && 4021 cpumask_intersects(cs->effective_xcpus, cpu_active_mask)) 4022 partcmd = partcmd_update; 4023 4024 if (partcmd >= 0) { 4025 update_parent_effective_cpumask(cs, partcmd, NULL, tmp); 4026 if (is_partition_valid(cs) || (partcmd == partcmd_invalidate)) { 4027 compute_effective_cpumask(&new_cpus, cs, parent); 4028 cpuset_force_rebuild(); 4029 } 4030 } 4031 4032 update_tasks: 4033 cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus); 4034 mems_updated = !nodes_equal(new_mems, cs->effective_mems); 4035 if (!cpus_updated && !mems_updated) 4036 goto unlock; /* Hotplug doesn't affect this cpuset */ 4037 4038 if (mems_updated) 4039 check_insane_mems_config(&new_mems); 4040 4041 if (is_in_v2_mode()) 4042 hotplug_update_tasks(cs, &new_cpus, &new_mems, 4043 cpus_updated, mems_updated); 4044 else 4045 cpuset1_hotplug_update_tasks(cs, &new_cpus, &new_mems, 4046 cpus_updated, mems_updated); 4047 4048 unlock: 4049 mutex_unlock(&cpuset_mutex); 4050 } 4051 4052 /** 4053 * cpuset_handle_hotplug - handle CPU/memory hot{,un}plug for a cpuset 4054 * 4055 * This function is called after either CPU or memory configuration has 4056 * changed and updates cpuset accordingly. The top_cpuset is always 4057 * synchronized to cpu_active_mask and N_MEMORY, which is necessary in 4058 * order to make cpusets transparent (of no affect) on systems that are 4059 * actively using CPU hotplug but making no active use of cpusets. 4060 * 4061 * Non-root cpusets are only affected by offlining. If any CPUs or memory 4062 * nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on 4063 * all descendants. 4064 * 4065 * Note that CPU offlining during suspend is ignored. We don't modify 4066 * cpusets across suspend/resume cycles at all. 4067 * 4068 * CPU / memory hotplug is handled synchronously. 4069 */ 4070 static void cpuset_handle_hotplug(void) 4071 { 4072 static DECLARE_WORK(hk_sd_work, hk_sd_workfn); 4073 static cpumask_t new_cpus; 4074 static nodemask_t new_mems; 4075 bool cpus_updated, mems_updated; 4076 bool on_dfl; 4077 struct tmpmasks tmp, *ptmp = NULL; 4078 4079 lockdep_assert_cpus_held(); 4080 mutex_lock(&cpuset_mutex); 4081 4082 on_dfl = is_in_v2_mode(); 4083 if (on_dfl && !alloc_tmpmasks(&tmp)) 4084 ptmp = &tmp; 4085 4086 /* fetch the available cpus/mems and find out which changed how */ 4087 cpumask_copy(&new_cpus, cpu_active_mask); 4088 new_mems = node_states[N_MEMORY]; 4089 4090 /* 4091 * If subpartitions_cpus is populated, it is likely that the check 4092 * below will produce a false positive on cpus_updated when the cpu 4093 * list isn't changed. It is extra work, but it is better to be safe. 4094 */ 4095 cpus_updated = !cpumask_equal(top_cpuset.effective_cpus, &new_cpus) || 4096 !cpumask_empty(subpartitions_cpus); 4097 mems_updated = !nodes_equal(top_cpuset.effective_mems, new_mems); 4098 4099 /* For v1, synchronize cpus_allowed to cpu_active_mask */ 4100 if (cpus_updated) { 4101 cpuset_force_rebuild(); 4102 spin_lock_irq(&callback_lock); 4103 if (!on_dfl) 4104 cpumask_copy(top_cpuset.cpus_allowed, &new_cpus); 4105 /* 4106 * Make sure that CPUs allocated to child partitions 4107 * do not show up in effective_cpus. If no CPU is left, 4108 * we clear the subpartitions_cpus & let the child partitions 4109 * fight for the CPUs again. 4110 */ 4111 if (!cpumask_empty(subpartitions_cpus)) { 4112 if (cpumask_subset(&new_cpus, subpartitions_cpus)) { 4113 cpumask_clear(subpartitions_cpus); 4114 } else { 4115 cpumask_andnot(&new_cpus, &new_cpus, 4116 subpartitions_cpus); 4117 } 4118 } 4119 cpumask_copy(top_cpuset.effective_cpus, &new_cpus); 4120 spin_unlock_irq(&callback_lock); 4121 /* we don't mess with cpumasks of tasks in top_cpuset */ 4122 } 4123 4124 /* synchronize mems_allowed to N_MEMORY */ 4125 if (mems_updated) { 4126 spin_lock_irq(&callback_lock); 4127 if (!on_dfl) 4128 top_cpuset.mems_allowed = new_mems; 4129 top_cpuset.effective_mems = new_mems; 4130 spin_unlock_irq(&callback_lock); 4131 cpuset_update_tasks_nodemask(&top_cpuset); 4132 } 4133 4134 mutex_unlock(&cpuset_mutex); 4135 4136 /* if cpus or mems changed, we need to propagate to descendants */ 4137 if (cpus_updated || mems_updated) { 4138 struct cpuset *cs; 4139 struct cgroup_subsys_state *pos_css; 4140 4141 rcu_read_lock(); 4142 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { 4143 if (cs == &top_cpuset || !css_tryget_online(&cs->css)) 4144 continue; 4145 rcu_read_unlock(); 4146 4147 cpuset_hotplug_update_tasks(cs, ptmp); 4148 4149 rcu_read_lock(); 4150 css_put(&cs->css); 4151 } 4152 rcu_read_unlock(); 4153 } 4154 4155 /* 4156 * rebuild_sched_domains() will always be called directly if needed 4157 * to make sure that newly added or removed CPU will be reflected in 4158 * the sched domains. However, if isolated partition invalidation 4159 * or recreation is being done (update_housekeeping set), a work item 4160 * will be queued to call housekeeping_update() to update the 4161 * corresponding housekeeping cpumasks after some slight delay. 4162 * 4163 * We rely on WORK_STRUCT_PENDING_BIT to not requeue a work item that 4164 * is still pending. Before the pending bit is cleared, the work data 4165 * is copied out and work item dequeued. So it is possible to queue 4166 * the work again before the hk_sd_workfn() is invoked to process the 4167 * previously queued work. Since hk_sd_workfn() doesn't use the work 4168 * item at all, this is not a problem. 4169 */ 4170 if (force_sd_rebuild) 4171 rebuild_sched_domains_cpuslocked(); 4172 if (update_housekeeping) 4173 queue_work(system_dfl_wq, &hk_sd_work); 4174 4175 free_tmpmasks(ptmp); 4176 } 4177 4178 void cpuset_update_active_cpus(void) 4179 { 4180 /* 4181 * We're inside cpu hotplug critical region which usually nests 4182 * inside cgroup synchronization. Bounce actual hotplug processing 4183 * to a work item to avoid reverse locking order. 4184 */ 4185 cpuset_handle_hotplug(); 4186 } 4187 4188 /* 4189 * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY]. 4190 * Call this routine anytime after node_states[N_MEMORY] changes. 4191 * See cpuset_update_active_cpus() for CPU hotplug handling. 4192 */ 4193 static int cpuset_track_online_nodes(struct notifier_block *self, 4194 unsigned long action, void *arg) 4195 { 4196 cpuset_handle_hotplug(); 4197 return NOTIFY_OK; 4198 } 4199 4200 /** 4201 * cpuset_init_smp - initialize cpus_allowed 4202 * 4203 * Description: Finish top cpuset after cpu, node maps are initialized 4204 */ 4205 void __init cpuset_init_smp(void) 4206 { 4207 /* 4208 * cpus_allowd/mems_allowed set to v2 values in the initial 4209 * cpuset_bind() call will be reset to v1 values in another 4210 * cpuset_bind() call when v1 cpuset is mounted. 4211 */ 4212 top_cpuset.old_mems_allowed = top_cpuset.mems_allowed; 4213 4214 cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask); 4215 top_cpuset.effective_mems = node_states[N_MEMORY]; 4216 4217 hotplug_node_notifier(cpuset_track_online_nodes, CPUSET_CALLBACK_PRI); 4218 4219 cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0); 4220 BUG_ON(!cpuset_migrate_mm_wq); 4221 } 4222 4223 /* 4224 * Return cpus_allowed mask from a task's cpuset. 4225 */ 4226 static void __cpuset_cpus_allowed_locked(struct task_struct *tsk, struct cpumask *pmask) 4227 { 4228 struct cpuset *cs; 4229 4230 cs = task_cs(tsk); 4231 if (cs != &top_cpuset) 4232 guarantee_active_cpus(tsk, pmask); 4233 /* 4234 * Tasks in the top cpuset won't get update to their cpumasks 4235 * when a hotplug online/offline event happens. So we include all 4236 * offline cpus in the allowed cpu list. 4237 */ 4238 if ((cs == &top_cpuset) || cpumask_empty(pmask)) { 4239 const struct cpumask *possible_mask = task_cpu_possible_mask(tsk); 4240 4241 /* 4242 * We first exclude cpus allocated to partitions. If there is no 4243 * allowable online cpu left, we fall back to all possible cpus. 4244 */ 4245 cpumask_andnot(pmask, possible_mask, subpartitions_cpus); 4246 if (!cpumask_intersects(pmask, cpu_active_mask)) 4247 cpumask_copy(pmask, possible_mask); 4248 } 4249 } 4250 4251 /** 4252 * cpuset_cpus_allowed_locked - return cpus_allowed mask from a task's cpuset. 4253 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed. 4254 * @pmask: pointer to struct cpumask variable to receive cpus_allowed set. 4255 * 4256 * Similir to cpuset_cpus_allowed() except that the caller must have acquired 4257 * cpuset_mutex. 4258 */ 4259 void cpuset_cpus_allowed_locked(struct task_struct *tsk, struct cpumask *pmask) 4260 { 4261 lockdep_assert_cpuset_lock_held(); 4262 __cpuset_cpus_allowed_locked(tsk, pmask); 4263 } 4264 4265 /** 4266 * cpuset_cpus_allowed - return cpus_allowed mask from a task's cpuset. 4267 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed. 4268 * @pmask: pointer to struct cpumask variable to receive cpus_allowed set. 4269 * 4270 * Description: Returns the cpumask_var_t cpus_allowed of the cpuset 4271 * attached to the specified @tsk. Guaranteed to return some non-empty 4272 * subset of cpu_active_mask, even if this means going outside the 4273 * tasks cpuset, except when the task is in the top cpuset. 4274 **/ 4275 4276 void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask) 4277 { 4278 unsigned long flags; 4279 4280 spin_lock_irqsave(&callback_lock, flags); 4281 __cpuset_cpus_allowed_locked(tsk, pmask); 4282 spin_unlock_irqrestore(&callback_lock, flags); 4283 } 4284 4285 /** 4286 * cpuset_cpus_allowed_fallback - final fallback before complete catastrophe. 4287 * @tsk: pointer to task_struct with which the scheduler is struggling 4288 * 4289 * Description: In the case that the scheduler cannot find an allowed cpu in 4290 * tsk->cpus_allowed, we fall back to task_cs(tsk)->cpus_allowed. In legacy 4291 * mode however, this value is the same as task_cs(tsk)->effective_cpus, 4292 * which will not contain a sane cpumask during cases such as cpu hotplugging. 4293 * This is the absolute last resort for the scheduler and it is only used if 4294 * _every_ other avenue has been traveled. 4295 * 4296 * Returns true if the affinity of @tsk was changed, false otherwise. 4297 **/ 4298 4299 bool cpuset_cpus_allowed_fallback(struct task_struct *tsk) 4300 { 4301 const struct cpumask *possible_mask = task_cpu_possible_mask(tsk); 4302 const struct cpumask *cs_mask; 4303 bool changed = false; 4304 4305 rcu_read_lock(); 4306 cs_mask = task_cs(tsk)->cpus_allowed; 4307 if (is_in_v2_mode() && cpumask_subset(cs_mask, possible_mask)) { 4308 set_cpus_allowed_force(tsk, cs_mask); 4309 changed = true; 4310 } 4311 rcu_read_unlock(); 4312 4313 /* 4314 * We own tsk->cpus_allowed, nobody can change it under us. 4315 * 4316 * But we used cs && cs->cpus_allowed lockless and thus can 4317 * race with cgroup_attach_task() or update_cpumask() and get 4318 * the wrong tsk->cpus_allowed. However, both cases imply the 4319 * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr() 4320 * which takes task_rq_lock(). 4321 * 4322 * If we are called after it dropped the lock we must see all 4323 * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary 4324 * set any mask even if it is not right from task_cs() pov, 4325 * the pending set_cpus_allowed_ptr() will fix things. 4326 * 4327 * select_fallback_rq() will fix things ups and set cpu_possible_mask 4328 * if required. 4329 */ 4330 return changed; 4331 } 4332 4333 /* 4334 * Returns the number of CPUs available for this cgroup. 4335 * 4336 * This only really works for cgroup-v2 where all the controllers are mounted 4337 * in the same hierarchy. If not cgroup-v2 or no cpuset controller is 4338 * configured it reverts to num_online_cpus(). 4339 */ 4340 int cpuset_num_cpus(struct cgroup *cgrp) 4341 { 4342 int nr = num_online_cpus(); 4343 struct cpuset *cs; 4344 4345 if (cpuset_v2()) { 4346 guard(rcu)(); 4347 cs = css_cs(cgroup_e_css(cgrp, &cpuset_cgrp_subsys)); 4348 if (cs) 4349 nr = cpumask_weight(cs->effective_cpus); 4350 } 4351 4352 return nr; 4353 } 4354 4355 void __init cpuset_init_current_mems_allowed(void) 4356 { 4357 nodes_setall(current->mems_allowed); 4358 } 4359 4360 /** 4361 * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset. 4362 * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed. 4363 * 4364 * Description: Returns the nodemask_t mems_allowed of the cpuset 4365 * attached to the specified @tsk. Guaranteed to return some non-empty 4366 * subset of node_states[N_MEMORY], even if this means going outside the 4367 * tasks cpuset. 4368 **/ 4369 4370 nodemask_t cpuset_mems_allowed(struct task_struct *tsk) 4371 { 4372 nodemask_t mask; 4373 unsigned long flags; 4374 4375 spin_lock_irqsave(&callback_lock, flags); 4376 guarantee_online_mems(task_cs(tsk), &mask); 4377 spin_unlock_irqrestore(&callback_lock, flags); 4378 4379 return mask; 4380 } 4381 4382 /** 4383 * cpuset_nodemask_valid_mems_allowed - check nodemask vs. current mems_allowed 4384 * @nodemask: the nodemask to be checked 4385 * 4386 * Are any of the nodes in the nodemask allowed in current->mems_allowed? 4387 * 4388 * A cpuset update rebinds the task's mempolicy and changes mems_allowed in 4389 * separate steps, so retry if an update raced with the check. 4390 */ 4391 int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask) 4392 { 4393 unsigned int seq; 4394 int ret; 4395 4396 do { 4397 seq = read_mems_allowed_begin(); 4398 ret = nodes_intersects(*nodemask, current->mems_allowed); 4399 } while (read_mems_allowed_retry(seq)); 4400 4401 return ret; 4402 } 4403 4404 /* 4405 * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or 4406 * mem_hardwall ancestor to the specified cpuset. Call holding 4407 * callback_lock. If no ancestor is mem_exclusive or mem_hardwall 4408 * (an unusual configuration), then returns the root cpuset. 4409 */ 4410 static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) 4411 { 4412 while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs)) 4413 cs = parent_cs(cs); 4414 return cs; 4415 } 4416 4417 /* 4418 * cpuset_current_node_allowed - Can current task allocate on a memory node? 4419 * @node: is this an allowed node? 4420 * @gfp_mask: memory allocation flags 4421 * 4422 * If we're in interrupt, yes, we can always allocate. If @node is set in 4423 * current's mems_allowed, yes. If it's not a __GFP_HARDWALL request and this 4424 * node is set in the nearest hardwalled cpuset ancestor to current's cpuset, 4425 * yes. If current has access to memory reserves as an oom victim, yes. 4426 * If the current task is PF_EXITING, yes. Otherwise, no. 4427 * 4428 * GFP_USER allocations are marked with the __GFP_HARDWALL bit, 4429 * and do not allow allocations outside the current tasks cpuset 4430 * unless the task has been OOM killed or is exiting. 4431 * GFP_KERNEL allocations are not so marked, so can escape to the 4432 * nearest enclosing hardwalled ancestor cpuset. 4433 * 4434 * Scanning up parent cpusets requires callback_lock. The 4435 * page allocator only calls here with __GFP_HARDWALL bit 4436 * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the 4437 * current tasks mems_allowed came up empty on the first pass over 4438 * the zonelist. So only GFP_KERNEL allocations, if all nodes in the 4439 * cpuset are short of memory, might require taking the callback_lock. 4440 * 4441 * The first call here from mm/page_alloc:get_page_from_freelist() 4442 * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets, 4443 * so no allocation on a node outside the cpuset is allowed (unless 4444 * in interrupt, of course). The PF_EXITING check must therefore 4445 * come before the __GFP_HARDWALL check, otherwise a dying task 4446 * would be blocked on the fast path. 4447 * 4448 * The second pass through get_page_from_freelist() doesn't even call here for 4449 * GFP_ATOMIC calls. That, and the checks below have the combined affect that: 4450 * in_interrupt - any node ok (current task context irrelevant) 4451 * GFP_ATOMIC - any node ok 4452 * tsk_is_oom_victim - any node ok 4453 * PF_EXITING - any node ok (let dying task exit quickly) 4454 * GFP_KERNEL - any node in enclosing hardwalled cpuset ok 4455 * GFP_USER - only nodes in current tasks mems allowed ok. 4456 */ 4457 bool cpuset_current_node_allowed(int node, gfp_t gfp_mask) 4458 { 4459 struct cpuset *cs; /* current cpuset ancestors */ 4460 bool allowed; /* is allocation in zone z allowed? */ 4461 unsigned long flags; 4462 4463 if (in_interrupt()) 4464 return true; 4465 if (node_isset(node, current->mems_allowed)) 4466 return true; 4467 /* 4468 * Allow tasks that have access to memory reserves because they have 4469 * been OOM killed to get memory anywhere. 4470 */ 4471 if (unlikely(tsk_is_oom_victim(current))) 4472 return true; 4473 if (current->flags & PF_EXITING) /* Let dying task have memory */ 4474 return true; 4475 if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */ 4476 return false; 4477 4478 if (cpuset_v2()) 4479 return true; 4480 4481 /* Not hardwall and node outside mems_allowed: scan up cpusets */ 4482 spin_lock_irqsave(&callback_lock, flags); 4483 4484 cs = nearest_hardwall_ancestor(task_cs(current)); 4485 allowed = node_isset(node, cs->mems_allowed); 4486 4487 spin_unlock_irqrestore(&callback_lock, flags); 4488 return allowed; 4489 } 4490 4491 /** 4492 * cpuset_nodes_allowed - return effective_mems mask from a cgroup cpuset. 4493 * @cgroup: pointer to struct cgroup. 4494 * @mask: pointer to struct nodemask_t to be returned. 4495 * 4496 * Returns effective_mems mask from a cgroup cpuset if it is cgroup v2 and 4497 * has cpuset subsys. Otherwise, returns node_states[N_MEMORY]. 4498 * 4499 * This function intentionally avoids taking the cpuset_mutex or callback_lock 4500 * when accessing effective_mems. This is because the obtained effective_mems 4501 * is stale immediately after the query anyway (e.g., effective_mems is updated 4502 * immediately after releasing the lock but before returning). 4503 * 4504 * As a result, returned @mask may be empty because cs->effective_mems can be 4505 * rebound during this call. Besides, nodes in @mask are not guaranteed to be 4506 * online due to hot plugins. Callers should check the mask for validity on 4507 * return based on its subsequent use. 4508 **/ 4509 void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask) 4510 { 4511 struct cgroup_subsys_state *css; 4512 struct cpuset *cs; 4513 4514 /* 4515 * In v1, mem_cgroup and cpuset are unlikely in the same hierarchy 4516 * and mems_allowed is likely to be empty even if we could get to it, 4517 * so return directly to avoid taking a global lock on the empty check. 4518 */ 4519 if (!cgroup || !cpuset_v2()) { 4520 nodes_copy(*mask, node_states[N_MEMORY]); 4521 return; 4522 } 4523 4524 css = cgroup_get_e_css(cgroup, &cpuset_cgrp_subsys); 4525 if (!css) { 4526 nodes_copy(*mask, node_states[N_MEMORY]); 4527 return; 4528 } 4529 4530 /* 4531 * The reference taken via cgroup_get_e_css is sufficient to 4532 * protect css, but it does not imply safe accesses to effective_mems. 4533 * 4534 * Normally, accessing effective_mems would require the cpuset_mutex 4535 * or callback_lock - but the correctness of this information is stale 4536 * immediately after the query anyway. We do not acquire the lock 4537 * during this process to save lock contention in exchange for racing 4538 * against mems_allowed rebinds. 4539 */ 4540 cs = container_of(css, struct cpuset, css); 4541 nodes_copy(*mask, cs->effective_mems); 4542 css_put(css); 4543 } 4544 4545 /** 4546 * cpuset_spread_node() - On which node to begin search for a page 4547 * @rotor: round robin rotor 4548 * 4549 * If a task is marked PFA_SPREAD_PAGE and a page cache allocation uses 4550 * cpuset_mem_spread_node() to determine where to start looking, spread the 4551 * starting node around the task's mems_allowed nodes instead of starting on 4552 * the local node. 4553 * 4554 * We don't have to worry about the returned node being offline 4555 * because "it can't happen", and even if it did, it would be ok. 4556 * 4557 * The routines calling guarantee_online_mems() are careful to 4558 * only set nodes in task->mems_allowed that are online. So it 4559 * should not be possible for the following code to return an 4560 * offline node. But if it did, that would be ok, as this routine 4561 * is not returning the node where the allocation must be, only 4562 * the node where the search should start. The zonelist used by 4563 * the allocator will include all nodes. If the slab allocator 4564 * is passed an offline node, it will fall back to the local node. 4565 * See kmem_cache_alloc_node(). 4566 */ 4567 static int cpuset_spread_node(int *rotor) 4568 { 4569 return *rotor = next_node_in(*rotor, current->mems_allowed); 4570 } 4571 4572 /** 4573 * cpuset_mem_spread_node() - On which node to begin search for a file page 4574 */ 4575 int cpuset_mem_spread_node(void) 4576 { 4577 if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE) 4578 current->cpuset_mem_spread_rotor = 4579 node_random(¤t->mems_allowed); 4580 4581 return cpuset_spread_node(¤t->cpuset_mem_spread_rotor); 4582 } 4583 4584 /** 4585 * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's? 4586 * @tsk1: pointer to task_struct of some task. 4587 * @tsk2: pointer to task_struct of some other task. 4588 * 4589 * Description: Return true if @tsk1's mems_allowed intersects the 4590 * mems_allowed of @tsk2. Used by the OOM killer to determine if 4591 * one of the task's memory usage might impact the memory available 4592 * to the other. 4593 **/ 4594 4595 int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, 4596 const struct task_struct *tsk2) 4597 { 4598 return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed); 4599 } 4600 4601 /** 4602 * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed 4603 * 4604 * Description: Prints current's name, cpuset name, and cached copy of its 4605 * mems_allowed to the kernel log. 4606 */ 4607 void cpuset_print_current_mems_allowed(void) 4608 { 4609 struct cgroup *cgrp; 4610 4611 rcu_read_lock(); 4612 4613 cgrp = task_cs(current)->css.cgroup; 4614 pr_cont(",cpuset="); 4615 pr_cont_cgroup_name(cgrp); 4616 pr_cont(",mems_allowed=%*pbl", 4617 nodemask_pr_args(¤t->mems_allowed)); 4618 4619 rcu_read_unlock(); 4620 } 4621 4622 /* Display task mems_allowed in /proc/<pid>/status file. */ 4623 void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task) 4624 { 4625 seq_printf(m, "Mems_allowed:\t%*pb\n", 4626 nodemask_pr_args(&task->mems_allowed)); 4627 seq_printf(m, "Mems_allowed_list:\t%*pbl\n", 4628 nodemask_pr_args(&task->mems_allowed)); 4629 } 4630