1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include "cgroup-internal.h" 4 #include "cpuset-internal.h" 5 6 /* 7 * Legacy hierarchy call to cgroup_transfer_tasks() is handled asynchrously 8 */ 9 struct cpuset_remove_tasks_struct { 10 struct work_struct work; 11 struct cpuset *cs; 12 }; 13 14 /* 15 * Frequency meter - How fast is some event occurring? 16 * 17 * These routines manage a digitally filtered, constant time based, 18 * event frequency meter. There are four routines: 19 * fmeter_init() - initialize a frequency meter. 20 * fmeter_markevent() - called each time the event happens. 21 * fmeter_getrate() - returns the recent rate of such events. 22 * fmeter_update() - internal routine used to update fmeter. 23 * 24 * A common data structure is passed to each of these routines, 25 * which is used to keep track of the state required to manage the 26 * frequency meter and its digital filter. 27 * 28 * The filter works on the number of events marked per unit time. 29 * The filter is single-pole low-pass recursive (IIR). The time unit 30 * is 1 second. Arithmetic is done using 32-bit integers scaled to 31 * simulate 3 decimal digits of precision (multiplied by 1000). 32 * 33 * With an FM_COEF of 933, and a time base of 1 second, the filter 34 * has a half-life of 10 seconds, meaning that if the events quit 35 * happening, then the rate returned from the fmeter_getrate() 36 * will be cut in half each 10 seconds, until it converges to zero. 37 * 38 * It is not worth doing a real infinitely recursive filter. If more 39 * than FM_MAXTICKS ticks have elapsed since the last filter event, 40 * just compute FM_MAXTICKS ticks worth, by which point the level 41 * will be stable. 42 * 43 * Limit the count of unprocessed events to FM_MAXCNT, so as to avoid 44 * arithmetic overflow in the fmeter_update() routine. 45 * 46 * Given the simple 32 bit integer arithmetic used, this meter works 47 * best for reporting rates between one per millisecond (msec) and 48 * one per 32 (approx) seconds. At constant rates faster than one 49 * per msec it maxes out at values just under 1,000,000. At constant 50 * rates between one per msec, and one per second it will stabilize 51 * to a value N*1000, where N is the rate of events per second. 52 * At constant rates between one per second and one per 32 seconds, 53 * it will be choppy, moving up on the seconds that have an event, 54 * and then decaying until the next event. At rates slower than 55 * about one in 32 seconds, it decays all the way back to zero between 56 * each event. 57 */ 58 59 #define FM_COEF 933 /* coefficient for half-life of 10 secs */ 60 #define FM_MAXTICKS ((u32)99) /* useless computing more ticks than this */ 61 #define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */ 62 #define FM_SCALE 1000 /* faux fixed point scale */ 63 64 /* Initialize a frequency meter */ 65 static void fmeter_init(struct fmeter *fmp) 66 { 67 fmp->cnt = 0; 68 fmp->val = 0; 69 fmp->time = 0; 70 spin_lock_init(&fmp->lock); 71 } 72 73 /* Internal meter update - process cnt events and update value */ 74 static void fmeter_update(struct fmeter *fmp) 75 { 76 time64_t now; 77 u32 ticks; 78 79 now = ktime_get_seconds(); 80 ticks = now - fmp->time; 81 82 if (ticks == 0) 83 return; 84 85 ticks = min(FM_MAXTICKS, ticks); 86 while (ticks-- > 0) 87 fmp->val = (FM_COEF * fmp->val) / FM_SCALE; 88 fmp->time = now; 89 90 fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE; 91 fmp->cnt = 0; 92 } 93 94 /* Process any previous ticks, then bump cnt by one (times scale). */ 95 static void fmeter_markevent(struct fmeter *fmp) 96 { 97 spin_lock(&fmp->lock); 98 fmeter_update(fmp); 99 fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE); 100 spin_unlock(&fmp->lock); 101 } 102 103 /* Process any previous ticks, then return current value. */ 104 static int fmeter_getrate(struct fmeter *fmp) 105 { 106 int val; 107 108 spin_lock(&fmp->lock); 109 fmeter_update(fmp); 110 val = fmp->val; 111 spin_unlock(&fmp->lock); 112 return val; 113 } 114 115 /* 116 * Collection of memory_pressure is suppressed unless 117 * this flag is enabled by writing "1" to the special 118 * cpuset file 'memory_pressure_enabled' in the root cpuset. 119 */ 120 121 int cpuset_memory_pressure_enabled __read_mostly; 122 123 /* 124 * __cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims. 125 * 126 * Keep a running average of the rate of synchronous (direct) 127 * page reclaim efforts initiated by tasks in each cpuset. 128 * 129 * This represents the rate at which some task in the cpuset 130 * ran low on memory on all nodes it was allowed to use, and 131 * had to enter the kernels page reclaim code in an effort to 132 * create more free memory by tossing clean pages or swapping 133 * or writing dirty pages. 134 * 135 * Display to user space in the per-cpuset read-only file 136 * "memory_pressure". Value displayed is an integer 137 * representing the recent rate of entry into the synchronous 138 * (direct) page reclaim by any task attached to the cpuset. 139 */ 140 141 void __cpuset_memory_pressure_bump(void) 142 { 143 rcu_read_lock(); 144 fmeter_markevent(&task_cs(current)->fmeter); 145 rcu_read_unlock(); 146 } 147 148 static int update_relax_domain_level(struct cpuset *cs, s64 val) 149 { 150 #ifdef CONFIG_SMP 151 if (val < -1 || val > sched_domain_level_max + 1) 152 return -EINVAL; 153 #endif 154 155 if (val != cs->relax_domain_level) { 156 cs->relax_domain_level = val; 157 if (!cpumask_empty(cs->cpus_allowed) && 158 is_sched_load_balance(cs)) 159 rebuild_sched_domains_locked(); 160 } 161 162 return 0; 163 } 164 165 static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft, 166 s64 val) 167 { 168 struct cpuset *cs = css_cs(css); 169 cpuset_filetype_t type = cft->private; 170 int retval = -ENODEV; 171 172 cpuset_full_lock(); 173 if (!is_cpuset_online(cs)) 174 goto out_unlock; 175 176 switch (type) { 177 case FILE_SCHED_RELAX_DOMAIN_LEVEL: 178 pr_info_once("cpuset.%s is deprecated\n", cft->name); 179 retval = update_relax_domain_level(cs, val); 180 break; 181 default: 182 retval = -EINVAL; 183 break; 184 } 185 out_unlock: 186 cpuset_full_unlock(); 187 return retval; 188 } 189 190 static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft) 191 { 192 struct cpuset *cs = css_cs(css); 193 cpuset_filetype_t type = cft->private; 194 195 switch (type) { 196 case FILE_SCHED_RELAX_DOMAIN_LEVEL: 197 return cs->relax_domain_level; 198 default: 199 BUG(); 200 } 201 202 /* Unreachable but makes gcc happy */ 203 return 0; 204 } 205 206 /* 207 * Update a task's spread flag if the cpuset's page spread flag is set. 208 * 209 * Call with callback_lock or cpuset_mutex held. The check can be skipped 210 * if on default hierarchy. 211 */ 212 void cpuset1_update_task_spread_flags(struct cpuset *cs, 213 struct task_struct *tsk) 214 { 215 if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) 216 return; 217 218 if (is_spread_page(cs)) 219 task_set_spread_page(tsk); 220 else 221 task_clear_spread_page(tsk); 222 } 223 224 /** 225 * cpuset1_update_tasks_flags - update the page spread flag of cpuset tasks 226 * @cs: the cpuset whose tasks need their page spread flag updated 227 * 228 * Iterate through each task of @cs updating its page spread flag. As this 229 * function is called with cpuset_mutex held, cpuset membership stays 230 * stable. 231 */ 232 static void cpuset1_update_tasks_flags(struct cpuset *cs) 233 { 234 struct css_task_iter it; 235 struct task_struct *task; 236 237 css_task_iter_start(&cs->css, 0, &it); 238 while ((task = css_task_iter_next(&it))) 239 cpuset1_update_task_spread_flags(cs, task); 240 css_task_iter_end(&it); 241 } 242 243 /* 244 * If CPU and/or memory hotplug handlers, below, unplug any CPUs 245 * or memory nodes, we need to walk over the cpuset hierarchy, 246 * removing that CPU or node from all cpusets. If this removes the 247 * last CPU or node from a cpuset, then move the tasks in the empty 248 * cpuset to its next-highest non-empty parent. 249 */ 250 static void remove_tasks_in_empty_cpuset(struct cpuset *cs) 251 { 252 struct cpuset *parent; 253 254 /* 255 * Find its next-highest non-empty parent, (top cpuset 256 * has online cpus, so can't be empty). 257 */ 258 parent = parent_cs(cs); 259 while (cpumask_empty(parent->cpus_allowed) || 260 nodes_empty(parent->mems_allowed)) 261 parent = parent_cs(parent); 262 263 if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) { 264 pr_err("cpuset: failed to transfer tasks out of empty cpuset "); 265 pr_cont_cgroup_name(cs->css.cgroup); 266 pr_cont("\n"); 267 } 268 } 269 270 static void cpuset_migrate_tasks_workfn(struct work_struct *work) 271 { 272 struct cpuset_remove_tasks_struct *s; 273 274 s = container_of(work, struct cpuset_remove_tasks_struct, work); 275 remove_tasks_in_empty_cpuset(s->cs); 276 css_put(&s->cs->css); 277 kfree(s); 278 } 279 280 void cpuset1_hotplug_update_tasks(struct cpuset *cs, 281 struct cpumask *new_cpus, nodemask_t *new_mems, 282 bool cpus_updated, bool mems_updated) 283 { 284 bool is_empty; 285 286 cpuset_callback_lock_irq(); 287 cpumask_copy(cs->cpus_allowed, new_cpus); 288 cpumask_copy(cs->effective_cpus, new_cpus); 289 cs->mems_allowed = *new_mems; 290 cs->effective_mems = *new_mems; 291 cpuset_callback_unlock_irq(); 292 293 /* 294 * Don't call cpuset_update_tasks_cpumask() if the cpuset becomes empty, 295 * as the tasks will be migrated to an ancestor. 296 */ 297 if (cpus_updated && !cpumask_empty(cs->cpus_allowed)) 298 cpuset_update_tasks_cpumask(cs, new_cpus); 299 if (mems_updated && !nodes_empty(cs->mems_allowed)) 300 cpuset_update_tasks_nodemask(cs); 301 302 is_empty = cpumask_empty(cs->cpus_allowed) || 303 nodes_empty(cs->mems_allowed); 304 305 /* 306 * Move tasks to the nearest ancestor with execution resources, 307 * This is full cgroup operation which will also call back into 308 * cpuset. Execute it asynchronously using workqueue. 309 */ 310 if (is_empty && cgroup_has_tasks(cs->css.cgroup) && 311 css_tryget_online(&cs->css)) { 312 struct cpuset_remove_tasks_struct *s; 313 314 s = kzalloc_obj(*s); 315 if (WARN_ON_ONCE(!s)) { 316 css_put(&cs->css); 317 return; 318 } 319 320 s->cs = cs; 321 INIT_WORK(&s->work, cpuset_migrate_tasks_workfn); 322 schedule_work(&s->work); 323 } 324 } 325 326 /* 327 * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q? 328 * 329 * One cpuset is a subset of another if all its allowed CPUs and 330 * Memory Nodes are a subset of the other, and its exclusive flags 331 * are only set if the other's are set. Call holding cpuset_mutex. 332 */ 333 334 static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q) 335 { 336 return cpumask_subset(p->cpus_allowed, q->cpus_allowed) && 337 nodes_subset(p->mems_allowed, q->mems_allowed) && 338 is_cpu_exclusive(p) <= is_cpu_exclusive(q) && 339 is_mem_exclusive(p) <= is_mem_exclusive(q); 340 } 341 342 /* 343 * cpuset1_validate_change() - Validate conditions specific to legacy (v1) 344 * behavior. 345 */ 346 int cpuset1_validate_change(struct cpuset *cur, struct cpuset *trial) 347 { 348 struct cgroup_subsys_state *css; 349 struct cpuset *c, *par; 350 int ret; 351 352 WARN_ON_ONCE(!rcu_read_lock_held()); 353 354 /* Each of our child cpusets must be a subset of us */ 355 ret = -EBUSY; 356 cpuset_for_each_child(c, css, cur) 357 if (!is_cpuset_subset(c, trial)) 358 goto out; 359 360 /* On legacy hierarchy, we must be a subset of our parent cpuset. */ 361 ret = -EACCES; 362 par = parent_cs(cur); 363 if (par && !is_cpuset_subset(trial, par)) 364 goto out; 365 366 /* 367 * Cpusets with tasks - existing or newly being attached - can't 368 * be changed to have empty cpus_allowed or mems_allowed. 369 */ 370 ret = -ENOSPC; 371 if (cpuset_is_populated(cur)) { 372 if (!cpumask_empty(cur->cpus_allowed) && 373 cpumask_empty(trial->cpus_allowed)) 374 goto out; 375 if (!nodes_empty(cur->mems_allowed) && 376 nodes_empty(trial->mems_allowed)) 377 goto out; 378 } 379 380 ret = 0; 381 out: 382 return ret; 383 } 384 385 /* 386 * cpuset1_cpus_excl_conflict() - Check if two cpusets have exclusive CPU conflicts 387 * to legacy (v1) 388 * @cs1: first cpuset to check 389 * @cs2: second cpuset to check 390 * 391 * Returns: true if CPU exclusivity conflict exists, false otherwise 392 * 393 * If either cpuset is CPU exclusive, their allowed CPUs cannot intersect. 394 */ 395 bool cpuset1_cpus_excl_conflict(struct cpuset *cs1, struct cpuset *cs2) 396 { 397 if (is_cpu_exclusive(cs1) || is_cpu_exclusive(cs2)) 398 return cpumask_intersects(cs1->cpus_allowed, 399 cs2->cpus_allowed); 400 401 return false; 402 } 403 404 #ifdef CONFIG_PROC_PID_CPUSET 405 /* 406 * proc_cpuset_show() 407 * - Print tasks cpuset path into seq_file. 408 * - Used for /proc/<pid>/cpuset. 409 */ 410 int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns, 411 struct pid *pid, struct task_struct *tsk) 412 { 413 char *buf; 414 struct cgroup_subsys_state *css; 415 int retval; 416 417 retval = -ENOMEM; 418 buf = kmalloc(PATH_MAX, GFP_KERNEL); 419 if (!buf) 420 goto out; 421 422 rcu_read_lock(); 423 spin_lock_irq(&css_set_lock); 424 css = task_css(tsk, cpuset_cgrp_id); 425 retval = cgroup_path_ns_locked(css->cgroup, buf, PATH_MAX, 426 current->nsproxy->cgroup_ns); 427 spin_unlock_irq(&css_set_lock); 428 rcu_read_unlock(); 429 430 if (retval == -E2BIG) 431 retval = -ENAMETOOLONG; 432 if (retval < 0) 433 goto out_free; 434 seq_puts(m, buf); 435 seq_putc(m, '\n'); 436 retval = 0; 437 out_free: 438 kfree(buf); 439 out: 440 return retval; 441 } 442 #endif /* CONFIG_PROC_PID_CPUSET */ 443 444 static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) 445 { 446 struct cpuset *cs = css_cs(css); 447 cpuset_filetype_t type = cft->private; 448 449 switch (type) { 450 case FILE_CPU_EXCLUSIVE: 451 return is_cpu_exclusive(cs); 452 case FILE_MEM_EXCLUSIVE: 453 return is_mem_exclusive(cs); 454 case FILE_MEM_HARDWALL: 455 return is_mem_hardwall(cs); 456 case FILE_SCHED_LOAD_BALANCE: 457 return is_sched_load_balance(cs); 458 case FILE_MEMORY_MIGRATE: 459 return is_memory_migrate(cs); 460 case FILE_MEMORY_PRESSURE_ENABLED: 461 return cpuset_memory_pressure_enabled; 462 case FILE_MEMORY_PRESSURE: 463 return fmeter_getrate(&cs->fmeter); 464 case FILE_SPREAD_PAGE: 465 return is_spread_page(cs); 466 case FILE_SPREAD_SLAB: 467 return is_spread_slab(cs); 468 default: 469 BUG(); 470 } 471 472 /* Unreachable but makes gcc happy */ 473 return 0; 474 } 475 476 /* 477 * cpuset_update_flag - read a 0 or a 1 in a file and update associated flag 478 * bit: the bit to update (see cpuset_flagbits_t) 479 * cs: the cpuset to update 480 * turning_on: whether the flag is being set or cleared 481 * 482 * Call with cpuset_mutex held. 483 */ 484 static int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, 485 int turning_on) 486 { 487 struct cpuset *trialcs; 488 int balance_flag_changed; 489 int spread_page_changed; 490 int err; 491 492 trialcs = dup_or_alloc_cpuset(cs); 493 if (!trialcs) 494 return -ENOMEM; 495 496 assign_bit(bit, &trialcs->flags, turning_on); 497 498 err = validate_change(cs, trialcs); 499 if (err < 0) 500 goto out; 501 502 balance_flag_changed = (is_sched_load_balance(cs) != 503 is_sched_load_balance(trialcs)); 504 505 spread_page_changed = is_spread_page(cs) != is_spread_page(trialcs); 506 507 cpuset_callback_lock_irq(); 508 cs->flags = trialcs->flags; 509 cpuset_callback_unlock_irq(); 510 511 if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) 512 rebuild_sched_domains_locked(); 513 514 if (spread_page_changed) 515 cpuset1_update_tasks_flags(cs); 516 out: 517 free_cpuset(trialcs); 518 return err; 519 } 520 521 static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft, 522 u64 val) 523 { 524 struct cpuset *cs = css_cs(css); 525 cpuset_filetype_t type = cft->private; 526 int retval = 0; 527 528 cpuset_full_lock(); 529 if (!is_cpuset_online(cs)) { 530 retval = -ENODEV; 531 goto out_unlock; 532 } 533 534 switch (type) { 535 case FILE_CPU_EXCLUSIVE: 536 retval = cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, val); 537 break; 538 case FILE_MEM_EXCLUSIVE: 539 pr_info_once("cpuset.%s is deprecated\n", cft->name); 540 retval = cpuset_update_flag(CS_MEM_EXCLUSIVE, cs, val); 541 break; 542 case FILE_MEM_HARDWALL: 543 pr_info_once("cpuset.%s is deprecated\n", cft->name); 544 retval = cpuset_update_flag(CS_MEM_HARDWALL, cs, val); 545 break; 546 case FILE_SCHED_LOAD_BALANCE: 547 pr_info_once("cpuset.%s is deprecated, use cpuset.cpus.partition instead\n", cft->name); 548 retval = cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, val); 549 break; 550 case FILE_MEMORY_MIGRATE: 551 pr_info_once("cpuset.%s is deprecated\n", cft->name); 552 retval = cpuset_update_flag(CS_MEMORY_MIGRATE, cs, val); 553 break; 554 case FILE_MEMORY_PRESSURE_ENABLED: 555 pr_info_once("cpuset.%s is deprecated, use memory.pressure with CONFIG_PSI instead\n", cft->name); 556 cpuset_memory_pressure_enabled = !!val; 557 break; 558 case FILE_SPREAD_PAGE: 559 pr_info_once("cpuset.%s is deprecated\n", cft->name); 560 retval = cpuset_update_flag(CS_SPREAD_PAGE, cs, val); 561 break; 562 case FILE_SPREAD_SLAB: 563 pr_warn_once("cpuset.%s is deprecated\n", cft->name); 564 retval = cpuset_update_flag(CS_SPREAD_SLAB, cs, val); 565 break; 566 default: 567 retval = -EINVAL; 568 break; 569 } 570 out_unlock: 571 cpuset_full_unlock(); 572 return retval; 573 } 574 575 void cpuset1_init(struct cpuset *cs) 576 { 577 fmeter_init(&cs->fmeter); 578 cs->relax_domain_level = -1; 579 } 580 581 void cpuset1_online_css(struct cgroup_subsys_state *css) 582 { 583 struct cpuset *tmp_cs; 584 struct cgroup_subsys_state *pos_css; 585 struct cpuset *cs = css_cs(css); 586 struct cpuset *parent = parent_cs(cs); 587 588 lockdep_assert_cpus_held(); 589 lockdep_assert_cpuset_lock_held(); 590 591 if (is_spread_page(parent)) 592 set_bit(CS_SPREAD_PAGE, &cs->flags); 593 if (is_spread_slab(parent)) 594 set_bit(CS_SPREAD_SLAB, &cs->flags); 595 596 if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags)) 597 return; 598 599 /* 600 * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is 601 * set. This flag handling is implemented in cgroup core for 602 * historical reasons - the flag may be specified during mount. 603 * 604 * Currently, if any sibling cpusets have exclusive cpus or mem, we 605 * refuse to clone the configuration - thereby refusing the task to 606 * be entered, and as a result refusing the sys_unshare() or 607 * clone() which initiated it. If this becomes a problem for some 608 * users who wish to allow that scenario, then this could be 609 * changed to grant parent->cpus_allowed-sibling_cpus_exclusive 610 * (and likewise for mems) to the new cgroup. 611 */ 612 rcu_read_lock(); 613 cpuset_for_each_child(tmp_cs, pos_css, parent) { 614 if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) { 615 rcu_read_unlock(); 616 return; 617 } 618 } 619 rcu_read_unlock(); 620 621 cpuset_callback_lock_irq(); 622 cs->mems_allowed = parent->mems_allowed; 623 cs->effective_mems = parent->mems_allowed; 624 cpumask_copy(cs->cpus_allowed, parent->cpus_allowed); 625 cpumask_copy(cs->effective_cpus, parent->cpus_allowed); 626 cpuset_callback_unlock_irq(); 627 } 628 629 /* 630 * Legacy-hierarchy handling when a cpuset is taken offline: if it had 631 * sched_load_balance enabled, turn it off so the scheduler domains are 632 * rebuilt. A no-op on the default hierarchy. 633 */ 634 void cpuset1_offline_css(struct cpuset *cs) 635 { 636 if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && 637 is_sched_load_balance(cs)) 638 cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); 639 } 640 641 static void 642 update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c) 643 { 644 if (dattr->relax_domain_level < c->relax_domain_level) 645 dattr->relax_domain_level = c->relax_domain_level; 646 } 647 648 static void update_domain_attr_tree(struct sched_domain_attr *dattr, 649 struct cpuset *root_cs) 650 { 651 struct cpuset *cp; 652 struct cgroup_subsys_state *pos_css; 653 654 rcu_read_lock(); 655 cpuset_for_each_descendant_pre(cp, pos_css, root_cs) { 656 /* skip the whole subtree if @cp doesn't have any CPU */ 657 if (cpumask_empty(cp->cpus_allowed)) { 658 pos_css = css_rightmost_descendant(pos_css); 659 continue; 660 } 661 662 if (is_sched_load_balance(cp)) 663 update_domain_attr(dattr, cp); 664 } 665 rcu_read_unlock(); 666 } 667 668 /* 669 * cpuset1_generate_sched_domains() 670 * 671 * Finding the best partition (set of domains): 672 * The double nested loops below over i, j scan over the load 673 * balanced cpusets (using the array of cpuset pointers in csa[]) 674 * looking for pairs of cpusets that have overlapping cpus_allowed 675 * and merging them using a union-find algorithm. 676 * 677 * The union of the cpus_allowed masks from the set of all cpusets 678 * having the same root then form the one element of the partition 679 * (one sched domain) to be passed to partition_sched_domains(). 680 */ 681 int cpuset1_generate_sched_domains(cpumask_var_t **domains, 682 struct sched_domain_attr **attributes) 683 { 684 struct cpuset *cp; /* top-down scan of cpusets */ 685 struct cpuset **csa; /* array of all cpuset ptrs */ 686 int csn; /* how many cpuset ptrs in csa so far */ 687 int i, j; /* indices for partition finding loops */ 688 cpumask_var_t *doms; /* resulting partition; i.e. sched domains */ 689 struct sched_domain_attr *dattr; /* attributes for custom domains */ 690 int ndoms = 0; /* number of sched domains in result */ 691 int nslot; /* next empty doms[] struct cpumask slot */ 692 struct cgroup_subsys_state *pos_css; 693 int nslot_update; 694 695 lockdep_assert_cpuset_lock_held(); 696 697 doms = NULL; 698 dattr = NULL; 699 csa = NULL; 700 701 /* Special case for the 99% of systems with one, full, sched domain */ 702 if (is_sched_load_balance(&top_cpuset)) { 703 ndoms = 1; 704 doms = alloc_sched_domains(ndoms); 705 if (!doms) 706 goto done; 707 708 dattr = kmalloc_obj(struct sched_domain_attr); 709 if (dattr) { 710 *dattr = SD_ATTR_INIT; 711 update_domain_attr_tree(dattr, &top_cpuset); 712 } 713 cpumask_and(doms[0], top_cpuset.effective_cpus, 714 housekeeping_cpumask(HK_TYPE_DOMAIN)); 715 716 goto done; 717 } 718 719 csa = kmalloc_objs(cp, nr_cpusets()); 720 if (!csa) 721 goto done; 722 csn = 0; 723 724 rcu_read_lock(); 725 cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) { 726 if (cp == &top_cpuset) 727 continue; 728 729 /* 730 * Continue traversing beyond @cp iff @cp has some CPUs and 731 * isn't load balancing. The former is obvious. The 732 * latter: All child cpusets contain a subset of the 733 * parent's cpus, so just skip them, and then we call 734 * update_domain_attr_tree() to calc relax_domain_level of 735 * the corresponding sched domain. 736 */ 737 if (!cpumask_empty(cp->cpus_allowed) && 738 !(is_sched_load_balance(cp) && 739 cpumask_intersects(cp->cpus_allowed, 740 housekeeping_cpumask(HK_TYPE_DOMAIN)))) 741 continue; 742 743 if (is_sched_load_balance(cp) && 744 !cpumask_empty(cp->effective_cpus)) 745 csa[csn++] = cp; 746 747 /* skip @cp's subtree */ 748 pos_css = css_rightmost_descendant(pos_css); 749 continue; 750 } 751 rcu_read_unlock(); 752 753 for (i = 0; i < csn; i++) 754 uf_node_init(&csa[i]->node); 755 756 /* Merge overlapping cpusets */ 757 for (i = 0; i < csn; i++) { 758 for (j = i + 1; j < csn; j++) { 759 if (cpusets_overlap(csa[i], csa[j])) 760 uf_union(&csa[i]->node, &csa[j]->node); 761 } 762 } 763 764 /* Count the total number of domains */ 765 for (i = 0; i < csn; i++) { 766 if (uf_find(&csa[i]->node) == &csa[i]->node) 767 ndoms++; 768 } 769 770 /* 771 * Now we know how many domains to create. 772 * Convert <csn, csa> to <ndoms, doms> and populate cpu masks. 773 */ 774 doms = alloc_sched_domains(ndoms); 775 if (!doms) 776 goto done; 777 778 /* 779 * The rest of the code, including the scheduler, can deal with 780 * dattr==NULL case. No need to abort if alloc fails. 781 */ 782 dattr = kmalloc_objs(struct sched_domain_attr, ndoms); 783 784 for (nslot = 0, i = 0; i < csn; i++) { 785 nslot_update = 0; 786 for (j = i; j < csn; j++) { 787 if (uf_find(&csa[j]->node) == &csa[i]->node) { 788 struct cpumask *dp = doms[nslot]; 789 790 if (i == j) { 791 nslot_update = 1; 792 cpumask_clear(dp); 793 if (dattr) 794 *(dattr + nslot) = SD_ATTR_INIT; 795 } 796 cpumask_or(dp, dp, csa[j]->effective_cpus); 797 cpumask_and(dp, dp, housekeeping_cpumask(HK_TYPE_DOMAIN)); 798 if (dattr) 799 update_domain_attr_tree(dattr + nslot, csa[j]); 800 } 801 } 802 if (nslot_update) 803 nslot++; 804 } 805 BUG_ON(nslot != ndoms); 806 807 done: 808 kfree(csa); 809 810 /* 811 * Fallback to the default domain if kmalloc() failed. 812 * See comments in partition_sched_domains(). 813 */ 814 if (doms == NULL) 815 ndoms = 1; 816 817 *domains = doms; 818 *attributes = dattr; 819 return ndoms; 820 } 821 822 /* 823 * for the common functions, 'private' gives the type of file 824 */ 825 826 struct cftype cpuset1_files[] = { 827 { 828 .name = "cpus", 829 .seq_show = cpuset_common_seq_show, 830 .write = cpuset_write_resmask, 831 .max_write_len = (100U + 6 * NR_CPUS), 832 .private = FILE_CPULIST, 833 }, 834 835 { 836 .name = "mems", 837 .seq_show = cpuset_common_seq_show, 838 .write = cpuset_write_resmask, 839 .max_write_len = (100U + 6 * MAX_NUMNODES), 840 .private = FILE_MEMLIST, 841 }, 842 843 { 844 .name = "effective_cpus", 845 .seq_show = cpuset_common_seq_show, 846 .private = FILE_EFFECTIVE_CPULIST, 847 }, 848 849 { 850 .name = "effective_mems", 851 .seq_show = cpuset_common_seq_show, 852 .private = FILE_EFFECTIVE_MEMLIST, 853 }, 854 855 { 856 .name = "cpu_exclusive", 857 .read_u64 = cpuset_read_u64, 858 .write_u64 = cpuset_write_u64, 859 .private = FILE_CPU_EXCLUSIVE, 860 }, 861 862 { 863 .name = "mem_exclusive", 864 .read_u64 = cpuset_read_u64, 865 .write_u64 = cpuset_write_u64, 866 .private = FILE_MEM_EXCLUSIVE, 867 }, 868 869 { 870 .name = "mem_hardwall", 871 .read_u64 = cpuset_read_u64, 872 .write_u64 = cpuset_write_u64, 873 .private = FILE_MEM_HARDWALL, 874 }, 875 876 { 877 .name = "sched_load_balance", 878 .read_u64 = cpuset_read_u64, 879 .write_u64 = cpuset_write_u64, 880 .private = FILE_SCHED_LOAD_BALANCE, 881 }, 882 883 { 884 .name = "sched_relax_domain_level", 885 .read_s64 = cpuset_read_s64, 886 .write_s64 = cpuset_write_s64, 887 .private = FILE_SCHED_RELAX_DOMAIN_LEVEL, 888 }, 889 890 { 891 .name = "memory_migrate", 892 .read_u64 = cpuset_read_u64, 893 .write_u64 = cpuset_write_u64, 894 .private = FILE_MEMORY_MIGRATE, 895 }, 896 897 { 898 .name = "memory_pressure", 899 .read_u64 = cpuset_read_u64, 900 .private = FILE_MEMORY_PRESSURE, 901 }, 902 903 { 904 .name = "memory_spread_page", 905 .read_u64 = cpuset_read_u64, 906 .write_u64 = cpuset_write_u64, 907 .private = FILE_SPREAD_PAGE, 908 }, 909 910 { 911 /* obsolete, may be removed in the future */ 912 .name = "memory_spread_slab", 913 .read_u64 = cpuset_read_u64, 914 .write_u64 = cpuset_write_u64, 915 .private = FILE_SPREAD_SLAB, 916 }, 917 918 { 919 .name = "memory_pressure_enabled", 920 .flags = CFTYPE_ONLY_ON_ROOT, 921 .read_u64 = cpuset_read_u64, 922 .write_u64 = cpuset_write_u64, 923 .private = FILE_MEMORY_PRESSURE_ENABLED, 924 }, 925 926 { } /* terminate */ 927 }; 928