1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) 4 * 5 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * 7 * Interactivity improvements by Mike Galbraith 8 * (C) 2007 Mike Galbraith <efault@gmx.de> 9 * 10 * Various enhancements by Dmitry Adamushko. 11 * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com> 12 * 13 * Group scheduling enhancements by Srivatsa Vaddagiri 14 * Copyright IBM Corporation, 2007 15 * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> 16 * 17 * Scaled math optimizations by Thomas Gleixner 18 * Copyright (C) 2007, Linutronix GmbH, Thomas Gleixner <tglx@kernel.org> 19 * 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 */ 23 #include <linux/energy_model.h> 24 #include <linux/mmap_lock.h> 25 #include <linux/hugetlb_inline.h> 26 #include <linux/jiffies.h> 27 #include <linux/mm_api.h> 28 #include <linux/highmem.h> 29 #include <linux/spinlock_api.h> 30 #include <linux/cpumask_api.h> 31 #include <linux/lockdep_api.h> 32 #include <linux/softirq.h> 33 #include <linux/refcount_api.h> 34 #include <linux/topology.h> 35 #include <linux/sched/clock.h> 36 #include <linux/sched/cond_resched.h> 37 #include <linux/sched/cputime.h> 38 #include <linux/sched/isolation.h> 39 #include <linux/sched/nohz.h> 40 #include <linux/sched/prio.h> 41 42 #include <linux/cpuidle.h> 43 #include <linux/interrupt.h> 44 #include <linux/memory-tiers.h> 45 #include <linux/mempolicy.h> 46 #include <linux/mutex_api.h> 47 #include <linux/profile.h> 48 #include <linux/psi.h> 49 #include <linux/ratelimit.h> 50 #include <linux/task_work.h> 51 #include <linux/rbtree_augmented.h> 52 53 #include <asm/switch_to.h> 54 55 #include <uapi/linux/sched/types.h> 56 57 #include "sched.h" 58 #include "stats.h" 59 #include "autogroup.h" 60 61 /* 62 * The initial- and re-scaling of tunables is configurable 63 * 64 * Options are: 65 * 66 * SCHED_TUNABLESCALING_NONE - unscaled, always *1 67 * SCHED_TUNABLESCALING_LOG - scaled logarithmically, *1+ilog(ncpus) 68 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus 69 * 70 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) 71 */ 72 unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; 73 74 /* 75 * Minimal preemption granularity for CPU-bound tasks: 76 * 77 * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) 78 */ 79 unsigned int sysctl_sched_base_slice = 700000ULL; 80 static unsigned int normalized_sysctl_sched_base_slice = 700000ULL; 81 82 __read_mostly unsigned int sysctl_sched_migration_cost = 500000UL; 83 84 static int __init setup_sched_thermal_decay_shift(char *str) 85 { 86 pr_warn("Ignoring the deprecated sched_thermal_decay_shift= option\n"); 87 return 1; 88 } 89 __setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift); 90 91 /* 92 * For asym packing, by default the lower numbered CPU has higher priority. 93 */ 94 int __weak arch_asym_cpu_priority(int cpu) 95 { 96 return -cpu; 97 } 98 99 /* 100 * The margin used when comparing utilization with CPU capacity. 101 * 102 * (default: ~20%) 103 */ 104 #define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024) 105 106 /* 107 * The margin used when comparing CPU capacities. 108 * is 'cap1' noticeably greater than 'cap2' 109 * 110 * (default: ~5%) 111 */ 112 #define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078) 113 114 #ifdef CONFIG_CFS_BANDWIDTH 115 /* 116 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool 117 * each time a cfs_rq requests quota. 118 * 119 * Note: in the case that the slice exceeds the runtime remaining (either due 120 * to consumption or the quota being specified to be smaller than the slice) 121 * we will always only issue the remaining available time. 122 * 123 * (default: 5 msec, units: microseconds) 124 */ 125 static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; 126 #endif 127 128 #ifdef CONFIG_NUMA_BALANCING 129 /* Restrict the NUMA promotion throughput (MB/s) for each target node. */ 130 static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; 131 #endif 132 133 #ifdef CONFIG_SYSCTL 134 static const struct ctl_table sched_fair_sysctls[] = { 135 #ifdef CONFIG_CFS_BANDWIDTH 136 { 137 .procname = "sched_cfs_bandwidth_slice_us", 138 .data = &sysctl_sched_cfs_bandwidth_slice, 139 .maxlen = sizeof(unsigned int), 140 .mode = 0644, 141 .proc_handler = proc_dointvec_minmax, 142 .extra1 = SYSCTL_ONE, 143 }, 144 #endif 145 #ifdef CONFIG_NUMA_BALANCING 146 { 147 .procname = "numa_balancing_promote_rate_limit_MBps", 148 .data = &sysctl_numa_balancing_promote_rate_limit, 149 .maxlen = sizeof(unsigned int), 150 .mode = 0644, 151 .proc_handler = proc_dointvec_minmax, 152 .extra1 = SYSCTL_ZERO, 153 }, 154 #endif /* CONFIG_NUMA_BALANCING */ 155 }; 156 157 static int __init sched_fair_sysctl_init(void) 158 { 159 register_sysctl_init("kernel", sched_fair_sysctls); 160 return 0; 161 } 162 late_initcall(sched_fair_sysctl_init); 163 #endif /* CONFIG_SYSCTL */ 164 165 static inline void update_load_add(struct load_weight *lw, unsigned long inc) 166 { 167 lw->weight += inc; 168 lw->inv_weight = 0; 169 } 170 171 static inline void update_load_sub(struct load_weight *lw, unsigned long dec) 172 { 173 lw->weight -= dec; 174 lw->inv_weight = 0; 175 } 176 177 static inline void update_load_set(struct load_weight *lw, unsigned long w) 178 { 179 lw->weight = w; 180 lw->inv_weight = 0; 181 } 182 183 /* 184 * Increase the granularity value when there are more CPUs, 185 * because with more CPUs the 'effective latency' as visible 186 * to users decreases. But the relationship is not linear, 187 * so pick a second-best guess by going with the log2 of the 188 * number of CPUs. 189 * 190 * This idea comes from the SD scheduler of Con Kolivas: 191 */ 192 static unsigned int get_update_sysctl_factor(void) 193 { 194 unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); 195 unsigned int factor; 196 197 switch (sysctl_sched_tunable_scaling) { 198 case SCHED_TUNABLESCALING_NONE: 199 factor = 1; 200 break; 201 case SCHED_TUNABLESCALING_LINEAR: 202 factor = cpus; 203 break; 204 case SCHED_TUNABLESCALING_LOG: 205 default: 206 factor = 1 + ilog2(cpus); 207 break; 208 } 209 210 return factor; 211 } 212 213 static void update_sysctl(void) 214 { 215 unsigned int factor = get_update_sysctl_factor(); 216 217 #define SET_SYSCTL(name) \ 218 (sysctl_##name = (factor) * normalized_sysctl_##name) 219 SET_SYSCTL(sched_base_slice); 220 #undef SET_SYSCTL 221 } 222 223 void __init sched_init_granularity(void) 224 { 225 update_sysctl(); 226 } 227 228 #define WMULT_CONST (~0U) 229 #define WMULT_SHIFT 32 230 231 static void __update_inv_weight(struct load_weight *lw) 232 { 233 unsigned long w; 234 235 if (likely(lw->inv_weight)) 236 return; 237 238 w = scale_load_down(lw->weight); 239 240 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 241 lw->inv_weight = 1; 242 else if (unlikely(!w)) 243 lw->inv_weight = WMULT_CONST; 244 else 245 lw->inv_weight = WMULT_CONST / w; 246 } 247 248 /* 249 * delta_exec * weight / lw.weight 250 * OR 251 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT 252 * 253 * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case 254 * we're guaranteed shift stays positive because inv_weight is guaranteed to 255 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. 256 * 257 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus 258 * weight/lw.weight <= 1, and therefore our shift will also be positive. 259 */ 260 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 261 { 262 u64 fact = scale_load_down(weight); 263 u32 fact_hi = (u32)(fact >> 32); 264 int shift = WMULT_SHIFT; 265 int fs; 266 267 __update_inv_weight(lw); 268 269 if (unlikely(fact_hi)) { 270 fs = fls(fact_hi); 271 shift -= fs; 272 fact >>= fs; 273 } 274 275 fact = mul_u32_u32(fact, lw->inv_weight); 276 277 fact_hi = (u32)(fact >> 32); 278 if (fact_hi) { 279 fs = fls(fact_hi); 280 shift -= fs; 281 fact >>= fs; 282 } 283 284 return mul_u64_u32_shr(delta_exec, fact, shift); 285 } 286 287 /* 288 * delta /= w 289 */ 290 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) 291 { 292 if (unlikely(se->load.weight != NICE_0_LOAD)) 293 delta = __calc_delta(delta, NICE_0_LOAD, &se->load); 294 295 return delta; 296 } 297 298 const struct sched_class fair_sched_class; 299 300 /************************************************************** 301 * CFS operations on generic schedulable entities: 302 */ 303 304 #ifdef CONFIG_FAIR_GROUP_SCHED 305 306 /* Walk up scheduling entities hierarchy */ 307 #define for_each_sched_entity(se) \ 308 for (; se; se = se->parent) 309 310 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 311 { 312 struct rq *rq = rq_of(cfs_rq); 313 int cpu = cpu_of(rq); 314 315 if (cfs_rq->on_list) 316 return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list; 317 318 cfs_rq->on_list = 1; 319 320 /* 321 * Ensure we either appear before our parent (if already 322 * enqueued) or force our parent to appear after us when it is 323 * enqueued. The fact that we always enqueue bottom-up 324 * reduces this to two cases and a special case for the root 325 * cfs_rq. Furthermore, it also means that we will always reset 326 * tmp_alone_branch either when the branch is connected 327 * to a tree or when we reach the top of the tree 328 */ 329 if (cfs_rq->tg->parent && 330 cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { 331 /* 332 * If parent is already on the list, we add the child 333 * just before. Thanks to circular linked property of 334 * the list, this means to put the child at the tail 335 * of the list that starts by parent. 336 */ 337 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 338 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); 339 /* 340 * The branch is now connected to its tree so we can 341 * reset tmp_alone_branch to the beginning of the 342 * list. 343 */ 344 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 345 return true; 346 } 347 348 if (!cfs_rq->tg->parent) { 349 /* 350 * cfs rq without parent should be put 351 * at the tail of the list. 352 */ 353 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 354 &rq->leaf_cfs_rq_list); 355 /* 356 * We have reach the top of a tree so we can reset 357 * tmp_alone_branch to the beginning of the list. 358 */ 359 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 360 return true; 361 } 362 363 /* 364 * The parent has not already been added so we want to 365 * make sure that it will be put after us. 366 * tmp_alone_branch points to the begin of the branch 367 * where we will add parent. 368 */ 369 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); 370 /* 371 * update tmp_alone_branch to points to the new begin 372 * of the branch 373 */ 374 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 375 return false; 376 } 377 378 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 379 { 380 if (cfs_rq->on_list) { 381 struct rq *rq = rq_of(cfs_rq); 382 383 /* 384 * With cfs_rq being unthrottled/throttled during an enqueue, 385 * it can happen the tmp_alone_branch points to the leaf that 386 * we finally want to delete. In this case, tmp_alone_branch moves 387 * to the prev element but it will point to rq->leaf_cfs_rq_list 388 * at the end of the enqueue. 389 */ 390 if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) 391 rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev; 392 393 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 394 cfs_rq->on_list = 0; 395 } 396 } 397 398 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 399 { 400 WARN_ON_ONCE(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list); 401 } 402 403 /* Iterate through all leaf cfs_rq's on a runqueue */ 404 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 405 list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \ 406 leaf_cfs_rq_list) 407 408 /* Do the two (enqueued) entities belong to the same group ? */ 409 static inline struct cfs_rq * 410 is_same_group(struct sched_entity *se, struct sched_entity *pse) 411 { 412 if (se->cfs_rq == pse->cfs_rq) 413 return se->cfs_rq; 414 415 return NULL; 416 } 417 418 static inline struct sched_entity *parent_entity(const struct sched_entity *se) 419 { 420 return se->parent; 421 } 422 423 static void 424 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 425 { 426 int se_depth, pse_depth; 427 428 /* 429 * preemption test can be made between sibling entities who are in the 430 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of 431 * both tasks until we find their ancestors who are siblings of common 432 * parent. 433 */ 434 435 /* First walk up until both entities are at same depth */ 436 se_depth = (*se)->depth; 437 pse_depth = (*pse)->depth; 438 439 while (se_depth > pse_depth) { 440 se_depth--; 441 *se = parent_entity(*se); 442 } 443 444 while (pse_depth > se_depth) { 445 pse_depth--; 446 *pse = parent_entity(*pse); 447 } 448 449 while (!is_same_group(*se, *pse)) { 450 *se = parent_entity(*se); 451 *pse = parent_entity(*pse); 452 } 453 } 454 455 static int tg_is_idle(struct task_group *tg) 456 { 457 return tg->idle > 0; 458 } 459 460 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 461 { 462 return cfs_rq->idle > 0; 463 } 464 465 static int se_is_idle(struct sched_entity *se) 466 { 467 if (entity_is_task(se)) 468 return task_has_idle_policy(task_of(se)); 469 return cfs_rq_is_idle(group_cfs_rq(se)); 470 } 471 472 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 473 474 #define for_each_sched_entity(se) \ 475 for (; se; se = NULL) 476 477 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 478 { 479 return true; 480 } 481 482 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 483 { 484 } 485 486 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 487 { 488 } 489 490 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 491 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos) 492 493 static inline struct sched_entity *parent_entity(struct sched_entity *se) 494 { 495 return NULL; 496 } 497 498 static inline void 499 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 500 { 501 } 502 503 static inline int tg_is_idle(struct task_group *tg) 504 { 505 return 0; 506 } 507 508 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 509 { 510 return 0; 511 } 512 513 static int se_is_idle(struct sched_entity *se) 514 { 515 return task_has_idle_policy(task_of(se)); 516 } 517 518 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 519 520 static __always_inline 521 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); 522 523 /************************************************************** 524 * Scheduling class tree data structure manipulation methods: 525 */ 526 527 extern void __BUILD_BUG_vruntime_cmp(void); 528 529 /* Use __builtin_strcmp() because of __HAVE_ARCH_STRCMP: */ 530 531 #define vruntime_cmp(A, CMP_STR, B) ({ \ 532 int __res = 0; \ 533 \ 534 if (!__builtin_strcmp(CMP_STR, "<")) { \ 535 __res = ((s64)((A)-(B)) < 0); \ 536 } else if (!__builtin_strcmp(CMP_STR, "<=")) { \ 537 __res = ((s64)((A)-(B)) <= 0); \ 538 } else if (!__builtin_strcmp(CMP_STR, ">")) { \ 539 __res = ((s64)((A)-(B)) > 0); \ 540 } else if (!__builtin_strcmp(CMP_STR, ">=")) { \ 541 __res = ((s64)((A)-(B)) >= 0); \ 542 } else { \ 543 /* Unknown operator throws linker error: */ \ 544 __BUILD_BUG_vruntime_cmp(); \ 545 } \ 546 \ 547 __res; \ 548 }) 549 550 extern void __BUILD_BUG_vruntime_op(void); 551 552 #define vruntime_op(A, OP_STR, B) ({ \ 553 s64 __res = 0; \ 554 \ 555 if (!__builtin_strcmp(OP_STR, "-")) { \ 556 __res = (s64)((A)-(B)); \ 557 } else { \ 558 /* Unknown operator throws linker error: */ \ 559 __BUILD_BUG_vruntime_op(); \ 560 } \ 561 \ 562 __res; \ 563 }) 564 565 566 static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime) 567 { 568 if (vruntime_cmp(vruntime, ">", max_vruntime)) 569 max_vruntime = vruntime; 570 571 return max_vruntime; 572 } 573 574 static inline __maybe_unused u64 min_vruntime(u64 min_vruntime, u64 vruntime) 575 { 576 if (vruntime_cmp(vruntime, "<", min_vruntime)) 577 min_vruntime = vruntime; 578 579 return min_vruntime; 580 } 581 582 static inline bool entity_before(const struct sched_entity *a, 583 const struct sched_entity *b) 584 { 585 /* 586 * Tiebreak on vruntime seems unnecessary since it can 587 * hardly happen. 588 */ 589 return vruntime_cmp(a->deadline, "<", b->deadline); 590 } 591 592 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) 593 { 594 return vruntime_op(se->vruntime, "-", cfs_rq->zero_vruntime); 595 } 596 597 #define __node_2_se(node) \ 598 rb_entry((node), struct sched_entity, run_node) 599 600 /* 601 * Compute virtual time from the per-task service numbers: 602 * 603 * Fair schedulers conserve lag: 604 * 605 * \Sum lag_i = 0 606 * 607 * Where lag_i is given by: 608 * 609 * lag_i = S - s_i = w_i * (V - v_i) 610 * 611 * Where S is the ideal service time and V is it's virtual time counterpart. 612 * Therefore: 613 * 614 * \Sum lag_i = 0 615 * \Sum w_i * (V - v_i) = 0 616 * \Sum (w_i * V - w_i * v_i) = 0 617 * 618 * From which we can solve an expression for V in v_i (which we have in 619 * se->vruntime): 620 * 621 * \Sum v_i * w_i \Sum v_i * w_i 622 * V = -------------- = -------------- 623 * \Sum w_i W 624 * 625 * Specifically, this is the weighted average of all entity virtual runtimes. 626 * 627 * [[ NOTE: this is only equal to the ideal scheduler under the condition 628 * that join/leave operations happen at lag_i = 0, otherwise the 629 * virtual time has non-contiguous motion equivalent to: 630 * 631 * V +-= lag_i / W 632 * 633 * Also see the comment in place_entity() that deals with this. ]] 634 * 635 * However, since v_i is u64, and the multiplication could easily overflow 636 * transform it into a relative form that uses smaller quantities: 637 * 638 * Substitute: v_i == (v_i - v0) + v0 639 * 640 * \Sum ((v_i - v0) + v0) * w_i \Sum (v_i - v0) * w_i 641 * V = ---------------------------- = --------------------- + v0 642 * W W 643 * 644 * Which we track using: 645 * 646 * v0 := cfs_rq->zero_vruntime 647 * \Sum (v_i - v0) * w_i := cfs_rq->sum_w_vruntime 648 * \Sum w_i := cfs_rq->sum_weight 649 * 650 * Since zero_vruntime closely tracks the per-task service, these 651 * deltas: (v_i - v0), will be in the order of the maximal (virtual) lag 652 * induced in the system due to quantisation. 653 * 654 * Also, we use scale_load_down() to reduce the size. 655 * 656 * As measured, the max (key * weight) value was ~44 bits for a kernel build. 657 */ 658 static void 659 sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) 660 { 661 unsigned long weight = scale_load_down(se->load.weight); 662 s64 key = entity_key(cfs_rq, se); 663 664 cfs_rq->sum_w_vruntime += key * weight; 665 cfs_rq->sum_weight += weight; 666 } 667 668 static void 669 sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) 670 { 671 unsigned long weight = scale_load_down(se->load.weight); 672 s64 key = entity_key(cfs_rq, se); 673 674 cfs_rq->sum_w_vruntime -= key * weight; 675 cfs_rq->sum_weight -= weight; 676 } 677 678 static inline 679 void sum_w_vruntime_update(struct cfs_rq *cfs_rq, s64 delta) 680 { 681 /* 682 * v' = v + d ==> sum_w_vruntime' = sum_runtime - d*sum_weight 683 */ 684 cfs_rq->sum_w_vruntime -= cfs_rq->sum_weight * delta; 685 } 686 687 /* 688 * Specifically: avg_runtime() + 0 must result in entity_eligible() := true 689 * For this to be so, the result of this function must have a left bias. 690 */ 691 u64 avg_vruntime(struct cfs_rq *cfs_rq) 692 { 693 struct sched_entity *curr = cfs_rq->curr; 694 s64 avg = cfs_rq->sum_w_vruntime; 695 long load = cfs_rq->sum_weight; 696 697 if (curr && curr->on_rq) { 698 unsigned long weight = scale_load_down(curr->load.weight); 699 700 avg += entity_key(cfs_rq, curr) * weight; 701 load += weight; 702 } 703 704 if (load) { 705 /* sign flips effective floor / ceiling */ 706 if (avg < 0) 707 avg -= (load - 1); 708 avg = div_s64(avg, load); 709 } 710 711 return cfs_rq->zero_vruntime + avg; 712 } 713 714 /* 715 * lag_i = S - s_i = w_i * (V - v_i) 716 * 717 * However, since V is approximated by the weighted average of all entities it 718 * is possible -- by addition/removal/reweight to the tree -- to move V around 719 * and end up with a larger lag than we started with. 720 * 721 * Limit this to either double the slice length with a minimum of TICK_NSEC 722 * since that is the timing granularity. 723 * 724 * EEVDF gives the following limit for a steady state system: 725 * 726 * -r_max < lag < max(r_max, q) 727 * 728 * XXX could add max_slice to the augmented data to track this. 729 */ 730 static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) 731 { 732 s64 vlag, limit; 733 734 WARN_ON_ONCE(!se->on_rq); 735 736 vlag = avg_vruntime(cfs_rq) - se->vruntime; 737 limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se); 738 739 se->vlag = clamp(vlag, -limit, limit); 740 } 741 742 /* 743 * Entity is eligible once it received less service than it ought to have, 744 * eg. lag >= 0. 745 * 746 * lag_i = S - s_i = w_i*(V - v_i) 747 * 748 * lag_i >= 0 -> V >= v_i 749 * 750 * \Sum (v_i - v)*w_i 751 * V = ------------------ + v 752 * \Sum w_i 753 * 754 * lag_i >= 0 -> \Sum (v_i - v)*w_i >= (v_i - v)*(\Sum w_i) 755 * 756 * Note: using 'avg_vruntime() > se->vruntime' is inaccurate due 757 * to the loss in precision caused by the division. 758 */ 759 static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) 760 { 761 struct sched_entity *curr = cfs_rq->curr; 762 s64 avg = cfs_rq->sum_w_vruntime; 763 long load = cfs_rq->sum_weight; 764 765 if (curr && curr->on_rq) { 766 unsigned long weight = scale_load_down(curr->load.weight); 767 768 avg += entity_key(cfs_rq, curr) * weight; 769 load += weight; 770 } 771 772 return avg >= vruntime_op(vruntime, "-", cfs_rq->zero_vruntime) * load; 773 } 774 775 int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) 776 { 777 return vruntime_eligible(cfs_rq, se->vruntime); 778 } 779 780 static void update_zero_vruntime(struct cfs_rq *cfs_rq) 781 { 782 u64 vruntime = avg_vruntime(cfs_rq); 783 s64 delta = vruntime_op(vruntime, "-", cfs_rq->zero_vruntime); 784 785 sum_w_vruntime_update(cfs_rq, delta); 786 787 cfs_rq->zero_vruntime = vruntime; 788 } 789 790 static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) 791 { 792 struct sched_entity *root = __pick_root_entity(cfs_rq); 793 struct sched_entity *curr = cfs_rq->curr; 794 u64 min_slice = ~0ULL; 795 796 if (curr && curr->on_rq) 797 min_slice = curr->slice; 798 799 if (root) 800 min_slice = min(min_slice, root->min_slice); 801 802 return min_slice; 803 } 804 805 static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) 806 { 807 return entity_before(__node_2_se(a), __node_2_se(b)); 808 } 809 810 static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node) 811 { 812 if (node) { 813 struct sched_entity *rse = __node_2_se(node); 814 815 if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime)) 816 se->min_vruntime = rse->min_vruntime; 817 } 818 } 819 820 static inline void __min_slice_update(struct sched_entity *se, struct rb_node *node) 821 { 822 if (node) { 823 struct sched_entity *rse = __node_2_se(node); 824 if (rse->min_slice < se->min_slice) 825 se->min_slice = rse->min_slice; 826 } 827 } 828 829 /* 830 * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) 831 */ 832 static inline bool min_vruntime_update(struct sched_entity *se, bool exit) 833 { 834 u64 old_min_vruntime = se->min_vruntime; 835 u64 old_min_slice = se->min_slice; 836 struct rb_node *node = &se->run_node; 837 838 se->min_vruntime = se->vruntime; 839 __min_vruntime_update(se, node->rb_right); 840 __min_vruntime_update(se, node->rb_left); 841 842 se->min_slice = se->slice; 843 __min_slice_update(se, node->rb_right); 844 __min_slice_update(se, node->rb_left); 845 846 return se->min_vruntime == old_min_vruntime && 847 se->min_slice == old_min_slice; 848 } 849 850 RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, 851 run_node, min_vruntime, min_vruntime_update); 852 853 /* 854 * Enqueue an entity into the rb-tree: 855 */ 856 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 857 { 858 sum_w_vruntime_add(cfs_rq, se); 859 update_zero_vruntime(cfs_rq); 860 se->min_vruntime = se->vruntime; 861 se->min_slice = se->slice; 862 rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 863 __entity_less, &min_vruntime_cb); 864 } 865 866 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 867 { 868 rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 869 &min_vruntime_cb); 870 sum_w_vruntime_sub(cfs_rq, se); 871 update_zero_vruntime(cfs_rq); 872 } 873 874 struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq) 875 { 876 struct rb_node *root = cfs_rq->tasks_timeline.rb_root.rb_node; 877 878 if (!root) 879 return NULL; 880 881 return __node_2_se(root); 882 } 883 884 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 885 { 886 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 887 888 if (!left) 889 return NULL; 890 891 return __node_2_se(left); 892 } 893 894 /* 895 * Set the vruntime up to which an entity can run before looking 896 * for another entity to pick. 897 * In case of run to parity, we use the shortest slice of the enqueued 898 * entities to set the protected period. 899 * When run to parity is disabled, we give a minimum quantum to the running 900 * entity to ensure progress. 901 */ 902 static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 903 { 904 u64 slice = normalized_sysctl_sched_base_slice; 905 u64 vprot = se->deadline; 906 907 if (sched_feat(RUN_TO_PARITY)) 908 slice = cfs_rq_min_slice(cfs_rq); 909 910 slice = min(slice, se->slice); 911 if (slice != se->slice) 912 vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); 913 914 se->vprot = vprot; 915 } 916 917 static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 918 { 919 u64 slice = cfs_rq_min_slice(cfs_rq); 920 921 se->vprot = min_vruntime(se->vprot, se->vruntime + calc_delta_fair(slice, se)); 922 } 923 924 static inline bool protect_slice(struct sched_entity *se) 925 { 926 return vruntime_cmp(se->vruntime, "<", se->vprot); 927 } 928 929 static inline void cancel_protect_slice(struct sched_entity *se) 930 { 931 if (protect_slice(se)) 932 se->vprot = se->vruntime; 933 } 934 935 /* 936 * Earliest Eligible Virtual Deadline First 937 * 938 * In order to provide latency guarantees for different request sizes 939 * EEVDF selects the best runnable task from two criteria: 940 * 941 * 1) the task must be eligible (must be owed service) 942 * 943 * 2) from those tasks that meet 1), we select the one 944 * with the earliest virtual deadline. 945 * 946 * We can do this in O(log n) time due to an augmented RB-tree. The 947 * tree keeps the entries sorted on deadline, but also functions as a 948 * heap based on the vruntime by keeping: 949 * 950 * se->min_vruntime = min(se->vruntime, se->{left,right}->min_vruntime) 951 * 952 * Which allows tree pruning through eligibility. 953 */ 954 static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq, bool protect) 955 { 956 struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; 957 struct sched_entity *se = __pick_first_entity(cfs_rq); 958 struct sched_entity *curr = cfs_rq->curr; 959 struct sched_entity *best = NULL; 960 961 /* 962 * We can safely skip eligibility check if there is only one entity 963 * in this cfs_rq, saving some cycles. 964 */ 965 if (cfs_rq->nr_queued == 1) 966 return curr && curr->on_rq ? curr : se; 967 968 /* 969 * Picking the ->next buddy will affect latency but not fairness. 970 */ 971 if (sched_feat(PICK_BUDDY) && 972 cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) { 973 /* ->next will never be delayed */ 974 WARN_ON_ONCE(cfs_rq->next->sched_delayed); 975 return cfs_rq->next; 976 } 977 978 if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr))) 979 curr = NULL; 980 981 if (curr && protect && protect_slice(curr)) 982 return curr; 983 984 /* Pick the leftmost entity if it's eligible */ 985 if (se && entity_eligible(cfs_rq, se)) { 986 best = se; 987 goto found; 988 } 989 990 /* Heap search for the EEVD entity */ 991 while (node) { 992 struct rb_node *left = node->rb_left; 993 994 /* 995 * Eligible entities in left subtree are always better 996 * choices, since they have earlier deadlines. 997 */ 998 if (left && vruntime_eligible(cfs_rq, 999 __node_2_se(left)->min_vruntime)) { 1000 node = left; 1001 continue; 1002 } 1003 1004 se = __node_2_se(node); 1005 1006 /* 1007 * The left subtree either is empty or has no eligible 1008 * entity, so check the current node since it is the one 1009 * with earliest deadline that might be eligible. 1010 */ 1011 if (entity_eligible(cfs_rq, se)) { 1012 best = se; 1013 break; 1014 } 1015 1016 node = node->rb_right; 1017 } 1018 found: 1019 if (!best || (curr && entity_before(curr, best))) 1020 best = curr; 1021 1022 return best; 1023 } 1024 1025 static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) 1026 { 1027 return __pick_eevdf(cfs_rq, true); 1028 } 1029 1030 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) 1031 { 1032 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); 1033 1034 if (!last) 1035 return NULL; 1036 1037 return __node_2_se(last); 1038 } 1039 1040 /************************************************************** 1041 * Scheduling class statistics methods: 1042 */ 1043 int sched_update_scaling(void) 1044 { 1045 unsigned int factor = get_update_sysctl_factor(); 1046 1047 #define WRT_SYSCTL(name) \ 1048 (normalized_sysctl_##name = sysctl_##name / (factor)) 1049 WRT_SYSCTL(sched_base_slice); 1050 #undef WRT_SYSCTL 1051 1052 return 0; 1053 } 1054 1055 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se); 1056 1057 /* 1058 * XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i 1059 * this is probably good enough. 1060 */ 1061 static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) 1062 { 1063 if (vruntime_cmp(se->vruntime, "<", se->deadline)) 1064 return false; 1065 1066 /* 1067 * For EEVDF the virtual time slope is determined by w_i (iow. 1068 * nice) while the request time r_i is determined by 1069 * sysctl_sched_base_slice. 1070 */ 1071 if (!se->custom_slice) 1072 se->slice = sysctl_sched_base_slice; 1073 1074 /* 1075 * EEVDF: vd_i = ve_i + r_i / w_i 1076 */ 1077 se->deadline = se->vruntime + calc_delta_fair(se->slice, se); 1078 1079 /* 1080 * The task has consumed its request, reschedule. 1081 */ 1082 return true; 1083 } 1084 1085 #include "pelt.h" 1086 1087 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); 1088 static unsigned long task_h_load(struct task_struct *p); 1089 static unsigned long capacity_of(int cpu); 1090 1091 /* Give new sched_entity start runnable values to heavy its load in infant time */ 1092 void init_entity_runnable_average(struct sched_entity *se) 1093 { 1094 struct sched_avg *sa = &se->avg; 1095 1096 memset(sa, 0, sizeof(*sa)); 1097 1098 /* 1099 * Tasks are initialized with full load to be seen as heavy tasks until 1100 * they get a chance to stabilize to their real load level. 1101 * Group entities are initialized with zero load to reflect the fact that 1102 * nothing has been attached to the task group yet. 1103 */ 1104 if (entity_is_task(se)) 1105 sa->load_avg = scale_load_down(se->load.weight); 1106 1107 /* when this task is enqueued, it will contribute to its cfs_rq's load_avg */ 1108 } 1109 1110 /* 1111 * With new tasks being created, their initial util_avgs are extrapolated 1112 * based on the cfs_rq's current util_avg: 1113 * 1114 * util_avg = cfs_rq->avg.util_avg / (cfs_rq->avg.load_avg + 1) 1115 * * se_weight(se) 1116 * 1117 * However, in many cases, the above util_avg does not give a desired 1118 * value. Moreover, the sum of the util_avgs may be divergent, such 1119 * as when the series is a harmonic series. 1120 * 1121 * To solve this problem, we also cap the util_avg of successive tasks to 1122 * only 1/2 of the left utilization budget: 1123 * 1124 * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n 1125 * 1126 * where n denotes the nth task and cpu_scale the CPU capacity. 1127 * 1128 * For example, for a CPU with 1024 of capacity, a simplest series from 1129 * the beginning would be like: 1130 * 1131 * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... 1132 * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... 1133 * 1134 * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) 1135 * if util_avg > util_avg_cap. 1136 */ 1137 void post_init_entity_util_avg(struct task_struct *p) 1138 { 1139 struct sched_entity *se = &p->se; 1140 struct cfs_rq *cfs_rq = cfs_rq_of(se); 1141 struct sched_avg *sa = &se->avg; 1142 long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq))); 1143 long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; 1144 1145 if (p->sched_class != &fair_sched_class) { 1146 /* 1147 * For !fair tasks do: 1148 * 1149 update_cfs_rq_load_avg(now, cfs_rq); 1150 attach_entity_load_avg(cfs_rq, se); 1151 switched_from_fair(rq, p); 1152 * 1153 * such that the next switched_to_fair() has the 1154 * expected state. 1155 */ 1156 se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq); 1157 return; 1158 } 1159 1160 if (cap > 0) { 1161 if (cfs_rq->avg.util_avg != 0) { 1162 sa->util_avg = cfs_rq->avg.util_avg * se_weight(se); 1163 sa->util_avg /= (cfs_rq->avg.load_avg + 1); 1164 1165 if (sa->util_avg > cap) 1166 sa->util_avg = cap; 1167 } else { 1168 sa->util_avg = cap; 1169 } 1170 } 1171 1172 sa->runnable_avg = sa->util_avg; 1173 } 1174 1175 static s64 update_se(struct rq *rq, struct sched_entity *se) 1176 { 1177 u64 now = rq_clock_task(rq); 1178 s64 delta_exec; 1179 1180 delta_exec = now - se->exec_start; 1181 if (unlikely(delta_exec <= 0)) 1182 return delta_exec; 1183 1184 se->exec_start = now; 1185 if (entity_is_task(se)) { 1186 struct task_struct *donor = task_of(se); 1187 struct task_struct *running = rq->curr; 1188 /* 1189 * If se is a task, we account the time against the running 1190 * task, as w/ proxy-exec they may not be the same. 1191 */ 1192 running->se.exec_start = now; 1193 running->se.sum_exec_runtime += delta_exec; 1194 1195 trace_sched_stat_runtime(running, delta_exec); 1196 account_group_exec_runtime(running, delta_exec); 1197 1198 /* cgroup time is always accounted against the donor */ 1199 cgroup_account_cputime(donor, delta_exec); 1200 } else { 1201 /* If not task, account the time against donor se */ 1202 se->sum_exec_runtime += delta_exec; 1203 } 1204 1205 if (schedstat_enabled()) { 1206 struct sched_statistics *stats; 1207 1208 stats = __schedstats_from_se(se); 1209 __schedstat_set(stats->exec_max, 1210 max(delta_exec, stats->exec_max)); 1211 } 1212 1213 return delta_exec; 1214 } 1215 1216 static void set_next_buddy(struct sched_entity *se); 1217 1218 /* 1219 * Used by other classes to account runtime. 1220 */ 1221 s64 update_curr_common(struct rq *rq) 1222 { 1223 return update_se(rq, &rq->donor->se); 1224 } 1225 1226 /* 1227 * Update the current task's runtime statistics. 1228 */ 1229 static void update_curr(struct cfs_rq *cfs_rq) 1230 { 1231 /* 1232 * Note: cfs_rq->curr corresponds to the task picked to 1233 * run (ie: rq->donor.se) which due to proxy-exec may 1234 * not necessarily be the actual task running 1235 * (rq->curr.se). This is easy to confuse! 1236 */ 1237 struct sched_entity *curr = cfs_rq->curr; 1238 struct rq *rq = rq_of(cfs_rq); 1239 s64 delta_exec; 1240 bool resched; 1241 1242 if (unlikely(!curr)) 1243 return; 1244 1245 delta_exec = update_se(rq, curr); 1246 if (unlikely(delta_exec <= 0)) 1247 return; 1248 1249 curr->vruntime += calc_delta_fair(delta_exec, curr); 1250 resched = update_deadline(cfs_rq, curr); 1251 1252 if (entity_is_task(curr)) { 1253 /* 1254 * If the fair_server is active, we need to account for the 1255 * fair_server time whether or not the task is running on 1256 * behalf of fair_server or not: 1257 * - If the task is running on behalf of fair_server, we need 1258 * to limit its time based on the assigned runtime. 1259 * - Fair task that runs outside of fair_server should account 1260 * against fair_server such that it can account for this time 1261 * and possibly avoid running this period. 1262 */ 1263 dl_server_update(&rq->fair_server, delta_exec); 1264 } 1265 1266 account_cfs_rq_runtime(cfs_rq, delta_exec); 1267 1268 if (cfs_rq->nr_queued == 1) 1269 return; 1270 1271 if (resched || !protect_slice(curr)) { 1272 resched_curr_lazy(rq); 1273 clear_buddies(cfs_rq, curr); 1274 } 1275 } 1276 1277 static void update_curr_fair(struct rq *rq) 1278 { 1279 update_curr(cfs_rq_of(&rq->donor->se)); 1280 } 1281 1282 static inline void 1283 update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1284 { 1285 struct sched_statistics *stats; 1286 struct task_struct *p = NULL; 1287 1288 if (!schedstat_enabled()) 1289 return; 1290 1291 stats = __schedstats_from_se(se); 1292 1293 if (entity_is_task(se)) 1294 p = task_of(se); 1295 1296 __update_stats_wait_start(rq_of(cfs_rq), p, stats); 1297 } 1298 1299 static inline void 1300 update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1301 { 1302 struct sched_statistics *stats; 1303 struct task_struct *p = NULL; 1304 1305 if (!schedstat_enabled()) 1306 return; 1307 1308 stats = __schedstats_from_se(se); 1309 1310 /* 1311 * When the sched_schedstat changes from 0 to 1, some sched se 1312 * maybe already in the runqueue, the se->statistics.wait_start 1313 * will be 0.So it will let the delta wrong. We need to avoid this 1314 * scenario. 1315 */ 1316 if (unlikely(!schedstat_val(stats->wait_start))) 1317 return; 1318 1319 if (entity_is_task(se)) 1320 p = task_of(se); 1321 1322 __update_stats_wait_end(rq_of(cfs_rq), p, stats); 1323 } 1324 1325 static inline void 1326 update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1327 { 1328 struct sched_statistics *stats; 1329 struct task_struct *tsk = NULL; 1330 1331 if (!schedstat_enabled()) 1332 return; 1333 1334 stats = __schedstats_from_se(se); 1335 1336 if (entity_is_task(se)) 1337 tsk = task_of(se); 1338 1339 __update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats); 1340 } 1341 1342 /* 1343 * Task is being enqueued - update stats: 1344 */ 1345 static inline void 1346 update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1347 { 1348 if (!schedstat_enabled()) 1349 return; 1350 1351 /* 1352 * Are we enqueueing a waiting task? (for current tasks 1353 * a dequeue/enqueue event is a NOP) 1354 */ 1355 if (se != cfs_rq->curr) 1356 update_stats_wait_start_fair(cfs_rq, se); 1357 1358 if (flags & ENQUEUE_WAKEUP) 1359 update_stats_enqueue_sleeper_fair(cfs_rq, se); 1360 } 1361 1362 static inline void 1363 update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1364 { 1365 1366 if (!schedstat_enabled()) 1367 return; 1368 1369 /* 1370 * Mark the end of the wait period if dequeueing a 1371 * waiting task: 1372 */ 1373 if (se != cfs_rq->curr) 1374 update_stats_wait_end_fair(cfs_rq, se); 1375 1376 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { 1377 struct task_struct *tsk = task_of(se); 1378 unsigned int state; 1379 1380 /* XXX racy against TTWU */ 1381 state = READ_ONCE(tsk->__state); 1382 if (state & TASK_INTERRUPTIBLE) 1383 __schedstat_set(tsk->stats.sleep_start, 1384 rq_clock(rq_of(cfs_rq))); 1385 if (state & TASK_UNINTERRUPTIBLE) 1386 __schedstat_set(tsk->stats.block_start, 1387 rq_clock(rq_of(cfs_rq))); 1388 } 1389 } 1390 1391 /* 1392 * We are picking a new current task - update its stats: 1393 */ 1394 static inline void 1395 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 1396 { 1397 /* 1398 * We are starting a new run period: 1399 */ 1400 se->exec_start = rq_clock_task(rq_of(cfs_rq)); 1401 } 1402 1403 /************************************************** 1404 * Scheduling class queueing methods: 1405 */ 1406 1407 static inline bool is_core_idle(int cpu) 1408 { 1409 #ifdef CONFIG_SCHED_SMT 1410 int sibling; 1411 1412 for_each_cpu(sibling, cpu_smt_mask(cpu)) { 1413 if (cpu == sibling) 1414 continue; 1415 1416 if (!idle_cpu(sibling)) 1417 return false; 1418 } 1419 #endif 1420 1421 return true; 1422 } 1423 1424 #ifdef CONFIG_NUMA 1425 #define NUMA_IMBALANCE_MIN 2 1426 1427 static inline long 1428 adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr) 1429 { 1430 /* 1431 * Allow a NUMA imbalance if busy CPUs is less than the maximum 1432 * threshold. Above this threshold, individual tasks may be contending 1433 * for both memory bandwidth and any shared HT resources. This is an 1434 * approximation as the number of running tasks may not be related to 1435 * the number of busy CPUs due to sched_setaffinity. 1436 */ 1437 if (dst_running > imb_numa_nr) 1438 return imbalance; 1439 1440 /* 1441 * Allow a small imbalance based on a simple pair of communicating 1442 * tasks that remain local when the destination is lightly loaded. 1443 */ 1444 if (imbalance <= NUMA_IMBALANCE_MIN) 1445 return 0; 1446 1447 return imbalance; 1448 } 1449 #endif /* CONFIG_NUMA */ 1450 1451 #ifdef CONFIG_NUMA_BALANCING 1452 /* 1453 * Approximate time to scan a full NUMA task in ms. The task scan period is 1454 * calculated based on the tasks virtual memory size and 1455 * numa_balancing_scan_size. 1456 */ 1457 unsigned int sysctl_numa_balancing_scan_period_min = 1000; 1458 unsigned int sysctl_numa_balancing_scan_period_max = 60000; 1459 1460 /* Portion of address space to scan in MB */ 1461 unsigned int sysctl_numa_balancing_scan_size = 256; 1462 1463 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ 1464 unsigned int sysctl_numa_balancing_scan_delay = 1000; 1465 1466 /* The page with hint page fault latency < threshold in ms is considered hot */ 1467 unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC; 1468 1469 struct numa_group { 1470 refcount_t refcount; 1471 1472 spinlock_t lock; /* nr_tasks, tasks */ 1473 int nr_tasks; 1474 pid_t gid; 1475 int active_nodes; 1476 1477 struct rcu_head rcu; 1478 unsigned long total_faults; 1479 unsigned long max_faults_cpu; 1480 /* 1481 * faults[] array is split into two regions: faults_mem and faults_cpu. 1482 * 1483 * Faults_cpu is used to decide whether memory should move 1484 * towards the CPU. As a consequence, these stats are weighted 1485 * more by CPU use than by memory faults. 1486 */ 1487 unsigned long faults[]; 1488 }; 1489 1490 /* 1491 * For functions that can be called in multiple contexts that permit reading 1492 * ->numa_group (see struct task_struct for locking rules). 1493 */ 1494 static struct numa_group *deref_task_numa_group(struct task_struct *p) 1495 { 1496 return rcu_dereference_check(p->numa_group, p == current || 1497 (lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu))); 1498 } 1499 1500 static struct numa_group *deref_curr_numa_group(struct task_struct *p) 1501 { 1502 return rcu_dereference_protected(p->numa_group, p == current); 1503 } 1504 1505 static inline unsigned long group_faults_priv(struct numa_group *ng); 1506 static inline unsigned long group_faults_shared(struct numa_group *ng); 1507 1508 static unsigned int task_nr_scan_windows(struct task_struct *p) 1509 { 1510 unsigned long rss = 0; 1511 unsigned long nr_scan_pages; 1512 1513 /* 1514 * Calculations based on RSS as non-present and empty pages are skipped 1515 * by the PTE scanner and NUMA hinting faults should be trapped based 1516 * on resident pages 1517 */ 1518 nr_scan_pages = MB_TO_PAGES(sysctl_numa_balancing_scan_size); 1519 rss = get_mm_rss(p->mm); 1520 if (!rss) 1521 rss = nr_scan_pages; 1522 1523 rss = round_up(rss, nr_scan_pages); 1524 return rss / nr_scan_pages; 1525 } 1526 1527 /* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ 1528 #define MAX_SCAN_WINDOW 2560 1529 1530 static unsigned int task_scan_min(struct task_struct *p) 1531 { 1532 unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); 1533 unsigned int scan, floor; 1534 unsigned int windows = 1; 1535 1536 if (scan_size < MAX_SCAN_WINDOW) 1537 windows = MAX_SCAN_WINDOW / scan_size; 1538 floor = 1000 / windows; 1539 1540 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); 1541 return max_t(unsigned int, floor, scan); 1542 } 1543 1544 static unsigned int task_scan_start(struct task_struct *p) 1545 { 1546 unsigned long smin = task_scan_min(p); 1547 unsigned long period = smin; 1548 struct numa_group *ng; 1549 1550 /* Scale the maximum scan period with the amount of shared memory. */ 1551 rcu_read_lock(); 1552 ng = rcu_dereference_all(p->numa_group); 1553 if (ng) { 1554 unsigned long shared = group_faults_shared(ng); 1555 unsigned long private = group_faults_priv(ng); 1556 1557 period *= refcount_read(&ng->refcount); 1558 period *= shared + 1; 1559 period /= private + shared + 1; 1560 } 1561 rcu_read_unlock(); 1562 1563 return max(smin, period); 1564 } 1565 1566 static unsigned int task_scan_max(struct task_struct *p) 1567 { 1568 unsigned long smin = task_scan_min(p); 1569 unsigned long smax; 1570 struct numa_group *ng; 1571 1572 /* Watch for min being lower than max due to floor calculations */ 1573 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); 1574 1575 /* Scale the maximum scan period with the amount of shared memory. */ 1576 ng = deref_curr_numa_group(p); 1577 if (ng) { 1578 unsigned long shared = group_faults_shared(ng); 1579 unsigned long private = group_faults_priv(ng); 1580 unsigned long period = smax; 1581 1582 period *= refcount_read(&ng->refcount); 1583 period *= shared + 1; 1584 period /= private + shared + 1; 1585 1586 smax = max(smax, period); 1587 } 1588 1589 return max(smin, smax); 1590 } 1591 1592 static void account_numa_enqueue(struct rq *rq, struct task_struct *p) 1593 { 1594 rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE); 1595 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 1596 } 1597 1598 static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 1599 { 1600 rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE); 1601 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 1602 } 1603 1604 /* Shared or private faults. */ 1605 #define NR_NUMA_HINT_FAULT_TYPES 2 1606 1607 /* Memory and CPU locality */ 1608 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) 1609 1610 /* Averaged statistics, and temporary buffers. */ 1611 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) 1612 1613 pid_t task_numa_group_id(struct task_struct *p) 1614 { 1615 struct numa_group *ng; 1616 pid_t gid = 0; 1617 1618 rcu_read_lock(); 1619 ng = rcu_dereference_all(p->numa_group); 1620 if (ng) 1621 gid = ng->gid; 1622 rcu_read_unlock(); 1623 1624 return gid; 1625 } 1626 1627 /* 1628 * The averaged statistics, shared & private, memory & CPU, 1629 * occupy the first half of the array. The second half of the 1630 * array is for current counters, which are averaged into the 1631 * first set by task_numa_placement. 1632 */ 1633 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) 1634 { 1635 return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 1636 } 1637 1638 static inline unsigned long task_faults(struct task_struct *p, int nid) 1639 { 1640 if (!p->numa_faults) 1641 return 0; 1642 1643 return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1644 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1645 } 1646 1647 static inline unsigned long group_faults(struct task_struct *p, int nid) 1648 { 1649 struct numa_group *ng = deref_task_numa_group(p); 1650 1651 if (!ng) 1652 return 0; 1653 1654 return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1655 ng->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1656 } 1657 1658 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 1659 { 1660 return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] + 1661 group->faults[task_faults_idx(NUMA_CPU, nid, 1)]; 1662 } 1663 1664 static inline unsigned long group_faults_priv(struct numa_group *ng) 1665 { 1666 unsigned long faults = 0; 1667 int node; 1668 1669 for_each_online_node(node) { 1670 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 1671 } 1672 1673 return faults; 1674 } 1675 1676 static inline unsigned long group_faults_shared(struct numa_group *ng) 1677 { 1678 unsigned long faults = 0; 1679 int node; 1680 1681 for_each_online_node(node) { 1682 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 1683 } 1684 1685 return faults; 1686 } 1687 1688 /* 1689 * A node triggering more than 1/3 as many NUMA faults as the maximum is 1690 * considered part of a numa group's pseudo-interleaving set. Migrations 1691 * between these nodes are slowed down, to allow things to settle down. 1692 */ 1693 #define ACTIVE_NODE_FRACTION 3 1694 1695 static bool numa_is_active_node(int nid, struct numa_group *ng) 1696 { 1697 return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; 1698 } 1699 1700 /* Handle placement on systems where not all nodes are directly connected. */ 1701 static unsigned long score_nearby_nodes(struct task_struct *p, int nid, 1702 int lim_dist, bool task) 1703 { 1704 unsigned long score = 0; 1705 int node, max_dist; 1706 1707 /* 1708 * All nodes are directly connected, and the same distance 1709 * from each other. No need for fancy placement algorithms. 1710 */ 1711 if (sched_numa_topology_type == NUMA_DIRECT) 1712 return 0; 1713 1714 /* sched_max_numa_distance may be changed in parallel. */ 1715 max_dist = READ_ONCE(sched_max_numa_distance); 1716 /* 1717 * This code is called for each node, introducing N^2 complexity, 1718 * which should be OK given the number of nodes rarely exceeds 8. 1719 */ 1720 for_each_online_node(node) { 1721 unsigned long faults; 1722 int dist = node_distance(nid, node); 1723 1724 /* 1725 * The furthest away nodes in the system are not interesting 1726 * for placement; nid was already counted. 1727 */ 1728 if (dist >= max_dist || node == nid) 1729 continue; 1730 1731 /* 1732 * On systems with a backplane NUMA topology, compare groups 1733 * of nodes, and move tasks towards the group with the most 1734 * memory accesses. When comparing two nodes at distance 1735 * "hoplimit", only nodes closer by than "hoplimit" are part 1736 * of each group. Skip other nodes. 1737 */ 1738 if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist) 1739 continue; 1740 1741 /* Add up the faults from nearby nodes. */ 1742 if (task) 1743 faults = task_faults(p, node); 1744 else 1745 faults = group_faults(p, node); 1746 1747 /* 1748 * On systems with a glueless mesh NUMA topology, there are 1749 * no fixed "groups of nodes". Instead, nodes that are not 1750 * directly connected bounce traffic through intermediate 1751 * nodes; a numa_group can occupy any set of nodes. 1752 * The further away a node is, the less the faults count. 1753 * This seems to result in good task placement. 1754 */ 1755 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 1756 faults *= (max_dist - dist); 1757 faults /= (max_dist - LOCAL_DISTANCE); 1758 } 1759 1760 score += faults; 1761 } 1762 1763 return score; 1764 } 1765 1766 /* 1767 * These return the fraction of accesses done by a particular task, or 1768 * task group, on a particular numa node. The group weight is given a 1769 * larger multiplier, in order to group tasks together that are almost 1770 * evenly spread out between numa nodes. 1771 */ 1772 static inline unsigned long task_weight(struct task_struct *p, int nid, 1773 int dist) 1774 { 1775 unsigned long faults, total_faults; 1776 1777 if (!p->numa_faults) 1778 return 0; 1779 1780 total_faults = p->total_numa_faults; 1781 1782 if (!total_faults) 1783 return 0; 1784 1785 faults = task_faults(p, nid); 1786 faults += score_nearby_nodes(p, nid, dist, true); 1787 1788 return 1000 * faults / total_faults; 1789 } 1790 1791 static inline unsigned long group_weight(struct task_struct *p, int nid, 1792 int dist) 1793 { 1794 struct numa_group *ng = deref_task_numa_group(p); 1795 unsigned long faults, total_faults; 1796 1797 if (!ng) 1798 return 0; 1799 1800 total_faults = ng->total_faults; 1801 1802 if (!total_faults) 1803 return 0; 1804 1805 faults = group_faults(p, nid); 1806 faults += score_nearby_nodes(p, nid, dist, false); 1807 1808 return 1000 * faults / total_faults; 1809 } 1810 1811 /* 1812 * If memory tiering mode is enabled, cpupid of slow memory page is 1813 * used to record scan time instead of CPU and PID. When tiering mode 1814 * is disabled at run time, the scan time (in cpupid) will be 1815 * interpreted as CPU and PID. So CPU needs to be checked to avoid to 1816 * access out of array bound. 1817 */ 1818 static inline bool cpupid_valid(int cpupid) 1819 { 1820 return cpupid_to_cpu(cpupid) < nr_cpu_ids; 1821 } 1822 1823 /* 1824 * For memory tiering mode, if there are enough free pages (more than 1825 * enough watermark defined here) in fast memory node, to take full 1826 * advantage of fast memory capacity, all recently accessed slow 1827 * memory pages will be migrated to fast memory node without 1828 * considering hot threshold. 1829 */ 1830 static bool pgdat_free_space_enough(struct pglist_data *pgdat) 1831 { 1832 int z; 1833 unsigned long enough_wmark; 1834 1835 enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT, 1836 pgdat->node_present_pages >> 4); 1837 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 1838 struct zone *zone = pgdat->node_zones + z; 1839 1840 if (!populated_zone(zone)) 1841 continue; 1842 1843 if (zone_watermark_ok(zone, 0, 1844 promo_wmark_pages(zone) + enough_wmark, 1845 ZONE_MOVABLE, 0)) 1846 return true; 1847 } 1848 return false; 1849 } 1850 1851 /* 1852 * For memory tiering mode, when page tables are scanned, the scan 1853 * time will be recorded in struct page in addition to make page 1854 * PROT_NONE for slow memory page. So when the page is accessed, in 1855 * hint page fault handler, the hint page fault latency is calculated 1856 * via, 1857 * 1858 * hint page fault latency = hint page fault time - scan time 1859 * 1860 * The smaller the hint page fault latency, the higher the possibility 1861 * for the page to be hot. 1862 */ 1863 static int numa_hint_fault_latency(struct folio *folio) 1864 { 1865 int last_time, time; 1866 1867 time = jiffies_to_msecs(jiffies); 1868 last_time = folio_xchg_access_time(folio, time); 1869 1870 return (time - last_time) & PAGE_ACCESS_TIME_MASK; 1871 } 1872 1873 /* 1874 * For memory tiering mode, too high promotion/demotion throughput may 1875 * hurt application latency. So we provide a mechanism to rate limit 1876 * the number of pages that are tried to be promoted. 1877 */ 1878 static bool numa_promotion_rate_limit(struct pglist_data *pgdat, 1879 unsigned long rate_limit, int nr) 1880 { 1881 unsigned long nr_cand; 1882 unsigned int now, start; 1883 1884 now = jiffies_to_msecs(jiffies); 1885 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr); 1886 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1887 start = pgdat->nbp_rl_start; 1888 if (now - start > MSEC_PER_SEC && 1889 cmpxchg(&pgdat->nbp_rl_start, start, now) == start) 1890 pgdat->nbp_rl_nr_cand = nr_cand; 1891 if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit) 1892 return true; 1893 return false; 1894 } 1895 1896 #define NUMA_MIGRATION_ADJUST_STEPS 16 1897 1898 static void numa_promotion_adjust_threshold(struct pglist_data *pgdat, 1899 unsigned long rate_limit, 1900 unsigned int ref_th) 1901 { 1902 unsigned int now, start, th_period, unit_th, th; 1903 unsigned long nr_cand, ref_cand, diff_cand; 1904 1905 now = jiffies_to_msecs(jiffies); 1906 th_period = sysctl_numa_balancing_scan_period_max; 1907 start = pgdat->nbp_th_start; 1908 if (now - start > th_period && 1909 cmpxchg(&pgdat->nbp_th_start, start, now) == start) { 1910 ref_cand = rate_limit * 1911 sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC; 1912 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1913 diff_cand = nr_cand - pgdat->nbp_th_nr_cand; 1914 unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS; 1915 th = pgdat->nbp_threshold ? : ref_th; 1916 if (diff_cand > ref_cand * 11 / 10) 1917 th = max(th - unit_th, unit_th); 1918 else if (diff_cand < ref_cand * 9 / 10) 1919 th = min(th + unit_th, ref_th * 2); 1920 pgdat->nbp_th_nr_cand = nr_cand; 1921 pgdat->nbp_threshold = th; 1922 } 1923 } 1924 1925 bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio, 1926 int src_nid, int dst_cpu) 1927 { 1928 struct numa_group *ng = deref_curr_numa_group(p); 1929 int dst_nid = cpu_to_node(dst_cpu); 1930 int last_cpupid, this_cpupid; 1931 1932 /* 1933 * Cannot migrate to memoryless nodes. 1934 */ 1935 if (!node_state(dst_nid, N_MEMORY)) 1936 return false; 1937 1938 /* 1939 * The pages in slow memory node should be migrated according 1940 * to hot/cold instead of private/shared. 1941 */ 1942 if (folio_use_access_time(folio)) { 1943 struct pglist_data *pgdat; 1944 unsigned long rate_limit; 1945 unsigned int latency, th, def_th; 1946 long nr = folio_nr_pages(folio); 1947 1948 pgdat = NODE_DATA(dst_nid); 1949 if (pgdat_free_space_enough(pgdat)) { 1950 /* workload changed, reset hot threshold */ 1951 pgdat->nbp_threshold = 0; 1952 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE_NRL, nr); 1953 return true; 1954 } 1955 1956 def_th = sysctl_numa_balancing_hot_threshold; 1957 rate_limit = MB_TO_PAGES(sysctl_numa_balancing_promote_rate_limit); 1958 numa_promotion_adjust_threshold(pgdat, rate_limit, def_th); 1959 1960 th = pgdat->nbp_threshold ? : def_th; 1961 latency = numa_hint_fault_latency(folio); 1962 if (latency >= th) 1963 return false; 1964 1965 return !numa_promotion_rate_limit(pgdat, rate_limit, nr); 1966 } 1967 1968 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); 1969 last_cpupid = folio_xchg_last_cpupid(folio, this_cpupid); 1970 1971 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 1972 !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid)) 1973 return false; 1974 1975 /* 1976 * Allow first faults or private faults to migrate immediately early in 1977 * the lifetime of a task. The magic number 4 is based on waiting for 1978 * two full passes of the "multi-stage node selection" test that is 1979 * executed below. 1980 */ 1981 if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) && 1982 (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid))) 1983 return true; 1984 1985 /* 1986 * Multi-stage node selection is used in conjunction with a periodic 1987 * migration fault to build a temporal task<->page relation. By using 1988 * a two-stage filter we remove short/unlikely relations. 1989 * 1990 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate 1991 * a task's usage of a particular page (n_p) per total usage of this 1992 * page (n_t) (in a given time-span) to a probability. 1993 * 1994 * Our periodic faults will sample this probability and getting the 1995 * same result twice in a row, given these samples are fully 1996 * independent, is then given by P(n)^2, provided our sample period 1997 * is sufficiently short compared to the usage pattern. 1998 * 1999 * This quadric squishes small probabilities, making it less likely we 2000 * act on an unlikely task<->page relation. 2001 */ 2002 if (!cpupid_pid_unset(last_cpupid) && 2003 cpupid_to_nid(last_cpupid) != dst_nid) 2004 return false; 2005 2006 /* Always allow migrate on private faults */ 2007 if (cpupid_match_pid(p, last_cpupid)) 2008 return true; 2009 2010 /* A shared fault, but p->numa_group has not been set up yet. */ 2011 if (!ng) 2012 return true; 2013 2014 /* 2015 * Destination node is much more heavily used than the source 2016 * node? Allow migration. 2017 */ 2018 if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * 2019 ACTIVE_NODE_FRACTION) 2020 return true; 2021 2022 /* 2023 * Distribute memory according to CPU & memory use on each node, 2024 * with 3/4 hysteresis to avoid unnecessary memory migrations: 2025 * 2026 * faults_cpu(dst) 3 faults_cpu(src) 2027 * --------------- * - > --------------- 2028 * faults_mem(dst) 4 faults_mem(src) 2029 */ 2030 return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > 2031 group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; 2032 } 2033 2034 /* 2035 * 'numa_type' describes the node at the moment of load balancing. 2036 */ 2037 enum numa_type { 2038 /* The node has spare capacity that can be used to run more tasks. */ 2039 node_has_spare = 0, 2040 /* 2041 * The node is fully used and the tasks don't compete for more CPU 2042 * cycles. Nevertheless, some tasks might wait before running. 2043 */ 2044 node_fully_busy, 2045 /* 2046 * The node is overloaded and can't provide expected CPU cycles to all 2047 * tasks. 2048 */ 2049 node_overloaded 2050 }; 2051 2052 /* Cached statistics for all CPUs within a node */ 2053 struct numa_stats { 2054 unsigned long load; 2055 unsigned long runnable; 2056 unsigned long util; 2057 /* Total compute capacity of CPUs on a node */ 2058 unsigned long compute_capacity; 2059 unsigned int nr_running; 2060 unsigned int weight; 2061 enum numa_type node_type; 2062 int idle_cpu; 2063 }; 2064 2065 struct task_numa_env { 2066 struct task_struct *p; 2067 2068 int src_cpu, src_nid; 2069 int dst_cpu, dst_nid; 2070 int imb_numa_nr; 2071 2072 struct numa_stats src_stats, dst_stats; 2073 2074 int imbalance_pct; 2075 int dist; 2076 2077 struct task_struct *best_task; 2078 long best_imp; 2079 int best_cpu; 2080 }; 2081 2082 static unsigned long cpu_load(struct rq *rq); 2083 static unsigned long cpu_runnable(struct rq *rq); 2084 2085 static inline enum 2086 numa_type numa_classify(unsigned int imbalance_pct, 2087 struct numa_stats *ns) 2088 { 2089 if ((ns->nr_running > ns->weight) && 2090 (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) || 2091 ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100)))) 2092 return node_overloaded; 2093 2094 if ((ns->nr_running < ns->weight) || 2095 (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) && 2096 ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100)))) 2097 return node_has_spare; 2098 2099 return node_fully_busy; 2100 } 2101 2102 #ifdef CONFIG_SCHED_SMT 2103 /* Forward declarations of select_idle_sibling helpers */ 2104 static inline bool test_idle_cores(int cpu); 2105 static inline int numa_idle_core(int idle_core, int cpu) 2106 { 2107 if (!static_branch_likely(&sched_smt_present) || 2108 idle_core >= 0 || !test_idle_cores(cpu)) 2109 return idle_core; 2110 2111 /* 2112 * Prefer cores instead of packing HT siblings 2113 * and triggering future load balancing. 2114 */ 2115 if (is_core_idle(cpu)) 2116 idle_core = cpu; 2117 2118 return idle_core; 2119 } 2120 #else /* !CONFIG_SCHED_SMT: */ 2121 static inline int numa_idle_core(int idle_core, int cpu) 2122 { 2123 return idle_core; 2124 } 2125 #endif /* !CONFIG_SCHED_SMT */ 2126 2127 /* 2128 * Gather all necessary information to make NUMA balancing placement 2129 * decisions that are compatible with standard load balancer. This 2130 * borrows code and logic from update_sg_lb_stats but sharing a 2131 * common implementation is impractical. 2132 */ 2133 static void update_numa_stats(struct task_numa_env *env, 2134 struct numa_stats *ns, int nid, 2135 bool find_idle) 2136 { 2137 int cpu, idle_core = -1; 2138 2139 memset(ns, 0, sizeof(*ns)); 2140 ns->idle_cpu = -1; 2141 2142 rcu_read_lock(); 2143 for_each_cpu(cpu, cpumask_of_node(nid)) { 2144 struct rq *rq = cpu_rq(cpu); 2145 2146 ns->load += cpu_load(rq); 2147 ns->runnable += cpu_runnable(rq); 2148 ns->util += cpu_util_cfs(cpu); 2149 ns->nr_running += rq->cfs.h_nr_runnable; 2150 ns->compute_capacity += capacity_of(cpu); 2151 2152 if (find_idle && idle_core < 0 && !rq->nr_running && idle_cpu(cpu)) { 2153 if (READ_ONCE(rq->numa_migrate_on) || 2154 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2155 continue; 2156 2157 if (ns->idle_cpu == -1) 2158 ns->idle_cpu = cpu; 2159 2160 idle_core = numa_idle_core(idle_core, cpu); 2161 } 2162 } 2163 rcu_read_unlock(); 2164 2165 ns->weight = cpumask_weight(cpumask_of_node(nid)); 2166 2167 ns->node_type = numa_classify(env->imbalance_pct, ns); 2168 2169 if (idle_core >= 0) 2170 ns->idle_cpu = idle_core; 2171 } 2172 2173 static void task_numa_assign(struct task_numa_env *env, 2174 struct task_struct *p, long imp) 2175 { 2176 struct rq *rq = cpu_rq(env->dst_cpu); 2177 2178 /* Check if run-queue part of active NUMA balance. */ 2179 if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) { 2180 int cpu; 2181 int start = env->dst_cpu; 2182 2183 /* Find alternative idle CPU. */ 2184 for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) { 2185 if (cpu == env->best_cpu || !idle_cpu(cpu) || 2186 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) { 2187 continue; 2188 } 2189 2190 env->dst_cpu = cpu; 2191 rq = cpu_rq(env->dst_cpu); 2192 if (!xchg(&rq->numa_migrate_on, 1)) 2193 goto assign; 2194 } 2195 2196 /* Failed to find an alternative idle CPU */ 2197 return; 2198 } 2199 2200 assign: 2201 /* 2202 * Clear previous best_cpu/rq numa-migrate flag, since task now 2203 * found a better CPU to move/swap. 2204 */ 2205 if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) { 2206 rq = cpu_rq(env->best_cpu); 2207 WRITE_ONCE(rq->numa_migrate_on, 0); 2208 } 2209 2210 if (env->best_task) 2211 put_task_struct(env->best_task); 2212 if (p) 2213 get_task_struct(p); 2214 2215 env->best_task = p; 2216 env->best_imp = imp; 2217 env->best_cpu = env->dst_cpu; 2218 } 2219 2220 static bool load_too_imbalanced(long src_load, long dst_load, 2221 struct task_numa_env *env) 2222 { 2223 long imb, old_imb; 2224 long orig_src_load, orig_dst_load; 2225 long src_capacity, dst_capacity; 2226 2227 /* 2228 * The load is corrected for the CPU capacity available on each node. 2229 * 2230 * src_load dst_load 2231 * ------------ vs --------- 2232 * src_capacity dst_capacity 2233 */ 2234 src_capacity = env->src_stats.compute_capacity; 2235 dst_capacity = env->dst_stats.compute_capacity; 2236 2237 imb = abs(dst_load * src_capacity - src_load * dst_capacity); 2238 2239 orig_src_load = env->src_stats.load; 2240 orig_dst_load = env->dst_stats.load; 2241 2242 old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity); 2243 2244 /* Would this change make things worse? */ 2245 return (imb > old_imb); 2246 } 2247 2248 /* 2249 * Maximum NUMA importance can be 1998 (2*999); 2250 * SMALLIMP @ 30 would be close to 1998/64. 2251 * Used to deter task migration. 2252 */ 2253 #define SMALLIMP 30 2254 2255 /* 2256 * This checks if the overall compute and NUMA accesses of the system would 2257 * be improved if the source tasks was migrated to the target dst_cpu taking 2258 * into account that it might be best if task running on the dst_cpu should 2259 * be exchanged with the source task 2260 */ 2261 static bool task_numa_compare(struct task_numa_env *env, 2262 long taskimp, long groupimp, bool maymove) 2263 { 2264 struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p); 2265 struct rq *dst_rq = cpu_rq(env->dst_cpu); 2266 long imp = p_ng ? groupimp : taskimp; 2267 struct task_struct *cur; 2268 long src_load, dst_load; 2269 int dist = env->dist; 2270 long moveimp = imp; 2271 long load; 2272 bool stopsearch = false; 2273 2274 if (READ_ONCE(dst_rq->numa_migrate_on)) 2275 return false; 2276 2277 rcu_read_lock(); 2278 cur = rcu_dereference_all(dst_rq->curr); 2279 if (cur && ((cur->flags & (PF_EXITING | PF_KTHREAD)) || 2280 !cur->mm)) 2281 cur = NULL; 2282 2283 /* 2284 * Because we have preemption enabled we can get migrated around and 2285 * end try selecting ourselves (current == env->p) as a swap candidate. 2286 */ 2287 if (cur == env->p) { 2288 stopsearch = true; 2289 goto unlock; 2290 } 2291 2292 if (!cur) { 2293 if (maymove && moveimp >= env->best_imp) 2294 goto assign; 2295 else 2296 goto unlock; 2297 } 2298 2299 /* Skip this swap candidate if cannot move to the source cpu. */ 2300 if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr)) 2301 goto unlock; 2302 2303 /* 2304 * Skip this swap candidate if it is not moving to its preferred 2305 * node and the best task is. 2306 */ 2307 if (env->best_task && 2308 env->best_task->numa_preferred_nid == env->src_nid && 2309 cur->numa_preferred_nid != env->src_nid) { 2310 goto unlock; 2311 } 2312 2313 /* 2314 * "imp" is the fault differential for the source task between the 2315 * source and destination node. Calculate the total differential for 2316 * the source task and potential destination task. The more negative 2317 * the value is, the more remote accesses that would be expected to 2318 * be incurred if the tasks were swapped. 2319 * 2320 * If dst and source tasks are in the same NUMA group, or not 2321 * in any group then look only at task weights. 2322 */ 2323 cur_ng = rcu_dereference_all(cur->numa_group); 2324 if (cur_ng == p_ng) { 2325 /* 2326 * Do not swap within a group or between tasks that have 2327 * no group if there is spare capacity. Swapping does 2328 * not address the load imbalance and helps one task at 2329 * the cost of punishing another. 2330 */ 2331 if (env->dst_stats.node_type == node_has_spare) 2332 goto unlock; 2333 2334 imp = taskimp + task_weight(cur, env->src_nid, dist) - 2335 task_weight(cur, env->dst_nid, dist); 2336 /* 2337 * Add some hysteresis to prevent swapping the 2338 * tasks within a group over tiny differences. 2339 */ 2340 if (cur_ng) 2341 imp -= imp / 16; 2342 } else { 2343 /* 2344 * Compare the group weights. If a task is all by itself 2345 * (not part of a group), use the task weight instead. 2346 */ 2347 if (cur_ng && p_ng) 2348 imp += group_weight(cur, env->src_nid, dist) - 2349 group_weight(cur, env->dst_nid, dist); 2350 else 2351 imp += task_weight(cur, env->src_nid, dist) - 2352 task_weight(cur, env->dst_nid, dist); 2353 } 2354 2355 /* Discourage picking a task already on its preferred node */ 2356 if (cur->numa_preferred_nid == env->dst_nid) 2357 imp -= imp / 16; 2358 2359 /* 2360 * Encourage picking a task that moves to its preferred node. 2361 * This potentially makes imp larger than it's maximum of 2362 * 1998 (see SMALLIMP and task_weight for why) but in this 2363 * case, it does not matter. 2364 */ 2365 if (cur->numa_preferred_nid == env->src_nid) 2366 imp += imp / 8; 2367 2368 if (maymove && moveimp > imp && moveimp > env->best_imp) { 2369 imp = moveimp; 2370 cur = NULL; 2371 goto assign; 2372 } 2373 2374 /* 2375 * Prefer swapping with a task moving to its preferred node over a 2376 * task that is not. 2377 */ 2378 if (env->best_task && cur->numa_preferred_nid == env->src_nid && 2379 env->best_task->numa_preferred_nid != env->src_nid) { 2380 goto assign; 2381 } 2382 2383 /* 2384 * If the NUMA importance is less than SMALLIMP, 2385 * task migration might only result in ping pong 2386 * of tasks and also hurt performance due to cache 2387 * misses. 2388 */ 2389 if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2) 2390 goto unlock; 2391 2392 /* 2393 * In the overloaded case, try and keep the load balanced. 2394 */ 2395 load = task_h_load(env->p) - task_h_load(cur); 2396 if (!load) 2397 goto assign; 2398 2399 dst_load = env->dst_stats.load + load; 2400 src_load = env->src_stats.load - load; 2401 2402 if (load_too_imbalanced(src_load, dst_load, env)) 2403 goto unlock; 2404 2405 assign: 2406 /* Evaluate an idle CPU for a task numa move. */ 2407 if (!cur) { 2408 int cpu = env->dst_stats.idle_cpu; 2409 2410 /* Nothing cached so current CPU went idle since the search. */ 2411 if (cpu < 0) 2412 cpu = env->dst_cpu; 2413 2414 /* 2415 * If the CPU is no longer truly idle and the previous best CPU 2416 * is, keep using it. 2417 */ 2418 if (!idle_cpu(cpu) && env->best_cpu >= 0 && 2419 idle_cpu(env->best_cpu)) { 2420 cpu = env->best_cpu; 2421 } 2422 2423 env->dst_cpu = cpu; 2424 } 2425 2426 task_numa_assign(env, cur, imp); 2427 2428 /* 2429 * If a move to idle is allowed because there is capacity or load 2430 * balance improves then stop the search. While a better swap 2431 * candidate may exist, a search is not free. 2432 */ 2433 if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu)) 2434 stopsearch = true; 2435 2436 /* 2437 * If a swap candidate must be identified and the current best task 2438 * moves its preferred node then stop the search. 2439 */ 2440 if (!maymove && env->best_task && 2441 env->best_task->numa_preferred_nid == env->src_nid) { 2442 stopsearch = true; 2443 } 2444 unlock: 2445 rcu_read_unlock(); 2446 2447 return stopsearch; 2448 } 2449 2450 static void task_numa_find_cpu(struct task_numa_env *env, 2451 long taskimp, long groupimp) 2452 { 2453 bool maymove = false; 2454 int cpu; 2455 2456 /* 2457 * If dst node has spare capacity, then check if there is an 2458 * imbalance that would be overruled by the load balancer. 2459 */ 2460 if (env->dst_stats.node_type == node_has_spare) { 2461 unsigned int imbalance; 2462 int src_running, dst_running; 2463 2464 /* 2465 * Would movement cause an imbalance? Note that if src has 2466 * more running tasks that the imbalance is ignored as the 2467 * move improves the imbalance from the perspective of the 2468 * CPU load balancer. 2469 * */ 2470 src_running = env->src_stats.nr_running - 1; 2471 dst_running = env->dst_stats.nr_running + 1; 2472 imbalance = max(0, dst_running - src_running); 2473 imbalance = adjust_numa_imbalance(imbalance, dst_running, 2474 env->imb_numa_nr); 2475 2476 /* Use idle CPU if there is no imbalance */ 2477 if (!imbalance) { 2478 maymove = true; 2479 if (env->dst_stats.idle_cpu >= 0) { 2480 env->dst_cpu = env->dst_stats.idle_cpu; 2481 task_numa_assign(env, NULL, 0); 2482 return; 2483 } 2484 } 2485 } else { 2486 long src_load, dst_load, load; 2487 /* 2488 * If the improvement from just moving env->p direction is better 2489 * than swapping tasks around, check if a move is possible. 2490 */ 2491 load = task_h_load(env->p); 2492 dst_load = env->dst_stats.load + load; 2493 src_load = env->src_stats.load - load; 2494 maymove = !load_too_imbalanced(src_load, dst_load, env); 2495 } 2496 2497 /* Skip CPUs if the source task cannot migrate */ 2498 for_each_cpu_and(cpu, cpumask_of_node(env->dst_nid), env->p->cpus_ptr) { 2499 env->dst_cpu = cpu; 2500 if (task_numa_compare(env, taskimp, groupimp, maymove)) 2501 break; 2502 } 2503 } 2504 2505 static int task_numa_migrate(struct task_struct *p) 2506 { 2507 struct task_numa_env env = { 2508 .p = p, 2509 2510 .src_cpu = task_cpu(p), 2511 .src_nid = task_node(p), 2512 2513 .imbalance_pct = 112, 2514 2515 .best_task = NULL, 2516 .best_imp = 0, 2517 .best_cpu = -1, 2518 }; 2519 unsigned long taskweight, groupweight; 2520 struct sched_domain *sd; 2521 long taskimp, groupimp; 2522 struct numa_group *ng; 2523 struct rq *best_rq; 2524 int nid, ret, dist; 2525 2526 /* 2527 * Pick the lowest SD_NUMA domain, as that would have the smallest 2528 * imbalance and would be the first to start moving tasks about. 2529 * 2530 * And we want to avoid any moving of tasks about, as that would create 2531 * random movement of tasks -- counter the numa conditions we're trying 2532 * to satisfy here. 2533 */ 2534 rcu_read_lock(); 2535 sd = rcu_dereference_all(per_cpu(sd_numa, env.src_cpu)); 2536 if (sd) { 2537 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; 2538 env.imb_numa_nr = sd->imb_numa_nr; 2539 } 2540 rcu_read_unlock(); 2541 2542 /* 2543 * Cpusets can break the scheduler domain tree into smaller 2544 * balance domains, some of which do not cross NUMA boundaries. 2545 * Tasks that are "trapped" in such domains cannot be migrated 2546 * elsewhere, so there is no point in (re)trying. 2547 */ 2548 if (unlikely(!sd)) { 2549 sched_setnuma(p, task_node(p)); 2550 return -EINVAL; 2551 } 2552 2553 env.dst_nid = p->numa_preferred_nid; 2554 dist = env.dist = node_distance(env.src_nid, env.dst_nid); 2555 taskweight = task_weight(p, env.src_nid, dist); 2556 groupweight = group_weight(p, env.src_nid, dist); 2557 update_numa_stats(&env, &env.src_stats, env.src_nid, false); 2558 taskimp = task_weight(p, env.dst_nid, dist) - taskweight; 2559 groupimp = group_weight(p, env.dst_nid, dist) - groupweight; 2560 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2561 2562 /* Try to find a spot on the preferred nid. */ 2563 task_numa_find_cpu(&env, taskimp, groupimp); 2564 2565 /* 2566 * Look at other nodes in these cases: 2567 * - there is no space available on the preferred_nid 2568 * - the task is part of a numa_group that is interleaved across 2569 * multiple NUMA nodes; in order to better consolidate the group, 2570 * we need to check other locations. 2571 */ 2572 ng = deref_curr_numa_group(p); 2573 if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) { 2574 for_each_node_state(nid, N_CPU) { 2575 if (nid == env.src_nid || nid == p->numa_preferred_nid) 2576 continue; 2577 2578 dist = node_distance(env.src_nid, env.dst_nid); 2579 if (sched_numa_topology_type == NUMA_BACKPLANE && 2580 dist != env.dist) { 2581 taskweight = task_weight(p, env.src_nid, dist); 2582 groupweight = group_weight(p, env.src_nid, dist); 2583 } 2584 2585 /* Only consider nodes where both task and groups benefit */ 2586 taskimp = task_weight(p, nid, dist) - taskweight; 2587 groupimp = group_weight(p, nid, dist) - groupweight; 2588 if (taskimp < 0 && groupimp < 0) 2589 continue; 2590 2591 env.dist = dist; 2592 env.dst_nid = nid; 2593 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2594 task_numa_find_cpu(&env, taskimp, groupimp); 2595 } 2596 } 2597 2598 /* 2599 * If the task is part of a workload that spans multiple NUMA nodes, 2600 * and is migrating into one of the workload's active nodes, remember 2601 * this node as the task's preferred numa node, so the workload can 2602 * settle down. 2603 * A task that migrated to a second choice node will be better off 2604 * trying for a better one later. Do not set the preferred node here. 2605 */ 2606 if (ng) { 2607 if (env.best_cpu == -1) 2608 nid = env.src_nid; 2609 else 2610 nid = cpu_to_node(env.best_cpu); 2611 2612 if (nid != p->numa_preferred_nid) 2613 sched_setnuma(p, nid); 2614 } 2615 2616 /* No better CPU than the current one was found. */ 2617 if (env.best_cpu == -1) { 2618 trace_sched_stick_numa(p, env.src_cpu, NULL, -1); 2619 return -EAGAIN; 2620 } 2621 2622 best_rq = cpu_rq(env.best_cpu); 2623 if (env.best_task == NULL) { 2624 ret = migrate_task_to(p, env.best_cpu); 2625 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2626 if (ret != 0) 2627 trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu); 2628 return ret; 2629 } 2630 2631 ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu); 2632 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2633 2634 if (ret != 0) 2635 trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu); 2636 put_task_struct(env.best_task); 2637 return ret; 2638 } 2639 2640 /* Attempt to migrate a task to a CPU on the preferred node. */ 2641 static void numa_migrate_preferred(struct task_struct *p) 2642 { 2643 unsigned long interval = HZ; 2644 2645 /* This task has no NUMA fault statistics yet */ 2646 if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults)) 2647 return; 2648 2649 /* Periodically retry migrating the task to the preferred node */ 2650 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); 2651 p->numa_migrate_retry = jiffies + interval; 2652 2653 /* Success if task is already running on preferred CPU */ 2654 if (task_node(p) == p->numa_preferred_nid) 2655 return; 2656 2657 /* Otherwise, try migrate to a CPU on the preferred node */ 2658 task_numa_migrate(p); 2659 } 2660 2661 /* 2662 * Find out how many nodes the workload is actively running on. Do this by 2663 * tracking the nodes from which NUMA hinting faults are triggered. This can 2664 * be different from the set of nodes where the workload's memory is currently 2665 * located. 2666 */ 2667 static void numa_group_count_active_nodes(struct numa_group *numa_group) 2668 { 2669 unsigned long faults, max_faults = 0; 2670 int nid, active_nodes = 0; 2671 2672 for_each_node_state(nid, N_CPU) { 2673 faults = group_faults_cpu(numa_group, nid); 2674 if (faults > max_faults) 2675 max_faults = faults; 2676 } 2677 2678 for_each_node_state(nid, N_CPU) { 2679 faults = group_faults_cpu(numa_group, nid); 2680 if (faults * ACTIVE_NODE_FRACTION > max_faults) 2681 active_nodes++; 2682 } 2683 2684 numa_group->max_faults_cpu = max_faults; 2685 numa_group->active_nodes = active_nodes; 2686 } 2687 2688 /* 2689 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS 2690 * increments. The more local the fault statistics are, the higher the scan 2691 * period will be for the next scan window. If local/(local+remote) ratio is 2692 * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) 2693 * the scan period will decrease. Aim for 70% local accesses. 2694 */ 2695 #define NUMA_PERIOD_SLOTS 10 2696 #define NUMA_PERIOD_THRESHOLD 7 2697 2698 /* 2699 * Increase the scan period (slow down scanning) if the majority of 2700 * our memory is already on our local node, or if the majority of 2701 * the page accesses are shared with other processes. 2702 * Otherwise, decrease the scan period. 2703 */ 2704 static void update_task_scan_period(struct task_struct *p, 2705 unsigned long shared, unsigned long private) 2706 { 2707 unsigned int period_slot; 2708 int lr_ratio, ps_ratio; 2709 int diff; 2710 2711 unsigned long remote = p->numa_faults_locality[0]; 2712 unsigned long local = p->numa_faults_locality[1]; 2713 2714 /* 2715 * If there were no record hinting faults then either the task is 2716 * completely idle or all activity is in areas that are not of interest 2717 * to automatic numa balancing. Related to that, if there were failed 2718 * migration then it implies we are migrating too quickly or the local 2719 * node is overloaded. In either case, scan slower 2720 */ 2721 if (local + shared == 0 || p->numa_faults_locality[2]) { 2722 p->numa_scan_period = min(p->numa_scan_period_max, 2723 p->numa_scan_period << 1); 2724 2725 p->mm->numa_next_scan = jiffies + 2726 msecs_to_jiffies(p->numa_scan_period); 2727 2728 return; 2729 } 2730 2731 /* 2732 * Prepare to scale scan period relative to the current period. 2733 * == NUMA_PERIOD_THRESHOLD scan period stays the same 2734 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) 2735 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) 2736 */ 2737 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); 2738 lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); 2739 ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); 2740 2741 if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { 2742 /* 2743 * Most memory accesses are local. There is no need to 2744 * do fast NUMA scanning, since memory is already local. 2745 */ 2746 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; 2747 if (!slot) 2748 slot = 1; 2749 diff = slot * period_slot; 2750 } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { 2751 /* 2752 * Most memory accesses are shared with other tasks. 2753 * There is no point in continuing fast NUMA scanning, 2754 * since other tasks may just move the memory elsewhere. 2755 */ 2756 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; 2757 if (!slot) 2758 slot = 1; 2759 diff = slot * period_slot; 2760 } else { 2761 /* 2762 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, 2763 * yet they are not on the local NUMA node. Speed up 2764 * NUMA scanning to get the memory moved over. 2765 */ 2766 int ratio = max(lr_ratio, ps_ratio); 2767 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; 2768 } 2769 2770 p->numa_scan_period = clamp(p->numa_scan_period + diff, 2771 task_scan_min(p), task_scan_max(p)); 2772 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2773 } 2774 2775 /* 2776 * Get the fraction of time the task has been running since the last 2777 * NUMA placement cycle. The scheduler keeps similar statistics, but 2778 * decays those on a 32ms period, which is orders of magnitude off 2779 * from the dozens-of-seconds NUMA balancing period. Use the scheduler 2780 * stats only if the task is so new there are no NUMA statistics yet. 2781 */ 2782 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) 2783 { 2784 u64 runtime, delta, now; 2785 /* Use the start of this time slice to avoid calculations. */ 2786 now = p->se.exec_start; 2787 runtime = p->se.sum_exec_runtime; 2788 2789 if (p->last_task_numa_placement) { 2790 delta = runtime - p->last_sum_exec_runtime; 2791 *period = now - p->last_task_numa_placement; 2792 2793 /* Avoid time going backwards, prevent potential divide error: */ 2794 if (unlikely((s64)*period < 0)) 2795 *period = 0; 2796 } else { 2797 delta = p->se.avg.load_sum; 2798 *period = LOAD_AVG_MAX; 2799 } 2800 2801 p->last_sum_exec_runtime = runtime; 2802 p->last_task_numa_placement = now; 2803 2804 return delta; 2805 } 2806 2807 /* 2808 * Determine the preferred nid for a task in a numa_group. This needs to 2809 * be done in a way that produces consistent results with group_weight, 2810 * otherwise workloads might not converge. 2811 */ 2812 static int preferred_group_nid(struct task_struct *p, int nid) 2813 { 2814 nodemask_t nodes; 2815 int dist; 2816 2817 /* Direct connections between all NUMA nodes. */ 2818 if (sched_numa_topology_type == NUMA_DIRECT) 2819 return nid; 2820 2821 /* 2822 * On a system with glueless mesh NUMA topology, group_weight 2823 * scores nodes according to the number of NUMA hinting faults on 2824 * both the node itself, and on nearby nodes. 2825 */ 2826 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 2827 unsigned long score, max_score = 0; 2828 int node, max_node = nid; 2829 2830 dist = sched_max_numa_distance; 2831 2832 for_each_node_state(node, N_CPU) { 2833 score = group_weight(p, node, dist); 2834 if (score > max_score) { 2835 max_score = score; 2836 max_node = node; 2837 } 2838 } 2839 return max_node; 2840 } 2841 2842 /* 2843 * Finding the preferred nid in a system with NUMA backplane 2844 * interconnect topology is more involved. The goal is to locate 2845 * tasks from numa_groups near each other in the system, and 2846 * untangle workloads from different sides of the system. This requires 2847 * searching down the hierarchy of node groups, recursively searching 2848 * inside the highest scoring group of nodes. The nodemask tricks 2849 * keep the complexity of the search down. 2850 */ 2851 nodes = node_states[N_CPU]; 2852 for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { 2853 unsigned long max_faults = 0; 2854 nodemask_t max_group = NODE_MASK_NONE; 2855 int a, b; 2856 2857 /* Are there nodes at this distance from each other? */ 2858 if (!find_numa_distance(dist)) 2859 continue; 2860 2861 for_each_node_mask(a, nodes) { 2862 unsigned long faults = 0; 2863 nodemask_t this_group; 2864 nodes_clear(this_group); 2865 2866 /* Sum group's NUMA faults; includes a==b case. */ 2867 for_each_node_mask(b, nodes) { 2868 if (node_distance(a, b) < dist) { 2869 faults += group_faults(p, b); 2870 node_set(b, this_group); 2871 node_clear(b, nodes); 2872 } 2873 } 2874 2875 /* Remember the top group. */ 2876 if (faults > max_faults) { 2877 max_faults = faults; 2878 max_group = this_group; 2879 /* 2880 * subtle: at the smallest distance there is 2881 * just one node left in each "group", the 2882 * winner is the preferred nid. 2883 */ 2884 nid = a; 2885 } 2886 } 2887 /* Next round, evaluate the nodes within max_group. */ 2888 if (!max_faults) 2889 break; 2890 nodes = max_group; 2891 } 2892 return nid; 2893 } 2894 2895 static void task_numa_placement(struct task_struct *p) 2896 __context_unsafe(/* conditional locking */) 2897 { 2898 int seq, nid, max_nid = NUMA_NO_NODE; 2899 unsigned long max_faults = 0; 2900 unsigned long fault_types[2] = { 0, 0 }; 2901 unsigned long total_faults; 2902 u64 runtime, period; 2903 spinlock_t *group_lock = NULL; 2904 struct numa_group *ng; 2905 2906 /* 2907 * The p->mm->numa_scan_seq field gets updated without 2908 * exclusive access. Use READ_ONCE() here to ensure 2909 * that the field is read in a single access: 2910 */ 2911 seq = READ_ONCE(p->mm->numa_scan_seq); 2912 if (p->numa_scan_seq == seq) 2913 return; 2914 p->numa_scan_seq = seq; 2915 p->numa_scan_period_max = task_scan_max(p); 2916 2917 total_faults = p->numa_faults_locality[0] + 2918 p->numa_faults_locality[1]; 2919 runtime = numa_get_avg_runtime(p, &period); 2920 2921 /* If the task is part of a group prevent parallel updates to group stats */ 2922 ng = deref_curr_numa_group(p); 2923 if (ng) { 2924 group_lock = &ng->lock; 2925 spin_lock_irq(group_lock); 2926 } 2927 2928 /* Find the node with the highest number of faults */ 2929 for_each_online_node(nid) { 2930 /* Keep track of the offsets in numa_faults array */ 2931 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; 2932 unsigned long faults = 0, group_faults = 0; 2933 int priv; 2934 2935 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { 2936 long diff, f_diff, f_weight; 2937 2938 mem_idx = task_faults_idx(NUMA_MEM, nid, priv); 2939 membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); 2940 cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); 2941 cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); 2942 2943 /* Decay existing window, copy faults since last scan */ 2944 diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; 2945 fault_types[priv] += p->numa_faults[membuf_idx]; 2946 p->numa_faults[membuf_idx] = 0; 2947 2948 /* 2949 * Normalize the faults_from, so all tasks in a group 2950 * count according to CPU use, instead of by the raw 2951 * number of faults. Tasks with little runtime have 2952 * little over-all impact on throughput, and thus their 2953 * faults are less important. 2954 */ 2955 f_weight = div64_u64(runtime << 16, period + 1); 2956 f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / 2957 (total_faults + 1); 2958 f_diff = f_weight - p->numa_faults[cpu_idx] / 2; 2959 p->numa_faults[cpubuf_idx] = 0; 2960 2961 p->numa_faults[mem_idx] += diff; 2962 p->numa_faults[cpu_idx] += f_diff; 2963 faults += p->numa_faults[mem_idx]; 2964 p->total_numa_faults += diff; 2965 if (ng) { 2966 /* 2967 * safe because we can only change our own group 2968 * 2969 * mem_idx represents the offset for a given 2970 * nid and priv in a specific region because it 2971 * is at the beginning of the numa_faults array. 2972 */ 2973 ng->faults[mem_idx] += diff; 2974 ng->faults[cpu_idx] += f_diff; 2975 ng->total_faults += diff; 2976 group_faults += ng->faults[mem_idx]; 2977 } 2978 } 2979 2980 if (!ng) { 2981 if (faults > max_faults) { 2982 max_faults = faults; 2983 max_nid = nid; 2984 } 2985 } else if (group_faults > max_faults) { 2986 max_faults = group_faults; 2987 max_nid = nid; 2988 } 2989 } 2990 2991 /* Cannot migrate task to CPU-less node */ 2992 max_nid = numa_nearest_node(max_nid, N_CPU); 2993 2994 if (ng) { 2995 numa_group_count_active_nodes(ng); 2996 spin_unlock_irq(group_lock); 2997 max_nid = preferred_group_nid(p, max_nid); 2998 } 2999 3000 if (max_faults) { 3001 /* Set the new preferred node */ 3002 if (max_nid != p->numa_preferred_nid) 3003 sched_setnuma(p, max_nid); 3004 } 3005 3006 update_task_scan_period(p, fault_types[0], fault_types[1]); 3007 } 3008 3009 static inline int get_numa_group(struct numa_group *grp) 3010 { 3011 return refcount_inc_not_zero(&grp->refcount); 3012 } 3013 3014 static inline void put_numa_group(struct numa_group *grp) 3015 { 3016 if (refcount_dec_and_test(&grp->refcount)) 3017 kfree_rcu(grp, rcu); 3018 } 3019 3020 static void task_numa_group(struct task_struct *p, int cpupid, int flags, 3021 int *priv) 3022 { 3023 struct numa_group *grp, *my_grp; 3024 struct task_struct *tsk; 3025 bool join = false; 3026 int cpu = cpupid_to_cpu(cpupid); 3027 int i; 3028 3029 if (unlikely(!deref_curr_numa_group(p))) { 3030 unsigned int size = sizeof(struct numa_group) + 3031 NR_NUMA_HINT_FAULT_STATS * 3032 nr_node_ids * sizeof(unsigned long); 3033 3034 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 3035 if (!grp) 3036 return; 3037 3038 refcount_set(&grp->refcount, 1); 3039 grp->active_nodes = 1; 3040 grp->max_faults_cpu = 0; 3041 spin_lock_init(&grp->lock); 3042 grp->gid = p->pid; 3043 3044 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3045 grp->faults[i] = p->numa_faults[i]; 3046 3047 grp->total_faults = p->total_numa_faults; 3048 3049 grp->nr_tasks++; 3050 rcu_assign_pointer(p->numa_group, grp); 3051 } 3052 3053 rcu_read_lock(); 3054 tsk = READ_ONCE(cpu_rq(cpu)->curr); 3055 3056 if (!cpupid_match_pid(tsk, cpupid)) 3057 goto no_join; 3058 3059 grp = rcu_dereference_all(tsk->numa_group); 3060 if (!grp) 3061 goto no_join; 3062 3063 my_grp = deref_curr_numa_group(p); 3064 if (grp == my_grp) 3065 goto no_join; 3066 3067 /* 3068 * Only join the other group if its bigger; if we're the bigger group, 3069 * the other task will join us. 3070 */ 3071 if (my_grp->nr_tasks > grp->nr_tasks) 3072 goto no_join; 3073 3074 /* 3075 * Tie-break on the grp address. 3076 */ 3077 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) 3078 goto no_join; 3079 3080 /* Always join threads in the same process. */ 3081 if (tsk->mm == current->mm) 3082 join = true; 3083 3084 /* Simple filter to avoid false positives due to PID collisions */ 3085 if (flags & TNF_SHARED) 3086 join = true; 3087 3088 /* Update priv based on whether false sharing was detected */ 3089 *priv = !join; 3090 3091 if (join && !get_numa_group(grp)) 3092 goto no_join; 3093 3094 rcu_read_unlock(); 3095 3096 if (!join) 3097 return; 3098 3099 WARN_ON_ONCE(irqs_disabled()); 3100 double_lock_irq(&my_grp->lock, &grp->lock); 3101 3102 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { 3103 my_grp->faults[i] -= p->numa_faults[i]; 3104 grp->faults[i] += p->numa_faults[i]; 3105 } 3106 my_grp->total_faults -= p->total_numa_faults; 3107 grp->total_faults += p->total_numa_faults; 3108 3109 my_grp->nr_tasks--; 3110 grp->nr_tasks++; 3111 3112 spin_unlock(&my_grp->lock); 3113 spin_unlock_irq(&grp->lock); 3114 3115 rcu_assign_pointer(p->numa_group, grp); 3116 3117 put_numa_group(my_grp); 3118 return; 3119 3120 no_join: 3121 rcu_read_unlock(); 3122 return; 3123 } 3124 3125 /* 3126 * Get rid of NUMA statistics associated with a task (either current or dead). 3127 * If @final is set, the task is dead and has reached refcount zero, so we can 3128 * safely free all relevant data structures. Otherwise, there might be 3129 * concurrent reads from places like load balancing and procfs, and we should 3130 * reset the data back to default state without freeing ->numa_faults. 3131 */ 3132 void task_numa_free(struct task_struct *p, bool final) 3133 { 3134 /* safe: p either is current or is being freed by current */ 3135 struct numa_group *grp = rcu_dereference_raw(p->numa_group); 3136 unsigned long *numa_faults = p->numa_faults; 3137 unsigned long flags; 3138 int i; 3139 3140 if (!numa_faults) 3141 return; 3142 3143 if (grp) { 3144 spin_lock_irqsave(&grp->lock, flags); 3145 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3146 grp->faults[i] -= p->numa_faults[i]; 3147 grp->total_faults -= p->total_numa_faults; 3148 3149 grp->nr_tasks--; 3150 spin_unlock_irqrestore(&grp->lock, flags); 3151 RCU_INIT_POINTER(p->numa_group, NULL); 3152 put_numa_group(grp); 3153 } 3154 3155 if (final) { 3156 p->numa_faults = NULL; 3157 kfree(numa_faults); 3158 } else { 3159 p->total_numa_faults = 0; 3160 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3161 numa_faults[i] = 0; 3162 } 3163 } 3164 3165 /* 3166 * Got a PROT_NONE fault for a page on @node. 3167 */ 3168 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) 3169 { 3170 struct task_struct *p = current; 3171 bool migrated = flags & TNF_MIGRATED; 3172 int cpu_node = task_node(current); 3173 int local = !!(flags & TNF_FAULT_LOCAL); 3174 struct numa_group *ng; 3175 int priv; 3176 3177 if (!static_branch_likely(&sched_numa_balancing)) 3178 return; 3179 3180 /* for example, ksmd faulting in a user's mm */ 3181 if (!p->mm) 3182 return; 3183 3184 /* 3185 * NUMA faults statistics are unnecessary for the slow memory 3186 * node for memory tiering mode. 3187 */ 3188 if (!node_is_toptier(mem_node) && 3189 (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING || 3190 !cpupid_valid(last_cpupid))) 3191 return; 3192 3193 /* Allocate buffer to track faults on a per-node basis */ 3194 if (unlikely(!p->numa_faults)) { 3195 int size = sizeof(*p->numa_faults) * 3196 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; 3197 3198 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); 3199 if (!p->numa_faults) 3200 return; 3201 3202 p->total_numa_faults = 0; 3203 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 3204 } 3205 3206 /* 3207 * First accesses are treated as private, otherwise consider accesses 3208 * to be private if the accessing pid has not changed 3209 */ 3210 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { 3211 priv = 1; 3212 } else { 3213 priv = cpupid_match_pid(p, last_cpupid); 3214 if (!priv && !(flags & TNF_NO_GROUP)) 3215 task_numa_group(p, last_cpupid, flags, &priv); 3216 } 3217 3218 /* 3219 * If a workload spans multiple NUMA nodes, a shared fault that 3220 * occurs wholly within the set of nodes that the workload is 3221 * actively using should be counted as local. This allows the 3222 * scan rate to slow down when a workload has settled down. 3223 */ 3224 ng = deref_curr_numa_group(p); 3225 if (!priv && !local && ng && ng->active_nodes > 1 && 3226 numa_is_active_node(cpu_node, ng) && 3227 numa_is_active_node(mem_node, ng)) 3228 local = 1; 3229 3230 /* 3231 * Retry to migrate task to preferred node periodically, in case it 3232 * previously failed, or the scheduler moved us. 3233 */ 3234 if (time_after(jiffies, p->numa_migrate_retry)) { 3235 task_numa_placement(p); 3236 numa_migrate_preferred(p); 3237 } 3238 3239 if (migrated) 3240 p->numa_pages_migrated += pages; 3241 if (flags & TNF_MIGRATE_FAIL) 3242 p->numa_faults_locality[2] += pages; 3243 3244 p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; 3245 p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; 3246 p->numa_faults_locality[local] += pages; 3247 } 3248 3249 static void reset_ptenuma_scan(struct task_struct *p) 3250 { 3251 /* 3252 * We only did a read acquisition of the mmap sem, so 3253 * p->mm->numa_scan_seq is written to without exclusive access 3254 * and the update is not guaranteed to be atomic. That's not 3255 * much of an issue though, since this is just used for 3256 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not 3257 * expensive, to avoid any form of compiler optimizations: 3258 */ 3259 WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 3260 p->mm->numa_scan_offset = 0; 3261 } 3262 3263 static bool vma_is_accessed(struct mm_struct *mm, struct vm_area_struct *vma) 3264 { 3265 unsigned long pids; 3266 /* 3267 * Allow unconditional access first two times, so that all the (pages) 3268 * of VMAs get prot_none fault introduced irrespective of accesses. 3269 * This is also done to avoid any side effect of task scanning 3270 * amplifying the unfairness of disjoint set of VMAs' access. 3271 */ 3272 if ((READ_ONCE(current->mm->numa_scan_seq) - vma->numab_state->start_scan_seq) < 2) 3273 return true; 3274 3275 pids = vma->numab_state->pids_active[0] | vma->numab_state->pids_active[1]; 3276 if (test_bit(hash_32(current->pid, ilog2(BITS_PER_LONG)), &pids)) 3277 return true; 3278 3279 /* 3280 * Complete a scan that has already started regardless of PID access, or 3281 * some VMAs may never be scanned in multi-threaded applications: 3282 */ 3283 if (mm->numa_scan_offset > vma->vm_start) { 3284 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_IGNORE_PID); 3285 return true; 3286 } 3287 3288 /* 3289 * This vma has not been accessed for a while, and if the number 3290 * the threads in the same process is low, which means no other 3291 * threads can help scan this vma, force a vma scan. 3292 */ 3293 if (READ_ONCE(mm->numa_scan_seq) > 3294 (vma->numab_state->prev_scan_seq + get_nr_threads(current))) 3295 return true; 3296 3297 return false; 3298 } 3299 3300 #define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay) 3301 3302 /* 3303 * The expensive part of numa migration is done from task_work context. 3304 * Triggered from task_tick_numa(). 3305 */ 3306 static void task_numa_work(struct callback_head *work) 3307 { 3308 unsigned long migrate, next_scan, now = jiffies; 3309 struct task_struct *p = current; 3310 struct mm_struct *mm = p->mm; 3311 u64 runtime = p->se.sum_exec_runtime; 3312 struct vm_area_struct *vma; 3313 unsigned long start, end; 3314 unsigned long nr_pte_updates = 0; 3315 long pages, virtpages; 3316 struct vma_iterator vmi; 3317 bool vma_pids_skipped; 3318 bool vma_pids_forced = false; 3319 3320 WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work)); 3321 3322 work->next = work; 3323 /* 3324 * Who cares about NUMA placement when they're dying. 3325 * 3326 * NOTE: make sure not to dereference p->mm before this check, 3327 * exit_task_work() happens _after_ exit_mm() so we could be called 3328 * without p->mm even though we still had it when we enqueued this 3329 * work. 3330 */ 3331 if (p->flags & PF_EXITING) 3332 return; 3333 3334 /* 3335 * Memory is pinned to only one NUMA node via cpuset.mems, naturally 3336 * no page can be migrated. 3337 */ 3338 if (cpusets_enabled() && nodes_weight(cpuset_current_mems_allowed) == 1) { 3339 trace_sched_skip_cpuset_numa(current, &cpuset_current_mems_allowed); 3340 return; 3341 } 3342 3343 if (!mm->numa_next_scan) { 3344 mm->numa_next_scan = now + 3345 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3346 } 3347 3348 /* 3349 * Enforce maximal scan/migration frequency.. 3350 */ 3351 migrate = mm->numa_next_scan; 3352 if (time_before(now, migrate)) 3353 return; 3354 3355 if (p->numa_scan_period == 0) { 3356 p->numa_scan_period_max = task_scan_max(p); 3357 p->numa_scan_period = task_scan_start(p); 3358 } 3359 3360 next_scan = now + msecs_to_jiffies(p->numa_scan_period); 3361 if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan)) 3362 return; 3363 3364 /* 3365 * Delay this task enough that another task of this mm will likely win 3366 * the next time around. 3367 */ 3368 p->node_stamp += 2 * TICK_NSEC; 3369 3370 pages = sysctl_numa_balancing_scan_size; 3371 pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 3372 virtpages = pages * 8; /* Scan up to this much virtual space */ 3373 if (!pages) 3374 return; 3375 3376 3377 if (!mmap_read_trylock(mm)) 3378 return; 3379 3380 /* 3381 * VMAs are skipped if the current PID has not trapped a fault within 3382 * the VMA recently. Allow scanning to be forced if there is no 3383 * suitable VMA remaining. 3384 */ 3385 vma_pids_skipped = false; 3386 3387 retry_pids: 3388 start = mm->numa_scan_offset; 3389 vma_iter_init(&vmi, mm, start); 3390 vma = vma_next(&vmi); 3391 if (!vma) { 3392 reset_ptenuma_scan(p); 3393 start = 0; 3394 vma_iter_set(&vmi, start); 3395 vma = vma_next(&vmi); 3396 } 3397 3398 for (; vma; vma = vma_next(&vmi)) { 3399 if (!vma_migratable(vma) || !vma_policy_mof(vma) || 3400 is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { 3401 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_UNSUITABLE); 3402 continue; 3403 } 3404 3405 /* 3406 * Shared library pages mapped by multiple processes are not 3407 * migrated as it is expected they are cache replicated. Avoid 3408 * hinting faults in read-only file-backed mappings or the vDSO 3409 * as migrating the pages will be of marginal benefit. 3410 */ 3411 if (!vma->vm_mm || 3412 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) { 3413 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SHARED_RO); 3414 continue; 3415 } 3416 3417 /* 3418 * Skip inaccessible VMAs to avoid any confusion between 3419 * PROT_NONE and NUMA hinting PTEs 3420 */ 3421 if (!vma_is_accessible(vma)) { 3422 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_INACCESSIBLE); 3423 continue; 3424 } 3425 3426 /* Initialise new per-VMA NUMAB state. */ 3427 if (!vma->numab_state) { 3428 struct vma_numab_state *ptr; 3429 3430 ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); 3431 if (!ptr) 3432 continue; 3433 3434 if (cmpxchg(&vma->numab_state, NULL, ptr)) { 3435 kfree(ptr); 3436 continue; 3437 } 3438 3439 vma->numab_state->start_scan_seq = mm->numa_scan_seq; 3440 3441 vma->numab_state->next_scan = now + 3442 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3443 3444 /* Reset happens after 4 times scan delay of scan start */ 3445 vma->numab_state->pids_active_reset = vma->numab_state->next_scan + 3446 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3447 3448 /* 3449 * Ensure prev_scan_seq does not match numa_scan_seq, 3450 * to prevent VMAs being skipped prematurely on the 3451 * first scan: 3452 */ 3453 vma->numab_state->prev_scan_seq = mm->numa_scan_seq - 1; 3454 } 3455 3456 /* 3457 * Scanning the VMAs of short lived tasks add more overhead. So 3458 * delay the scan for new VMAs. 3459 */ 3460 if (mm->numa_scan_seq && time_before(jiffies, 3461 vma->numab_state->next_scan)) { 3462 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SCAN_DELAY); 3463 continue; 3464 } 3465 3466 /* RESET access PIDs regularly for old VMAs. */ 3467 if (mm->numa_scan_seq && 3468 time_after(jiffies, vma->numab_state->pids_active_reset)) { 3469 vma->numab_state->pids_active_reset = vma->numab_state->pids_active_reset + 3470 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3471 vma->numab_state->pids_active[0] = READ_ONCE(vma->numab_state->pids_active[1]); 3472 vma->numab_state->pids_active[1] = 0; 3473 } 3474 3475 /* Do not rescan VMAs twice within the same sequence. */ 3476 if (vma->numab_state->prev_scan_seq == mm->numa_scan_seq) { 3477 mm->numa_scan_offset = vma->vm_end; 3478 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SEQ_COMPLETED); 3479 continue; 3480 } 3481 3482 /* 3483 * Do not scan the VMA if task has not accessed it, unless no other 3484 * VMA candidate exists. 3485 */ 3486 if (!vma_pids_forced && !vma_is_accessed(mm, vma)) { 3487 vma_pids_skipped = true; 3488 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_PID_INACTIVE); 3489 continue; 3490 } 3491 3492 do { 3493 start = max(start, vma->vm_start); 3494 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 3495 end = min(end, vma->vm_end); 3496 nr_pte_updates = change_prot_numa(vma, start, end); 3497 3498 /* 3499 * Try to scan sysctl_numa_balancing_size worth of 3500 * hpages that have at least one present PTE that 3501 * is not already PTE-numa. If the VMA contains 3502 * areas that are unused or already full of prot_numa 3503 * PTEs, scan up to virtpages, to skip through those 3504 * areas faster. 3505 */ 3506 if (nr_pte_updates) 3507 pages -= (end - start) >> PAGE_SHIFT; 3508 virtpages -= (end - start) >> PAGE_SHIFT; 3509 3510 start = end; 3511 if (pages <= 0 || virtpages <= 0) 3512 goto out; 3513 3514 cond_resched(); 3515 } while (end != vma->vm_end); 3516 3517 /* VMA scan is complete, do not scan until next sequence. */ 3518 vma->numab_state->prev_scan_seq = mm->numa_scan_seq; 3519 3520 /* 3521 * Only force scan within one VMA at a time, to limit the 3522 * cost of scanning a potentially uninteresting VMA. 3523 */ 3524 if (vma_pids_forced) 3525 break; 3526 } 3527 3528 /* 3529 * If no VMAs are remaining and VMAs were skipped due to the PID 3530 * not accessing the VMA previously, then force a scan to ensure 3531 * forward progress: 3532 */ 3533 if (!vma && !vma_pids_forced && vma_pids_skipped) { 3534 vma_pids_forced = true; 3535 goto retry_pids; 3536 } 3537 3538 out: 3539 /* 3540 * It is possible to reach the end of the VMA list but the last few 3541 * VMAs are not guaranteed to the vma_migratable. If they are not, we 3542 * would find the !migratable VMA on the next scan but not reset the 3543 * scanner to the start so check it now. 3544 */ 3545 if (vma) 3546 mm->numa_scan_offset = start; 3547 else 3548 reset_ptenuma_scan(p); 3549 mmap_read_unlock(mm); 3550 3551 /* 3552 * Make sure tasks use at least 32x as much time to run other code 3553 * than they used here, to limit NUMA PTE scanning overhead to 3% max. 3554 * Usually update_task_scan_period slows down scanning enough; on an 3555 * overloaded system we need to limit overhead on a per task basis. 3556 */ 3557 if (unlikely(p->se.sum_exec_runtime != runtime)) { 3558 u64 diff = p->se.sum_exec_runtime - runtime; 3559 p->node_stamp += 32 * diff; 3560 } 3561 } 3562 3563 void init_numa_balancing(u64 clone_flags, struct task_struct *p) 3564 { 3565 int mm_users = 0; 3566 struct mm_struct *mm = p->mm; 3567 3568 if (mm) { 3569 mm_users = atomic_read(&mm->mm_users); 3570 if (mm_users == 1) { 3571 mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3572 mm->numa_scan_seq = 0; 3573 } 3574 } 3575 p->node_stamp = 0; 3576 p->numa_scan_seq = mm ? mm->numa_scan_seq : 0; 3577 p->numa_scan_period = sysctl_numa_balancing_scan_delay; 3578 p->numa_migrate_retry = 0; 3579 /* Protect against double add, see task_tick_numa and task_numa_work */ 3580 p->numa_work.next = &p->numa_work; 3581 p->numa_faults = NULL; 3582 p->numa_pages_migrated = 0; 3583 p->total_numa_faults = 0; 3584 RCU_INIT_POINTER(p->numa_group, NULL); 3585 p->last_task_numa_placement = 0; 3586 p->last_sum_exec_runtime = 0; 3587 3588 init_task_work(&p->numa_work, task_numa_work); 3589 3590 /* New address space, reset the preferred nid */ 3591 if (!(clone_flags & CLONE_VM)) { 3592 p->numa_preferred_nid = NUMA_NO_NODE; 3593 return; 3594 } 3595 3596 /* 3597 * New thread, keep existing numa_preferred_nid which should be copied 3598 * already by arch_dup_task_struct but stagger when scans start. 3599 */ 3600 if (mm) { 3601 unsigned int delay; 3602 3603 delay = min_t(unsigned int, task_scan_max(current), 3604 current->numa_scan_period * mm_users * NSEC_PER_MSEC); 3605 delay += 2 * TICK_NSEC; 3606 p->node_stamp = delay; 3607 } 3608 } 3609 3610 /* 3611 * Drive the periodic memory faults.. 3612 */ 3613 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3614 { 3615 struct callback_head *work = &curr->numa_work; 3616 u64 period, now; 3617 3618 /* 3619 * We don't care about NUMA placement if we don't have memory. 3620 */ 3621 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work) 3622 return; 3623 3624 /* 3625 * Using runtime rather than walltime has the dual advantage that 3626 * we (mostly) drive the selection from busy threads and that the 3627 * task needs to have done some actual work before we bother with 3628 * NUMA placement. 3629 */ 3630 now = curr->se.sum_exec_runtime; 3631 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; 3632 3633 if (now > curr->node_stamp + period) { 3634 if (!curr->node_stamp) 3635 curr->numa_scan_period = task_scan_start(curr); 3636 curr->node_stamp += period; 3637 3638 if (!time_before(jiffies, curr->mm->numa_next_scan)) 3639 task_work_add(curr, work, TWA_RESUME); 3640 } 3641 } 3642 3643 static void update_scan_period(struct task_struct *p, int new_cpu) 3644 { 3645 int src_nid = cpu_to_node(task_cpu(p)); 3646 int dst_nid = cpu_to_node(new_cpu); 3647 3648 if (!static_branch_likely(&sched_numa_balancing)) 3649 return; 3650 3651 if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING)) 3652 return; 3653 3654 if (src_nid == dst_nid) 3655 return; 3656 3657 /* 3658 * Allow resets if faults have been trapped before one scan 3659 * has completed. This is most likely due to a new task that 3660 * is pulled cross-node due to wakeups or load balancing. 3661 */ 3662 if (p->numa_scan_seq) { 3663 /* 3664 * Avoid scan adjustments if moving to the preferred 3665 * node or if the task was not previously running on 3666 * the preferred node. 3667 */ 3668 if (dst_nid == p->numa_preferred_nid || 3669 (p->numa_preferred_nid != NUMA_NO_NODE && 3670 src_nid != p->numa_preferred_nid)) 3671 return; 3672 } 3673 3674 p->numa_scan_period = task_scan_start(p); 3675 } 3676 3677 #else /* !CONFIG_NUMA_BALANCING: */ 3678 3679 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3680 { 3681 } 3682 3683 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) 3684 { 3685 } 3686 3687 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) 3688 { 3689 } 3690 3691 static inline void update_scan_period(struct task_struct *p, int new_cpu) 3692 { 3693 } 3694 3695 #endif /* !CONFIG_NUMA_BALANCING */ 3696 3697 static void 3698 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3699 { 3700 update_load_add(&cfs_rq->load, se->load.weight); 3701 if (entity_is_task(se)) { 3702 struct rq *rq = rq_of(cfs_rq); 3703 3704 account_numa_enqueue(rq, task_of(se)); 3705 list_add(&se->group_node, &rq->cfs_tasks); 3706 } 3707 cfs_rq->nr_queued++; 3708 } 3709 3710 static void 3711 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3712 { 3713 update_load_sub(&cfs_rq->load, se->load.weight); 3714 if (entity_is_task(se)) { 3715 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); 3716 list_del_init(&se->group_node); 3717 } 3718 cfs_rq->nr_queued--; 3719 } 3720 3721 /* 3722 * Signed add and clamp on underflow. 3723 * 3724 * Explicitly do a load-store to ensure the intermediate value never hits 3725 * memory. This allows lockless observations without ever seeing the negative 3726 * values. 3727 */ 3728 #define add_positive(_ptr, _val) do { \ 3729 typeof(_ptr) ptr = (_ptr); \ 3730 __signed_scalar_typeof(*ptr) val = (_val); \ 3731 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3732 \ 3733 res = var + val; \ 3734 \ 3735 if (val < 0 && res > var) \ 3736 res = 0; \ 3737 \ 3738 WRITE_ONCE(*ptr, res); \ 3739 } while (0) 3740 3741 /* 3742 * Remove and clamp on negative, from a local variable. 3743 * 3744 * A variant of sub_positive(), which does not use explicit load-store 3745 * and is thus optimized for local variable updates. 3746 */ 3747 #define lsub_positive(_ptr, _val) do { \ 3748 typeof(_ptr) ptr = (_ptr); \ 3749 *ptr -= min_t(typeof(*ptr), *ptr, _val); \ 3750 } while (0) 3751 3752 3753 /* 3754 * Because of rounding, se->util_sum might ends up being +1 more than 3755 * cfs->util_sum. Although this is not a problem by itself, detaching 3756 * a lot of tasks with the rounding problem between 2 updates of 3757 * util_avg (~1ms) can make cfs->util_sum becoming null whereas 3758 * cfs_util_avg is not. 3759 * 3760 * Check that util_sum is still above its lower bound for the new 3761 * util_avg. Given that period_contrib might have moved since the last 3762 * sync, we are only sure that util_sum must be above or equal to 3763 * util_avg * minimum possible divider 3764 */ 3765 #define __update_sa(sa, name, delta_avg, delta_sum) do { \ 3766 add_positive(&(sa)->name##_avg, delta_avg); \ 3767 add_positive(&(sa)->name##_sum, delta_sum); \ 3768 (sa)->name##_sum = max_t(typeof((sa)->name##_sum), \ 3769 (sa)->name##_sum, \ 3770 (sa)->name##_avg * PELT_MIN_DIVIDER); \ 3771 } while (0) 3772 3773 static inline void 3774 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3775 { 3776 __update_sa(&cfs_rq->avg, load, se->avg.load_avg, 3777 se_weight(se) * se->avg.load_sum); 3778 } 3779 3780 static inline void 3781 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3782 { 3783 __update_sa(&cfs_rq->avg, load, -se->avg.load_avg, 3784 se_weight(se) * -se->avg.load_sum); 3785 } 3786 3787 static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags); 3788 3789 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 3790 unsigned long weight) 3791 { 3792 bool curr = cfs_rq->curr == se; 3793 3794 if (se->on_rq) { 3795 /* commit outstanding execution time */ 3796 update_curr(cfs_rq); 3797 update_entity_lag(cfs_rq, se); 3798 se->deadline -= se->vruntime; 3799 se->rel_deadline = 1; 3800 cfs_rq->nr_queued--; 3801 if (!curr) 3802 __dequeue_entity(cfs_rq, se); 3803 update_load_sub(&cfs_rq->load, se->load.weight); 3804 } 3805 dequeue_load_avg(cfs_rq, se); 3806 3807 /* 3808 * Because we keep se->vlag = V - v_i, while: lag_i = w_i*(V - v_i), 3809 * we need to scale se->vlag when w_i changes. 3810 */ 3811 se->vlag = div_s64(se->vlag * se->load.weight, weight); 3812 if (se->rel_deadline) 3813 se->deadline = div_s64(se->deadline * se->load.weight, weight); 3814 3815 update_load_set(&se->load, weight); 3816 3817 do { 3818 u32 divider = get_pelt_divider(&se->avg); 3819 3820 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 3821 } while (0); 3822 3823 enqueue_load_avg(cfs_rq, se); 3824 if (se->on_rq) { 3825 place_entity(cfs_rq, se, 0); 3826 update_load_add(&cfs_rq->load, se->load.weight); 3827 if (!curr) 3828 __enqueue_entity(cfs_rq, se); 3829 cfs_rq->nr_queued++; 3830 } 3831 } 3832 3833 static void reweight_task_fair(struct rq *rq, struct task_struct *p, 3834 const struct load_weight *lw) 3835 { 3836 struct sched_entity *se = &p->se; 3837 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3838 struct load_weight *load = &se->load; 3839 3840 reweight_entity(cfs_rq, se, lw->weight); 3841 load->inv_weight = lw->inv_weight; 3842 } 3843 3844 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); 3845 3846 #ifdef CONFIG_FAIR_GROUP_SCHED 3847 /* 3848 * All this does is approximate the hierarchical proportion which includes that 3849 * global sum we all love to hate. 3850 * 3851 * That is, the weight of a group entity, is the proportional share of the 3852 * group weight based on the group runqueue weights. That is: 3853 * 3854 * tg->weight * grq->load.weight 3855 * ge->load.weight = ----------------------------- (1) 3856 * \Sum grq->load.weight 3857 * 3858 * Now, because computing that sum is prohibitively expensive to compute (been 3859 * there, done that) we approximate it with this average stuff. The average 3860 * moves slower and therefore the approximation is cheaper and more stable. 3861 * 3862 * So instead of the above, we substitute: 3863 * 3864 * grq->load.weight -> grq->avg.load_avg (2) 3865 * 3866 * which yields the following: 3867 * 3868 * tg->weight * grq->avg.load_avg 3869 * ge->load.weight = ------------------------------ (3) 3870 * tg->load_avg 3871 * 3872 * Where: tg->load_avg ~= \Sum grq->avg.load_avg 3873 * 3874 * That is shares_avg, and it is right (given the approximation (2)). 3875 * 3876 * The problem with it is that because the average is slow -- it was designed 3877 * to be exactly that of course -- this leads to transients in boundary 3878 * conditions. In specific, the case where the group was idle and we start the 3879 * one task. It takes time for our CPU's grq->avg.load_avg to build up, 3880 * yielding bad latency etc.. 3881 * 3882 * Now, in that special case (1) reduces to: 3883 * 3884 * tg->weight * grq->load.weight 3885 * ge->load.weight = ----------------------------- = tg->weight (4) 3886 * grp->load.weight 3887 * 3888 * That is, the sum collapses because all other CPUs are idle; the UP scenario. 3889 * 3890 * So what we do is modify our approximation (3) to approach (4) in the (near) 3891 * UP case, like: 3892 * 3893 * ge->load.weight = 3894 * 3895 * tg->weight * grq->load.weight 3896 * --------------------------------------------------- (5) 3897 * tg->load_avg - grq->avg.load_avg + grq->load.weight 3898 * 3899 * But because grq->load.weight can drop to 0, resulting in a divide by zero, 3900 * we need to use grq->avg.load_avg as its lower bound, which then gives: 3901 * 3902 * 3903 * tg->weight * grq->load.weight 3904 * ge->load.weight = ----------------------------- (6) 3905 * tg_load_avg' 3906 * 3907 * Where: 3908 * 3909 * tg_load_avg' = tg->load_avg - grq->avg.load_avg + 3910 * max(grq->load.weight, grq->avg.load_avg) 3911 * 3912 * And that is shares_weight and is icky. In the (near) UP case it approaches 3913 * (4) while in the normal case it approaches (3). It consistently 3914 * overestimates the ge->load.weight and therefore: 3915 * 3916 * \Sum ge->load.weight >= tg->weight 3917 * 3918 * hence icky! 3919 */ 3920 static long calc_group_shares(struct cfs_rq *cfs_rq) 3921 { 3922 long tg_weight, tg_shares, load, shares; 3923 struct task_group *tg = cfs_rq->tg; 3924 3925 tg_shares = READ_ONCE(tg->shares); 3926 3927 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 3928 3929 tg_weight = atomic_long_read(&tg->load_avg); 3930 3931 /* Ensure tg_weight >= load */ 3932 tg_weight -= cfs_rq->tg_load_avg_contrib; 3933 tg_weight += load; 3934 3935 shares = (tg_shares * load); 3936 if (tg_weight) 3937 shares /= tg_weight; 3938 3939 /* 3940 * MIN_SHARES has to be unscaled here to support per-CPU partitioning 3941 * of a group with small tg->shares value. It is a floor value which is 3942 * assigned as a minimum load.weight to the sched_entity representing 3943 * the group on a CPU. 3944 * 3945 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 3946 * on an 8-core system with 8 tasks each runnable on one CPU shares has 3947 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In 3948 * case no task is runnable on a CPU MIN_SHARES=2 should be returned 3949 * instead of 0. 3950 */ 3951 return clamp_t(long, shares, MIN_SHARES, tg_shares); 3952 } 3953 3954 /* 3955 * Recomputes the group entity based on the current state of its group 3956 * runqueue. 3957 */ 3958 static void update_cfs_group(struct sched_entity *se) 3959 { 3960 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 3961 long shares; 3962 3963 /* 3964 * When a group becomes empty, preserve its weight. This matters for 3965 * DELAY_DEQUEUE. 3966 */ 3967 if (!gcfs_rq || !gcfs_rq->load.weight) 3968 return; 3969 3970 shares = calc_group_shares(gcfs_rq); 3971 if (unlikely(se->load.weight != shares)) 3972 reweight_entity(cfs_rq_of(se), se, shares); 3973 } 3974 3975 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 3976 static inline void update_cfs_group(struct sched_entity *se) 3977 { 3978 } 3979 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 3980 3981 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) 3982 { 3983 struct rq *rq = rq_of(cfs_rq); 3984 3985 if (&rq->cfs == cfs_rq) { 3986 /* 3987 * There are a few boundary cases this might miss but it should 3988 * get called often enough that that should (hopefully) not be 3989 * a real problem. 3990 * 3991 * It will not get called when we go idle, because the idle 3992 * thread is a different class (!fair), nor will the utilization 3993 * number include things like RT tasks. 3994 * 3995 * As is, the util number is not freq-invariant (we'd have to 3996 * implement arch_scale_freq_capacity() for that). 3997 * 3998 * See cpu_util_cfs(). 3999 */ 4000 cpufreq_update_util(rq, flags); 4001 } 4002 } 4003 4004 static inline bool load_avg_is_decayed(struct sched_avg *sa) 4005 { 4006 if (sa->load_sum) 4007 return false; 4008 4009 if (sa->util_sum) 4010 return false; 4011 4012 if (sa->runnable_sum) 4013 return false; 4014 4015 /* 4016 * _avg must be null when _sum are null because _avg = _sum / divider 4017 * Make sure that rounding and/or propagation of PELT values never 4018 * break this. 4019 */ 4020 WARN_ON_ONCE(sa->load_avg || 4021 sa->util_avg || 4022 sa->runnable_avg); 4023 4024 return true; 4025 } 4026 4027 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 4028 { 4029 return u64_u32_load_copy(cfs_rq->avg.last_update_time, 4030 cfs_rq->last_update_time_copy); 4031 } 4032 #ifdef CONFIG_FAIR_GROUP_SCHED 4033 /* 4034 * Because list_add_leaf_cfs_rq always places a child cfs_rq on the list 4035 * immediately before a parent cfs_rq, and cfs_rqs are removed from the list 4036 * bottom-up, we only have to test whether the cfs_rq before us on the list 4037 * is our child. 4038 * If cfs_rq is not on the list, test whether a child needs its to be added to 4039 * connect a branch to the tree * (see list_add_leaf_cfs_rq() for details). 4040 */ 4041 static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq) 4042 { 4043 struct cfs_rq *prev_cfs_rq; 4044 struct list_head *prev; 4045 struct rq *rq = rq_of(cfs_rq); 4046 4047 if (cfs_rq->on_list) { 4048 prev = cfs_rq->leaf_cfs_rq_list.prev; 4049 } else { 4050 prev = rq->tmp_alone_branch; 4051 } 4052 4053 if (prev == &rq->leaf_cfs_rq_list) 4054 return false; 4055 4056 prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list); 4057 4058 return (prev_cfs_rq->tg->parent == cfs_rq->tg); 4059 } 4060 4061 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 4062 { 4063 if (cfs_rq->load.weight) 4064 return false; 4065 4066 if (!load_avg_is_decayed(&cfs_rq->avg)) 4067 return false; 4068 4069 if (child_cfs_rq_on_list(cfs_rq)) 4070 return false; 4071 4072 if (cfs_rq->tg_load_avg_contrib) 4073 return false; 4074 4075 return true; 4076 } 4077 4078 /** 4079 * update_tg_load_avg - update the tg's load avg 4080 * @cfs_rq: the cfs_rq whose avg changed 4081 * 4082 * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. 4083 * However, because tg->load_avg is a global value there are performance 4084 * considerations. 4085 * 4086 * In order to avoid having to look at the other cfs_rq's, we use a 4087 * differential update where we store the last value we propagated. This in 4088 * turn allows skipping updates if the differential is 'small'. 4089 * 4090 * Updating tg's load_avg is necessary before update_cfs_share(). 4091 */ 4092 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) 4093 { 4094 long delta; 4095 u64 now; 4096 4097 /* 4098 * No need to update load_avg for root_task_group as it is not used. 4099 */ 4100 if (cfs_rq->tg == &root_task_group) 4101 return; 4102 4103 /* rq has been offline and doesn't contribute to the share anymore: */ 4104 if (!cpu_active(cpu_of(rq_of(cfs_rq)))) 4105 return; 4106 4107 /* 4108 * For migration heavy workloads, access to tg->load_avg can be 4109 * unbound. Limit the update rate to at most once per ms. 4110 */ 4111 now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); 4112 if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) 4113 return; 4114 4115 delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 4116 if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 4117 atomic_long_add(delta, &cfs_rq->tg->load_avg); 4118 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 4119 cfs_rq->last_update_tg_load_avg = now; 4120 } 4121 } 4122 4123 static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) 4124 { 4125 long delta; 4126 u64 now; 4127 4128 /* 4129 * No need to update load_avg for root_task_group, as it is not used. 4130 */ 4131 if (cfs_rq->tg == &root_task_group) 4132 return; 4133 4134 now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); 4135 delta = 0 - cfs_rq->tg_load_avg_contrib; 4136 atomic_long_add(delta, &cfs_rq->tg->load_avg); 4137 cfs_rq->tg_load_avg_contrib = 0; 4138 cfs_rq->last_update_tg_load_avg = now; 4139 } 4140 4141 /* CPU offline callback: */ 4142 static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq) 4143 { 4144 struct task_group *tg; 4145 4146 lockdep_assert_rq_held(rq); 4147 4148 /* 4149 * The rq clock has already been updated in 4150 * set_rq_offline(), so we should skip updating 4151 * the rq clock again in unthrottle_cfs_rq(). 4152 */ 4153 rq_clock_start_loop_update(rq); 4154 4155 rcu_read_lock(); 4156 list_for_each_entry_rcu(tg, &task_groups, list) { 4157 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4158 4159 clear_tg_load_avg(cfs_rq); 4160 } 4161 rcu_read_unlock(); 4162 4163 rq_clock_stop_loop_update(rq); 4164 } 4165 4166 /* 4167 * Called within set_task_rq() right before setting a task's CPU. The 4168 * caller only guarantees p->pi_lock is held; no other assumptions, 4169 * including the state of rq->lock, should be made. 4170 */ 4171 void set_task_rq_fair(struct sched_entity *se, 4172 struct cfs_rq *prev, struct cfs_rq *next) 4173 { 4174 u64 p_last_update_time; 4175 u64 n_last_update_time; 4176 4177 if (!sched_feat(ATTACH_AGE_LOAD)) 4178 return; 4179 4180 /* 4181 * We are supposed to update the task to "current" time, then its up to 4182 * date and ready to go to new CPU/cfs_rq. But we have difficulty in 4183 * getting what current time is, so simply throw away the out-of-date 4184 * time. This will result in the wakee task is less decayed, but giving 4185 * the wakee more load sounds not bad. 4186 */ 4187 if (!(se->avg.last_update_time && prev)) 4188 return; 4189 4190 p_last_update_time = cfs_rq_last_update_time(prev); 4191 n_last_update_time = cfs_rq_last_update_time(next); 4192 4193 __update_load_avg_blocked_se(p_last_update_time, se); 4194 se->avg.last_update_time = n_last_update_time; 4195 } 4196 4197 /* 4198 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to 4199 * propagate its contribution. The key to this propagation is the invariant 4200 * that for each group: 4201 * 4202 * ge->avg == grq->avg (1) 4203 * 4204 * _IFF_ we look at the pure running and runnable sums. Because they 4205 * represent the very same entity, just at different points in the hierarchy. 4206 * 4207 * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial 4208 * and simply copies the running/runnable sum over (but still wrong, because 4209 * the group entity and group rq do not have their PELT windows aligned). 4210 * 4211 * However, update_tg_cfs_load() is more complex. So we have: 4212 * 4213 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2) 4214 * 4215 * And since, like util, the runnable part should be directly transferable, 4216 * the following would _appear_ to be the straight forward approach: 4217 * 4218 * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3) 4219 * 4220 * And per (1) we have: 4221 * 4222 * ge->avg.runnable_avg == grq->avg.runnable_avg 4223 * 4224 * Which gives: 4225 * 4226 * ge->load.weight * grq->avg.load_avg 4227 * ge->avg.load_avg = ----------------------------------- (4) 4228 * grq->load.weight 4229 * 4230 * Except that is wrong! 4231 * 4232 * Because while for entities historical weight is not important and we 4233 * really only care about our future and therefore can consider a pure 4234 * runnable sum, runqueues can NOT do this. 4235 * 4236 * We specifically want runqueues to have a load_avg that includes 4237 * historical weights. Those represent the blocked load, the load we expect 4238 * to (shortly) return to us. This only works by keeping the weights as 4239 * integral part of the sum. We therefore cannot decompose as per (3). 4240 * 4241 * Another reason this doesn't work is that runnable isn't a 0-sum entity. 4242 * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the 4243 * rq itself is runnable anywhere between 2/3 and 1 depending on how the 4244 * runnable section of these tasks overlap (or not). If they were to perfectly 4245 * align the rq as a whole would be runnable 2/3 of the time. If however we 4246 * always have at least 1 runnable task, the rq as a whole is always runnable. 4247 * 4248 * So we'll have to approximate.. :/ 4249 * 4250 * Given the constraint: 4251 * 4252 * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX 4253 * 4254 * We can construct a rule that adds runnable to a rq by assuming minimal 4255 * overlap. 4256 * 4257 * On removal, we'll assume each task is equally runnable; which yields: 4258 * 4259 * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight 4260 * 4261 * XXX: only do this for the part of runnable > running ? 4262 * 4263 */ 4264 static inline void 4265 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4266 { 4267 long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg; 4268 u32 new_sum, divider; 4269 4270 /* Nothing to update */ 4271 if (!delta_avg) 4272 return; 4273 4274 /* 4275 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4276 * See ___update_load_avg() for details. 4277 */ 4278 divider = get_pelt_divider(&cfs_rq->avg); 4279 4280 /* Set new sched_entity's utilization */ 4281 se->avg.util_avg = gcfs_rq->avg.util_avg; 4282 new_sum = se->avg.util_avg * divider; 4283 delta_sum = (long)new_sum - (long)se->avg.util_sum; 4284 se->avg.util_sum = new_sum; 4285 4286 /* Update parent cfs_rq utilization */ 4287 __update_sa(&cfs_rq->avg, util, delta_avg, delta_sum); 4288 } 4289 4290 static inline void 4291 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4292 { 4293 long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; 4294 u32 new_sum, divider; 4295 4296 /* Nothing to update */ 4297 if (!delta_avg) 4298 return; 4299 4300 /* 4301 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4302 * See ___update_load_avg() for details. 4303 */ 4304 divider = get_pelt_divider(&cfs_rq->avg); 4305 4306 /* Set new sched_entity's runnable */ 4307 se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; 4308 new_sum = se->avg.runnable_avg * divider; 4309 delta_sum = (long)new_sum - (long)se->avg.runnable_sum; 4310 se->avg.runnable_sum = new_sum; 4311 4312 /* Update parent cfs_rq runnable */ 4313 __update_sa(&cfs_rq->avg, runnable, delta_avg, delta_sum); 4314 } 4315 4316 static inline void 4317 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4318 { 4319 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 4320 unsigned long load_avg; 4321 u64 load_sum = 0; 4322 s64 delta_sum; 4323 u32 divider; 4324 4325 if (!runnable_sum) 4326 return; 4327 4328 gcfs_rq->prop_runnable_sum = 0; 4329 4330 /* 4331 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4332 * See ___update_load_avg() for details. 4333 */ 4334 divider = get_pelt_divider(&cfs_rq->avg); 4335 4336 if (runnable_sum >= 0) { 4337 /* 4338 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until 4339 * the CPU is saturated running == runnable. 4340 */ 4341 runnable_sum += se->avg.load_sum; 4342 runnable_sum = min_t(long, runnable_sum, divider); 4343 } else { 4344 /* 4345 * Estimate the new unweighted runnable_sum of the gcfs_rq by 4346 * assuming all tasks are equally runnable. 4347 */ 4348 if (scale_load_down(gcfs_rq->load.weight)) { 4349 load_sum = div_u64(gcfs_rq->avg.load_sum, 4350 scale_load_down(gcfs_rq->load.weight)); 4351 } 4352 4353 /* But make sure to not inflate se's runnable */ 4354 runnable_sum = min(se->avg.load_sum, load_sum); 4355 } 4356 4357 /* 4358 * runnable_sum can't be lower than running_sum 4359 * Rescale running sum to be in the same range as runnable sum 4360 * running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT] 4361 * runnable_sum is in [0 : LOAD_AVG_MAX] 4362 */ 4363 running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 4364 runnable_sum = max(runnable_sum, running_sum); 4365 4366 load_sum = se_weight(se) * runnable_sum; 4367 load_avg = div_u64(load_sum, divider); 4368 4369 delta_avg = load_avg - se->avg.load_avg; 4370 if (!delta_avg) 4371 return; 4372 4373 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 4374 4375 se->avg.load_sum = runnable_sum; 4376 se->avg.load_avg = load_avg; 4377 __update_sa(&cfs_rq->avg, load, delta_avg, delta_sum); 4378 } 4379 4380 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) 4381 { 4382 cfs_rq->propagate = 1; 4383 cfs_rq->prop_runnable_sum += runnable_sum; 4384 } 4385 4386 /* Update task and its cfs_rq load average */ 4387 static inline int propagate_entity_load_avg(struct sched_entity *se) 4388 { 4389 struct cfs_rq *cfs_rq, *gcfs_rq; 4390 4391 if (entity_is_task(se)) 4392 return 0; 4393 4394 gcfs_rq = group_cfs_rq(se); 4395 if (!gcfs_rq->propagate) 4396 return 0; 4397 4398 gcfs_rq->propagate = 0; 4399 4400 cfs_rq = cfs_rq_of(se); 4401 4402 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 4403 4404 update_tg_cfs_util(cfs_rq, se, gcfs_rq); 4405 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); 4406 update_tg_cfs_load(cfs_rq, se, gcfs_rq); 4407 4408 trace_pelt_cfs_tp(cfs_rq); 4409 trace_pelt_se_tp(se); 4410 4411 return 1; 4412 } 4413 4414 /* 4415 * Check if we need to update the load and the utilization of a blocked 4416 * group_entity: 4417 */ 4418 static inline bool skip_blocked_update(struct sched_entity *se) 4419 { 4420 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 4421 4422 /* 4423 * If sched_entity still have not zero load or utilization, we have to 4424 * decay it: 4425 */ 4426 if (se->avg.load_avg || se->avg.util_avg) 4427 return false; 4428 4429 /* 4430 * If there is a pending propagation, we have to update the load and 4431 * the utilization of the sched_entity: 4432 */ 4433 if (gcfs_rq->propagate) 4434 return false; 4435 4436 /* 4437 * Otherwise, the load and the utilization of the sched_entity is 4438 * already zero and there is no pending propagation, so it will be a 4439 * waste of time to try to decay it: 4440 */ 4441 return true; 4442 } 4443 4444 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 4445 4446 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {} 4447 4448 static inline void clear_tg_offline_cfs_rqs(struct rq *rq) {} 4449 4450 static inline int propagate_entity_load_avg(struct sched_entity *se) 4451 { 4452 return 0; 4453 } 4454 4455 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} 4456 4457 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 4458 4459 #ifdef CONFIG_NO_HZ_COMMON 4460 static inline void migrate_se_pelt_lag(struct sched_entity *se) 4461 { 4462 u64 throttled = 0, now, lut; 4463 struct cfs_rq *cfs_rq; 4464 struct rq *rq; 4465 bool is_idle; 4466 4467 if (load_avg_is_decayed(&se->avg)) 4468 return; 4469 4470 cfs_rq = cfs_rq_of(se); 4471 rq = rq_of(cfs_rq); 4472 4473 rcu_read_lock(); 4474 is_idle = is_idle_task(rcu_dereference_all(rq->curr)); 4475 rcu_read_unlock(); 4476 4477 /* 4478 * The lag estimation comes with a cost we don't want to pay all the 4479 * time. Hence, limiting to the case where the source CPU is idle and 4480 * we know we are at the greatest risk to have an outdated clock. 4481 */ 4482 if (!is_idle) 4483 return; 4484 4485 /* 4486 * Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where: 4487 * 4488 * last_update_time (the cfs_rq's last_update_time) 4489 * = cfs_rq_clock_pelt()@cfs_rq_idle 4490 * = rq_clock_pelt()@cfs_rq_idle 4491 * - cfs->throttled_clock_pelt_time@cfs_rq_idle 4492 * 4493 * cfs_idle_lag (delta between rq's update and cfs_rq's update) 4494 * = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle 4495 * 4496 * rq_idle_lag (delta between now and rq's update) 4497 * = sched_clock_cpu() - rq_clock()@rq_idle 4498 * 4499 * We can then write: 4500 * 4501 * now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time + 4502 * sched_clock_cpu() - rq_clock()@rq_idle 4503 * Where: 4504 * rq_clock_pelt()@rq_idle is rq->clock_pelt_idle 4505 * rq_clock()@rq_idle is rq->clock_idle 4506 * cfs->throttled_clock_pelt_time@cfs_rq_idle 4507 * is cfs_rq->throttled_pelt_idle 4508 */ 4509 4510 #ifdef CONFIG_CFS_BANDWIDTH 4511 throttled = u64_u32_load(cfs_rq->throttled_pelt_idle); 4512 /* The clock has been stopped for throttling */ 4513 if (throttled == U64_MAX) 4514 return; 4515 #endif 4516 now = u64_u32_load(rq->clock_pelt_idle); 4517 /* 4518 * Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case 4519 * is observed the old clock_pelt_idle value and the new clock_idle, 4520 * which lead to an underestimation. The opposite would lead to an 4521 * overestimation. 4522 */ 4523 smp_rmb(); 4524 lut = cfs_rq_last_update_time(cfs_rq); 4525 4526 now -= throttled; 4527 if (now < lut) 4528 /* 4529 * cfs_rq->avg.last_update_time is more recent than our 4530 * estimation, let's use it. 4531 */ 4532 now = lut; 4533 else 4534 now += sched_clock_cpu(cpu_of(rq)) - u64_u32_load(rq->clock_idle); 4535 4536 __update_load_avg_blocked_se(now, se); 4537 } 4538 #else /* !CONFIG_NO_HZ_COMMON: */ 4539 static void migrate_se_pelt_lag(struct sched_entity *se) {} 4540 #endif /* !CONFIG_NO_HZ_COMMON */ 4541 4542 /** 4543 * update_cfs_rq_load_avg - update the cfs_rq's load/util averages 4544 * @now: current time, as per cfs_rq_clock_pelt() 4545 * @cfs_rq: cfs_rq to update 4546 * 4547 * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) 4548 * avg. The immediate corollary is that all (fair) tasks must be attached. 4549 * 4550 * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. 4551 * 4552 * Return: true if the load decayed or we removed load. 4553 * 4554 * Since both these conditions indicate a changed cfs_rq->avg.load we should 4555 * call update_tg_load_avg() when this function returns true. 4556 */ 4557 static inline int 4558 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 4559 { 4560 unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0; 4561 struct sched_avg *sa = &cfs_rq->avg; 4562 int decayed = 0; 4563 4564 if (cfs_rq->removed.nr) { 4565 unsigned long r; 4566 u32 divider = get_pelt_divider(&cfs_rq->avg); 4567 4568 raw_spin_lock(&cfs_rq->removed.lock); 4569 swap(cfs_rq->removed.util_avg, removed_util); 4570 swap(cfs_rq->removed.load_avg, removed_load); 4571 swap(cfs_rq->removed.runnable_avg, removed_runnable); 4572 cfs_rq->removed.nr = 0; 4573 raw_spin_unlock(&cfs_rq->removed.lock); 4574 4575 r = removed_load; 4576 __update_sa(sa, load, -r, -r*divider); 4577 4578 r = removed_util; 4579 __update_sa(sa, util, -r, -r*divider); 4580 4581 r = removed_runnable; 4582 __update_sa(sa, runnable, -r, -r*divider); 4583 4584 /* 4585 * removed_runnable is the unweighted version of removed_load so we 4586 * can use it to estimate removed_load_sum. 4587 */ 4588 add_tg_cfs_propagate(cfs_rq, 4589 -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT); 4590 4591 decayed = 1; 4592 } 4593 4594 decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 4595 u64_u32_store_copy(sa->last_update_time, 4596 cfs_rq->last_update_time_copy, 4597 sa->last_update_time); 4598 return decayed; 4599 } 4600 4601 /** 4602 * attach_entity_load_avg - attach this entity to its cfs_rq load avg 4603 * @cfs_rq: cfs_rq to attach to 4604 * @se: sched_entity to attach 4605 * 4606 * Must call update_cfs_rq_load_avg() before this, since we rely on 4607 * cfs_rq->avg.last_update_time being current. 4608 */ 4609 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4610 { 4611 /* 4612 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4613 * See ___update_load_avg() for details. 4614 */ 4615 u32 divider = get_pelt_divider(&cfs_rq->avg); 4616 4617 /* 4618 * When we attach the @se to the @cfs_rq, we must align the decay 4619 * window because without that, really weird and wonderful things can 4620 * happen. 4621 * 4622 * XXX illustrate 4623 */ 4624 se->avg.last_update_time = cfs_rq->avg.last_update_time; 4625 se->avg.period_contrib = cfs_rq->avg.period_contrib; 4626 4627 /* 4628 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new 4629 * period_contrib. This isn't strictly correct, but since we're 4630 * entirely outside of the PELT hierarchy, nobody cares if we truncate 4631 * _sum a little. 4632 */ 4633 se->avg.util_sum = se->avg.util_avg * divider; 4634 4635 se->avg.runnable_sum = se->avg.runnable_avg * divider; 4636 4637 se->avg.load_sum = se->avg.load_avg * divider; 4638 if (se_weight(se) < se->avg.load_sum) 4639 se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se)); 4640 else 4641 se->avg.load_sum = 1; 4642 4643 enqueue_load_avg(cfs_rq, se); 4644 cfs_rq->avg.util_avg += se->avg.util_avg; 4645 cfs_rq->avg.util_sum += se->avg.util_sum; 4646 cfs_rq->avg.runnable_avg += se->avg.runnable_avg; 4647 cfs_rq->avg.runnable_sum += se->avg.runnable_sum; 4648 4649 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); 4650 4651 cfs_rq_util_change(cfs_rq, 0); 4652 4653 trace_pelt_cfs_tp(cfs_rq); 4654 } 4655 4656 /** 4657 * detach_entity_load_avg - detach this entity from its cfs_rq load avg 4658 * @cfs_rq: cfs_rq to detach from 4659 * @se: sched_entity to detach 4660 * 4661 * Must call update_cfs_rq_load_avg() before this, since we rely on 4662 * cfs_rq->avg.last_update_time being current. 4663 */ 4664 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4665 { 4666 dequeue_load_avg(cfs_rq, se); 4667 __update_sa(&cfs_rq->avg, util, -se->avg.util_avg, -se->avg.util_sum); 4668 __update_sa(&cfs_rq->avg, runnable, -se->avg.runnable_avg, -se->avg.runnable_sum); 4669 4670 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); 4671 4672 cfs_rq_util_change(cfs_rq, 0); 4673 4674 trace_pelt_cfs_tp(cfs_rq); 4675 } 4676 4677 /* 4678 * Optional action to be done while updating the load average 4679 */ 4680 #define UPDATE_TG 0x1 4681 #define SKIP_AGE_LOAD 0x2 4682 #define DO_ATTACH 0x4 4683 #define DO_DETACH 0x8 4684 4685 /* Update task and its cfs_rq load average */ 4686 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4687 { 4688 u64 now = cfs_rq_clock_pelt(cfs_rq); 4689 int decayed; 4690 4691 /* 4692 * Track task load average for carrying it to new CPU after migrated, and 4693 * track group sched_entity load average for task_h_load calculation in migration 4694 */ 4695 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 4696 __update_load_avg_se(now, cfs_rq, se); 4697 4698 decayed = update_cfs_rq_load_avg(now, cfs_rq); 4699 decayed |= propagate_entity_load_avg(se); 4700 4701 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 4702 4703 /* 4704 * DO_ATTACH means we're here from enqueue_entity(). 4705 * !last_update_time means we've passed through 4706 * migrate_task_rq_fair() indicating we migrated. 4707 * 4708 * IOW we're enqueueing a task on a new CPU. 4709 */ 4710 attach_entity_load_avg(cfs_rq, se); 4711 update_tg_load_avg(cfs_rq); 4712 4713 } else if (flags & DO_DETACH) { 4714 /* 4715 * DO_DETACH means we're here from dequeue_entity() 4716 * and we are migrating task out of the CPU. 4717 */ 4718 detach_entity_load_avg(cfs_rq, se); 4719 update_tg_load_avg(cfs_rq); 4720 } else if (decayed) { 4721 cfs_rq_util_change(cfs_rq, 0); 4722 4723 if (flags & UPDATE_TG) 4724 update_tg_load_avg(cfs_rq); 4725 } 4726 } 4727 4728 /* 4729 * Synchronize entity load avg of dequeued entity without locking 4730 * the previous rq. 4731 */ 4732 static void sync_entity_load_avg(struct sched_entity *se) 4733 { 4734 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4735 u64 last_update_time; 4736 4737 last_update_time = cfs_rq_last_update_time(cfs_rq); 4738 __update_load_avg_blocked_se(last_update_time, se); 4739 } 4740 4741 /* 4742 * Task first catches up with cfs_rq, and then subtract 4743 * itself from the cfs_rq (task must be off the queue now). 4744 */ 4745 static void remove_entity_load_avg(struct sched_entity *se) 4746 { 4747 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4748 unsigned long flags; 4749 4750 /* 4751 * tasks cannot exit without having gone through wake_up_new_task() -> 4752 * enqueue_task_fair() which will have added things to the cfs_rq, 4753 * so we can remove unconditionally. 4754 */ 4755 4756 sync_entity_load_avg(se); 4757 4758 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); 4759 ++cfs_rq->removed.nr; 4760 cfs_rq->removed.util_avg += se->avg.util_avg; 4761 cfs_rq->removed.load_avg += se->avg.load_avg; 4762 cfs_rq->removed.runnable_avg += se->avg.runnable_avg; 4763 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); 4764 } 4765 4766 static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq) 4767 { 4768 return cfs_rq->avg.runnable_avg; 4769 } 4770 4771 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 4772 { 4773 return cfs_rq->avg.load_avg; 4774 } 4775 4776 static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) 4777 __must_hold(__rq_lockp(this_rq)); 4778 4779 static inline unsigned long task_util(struct task_struct *p) 4780 { 4781 return READ_ONCE(p->se.avg.util_avg); 4782 } 4783 4784 static inline unsigned long task_runnable(struct task_struct *p) 4785 { 4786 return READ_ONCE(p->se.avg.runnable_avg); 4787 } 4788 4789 static inline unsigned long _task_util_est(struct task_struct *p) 4790 { 4791 return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED; 4792 } 4793 4794 static inline unsigned long task_util_est(struct task_struct *p) 4795 { 4796 return max(task_util(p), _task_util_est(p)); 4797 } 4798 4799 static inline void util_est_enqueue(struct cfs_rq *cfs_rq, 4800 struct task_struct *p) 4801 { 4802 unsigned int enqueued; 4803 4804 if (!sched_feat(UTIL_EST)) 4805 return; 4806 4807 /* Update root cfs_rq's estimated utilization */ 4808 enqueued = cfs_rq->avg.util_est; 4809 enqueued += _task_util_est(p); 4810 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); 4811 4812 trace_sched_util_est_cfs_tp(cfs_rq); 4813 } 4814 4815 static inline void util_est_dequeue(struct cfs_rq *cfs_rq, 4816 struct task_struct *p) 4817 { 4818 unsigned int enqueued; 4819 4820 if (!sched_feat(UTIL_EST)) 4821 return; 4822 4823 /* Update root cfs_rq's estimated utilization */ 4824 enqueued = cfs_rq->avg.util_est; 4825 enqueued -= min_t(unsigned int, enqueued, _task_util_est(p)); 4826 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); 4827 4828 trace_sched_util_est_cfs_tp(cfs_rq); 4829 } 4830 4831 #define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) 4832 4833 static inline void util_est_update(struct cfs_rq *cfs_rq, 4834 struct task_struct *p, 4835 bool task_sleep) 4836 { 4837 unsigned int ewma, dequeued, last_ewma_diff; 4838 4839 if (!sched_feat(UTIL_EST)) 4840 return; 4841 4842 /* 4843 * Skip update of task's estimated utilization when the task has not 4844 * yet completed an activation, e.g. being migrated. 4845 */ 4846 if (!task_sleep) 4847 return; 4848 4849 /* Get current estimate of utilization */ 4850 ewma = READ_ONCE(p->se.avg.util_est); 4851 4852 /* 4853 * If the PELT values haven't changed since enqueue time, 4854 * skip the util_est update. 4855 */ 4856 if (ewma & UTIL_AVG_UNCHANGED) 4857 return; 4858 4859 /* Get utilization at dequeue */ 4860 dequeued = task_util(p); 4861 4862 /* 4863 * Reset EWMA on utilization increases, the moving average is used only 4864 * to smooth utilization decreases. 4865 */ 4866 if (ewma <= dequeued) { 4867 ewma = dequeued; 4868 goto done; 4869 } 4870 4871 /* 4872 * Skip update of task's estimated utilization when its members are 4873 * already ~1% close to its last activation value. 4874 */ 4875 last_ewma_diff = ewma - dequeued; 4876 if (last_ewma_diff < UTIL_EST_MARGIN) 4877 goto done; 4878 4879 /* 4880 * To avoid underestimate of task utilization, skip updates of EWMA if 4881 * we cannot grant that thread got all CPU time it wanted. 4882 */ 4883 if ((dequeued + UTIL_EST_MARGIN) < task_runnable(p)) 4884 goto done; 4885 4886 4887 /* 4888 * Update Task's estimated utilization 4889 * 4890 * When *p completes an activation we can consolidate another sample 4891 * of the task size. This is done by using this value to update the 4892 * Exponential Weighted Moving Average (EWMA): 4893 * 4894 * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) 4895 * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) 4896 * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) 4897 * = w * ( -last_ewma_diff ) + ewma(t-1) 4898 * = w * (-last_ewma_diff + ewma(t-1) / w) 4899 * 4900 * Where 'w' is the weight of new samples, which is configured to be 4901 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) 4902 */ 4903 ewma <<= UTIL_EST_WEIGHT_SHIFT; 4904 ewma -= last_ewma_diff; 4905 ewma >>= UTIL_EST_WEIGHT_SHIFT; 4906 done: 4907 ewma |= UTIL_AVG_UNCHANGED; 4908 WRITE_ONCE(p->se.avg.util_est, ewma); 4909 4910 trace_sched_util_est_se_tp(&p->se); 4911 } 4912 4913 static inline unsigned long get_actual_cpu_capacity(int cpu) 4914 { 4915 unsigned long capacity = arch_scale_cpu_capacity(cpu); 4916 4917 capacity -= max(hw_load_avg(cpu_rq(cpu)), cpufreq_get_pressure(cpu)); 4918 4919 return capacity; 4920 } 4921 4922 static inline int util_fits_cpu(unsigned long util, 4923 unsigned long uclamp_min, 4924 unsigned long uclamp_max, 4925 int cpu) 4926 { 4927 unsigned long capacity = capacity_of(cpu); 4928 unsigned long capacity_orig; 4929 bool fits, uclamp_max_fits; 4930 4931 /* 4932 * Check if the real util fits without any uclamp boost/cap applied. 4933 */ 4934 fits = fits_capacity(util, capacity); 4935 4936 if (!uclamp_is_used()) 4937 return fits; 4938 4939 /* 4940 * We must use arch_scale_cpu_capacity() for comparing against uclamp_min and 4941 * uclamp_max. We only care about capacity pressure (by using 4942 * capacity_of()) for comparing against the real util. 4943 * 4944 * If a task is boosted to 1024 for example, we don't want a tiny 4945 * pressure to skew the check whether it fits a CPU or not. 4946 * 4947 * Similarly if a task is capped to arch_scale_cpu_capacity(little_cpu), it 4948 * should fit a little cpu even if there's some pressure. 4949 * 4950 * Only exception is for HW or cpufreq pressure since it has a direct impact 4951 * on available OPP of the system. 4952 * 4953 * We honour it for uclamp_min only as a drop in performance level 4954 * could result in not getting the requested minimum performance level. 4955 * 4956 * For uclamp_max, we can tolerate a drop in performance level as the 4957 * goal is to cap the task. So it's okay if it's getting less. 4958 */ 4959 capacity_orig = arch_scale_cpu_capacity(cpu); 4960 4961 /* 4962 * We want to force a task to fit a cpu as implied by uclamp_max. 4963 * But we do have some corner cases to cater for.. 4964 * 4965 * 4966 * C=z 4967 * | ___ 4968 * | C=y | | 4969 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 4970 * | C=x | | | | 4971 * | ___ | | | | 4972 * | | | | | | | (util somewhere in this region) 4973 * | | | | | | | 4974 * | | | | | | | 4975 * +---------------------------------------- 4976 * CPU0 CPU1 CPU2 4977 * 4978 * In the above example if a task is capped to a specific performance 4979 * point, y, then when: 4980 * 4981 * * util = 80% of x then it does not fit on CPU0 and should migrate 4982 * to CPU1 4983 * * util = 80% of y then it is forced to fit on CPU1 to honour 4984 * uclamp_max request. 4985 * 4986 * which is what we're enforcing here. A task always fits if 4987 * uclamp_max <= capacity_orig. But when uclamp_max > capacity_orig, 4988 * the normal upmigration rules should withhold still. 4989 * 4990 * Only exception is when we are on max capacity, then we need to be 4991 * careful not to block overutilized state. This is so because: 4992 * 4993 * 1. There's no concept of capping at max_capacity! We can't go 4994 * beyond this performance level anyway. 4995 * 2. The system is being saturated when we're operating near 4996 * max capacity, it doesn't make sense to block overutilized. 4997 */ 4998 uclamp_max_fits = (capacity_orig == SCHED_CAPACITY_SCALE) && (uclamp_max == SCHED_CAPACITY_SCALE); 4999 uclamp_max_fits = !uclamp_max_fits && (uclamp_max <= capacity_orig); 5000 fits = fits || uclamp_max_fits; 5001 5002 /* 5003 * 5004 * C=z 5005 * | ___ (region a, capped, util >= uclamp_max) 5006 * | C=y | | 5007 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 5008 * | C=x | | | | 5009 * | ___ | | | | (region b, uclamp_min <= util <= uclamp_max) 5010 * |_ _ _|_ _|_ _ _ _| _ | _ _ _| _ | _ _ _ _ _ uclamp_min 5011 * | | | | | | | 5012 * | | | | | | | (region c, boosted, util < uclamp_min) 5013 * +---------------------------------------- 5014 * CPU0 CPU1 CPU2 5015 * 5016 * a) If util > uclamp_max, then we're capped, we don't care about 5017 * actual fitness value here. We only care if uclamp_max fits 5018 * capacity without taking margin/pressure into account. 5019 * See comment above. 5020 * 5021 * b) If uclamp_min <= util <= uclamp_max, then the normal 5022 * fits_capacity() rules apply. Except we need to ensure that we 5023 * enforce we remain within uclamp_max, see comment above. 5024 * 5025 * c) If util < uclamp_min, then we are boosted. Same as (b) but we 5026 * need to take into account the boosted value fits the CPU without 5027 * taking margin/pressure into account. 5028 * 5029 * Cases (a) and (b) are handled in the 'fits' variable already. We 5030 * just need to consider an extra check for case (c) after ensuring we 5031 * handle the case uclamp_min > uclamp_max. 5032 */ 5033 uclamp_min = min(uclamp_min, uclamp_max); 5034 if (fits && (util < uclamp_min) && 5035 (uclamp_min > get_actual_cpu_capacity(cpu))) 5036 return -1; 5037 5038 return fits; 5039 } 5040 5041 static inline int task_fits_cpu(struct task_struct *p, int cpu) 5042 { 5043 unsigned long uclamp_min = uclamp_eff_value(p, UCLAMP_MIN); 5044 unsigned long uclamp_max = uclamp_eff_value(p, UCLAMP_MAX); 5045 unsigned long util = task_util_est(p); 5046 /* 5047 * Return true only if the cpu fully fits the task requirements, which 5048 * include the utilization but also the performance hints. 5049 */ 5050 return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0); 5051 } 5052 5053 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) 5054 { 5055 int cpu = cpu_of(rq); 5056 5057 if (!sched_asym_cpucap_active()) 5058 return; 5059 5060 /* 5061 * Affinity allows us to go somewhere higher? Or are we on biggest 5062 * available CPU already? Or do we fit into this CPU ? 5063 */ 5064 if (!p || (p->nr_cpus_allowed == 1) || 5065 (arch_scale_cpu_capacity(cpu) == p->max_allowed_capacity) || 5066 task_fits_cpu(p, cpu)) { 5067 5068 rq->misfit_task_load = 0; 5069 return; 5070 } 5071 5072 /* 5073 * Make sure that misfit_task_load will not be null even if 5074 * task_h_load() returns 0. 5075 */ 5076 rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); 5077 } 5078 5079 void __setparam_fair(struct task_struct *p, const struct sched_attr *attr) 5080 { 5081 struct sched_entity *se = &p->se; 5082 5083 p->static_prio = NICE_TO_PRIO(attr->sched_nice); 5084 if (attr->sched_runtime) { 5085 se->custom_slice = 1; 5086 se->slice = clamp_t(u64, attr->sched_runtime, 5087 NSEC_PER_MSEC/10, /* HZ=1000 * 10 */ 5088 NSEC_PER_MSEC*100); /* HZ=100 / 10 */ 5089 } else { 5090 se->custom_slice = 0; 5091 se->slice = sysctl_sched_base_slice; 5092 } 5093 } 5094 5095 static void 5096 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5097 { 5098 u64 vslice, vruntime = avg_vruntime(cfs_rq); 5099 s64 lag = 0; 5100 5101 if (!se->custom_slice) 5102 se->slice = sysctl_sched_base_slice; 5103 vslice = calc_delta_fair(se->slice, se); 5104 5105 /* 5106 * Due to how V is constructed as the weighted average of entities, 5107 * adding tasks with positive lag, or removing tasks with negative lag 5108 * will move 'time' backwards, this can screw around with the lag of 5109 * other tasks. 5110 * 5111 * EEVDF: placement strategy #1 / #2 5112 */ 5113 if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) { 5114 struct sched_entity *curr = cfs_rq->curr; 5115 unsigned long load; 5116 5117 lag = se->vlag; 5118 5119 /* 5120 * If we want to place a task and preserve lag, we have to 5121 * consider the effect of the new entity on the weighted 5122 * average and compensate for this, otherwise lag can quickly 5123 * evaporate. 5124 * 5125 * Lag is defined as: 5126 * 5127 * lag_i = S - s_i = w_i * (V - v_i) 5128 * 5129 * To avoid the 'w_i' term all over the place, we only track 5130 * the virtual lag: 5131 * 5132 * vl_i = V - v_i <=> v_i = V - vl_i 5133 * 5134 * And we take V to be the weighted average of all v: 5135 * 5136 * V = (\Sum w_j*v_j) / W 5137 * 5138 * Where W is: \Sum w_j 5139 * 5140 * Then, the weighted average after adding an entity with lag 5141 * vl_i is given by: 5142 * 5143 * V' = (\Sum w_j*v_j + w_i*v_i) / (W + w_i) 5144 * = (W*V + w_i*(V - vl_i)) / (W + w_i) 5145 * = (W*V + w_i*V - w_i*vl_i) / (W + w_i) 5146 * = (V*(W + w_i) - w_i*vl_i) / (W + w_i) 5147 * = V - w_i*vl_i / (W + w_i) 5148 * 5149 * And the actual lag after adding an entity with vl_i is: 5150 * 5151 * vl'_i = V' - v_i 5152 * = V - w_i*vl_i / (W + w_i) - (V - vl_i) 5153 * = vl_i - w_i*vl_i / (W + w_i) 5154 * 5155 * Which is strictly less than vl_i. So in order to preserve lag 5156 * we should inflate the lag before placement such that the 5157 * effective lag after placement comes out right. 5158 * 5159 * As such, invert the above relation for vl'_i to get the vl_i 5160 * we need to use such that the lag after placement is the lag 5161 * we computed before dequeue. 5162 * 5163 * vl'_i = vl_i - w_i*vl_i / (W + w_i) 5164 * = ((W + w_i)*vl_i - w_i*vl_i) / (W + w_i) 5165 * 5166 * (W + w_i)*vl'_i = (W + w_i)*vl_i - w_i*vl_i 5167 * = W*vl_i 5168 * 5169 * vl_i = (W + w_i)*vl'_i / W 5170 */ 5171 load = cfs_rq->sum_weight; 5172 if (curr && curr->on_rq) 5173 load += scale_load_down(curr->load.weight); 5174 5175 lag *= load + scale_load_down(se->load.weight); 5176 if (WARN_ON_ONCE(!load)) 5177 load = 1; 5178 lag = div_s64(lag, load); 5179 } 5180 5181 se->vruntime = vruntime - lag; 5182 5183 if (se->rel_deadline) { 5184 se->deadline += se->vruntime; 5185 se->rel_deadline = 0; 5186 return; 5187 } 5188 5189 /* 5190 * When joining the competition; the existing tasks will be, 5191 * on average, halfway through their slice, as such start tasks 5192 * off with half a slice to ease into the competition. 5193 */ 5194 if (sched_feat(PLACE_DEADLINE_INITIAL) && (flags & ENQUEUE_INITIAL)) 5195 vslice /= 2; 5196 5197 /* 5198 * EEVDF: vd_i = ve_i + r_i/w_i 5199 */ 5200 se->deadline = se->vruntime + vslice; 5201 } 5202 5203 static void check_enqueue_throttle(struct cfs_rq *cfs_rq); 5204 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); 5205 5206 static void 5207 requeue_delayed_entity(struct sched_entity *se); 5208 5209 static void 5210 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5211 { 5212 bool curr = cfs_rq->curr == se; 5213 5214 /* 5215 * If we're the current task, we must renormalise before calling 5216 * update_curr(). 5217 */ 5218 if (curr) 5219 place_entity(cfs_rq, se, flags); 5220 5221 update_curr(cfs_rq); 5222 5223 /* 5224 * When enqueuing a sched_entity, we must: 5225 * - Update loads to have both entity and cfs_rq synced with now. 5226 * - For group_entity, update its runnable_weight to reflect the new 5227 * h_nr_runnable of its group cfs_rq. 5228 * - For group_entity, update its weight to reflect the new share of 5229 * its group cfs_rq 5230 * - Add its new weight to cfs_rq->load.weight 5231 */ 5232 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); 5233 se_update_runnable(se); 5234 /* 5235 * XXX update_load_avg() above will have attached us to the pelt sum; 5236 * but update_cfs_group() here will re-adjust the weight and have to 5237 * undo/redo all that. Seems wasteful. 5238 */ 5239 update_cfs_group(se); 5240 5241 /* 5242 * XXX now that the entity has been re-weighted, and it's lag adjusted, 5243 * we can place the entity. 5244 */ 5245 if (!curr) 5246 place_entity(cfs_rq, se, flags); 5247 5248 account_entity_enqueue(cfs_rq, se); 5249 5250 /* Entity has migrated, no longer consider this task hot */ 5251 if (flags & ENQUEUE_MIGRATED) 5252 se->exec_start = 0; 5253 5254 check_schedstat_required(); 5255 update_stats_enqueue_fair(cfs_rq, se, flags); 5256 if (!curr) 5257 __enqueue_entity(cfs_rq, se); 5258 se->on_rq = 1; 5259 5260 if (cfs_rq->nr_queued == 1) { 5261 check_enqueue_throttle(cfs_rq); 5262 list_add_leaf_cfs_rq(cfs_rq); 5263 #ifdef CONFIG_CFS_BANDWIDTH 5264 if (cfs_rq->pelt_clock_throttled) { 5265 struct rq *rq = rq_of(cfs_rq); 5266 5267 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 5268 cfs_rq->throttled_clock_pelt; 5269 cfs_rq->pelt_clock_throttled = 0; 5270 } 5271 #endif 5272 } 5273 } 5274 5275 static void __clear_buddies_next(struct sched_entity *se) 5276 { 5277 for_each_sched_entity(se) { 5278 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5279 if (cfs_rq->next != se) 5280 break; 5281 5282 cfs_rq->next = NULL; 5283 } 5284 } 5285 5286 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 5287 { 5288 if (cfs_rq->next == se) 5289 __clear_buddies_next(se); 5290 } 5291 5292 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5293 5294 static void set_delayed(struct sched_entity *se) 5295 { 5296 se->sched_delayed = 1; 5297 5298 /* 5299 * Delayed se of cfs_rq have no tasks queued on them. 5300 * Do not adjust h_nr_runnable since dequeue_entities() 5301 * will account it for blocked tasks. 5302 */ 5303 if (!entity_is_task(se)) 5304 return; 5305 5306 for_each_sched_entity(se) { 5307 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5308 5309 cfs_rq->h_nr_runnable--; 5310 } 5311 } 5312 5313 static void clear_delayed(struct sched_entity *se) 5314 { 5315 se->sched_delayed = 0; 5316 5317 /* 5318 * Delayed se of cfs_rq have no tasks queued on them. 5319 * Do not adjust h_nr_runnable since a dequeue has 5320 * already accounted for it or an enqueue of a task 5321 * below it will account for it in enqueue_task_fair(). 5322 */ 5323 if (!entity_is_task(se)) 5324 return; 5325 5326 for_each_sched_entity(se) { 5327 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5328 5329 cfs_rq->h_nr_runnable++; 5330 } 5331 } 5332 5333 static inline void finish_delayed_dequeue_entity(struct sched_entity *se) 5334 { 5335 clear_delayed(se); 5336 if (sched_feat(DELAY_ZERO) && se->vlag > 0) 5337 se->vlag = 0; 5338 } 5339 5340 static bool 5341 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5342 { 5343 bool sleep = flags & DEQUEUE_SLEEP; 5344 int action = UPDATE_TG; 5345 5346 update_curr(cfs_rq); 5347 clear_buddies(cfs_rq, se); 5348 5349 if (flags & DEQUEUE_DELAYED) { 5350 WARN_ON_ONCE(!se->sched_delayed); 5351 } else { 5352 bool delay = sleep; 5353 /* 5354 * DELAY_DEQUEUE relies on spurious wakeups, special task 5355 * states must not suffer spurious wakeups, excempt them. 5356 */ 5357 if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) 5358 delay = false; 5359 5360 WARN_ON_ONCE(delay && se->sched_delayed); 5361 5362 if (sched_feat(DELAY_DEQUEUE) && delay && 5363 !entity_eligible(cfs_rq, se)) { 5364 update_load_avg(cfs_rq, se, 0); 5365 set_delayed(se); 5366 return false; 5367 } 5368 } 5369 5370 if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) 5371 action |= DO_DETACH; 5372 5373 /* 5374 * When dequeuing a sched_entity, we must: 5375 * - Update loads to have both entity and cfs_rq synced with now. 5376 * - For group_entity, update its runnable_weight to reflect the new 5377 * h_nr_runnable of its group cfs_rq. 5378 * - Subtract its previous weight from cfs_rq->load.weight. 5379 * - For group entity, update its weight to reflect the new share 5380 * of its group cfs_rq. 5381 */ 5382 update_load_avg(cfs_rq, se, action); 5383 se_update_runnable(se); 5384 5385 update_stats_dequeue_fair(cfs_rq, se, flags); 5386 5387 update_entity_lag(cfs_rq, se); 5388 if (sched_feat(PLACE_REL_DEADLINE) && !sleep) { 5389 se->deadline -= se->vruntime; 5390 se->rel_deadline = 1; 5391 } 5392 5393 if (se != cfs_rq->curr) 5394 __dequeue_entity(cfs_rq, se); 5395 se->on_rq = 0; 5396 account_entity_dequeue(cfs_rq, se); 5397 5398 /* return excess runtime on last dequeue */ 5399 return_cfs_rq_runtime(cfs_rq); 5400 5401 update_cfs_group(se); 5402 5403 if (flags & DEQUEUE_DELAYED) 5404 finish_delayed_dequeue_entity(se); 5405 5406 if (cfs_rq->nr_queued == 0) { 5407 update_idle_cfs_rq_clock_pelt(cfs_rq); 5408 #ifdef CONFIG_CFS_BANDWIDTH 5409 if (throttled_hierarchy(cfs_rq)) { 5410 struct rq *rq = rq_of(cfs_rq); 5411 5412 list_del_leaf_cfs_rq(cfs_rq); 5413 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 5414 cfs_rq->pelt_clock_throttled = 1; 5415 } 5416 #endif 5417 } 5418 5419 return true; 5420 } 5421 5422 static void 5423 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 5424 { 5425 clear_buddies(cfs_rq, se); 5426 5427 /* 'current' is not kept within the tree. */ 5428 if (se->on_rq) { 5429 /* 5430 * Any task has to be enqueued before it get to execute on 5431 * a CPU. So account for the time it spent waiting on the 5432 * runqueue. 5433 */ 5434 update_stats_wait_end_fair(cfs_rq, se); 5435 __dequeue_entity(cfs_rq, se); 5436 update_load_avg(cfs_rq, se, UPDATE_TG); 5437 5438 set_protect_slice(cfs_rq, se); 5439 } 5440 5441 update_stats_curr_start(cfs_rq, se); 5442 WARN_ON_ONCE(cfs_rq->curr); 5443 cfs_rq->curr = se; 5444 5445 /* 5446 * Track our maximum slice length, if the CPU's load is at 5447 * least twice that of our own weight (i.e. don't track it 5448 * when there are only lesser-weight tasks around): 5449 */ 5450 if (schedstat_enabled() && 5451 rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) { 5452 struct sched_statistics *stats; 5453 5454 stats = __schedstats_from_se(se); 5455 __schedstat_set(stats->slice_max, 5456 max((u64)stats->slice_max, 5457 se->sum_exec_runtime - se->prev_sum_exec_runtime)); 5458 } 5459 5460 se->prev_sum_exec_runtime = se->sum_exec_runtime; 5461 } 5462 5463 static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags); 5464 5465 /* 5466 * Pick the next process, keeping these things in mind, in this order: 5467 * 1) keep things fair between processes/task groups 5468 * 2) pick the "next" process, since someone really wants that to run 5469 * 3) pick the "last" process, for cache locality 5470 * 4) do not run the "skip" process, if something else is available 5471 */ 5472 static struct sched_entity * 5473 pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq) 5474 { 5475 struct sched_entity *se; 5476 5477 se = pick_eevdf(cfs_rq); 5478 if (se->sched_delayed) { 5479 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 5480 /* 5481 * Must not reference @se again, see __block_task(). 5482 */ 5483 return NULL; 5484 } 5485 return se; 5486 } 5487 5488 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5489 5490 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) 5491 { 5492 /* 5493 * If still on the runqueue then deactivate_task() 5494 * was not called and update_curr() has to be done: 5495 */ 5496 if (prev->on_rq) 5497 update_curr(cfs_rq); 5498 5499 /* throttle cfs_rqs exceeding runtime */ 5500 check_cfs_rq_runtime(cfs_rq); 5501 5502 if (prev->on_rq) { 5503 update_stats_wait_start_fair(cfs_rq, prev); 5504 /* Put 'current' back into the tree. */ 5505 __enqueue_entity(cfs_rq, prev); 5506 /* in !on_rq case, update occurred at dequeue */ 5507 update_load_avg(cfs_rq, prev, 0); 5508 } 5509 WARN_ON_ONCE(cfs_rq->curr != prev); 5510 cfs_rq->curr = NULL; 5511 } 5512 5513 static void 5514 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 5515 { 5516 /* 5517 * Update run-time statistics of the 'current'. 5518 */ 5519 update_curr(cfs_rq); 5520 5521 /* 5522 * Ensure that runnable average is periodically updated. 5523 */ 5524 update_load_avg(cfs_rq, curr, UPDATE_TG); 5525 update_cfs_group(curr); 5526 5527 #ifdef CONFIG_SCHED_HRTICK 5528 /* 5529 * queued ticks are scheduled to match the slice, so don't bother 5530 * validating it and just reschedule. 5531 */ 5532 if (queued) { 5533 resched_curr_lazy(rq_of(cfs_rq)); 5534 return; 5535 } 5536 #endif 5537 } 5538 5539 5540 /************************************************** 5541 * CFS bandwidth control machinery 5542 */ 5543 5544 #ifdef CONFIG_CFS_BANDWIDTH 5545 5546 #ifdef CONFIG_JUMP_LABEL 5547 static struct static_key __cfs_bandwidth_used; 5548 5549 static inline bool cfs_bandwidth_used(void) 5550 { 5551 return static_key_false(&__cfs_bandwidth_used); 5552 } 5553 5554 void cfs_bandwidth_usage_inc(void) 5555 { 5556 static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used); 5557 } 5558 5559 void cfs_bandwidth_usage_dec(void) 5560 { 5561 static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); 5562 } 5563 #else /* !CONFIG_JUMP_LABEL: */ 5564 static bool cfs_bandwidth_used(void) 5565 { 5566 return true; 5567 } 5568 5569 void cfs_bandwidth_usage_inc(void) {} 5570 void cfs_bandwidth_usage_dec(void) {} 5571 #endif /* !CONFIG_JUMP_LABEL */ 5572 5573 static inline u64 sched_cfs_bandwidth_slice(void) 5574 { 5575 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; 5576 } 5577 5578 /* 5579 * Replenish runtime according to assigned quota. We use sched_clock_cpu 5580 * directly instead of rq->clock to avoid adding additional synchronization 5581 * around rq->lock. 5582 * 5583 * requires cfs_b->lock 5584 */ 5585 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) 5586 { 5587 s64 runtime; 5588 5589 if (unlikely(cfs_b->quota == RUNTIME_INF)) 5590 return; 5591 5592 cfs_b->runtime += cfs_b->quota; 5593 runtime = cfs_b->runtime_snap - cfs_b->runtime; 5594 if (runtime > 0) { 5595 cfs_b->burst_time += runtime; 5596 cfs_b->nr_burst++; 5597 } 5598 5599 cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst); 5600 cfs_b->runtime_snap = cfs_b->runtime; 5601 } 5602 5603 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 5604 { 5605 return &tg->cfs_bandwidth; 5606 } 5607 5608 /* returns 0 on failure to allocate runtime */ 5609 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, 5610 struct cfs_rq *cfs_rq, u64 target_runtime) 5611 { 5612 u64 min_amount, amount = 0; 5613 5614 lockdep_assert_held(&cfs_b->lock); 5615 5616 /* note: this is a positive sum as runtime_remaining <= 0 */ 5617 min_amount = target_runtime - cfs_rq->runtime_remaining; 5618 5619 if (cfs_b->quota == RUNTIME_INF) 5620 amount = min_amount; 5621 else { 5622 start_cfs_bandwidth(cfs_b); 5623 5624 if (cfs_b->runtime > 0) { 5625 amount = min(cfs_b->runtime, min_amount); 5626 cfs_b->runtime -= amount; 5627 cfs_b->idle = 0; 5628 } 5629 } 5630 5631 cfs_rq->runtime_remaining += amount; 5632 5633 return cfs_rq->runtime_remaining > 0; 5634 } 5635 5636 /* returns 0 on failure to allocate runtime */ 5637 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5638 { 5639 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5640 int ret; 5641 5642 raw_spin_lock(&cfs_b->lock); 5643 ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); 5644 raw_spin_unlock(&cfs_b->lock); 5645 5646 return ret; 5647 } 5648 5649 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5650 { 5651 /* dock delta_exec before expiring quota (as it could span periods) */ 5652 cfs_rq->runtime_remaining -= delta_exec; 5653 5654 if (likely(cfs_rq->runtime_remaining > 0)) 5655 return; 5656 5657 if (cfs_rq->throttled) 5658 return; 5659 /* 5660 * if we're unable to extend our runtime we resched so that the active 5661 * hierarchy can be throttled 5662 */ 5663 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) 5664 resched_curr(rq_of(cfs_rq)); 5665 } 5666 5667 static __always_inline 5668 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5669 { 5670 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) 5671 return; 5672 5673 __account_cfs_rq_runtime(cfs_rq, delta_exec); 5674 } 5675 5676 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 5677 { 5678 return cfs_bandwidth_used() && cfs_rq->throttled; 5679 } 5680 5681 static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq) 5682 { 5683 return cfs_bandwidth_used() && cfs_rq->pelt_clock_throttled; 5684 } 5685 5686 /* check whether cfs_rq, or any parent, is throttled */ 5687 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 5688 { 5689 return cfs_bandwidth_used() && cfs_rq->throttle_count; 5690 } 5691 5692 static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu) 5693 { 5694 return throttled_hierarchy(task_group(p)->cfs_rq[dst_cpu]); 5695 } 5696 5697 static inline bool task_is_throttled(struct task_struct *p) 5698 { 5699 return cfs_bandwidth_used() && p->throttled; 5700 } 5701 5702 static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags); 5703 static void throttle_cfs_rq_work(struct callback_head *work) 5704 { 5705 struct task_struct *p = container_of(work, struct task_struct, sched_throttle_work); 5706 struct sched_entity *se; 5707 struct cfs_rq *cfs_rq; 5708 struct rq *rq; 5709 5710 WARN_ON_ONCE(p != current); 5711 p->sched_throttle_work.next = &p->sched_throttle_work; 5712 5713 /* 5714 * If task is exiting, then there won't be a return to userspace, so we 5715 * don't have to bother with any of this. 5716 */ 5717 if ((p->flags & PF_EXITING)) 5718 return; 5719 5720 scoped_guard(task_rq_lock, p) { 5721 se = &p->se; 5722 cfs_rq = cfs_rq_of(se); 5723 5724 /* Raced, forget */ 5725 if (p->sched_class != &fair_sched_class) 5726 return; 5727 5728 /* 5729 * If not in limbo, then either replenish has happened or this 5730 * task got migrated out of the throttled cfs_rq, move along. 5731 */ 5732 if (!cfs_rq->throttle_count) 5733 return; 5734 rq = scope.rq; 5735 update_rq_clock(rq); 5736 WARN_ON_ONCE(p->throttled || !list_empty(&p->throttle_node)); 5737 dequeue_task_fair(rq, p, DEQUEUE_SLEEP | DEQUEUE_THROTTLE); 5738 list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list); 5739 /* 5740 * Must not set throttled before dequeue or dequeue will 5741 * mistakenly regard this task as an already throttled one. 5742 */ 5743 p->throttled = true; 5744 resched_curr(rq); 5745 } 5746 } 5747 5748 void init_cfs_throttle_work(struct task_struct *p) 5749 { 5750 init_task_work(&p->sched_throttle_work, throttle_cfs_rq_work); 5751 /* Protect against double add, see throttle_cfs_rq() and throttle_cfs_rq_work() */ 5752 p->sched_throttle_work.next = &p->sched_throttle_work; 5753 INIT_LIST_HEAD(&p->throttle_node); 5754 } 5755 5756 /* 5757 * Task is throttled and someone wants to dequeue it again: 5758 * it could be sched/core when core needs to do things like 5759 * task affinity change, task group change, task sched class 5760 * change etc. and in these cases, DEQUEUE_SLEEP is not set; 5761 * or the task is blocked after throttled due to freezer etc. 5762 * and in these cases, DEQUEUE_SLEEP is set. 5763 */ 5764 static void detach_task_cfs_rq(struct task_struct *p); 5765 static void dequeue_throttled_task(struct task_struct *p, int flags) 5766 { 5767 WARN_ON_ONCE(p->se.on_rq); 5768 list_del_init(&p->throttle_node); 5769 5770 /* task blocked after throttled */ 5771 if (flags & DEQUEUE_SLEEP) { 5772 p->throttled = false; 5773 return; 5774 } 5775 5776 /* 5777 * task is migrating off its old cfs_rq, detach 5778 * the task's load from its old cfs_rq. 5779 */ 5780 if (task_on_rq_migrating(p)) 5781 detach_task_cfs_rq(p); 5782 } 5783 5784 static bool enqueue_throttled_task(struct task_struct *p) 5785 { 5786 struct cfs_rq *cfs_rq = cfs_rq_of(&p->se); 5787 5788 /* @p should have gone through dequeue_throttled_task() first */ 5789 WARN_ON_ONCE(!list_empty(&p->throttle_node)); 5790 5791 /* 5792 * If the throttled task @p is enqueued to a throttled cfs_rq, 5793 * take the fast path by directly putting the task on the 5794 * target cfs_rq's limbo list. 5795 * 5796 * Do not do that when @p is current because the following race can 5797 * cause @p's group_node to be incorectly re-insterted in its rq's 5798 * cfs_tasks list, despite being throttled: 5799 * 5800 * cpuX cpuY 5801 * p ret2user 5802 * throttle_cfs_rq_work() sched_move_task(p) 5803 * LOCK task_rq_lock 5804 * dequeue_task_fair(p) 5805 * UNLOCK task_rq_lock 5806 * LOCK task_rq_lock 5807 * task_current_donor(p) == true 5808 * task_on_rq_queued(p) == true 5809 * dequeue_task(p) 5810 * put_prev_task(p) 5811 * sched_change_group() 5812 * enqueue_task(p) -> p's new cfs_rq 5813 * is throttled, go 5814 * fast path and skip 5815 * actual enqueue 5816 * set_next_task(p) 5817 * list_move(&se->group_node, &rq->cfs_tasks); // bug 5818 * schedule() 5819 * 5820 * In the above race case, @p current cfs_rq is in the same rq as 5821 * its previous cfs_rq because sched_move_task() only moves a task 5822 * to a different group from the same rq, so we can use its current 5823 * cfs_rq to derive rq and test if the task is current. 5824 */ 5825 if (throttled_hierarchy(cfs_rq) && 5826 !task_current_donor(rq_of(cfs_rq), p)) { 5827 list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list); 5828 return true; 5829 } 5830 5831 /* we can't take the fast path, do an actual enqueue*/ 5832 p->throttled = false; 5833 return false; 5834 } 5835 5836 static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags); 5837 static int tg_unthrottle_up(struct task_group *tg, void *data) 5838 { 5839 struct rq *rq = data; 5840 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5841 struct task_struct *p, *tmp; 5842 5843 if (--cfs_rq->throttle_count) 5844 return 0; 5845 5846 if (cfs_rq->pelt_clock_throttled) { 5847 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 5848 cfs_rq->throttled_clock_pelt; 5849 cfs_rq->pelt_clock_throttled = 0; 5850 } 5851 5852 if (cfs_rq->throttled_clock_self) { 5853 u64 delta = rq_clock(rq) - cfs_rq->throttled_clock_self; 5854 5855 cfs_rq->throttled_clock_self = 0; 5856 5857 if (WARN_ON_ONCE((s64)delta < 0)) 5858 delta = 0; 5859 5860 cfs_rq->throttled_clock_self_time += delta; 5861 } 5862 5863 /* Re-enqueue the tasks that have been throttled at this level. */ 5864 list_for_each_entry_safe(p, tmp, &cfs_rq->throttled_limbo_list, throttle_node) { 5865 list_del_init(&p->throttle_node); 5866 p->throttled = false; 5867 enqueue_task_fair(rq_of(cfs_rq), p, ENQUEUE_WAKEUP); 5868 } 5869 5870 /* Add cfs_rq with load or one or more already running entities to the list */ 5871 if (!cfs_rq_is_decayed(cfs_rq)) 5872 list_add_leaf_cfs_rq(cfs_rq); 5873 5874 return 0; 5875 } 5876 5877 static inline bool task_has_throttle_work(struct task_struct *p) 5878 { 5879 return p->sched_throttle_work.next != &p->sched_throttle_work; 5880 } 5881 5882 static inline void task_throttle_setup_work(struct task_struct *p) 5883 { 5884 if (task_has_throttle_work(p)) 5885 return; 5886 5887 /* 5888 * Kthreads and exiting tasks don't return to userspace, so adding the 5889 * work is pointless 5890 */ 5891 if ((p->flags & (PF_EXITING | PF_KTHREAD))) 5892 return; 5893 5894 task_work_add(p, &p->sched_throttle_work, TWA_RESUME); 5895 } 5896 5897 static void record_throttle_clock(struct cfs_rq *cfs_rq) 5898 { 5899 struct rq *rq = rq_of(cfs_rq); 5900 5901 if (cfs_rq_throttled(cfs_rq) && !cfs_rq->throttled_clock) 5902 cfs_rq->throttled_clock = rq_clock(rq); 5903 5904 if (!cfs_rq->throttled_clock_self) 5905 cfs_rq->throttled_clock_self = rq_clock(rq); 5906 } 5907 5908 static int tg_throttle_down(struct task_group *tg, void *data) 5909 { 5910 struct rq *rq = data; 5911 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5912 5913 if (cfs_rq->throttle_count++) 5914 return 0; 5915 5916 /* 5917 * For cfs_rqs that still have entities enqueued, PELT clock 5918 * stop happens at dequeue time when all entities are dequeued. 5919 */ 5920 if (!cfs_rq->nr_queued) { 5921 list_del_leaf_cfs_rq(cfs_rq); 5922 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 5923 cfs_rq->pelt_clock_throttled = 1; 5924 } 5925 5926 WARN_ON_ONCE(cfs_rq->throttled_clock_self); 5927 WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_limbo_list)); 5928 return 0; 5929 } 5930 5931 static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) 5932 { 5933 struct rq *rq = rq_of(cfs_rq); 5934 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5935 int dequeue = 1; 5936 5937 raw_spin_lock(&cfs_b->lock); 5938 /* This will start the period timer if necessary */ 5939 if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) { 5940 /* 5941 * We have raced with bandwidth becoming available, and if we 5942 * actually throttled the timer might not unthrottle us for an 5943 * entire period. We additionally needed to make sure that any 5944 * subsequent check_cfs_rq_runtime calls agree not to throttle 5945 * us, as we may commit to do cfs put_prev+pick_next, so we ask 5946 * for 1ns of runtime rather than just check cfs_b. 5947 */ 5948 dequeue = 0; 5949 } else { 5950 list_add_tail_rcu(&cfs_rq->throttled_list, 5951 &cfs_b->throttled_cfs_rq); 5952 } 5953 raw_spin_unlock(&cfs_b->lock); 5954 5955 if (!dequeue) 5956 return false; /* Throttle no longer required. */ 5957 5958 /* freeze hierarchy runnable averages while throttled */ 5959 rcu_read_lock(); 5960 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); 5961 rcu_read_unlock(); 5962 5963 /* 5964 * Note: distribution will already see us throttled via the 5965 * throttled-list. rq->lock protects completion. 5966 */ 5967 cfs_rq->throttled = 1; 5968 WARN_ON_ONCE(cfs_rq->throttled_clock); 5969 return true; 5970 } 5971 5972 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) 5973 { 5974 struct rq *rq = rq_of(cfs_rq); 5975 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5976 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 5977 5978 /* 5979 * It's possible we are called with runtime_remaining < 0 due to things 5980 * like async unthrottled us with a positive runtime_remaining but other 5981 * still running entities consumed those runtime before we reached here. 5982 * 5983 * We can't unthrottle this cfs_rq without any runtime remaining because 5984 * any enqueue in tg_unthrottle_up() will immediately trigger a throttle, 5985 * which is not supposed to happen on unthrottle path. 5986 */ 5987 if (cfs_rq->runtime_enabled && cfs_rq->runtime_remaining <= 0) 5988 return; 5989 5990 cfs_rq->throttled = 0; 5991 5992 update_rq_clock(rq); 5993 5994 raw_spin_lock(&cfs_b->lock); 5995 if (cfs_rq->throttled_clock) { 5996 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; 5997 cfs_rq->throttled_clock = 0; 5998 } 5999 list_del_rcu(&cfs_rq->throttled_list); 6000 raw_spin_unlock(&cfs_b->lock); 6001 6002 /* update hierarchical throttle state */ 6003 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); 6004 6005 if (!cfs_rq->load.weight) { 6006 if (!cfs_rq->on_list) 6007 return; 6008 /* 6009 * Nothing to run but something to decay (on_list)? 6010 * Complete the branch. 6011 */ 6012 for_each_sched_entity(se) { 6013 if (list_add_leaf_cfs_rq(cfs_rq_of(se))) 6014 break; 6015 } 6016 } 6017 6018 assert_list_leaf_cfs_rq(rq); 6019 6020 /* Determine whether we need to wake up potentially idle CPU: */ 6021 if (rq->curr == rq->idle && rq->cfs.nr_queued) 6022 resched_curr(rq); 6023 } 6024 6025 static void __cfsb_csd_unthrottle(void *arg) 6026 { 6027 struct cfs_rq *cursor, *tmp; 6028 struct rq *rq = arg; 6029 struct rq_flags rf; 6030 6031 rq_lock(rq, &rf); 6032 6033 /* 6034 * Iterating over the list can trigger several call to 6035 * update_rq_clock() in unthrottle_cfs_rq(). 6036 * Do it once and skip the potential next ones. 6037 */ 6038 update_rq_clock(rq); 6039 rq_clock_start_loop_update(rq); 6040 6041 /* 6042 * Since we hold rq lock we're safe from concurrent manipulation of 6043 * the CSD list. However, this RCU critical section annotates the 6044 * fact that we pair with sched_free_group_rcu(), so that we cannot 6045 * race with group being freed in the window between removing it 6046 * from the list and advancing to the next entry in the list. 6047 */ 6048 rcu_read_lock(); 6049 6050 list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list, 6051 throttled_csd_list) { 6052 list_del_init(&cursor->throttled_csd_list); 6053 6054 if (cfs_rq_throttled(cursor)) 6055 unthrottle_cfs_rq(cursor); 6056 } 6057 6058 rcu_read_unlock(); 6059 6060 rq_clock_stop_loop_update(rq); 6061 rq_unlock(rq, &rf); 6062 } 6063 6064 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 6065 { 6066 struct rq *rq = rq_of(cfs_rq); 6067 bool first; 6068 6069 if (rq == this_rq()) { 6070 unthrottle_cfs_rq(cfs_rq); 6071 return; 6072 } 6073 6074 /* Already enqueued */ 6075 if (WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_csd_list))) 6076 return; 6077 6078 first = list_empty(&rq->cfsb_csd_list); 6079 list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list); 6080 if (first) 6081 smp_call_function_single_async(cpu_of(rq), &rq->cfsb_csd); 6082 } 6083 6084 static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 6085 { 6086 lockdep_assert_rq_held(rq_of(cfs_rq)); 6087 6088 if (WARN_ON_ONCE(!cfs_rq_throttled(cfs_rq) || 6089 cfs_rq->runtime_remaining <= 0)) 6090 return; 6091 6092 __unthrottle_cfs_rq_async(cfs_rq); 6093 } 6094 6095 static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) 6096 { 6097 int this_cpu = smp_processor_id(); 6098 u64 runtime, remaining = 1; 6099 bool throttled = false; 6100 struct cfs_rq *cfs_rq, *tmp; 6101 struct rq_flags rf; 6102 struct rq *rq; 6103 LIST_HEAD(local_unthrottle); 6104 6105 rcu_read_lock(); 6106 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, 6107 throttled_list) { 6108 rq = rq_of(cfs_rq); 6109 6110 if (!remaining) { 6111 throttled = true; 6112 break; 6113 } 6114 6115 rq_lock_irqsave(rq, &rf); 6116 if (!cfs_rq_throttled(cfs_rq)) 6117 goto next; 6118 6119 /* Already queued for async unthrottle */ 6120 if (!list_empty(&cfs_rq->throttled_csd_list)) 6121 goto next; 6122 6123 /* By the above checks, this should never be true */ 6124 WARN_ON_ONCE(cfs_rq->runtime_remaining > 0); 6125 6126 raw_spin_lock(&cfs_b->lock); 6127 runtime = -cfs_rq->runtime_remaining + 1; 6128 if (runtime > cfs_b->runtime) 6129 runtime = cfs_b->runtime; 6130 cfs_b->runtime -= runtime; 6131 remaining = cfs_b->runtime; 6132 raw_spin_unlock(&cfs_b->lock); 6133 6134 cfs_rq->runtime_remaining += runtime; 6135 6136 /* we check whether we're throttled above */ 6137 if (cfs_rq->runtime_remaining > 0) { 6138 if (cpu_of(rq) != this_cpu) { 6139 unthrottle_cfs_rq_async(cfs_rq); 6140 } else { 6141 /* 6142 * We currently only expect to be unthrottling 6143 * a single cfs_rq locally. 6144 */ 6145 WARN_ON_ONCE(!list_empty(&local_unthrottle)); 6146 list_add_tail(&cfs_rq->throttled_csd_list, 6147 &local_unthrottle); 6148 } 6149 } else { 6150 throttled = true; 6151 } 6152 6153 next: 6154 rq_unlock_irqrestore(rq, &rf); 6155 } 6156 6157 list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle, 6158 throttled_csd_list) { 6159 struct rq *rq = rq_of(cfs_rq); 6160 6161 rq_lock_irqsave(rq, &rf); 6162 6163 list_del_init(&cfs_rq->throttled_csd_list); 6164 6165 if (cfs_rq_throttled(cfs_rq)) 6166 unthrottle_cfs_rq(cfs_rq); 6167 6168 rq_unlock_irqrestore(rq, &rf); 6169 } 6170 WARN_ON_ONCE(!list_empty(&local_unthrottle)); 6171 6172 rcu_read_unlock(); 6173 6174 return throttled; 6175 } 6176 6177 /* 6178 * Responsible for refilling a task_group's bandwidth and unthrottling its 6179 * cfs_rqs as appropriate. If there has been no activity within the last 6180 * period the timer is deactivated until scheduling resumes; cfs_b->idle is 6181 * used to track this state. 6182 */ 6183 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) 6184 __must_hold(&cfs_b->lock) 6185 { 6186 int throttled; 6187 6188 /* no need to continue the timer with no bandwidth constraint */ 6189 if (cfs_b->quota == RUNTIME_INF) 6190 goto out_deactivate; 6191 6192 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 6193 cfs_b->nr_periods += overrun; 6194 6195 /* Refill extra burst quota even if cfs_b->idle */ 6196 __refill_cfs_bandwidth_runtime(cfs_b); 6197 6198 /* 6199 * idle depends on !throttled (for the case of a large deficit), and if 6200 * we're going inactive then everything else can be deferred 6201 */ 6202 if (cfs_b->idle && !throttled) 6203 goto out_deactivate; 6204 6205 if (!throttled) { 6206 /* mark as potentially idle for the upcoming period */ 6207 cfs_b->idle = 1; 6208 return 0; 6209 } 6210 6211 /* account preceding periods in which throttling occurred */ 6212 cfs_b->nr_throttled += overrun; 6213 6214 /* 6215 * This check is repeated as we release cfs_b->lock while we unthrottle. 6216 */ 6217 while (throttled && cfs_b->runtime > 0) { 6218 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6219 /* we can't nest cfs_b->lock while distributing bandwidth */ 6220 throttled = distribute_cfs_runtime(cfs_b); 6221 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6222 } 6223 6224 /* 6225 * While we are ensured activity in the period following an 6226 * unthrottle, this also covers the case in which the new bandwidth is 6227 * insufficient to cover the existing bandwidth deficit. (Forcing the 6228 * timer to remain active while there are any throttled entities.) 6229 */ 6230 cfs_b->idle = 0; 6231 6232 return 0; 6233 6234 out_deactivate: 6235 return 1; 6236 } 6237 6238 /* a cfs_rq won't donate quota below this amount */ 6239 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; 6240 /* minimum remaining period time to redistribute slack quota */ 6241 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; 6242 /* how long we wait to gather additional slack before distributing */ 6243 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; 6244 6245 /* 6246 * Are we near the end of the current quota period? 6247 * 6248 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the 6249 * hrtimer base being cleared by hrtimer_start. In the case of 6250 * migrate_hrtimers, base is never cleared, so we are fine. 6251 */ 6252 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) 6253 { 6254 struct hrtimer *refresh_timer = &cfs_b->period_timer; 6255 s64 remaining; 6256 6257 /* if the call-back is running a quota refresh is already occurring */ 6258 if (hrtimer_callback_running(refresh_timer)) 6259 return 1; 6260 6261 /* is a quota refresh about to occur? */ 6262 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); 6263 if (remaining < (s64)min_expire) 6264 return 1; 6265 6266 return 0; 6267 } 6268 6269 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) 6270 { 6271 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; 6272 6273 /* if there's a quota refresh soon don't bother with slack */ 6274 if (runtime_refresh_within(cfs_b, min_left)) 6275 return; 6276 6277 /* don't push forwards an existing deferred unthrottle */ 6278 if (cfs_b->slack_started) 6279 return; 6280 cfs_b->slack_started = true; 6281 6282 hrtimer_start(&cfs_b->slack_timer, 6283 ns_to_ktime(cfs_bandwidth_slack_period), 6284 HRTIMER_MODE_REL); 6285 } 6286 6287 /* we know any runtime found here is valid as update_curr() precedes return */ 6288 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6289 { 6290 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 6291 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; 6292 6293 if (slack_runtime <= 0) 6294 return; 6295 6296 raw_spin_lock(&cfs_b->lock); 6297 if (cfs_b->quota != RUNTIME_INF) { 6298 cfs_b->runtime += slack_runtime; 6299 6300 /* we are under rq->lock, defer unthrottling using a timer */ 6301 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && 6302 !list_empty(&cfs_b->throttled_cfs_rq)) 6303 start_cfs_slack_bandwidth(cfs_b); 6304 } 6305 raw_spin_unlock(&cfs_b->lock); 6306 6307 /* even if it's not valid for return we don't want to try again */ 6308 cfs_rq->runtime_remaining -= slack_runtime; 6309 } 6310 6311 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6312 { 6313 if (!cfs_bandwidth_used()) 6314 return; 6315 6316 if (!cfs_rq->runtime_enabled || cfs_rq->nr_queued) 6317 return; 6318 6319 __return_cfs_rq_runtime(cfs_rq); 6320 } 6321 6322 /* 6323 * This is done with a timer (instead of inline with bandwidth return) since 6324 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. 6325 */ 6326 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 6327 { 6328 u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); 6329 unsigned long flags; 6330 6331 /* confirm we're still not at a refresh boundary */ 6332 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6333 cfs_b->slack_started = false; 6334 6335 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { 6336 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6337 return; 6338 } 6339 6340 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) 6341 runtime = cfs_b->runtime; 6342 6343 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6344 6345 if (!runtime) 6346 return; 6347 6348 distribute_cfs_runtime(cfs_b); 6349 } 6350 6351 /* 6352 * When a group wakes up we want to make sure that its quota is not already 6353 * expired/exceeded, otherwise it may be allowed to steal additional ticks of 6354 * runtime as update_curr() throttling can not trigger until it's on-rq. 6355 */ 6356 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) 6357 { 6358 if (!cfs_bandwidth_used()) 6359 return; 6360 6361 /* an active group must be handled by the update_curr()->put() path */ 6362 if (!cfs_rq->runtime_enabled || cfs_rq->curr) 6363 return; 6364 6365 /* ensure the group is not already throttled */ 6366 if (cfs_rq_throttled(cfs_rq)) 6367 return; 6368 6369 /* update runtime allocation */ 6370 account_cfs_rq_runtime(cfs_rq, 0); 6371 if (cfs_rq->runtime_remaining <= 0) 6372 throttle_cfs_rq(cfs_rq); 6373 } 6374 6375 static void sync_throttle(struct task_group *tg, int cpu) 6376 { 6377 struct cfs_rq *pcfs_rq, *cfs_rq; 6378 6379 if (!cfs_bandwidth_used()) 6380 return; 6381 6382 if (!tg->parent) 6383 return; 6384 6385 cfs_rq = tg->cfs_rq[cpu]; 6386 pcfs_rq = tg->parent->cfs_rq[cpu]; 6387 6388 cfs_rq->throttle_count = pcfs_rq->throttle_count; 6389 cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu)); 6390 6391 /* 6392 * It is not enough to sync the "pelt_clock_throttled" indicator 6393 * with the parent cfs_rq when the hierarchy is not queued. 6394 * Always join a throttled hierarchy with PELT clock throttled 6395 * and leaf it to the first enqueue, or distribution to 6396 * unthrottle the PELT clock. 6397 */ 6398 if (cfs_rq->throttle_count) 6399 cfs_rq->pelt_clock_throttled = 1; 6400 } 6401 6402 /* conditionally throttle active cfs_rq's from put_prev_entity() */ 6403 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6404 { 6405 if (!cfs_bandwidth_used()) 6406 return false; 6407 6408 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) 6409 return false; 6410 6411 /* 6412 * it's possible for a throttled entity to be forced into a running 6413 * state (e.g. set_curr_task), in this case we're finished. 6414 */ 6415 if (cfs_rq_throttled(cfs_rq)) 6416 return true; 6417 6418 return throttle_cfs_rq(cfs_rq); 6419 } 6420 6421 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) 6422 { 6423 struct cfs_bandwidth *cfs_b = 6424 container_of(timer, struct cfs_bandwidth, slack_timer); 6425 6426 do_sched_cfs_slack_timer(cfs_b); 6427 6428 return HRTIMER_NORESTART; 6429 } 6430 6431 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) 6432 { 6433 struct cfs_bandwidth *cfs_b = 6434 container_of(timer, struct cfs_bandwidth, period_timer); 6435 unsigned long flags; 6436 int overrun; 6437 int idle = 0; 6438 int count = 0; 6439 6440 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6441 for (;;) { 6442 overrun = hrtimer_forward_now(timer, cfs_b->period); 6443 if (!overrun) 6444 break; 6445 6446 idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); 6447 6448 if (++count > 3) { 6449 u64 new, old = ktime_to_ns(cfs_b->period); 6450 6451 /* 6452 * Grow period by a factor of 2 to avoid losing precision. 6453 * Precision loss in the quota/period ratio can cause __cfs_schedulable 6454 * to fail. 6455 */ 6456 new = old * 2; 6457 if (new < max_bw_quota_period_us * NSEC_PER_USEC) { 6458 cfs_b->period = ns_to_ktime(new); 6459 cfs_b->quota *= 2; 6460 cfs_b->burst *= 2; 6461 6462 pr_warn_ratelimited( 6463 "cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6464 smp_processor_id(), 6465 div_u64(new, NSEC_PER_USEC), 6466 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6467 } else { 6468 pr_warn_ratelimited( 6469 "cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6470 smp_processor_id(), 6471 div_u64(old, NSEC_PER_USEC), 6472 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6473 } 6474 6475 /* reset count so we don't come right back in here */ 6476 count = 0; 6477 } 6478 } 6479 if (idle) 6480 cfs_b->period_active = 0; 6481 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6482 6483 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 6484 } 6485 6486 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) 6487 { 6488 raw_spin_lock_init(&cfs_b->lock); 6489 cfs_b->runtime = 0; 6490 cfs_b->quota = RUNTIME_INF; 6491 cfs_b->period = us_to_ktime(default_bw_period_us()); 6492 cfs_b->burst = 0; 6493 cfs_b->hierarchical_quota = parent ? parent->hierarchical_quota : RUNTIME_INF; 6494 6495 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); 6496 hrtimer_setup(&cfs_b->period_timer, sched_cfs_period_timer, CLOCK_MONOTONIC, 6497 HRTIMER_MODE_ABS_PINNED); 6498 6499 /* Add a random offset so that timers interleave */ 6500 hrtimer_set_expires(&cfs_b->period_timer, 6501 get_random_u32_below(cfs_b->period)); 6502 hrtimer_setup(&cfs_b->slack_timer, sched_cfs_slack_timer, CLOCK_MONOTONIC, 6503 HRTIMER_MODE_REL); 6504 cfs_b->slack_started = false; 6505 } 6506 6507 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6508 { 6509 cfs_rq->runtime_enabled = 0; 6510 INIT_LIST_HEAD(&cfs_rq->throttled_list); 6511 INIT_LIST_HEAD(&cfs_rq->throttled_csd_list); 6512 INIT_LIST_HEAD(&cfs_rq->throttled_limbo_list); 6513 } 6514 6515 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6516 { 6517 lockdep_assert_held(&cfs_b->lock); 6518 6519 if (cfs_b->period_active) 6520 return; 6521 6522 cfs_b->period_active = 1; 6523 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); 6524 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); 6525 } 6526 6527 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6528 { 6529 int __maybe_unused i; 6530 6531 /* init_cfs_bandwidth() was not called */ 6532 if (!cfs_b->throttled_cfs_rq.next) 6533 return; 6534 6535 hrtimer_cancel(&cfs_b->period_timer); 6536 hrtimer_cancel(&cfs_b->slack_timer); 6537 6538 /* 6539 * It is possible that we still have some cfs_rq's pending on a CSD 6540 * list, though this race is very rare. In order for this to occur, we 6541 * must have raced with the last task leaving the group while there 6542 * exist throttled cfs_rq(s), and the period_timer must have queued the 6543 * CSD item but the remote cpu has not yet processed it. To handle this, 6544 * we can simply flush all pending CSD work inline here. We're 6545 * guaranteed at this point that no additional cfs_rq of this group can 6546 * join a CSD list. 6547 */ 6548 for_each_possible_cpu(i) { 6549 struct rq *rq = cpu_rq(i); 6550 unsigned long flags; 6551 6552 if (list_empty(&rq->cfsb_csd_list)) 6553 continue; 6554 6555 local_irq_save(flags); 6556 __cfsb_csd_unthrottle(rq); 6557 local_irq_restore(flags); 6558 } 6559 } 6560 6561 /* 6562 * Both these CPU hotplug callbacks race against unregister_fair_sched_group() 6563 * 6564 * The race is harmless, since modifying bandwidth settings of unhooked group 6565 * bits doesn't do much. 6566 */ 6567 6568 /* cpu online callback */ 6569 static void __maybe_unused update_runtime_enabled(struct rq *rq) 6570 { 6571 struct task_group *tg; 6572 6573 lockdep_assert_rq_held(rq); 6574 6575 rcu_read_lock(); 6576 list_for_each_entry_rcu(tg, &task_groups, list) { 6577 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 6578 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6579 6580 raw_spin_lock(&cfs_b->lock); 6581 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; 6582 raw_spin_unlock(&cfs_b->lock); 6583 } 6584 rcu_read_unlock(); 6585 } 6586 6587 /* cpu offline callback */ 6588 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) 6589 { 6590 struct task_group *tg; 6591 6592 lockdep_assert_rq_held(rq); 6593 6594 // Do not unthrottle for an active CPU 6595 if (cpumask_test_cpu(cpu_of(rq), cpu_active_mask)) 6596 return; 6597 6598 /* 6599 * The rq clock has already been updated in the 6600 * set_rq_offline(), so we should skip updating 6601 * the rq clock again in unthrottle_cfs_rq(). 6602 */ 6603 rq_clock_start_loop_update(rq); 6604 6605 rcu_read_lock(); 6606 list_for_each_entry_rcu(tg, &task_groups, list) { 6607 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6608 6609 if (!cfs_rq->runtime_enabled) 6610 continue; 6611 6612 /* 6613 * Offline rq is schedulable till CPU is completely disabled 6614 * in take_cpu_down(), so we prevent new cfs throttling here. 6615 */ 6616 cfs_rq->runtime_enabled = 0; 6617 6618 if (!cfs_rq_throttled(cfs_rq)) 6619 continue; 6620 6621 /* 6622 * clock_task is not advancing so we just need to make sure 6623 * there's some valid quota amount 6624 */ 6625 cfs_rq->runtime_remaining = 1; 6626 unthrottle_cfs_rq(cfs_rq); 6627 } 6628 rcu_read_unlock(); 6629 6630 rq_clock_stop_loop_update(rq); 6631 } 6632 6633 bool cfs_task_bw_constrained(struct task_struct *p) 6634 { 6635 struct cfs_rq *cfs_rq = task_cfs_rq(p); 6636 6637 if (!cfs_bandwidth_used()) 6638 return false; 6639 6640 if (cfs_rq->runtime_enabled || 6641 tg_cfs_bandwidth(cfs_rq->tg)->hierarchical_quota != RUNTIME_INF) 6642 return true; 6643 6644 return false; 6645 } 6646 6647 #ifdef CONFIG_NO_HZ_FULL 6648 /* called from pick_next_task_fair() */ 6649 static void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) 6650 { 6651 int cpu = cpu_of(rq); 6652 6653 if (!cfs_bandwidth_used()) 6654 return; 6655 6656 if (!tick_nohz_full_cpu(cpu)) 6657 return; 6658 6659 if (rq->nr_running != 1) 6660 return; 6661 6662 /* 6663 * We know there is only one task runnable and we've just picked it. The 6664 * normal enqueue path will have cleared TICK_DEP_BIT_SCHED if we will 6665 * be otherwise able to stop the tick. Just need to check if we are using 6666 * bandwidth control. 6667 */ 6668 if (cfs_task_bw_constrained(p)) 6669 tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED); 6670 } 6671 #endif /* CONFIG_NO_HZ_FULL */ 6672 6673 #else /* !CONFIG_CFS_BANDWIDTH: */ 6674 6675 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} 6676 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } 6677 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 6678 static inline void sync_throttle(struct task_group *tg, int cpu) {} 6679 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6680 static void task_throttle_setup_work(struct task_struct *p) {} 6681 static bool task_is_throttled(struct task_struct *p) { return false; } 6682 static void dequeue_throttled_task(struct task_struct *p, int flags) {} 6683 static bool enqueue_throttled_task(struct task_struct *p) { return false; } 6684 static void record_throttle_clock(struct cfs_rq *cfs_rq) {} 6685 6686 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 6687 { 6688 return 0; 6689 } 6690 6691 static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq) 6692 { 6693 return false; 6694 } 6695 6696 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 6697 { 6698 return 0; 6699 } 6700 6701 static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu) 6702 { 6703 return 0; 6704 } 6705 6706 #ifdef CONFIG_FAIR_GROUP_SCHED 6707 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) {} 6708 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6709 #endif 6710 6711 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 6712 { 6713 return NULL; 6714 } 6715 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 6716 static inline void update_runtime_enabled(struct rq *rq) {} 6717 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} 6718 #ifdef CONFIG_CGROUP_SCHED 6719 bool cfs_task_bw_constrained(struct task_struct *p) 6720 { 6721 return false; 6722 } 6723 #endif 6724 #endif /* !CONFIG_CFS_BANDWIDTH */ 6725 6726 #if !defined(CONFIG_CFS_BANDWIDTH) || !defined(CONFIG_NO_HZ_FULL) 6727 static inline void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) {} 6728 #endif 6729 6730 /************************************************** 6731 * CFS operations on tasks: 6732 */ 6733 6734 #ifdef CONFIG_SCHED_HRTICK 6735 static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 6736 { 6737 struct sched_entity *se = &p->se; 6738 6739 WARN_ON_ONCE(task_rq(p) != rq); 6740 6741 if (rq->cfs.h_nr_queued > 1) { 6742 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; 6743 u64 slice = se->slice; 6744 s64 delta = slice - ran; 6745 6746 if (delta < 0) { 6747 if (task_current_donor(rq, p)) 6748 resched_curr(rq); 6749 return; 6750 } 6751 hrtick_start(rq, delta); 6752 } 6753 } 6754 6755 /* 6756 * called from enqueue/dequeue and updates the hrtick when the 6757 * current task is from our class and nr_running is low enough 6758 * to matter. 6759 */ 6760 static void hrtick_update(struct rq *rq) 6761 { 6762 struct task_struct *donor = rq->donor; 6763 6764 if (!hrtick_enabled_fair(rq) || donor->sched_class != &fair_sched_class) 6765 return; 6766 6767 hrtick_start_fair(rq, donor); 6768 } 6769 #else /* !CONFIG_SCHED_HRTICK: */ 6770 static inline void 6771 hrtick_start_fair(struct rq *rq, struct task_struct *p) 6772 { 6773 } 6774 6775 static inline void hrtick_update(struct rq *rq) 6776 { 6777 } 6778 #endif /* !CONFIG_SCHED_HRTICK */ 6779 6780 static inline bool cpu_overutilized(int cpu) 6781 { 6782 unsigned long rq_util_min, rq_util_max; 6783 6784 if (!sched_energy_enabled()) 6785 return false; 6786 6787 rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN); 6788 rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX); 6789 6790 /* Return true only if the utilization doesn't fit CPU's capacity */ 6791 return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu); 6792 } 6793 6794 /* 6795 * overutilized value make sense only if EAS is enabled 6796 */ 6797 static inline bool is_rd_overutilized(struct root_domain *rd) 6798 { 6799 return !sched_energy_enabled() || READ_ONCE(rd->overutilized); 6800 } 6801 6802 static inline void set_rd_overutilized(struct root_domain *rd, bool flag) 6803 { 6804 if (!sched_energy_enabled()) 6805 return; 6806 6807 WRITE_ONCE(rd->overutilized, flag); 6808 trace_sched_overutilized_tp(rd, flag); 6809 } 6810 6811 static inline void check_update_overutilized_status(struct rq *rq) 6812 { 6813 /* 6814 * overutilized field is used for load balancing decisions only 6815 * if energy aware scheduler is being used 6816 */ 6817 6818 if (!is_rd_overutilized(rq->rd) && cpu_overutilized(rq->cpu)) 6819 set_rd_overutilized(rq->rd, 1); 6820 } 6821 6822 /* Runqueue only has SCHED_IDLE tasks enqueued */ 6823 static int sched_idle_rq(struct rq *rq) 6824 { 6825 return unlikely(rq->nr_running == rq->cfs.h_nr_idle && 6826 rq->nr_running); 6827 } 6828 6829 static int sched_idle_cpu(int cpu) 6830 { 6831 return sched_idle_rq(cpu_rq(cpu)); 6832 } 6833 6834 static void 6835 requeue_delayed_entity(struct sched_entity *se) 6836 { 6837 struct cfs_rq *cfs_rq = cfs_rq_of(se); 6838 6839 /* 6840 * se->sched_delayed should imply: se->on_rq == 1. 6841 * Because a delayed entity is one that is still on 6842 * the runqueue competing until elegibility. 6843 */ 6844 WARN_ON_ONCE(!se->sched_delayed); 6845 WARN_ON_ONCE(!se->on_rq); 6846 6847 if (sched_feat(DELAY_ZERO)) { 6848 update_entity_lag(cfs_rq, se); 6849 if (se->vlag > 0) { 6850 cfs_rq->nr_queued--; 6851 if (se != cfs_rq->curr) 6852 __dequeue_entity(cfs_rq, se); 6853 se->vlag = 0; 6854 place_entity(cfs_rq, se, 0); 6855 if (se != cfs_rq->curr) 6856 __enqueue_entity(cfs_rq, se); 6857 cfs_rq->nr_queued++; 6858 } 6859 } 6860 6861 update_load_avg(cfs_rq, se, 0); 6862 clear_delayed(se); 6863 } 6864 6865 /* 6866 * The enqueue_task method is called before nr_running is 6867 * increased. Here we update the fair scheduling stats and 6868 * then put the task into the rbtree: 6869 */ 6870 static void 6871 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 6872 { 6873 struct cfs_rq *cfs_rq; 6874 struct sched_entity *se = &p->se; 6875 int h_nr_idle = task_has_idle_policy(p); 6876 int h_nr_runnable = 1; 6877 int task_new = !(flags & ENQUEUE_WAKEUP); 6878 int rq_h_nr_queued = rq->cfs.h_nr_queued; 6879 u64 slice = 0; 6880 6881 if (task_is_throttled(p) && enqueue_throttled_task(p)) 6882 return; 6883 6884 /* 6885 * The code below (indirectly) updates schedutil which looks at 6886 * the cfs_rq utilization to select a frequency. 6887 * Let's add the task's estimated utilization to the cfs_rq's 6888 * estimated utilization, before we update schedutil. 6889 */ 6890 if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED)) 6891 util_est_enqueue(&rq->cfs, p); 6892 6893 if (flags & ENQUEUE_DELAYED) { 6894 requeue_delayed_entity(se); 6895 return; 6896 } 6897 6898 /* 6899 * If in_iowait is set, the code below may not trigger any cpufreq 6900 * utilization updates, so do it here explicitly with the IOWAIT flag 6901 * passed. 6902 */ 6903 if (p->in_iowait) 6904 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); 6905 6906 if (task_new && se->sched_delayed) 6907 h_nr_runnable = 0; 6908 6909 for_each_sched_entity(se) { 6910 if (se->on_rq) { 6911 if (se->sched_delayed) 6912 requeue_delayed_entity(se); 6913 break; 6914 } 6915 cfs_rq = cfs_rq_of(se); 6916 6917 /* 6918 * Basically set the slice of group entries to the min_slice of 6919 * their respective cfs_rq. This ensures the group can service 6920 * its entities in the desired time-frame. 6921 */ 6922 if (slice) { 6923 se->slice = slice; 6924 se->custom_slice = 1; 6925 } 6926 enqueue_entity(cfs_rq, se, flags); 6927 slice = cfs_rq_min_slice(cfs_rq); 6928 6929 cfs_rq->h_nr_runnable += h_nr_runnable; 6930 cfs_rq->h_nr_queued++; 6931 cfs_rq->h_nr_idle += h_nr_idle; 6932 6933 if (cfs_rq_is_idle(cfs_rq)) 6934 h_nr_idle = 1; 6935 6936 flags = ENQUEUE_WAKEUP; 6937 } 6938 6939 for_each_sched_entity(se) { 6940 cfs_rq = cfs_rq_of(se); 6941 6942 update_load_avg(cfs_rq, se, UPDATE_TG); 6943 se_update_runnable(se); 6944 update_cfs_group(se); 6945 6946 se->slice = slice; 6947 if (se != cfs_rq->curr) 6948 min_vruntime_cb_propagate(&se->run_node, NULL); 6949 slice = cfs_rq_min_slice(cfs_rq); 6950 6951 cfs_rq->h_nr_runnable += h_nr_runnable; 6952 cfs_rq->h_nr_queued++; 6953 cfs_rq->h_nr_idle += h_nr_idle; 6954 6955 if (cfs_rq_is_idle(cfs_rq)) 6956 h_nr_idle = 1; 6957 } 6958 6959 if (!rq_h_nr_queued && rq->cfs.h_nr_queued) 6960 dl_server_start(&rq->fair_server); 6961 6962 /* At this point se is NULL and we are at root level*/ 6963 add_nr_running(rq, 1); 6964 6965 /* 6966 * Since new tasks are assigned an initial util_avg equal to 6967 * half of the spare capacity of their CPU, tiny tasks have the 6968 * ability to cross the overutilized threshold, which will 6969 * result in the load balancer ruining all the task placement 6970 * done by EAS. As a way to mitigate that effect, do not account 6971 * for the first enqueue operation of new tasks during the 6972 * overutilized flag detection. 6973 * 6974 * A better way of solving this problem would be to wait for 6975 * the PELT signals of tasks to converge before taking them 6976 * into account, but that is not straightforward to implement, 6977 * and the following generally works well enough in practice. 6978 */ 6979 if (!task_new) 6980 check_update_overutilized_status(rq); 6981 6982 assert_list_leaf_cfs_rq(rq); 6983 6984 hrtick_update(rq); 6985 } 6986 6987 /* 6988 * Basically dequeue_task_fair(), except it can deal with dequeue_entity() 6989 * failing half-way through and resume the dequeue later. 6990 * 6991 * Returns: 6992 * -1 - dequeue delayed 6993 * 0 - dequeue throttled 6994 * 1 - dequeue complete 6995 */ 6996 static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) 6997 { 6998 bool was_sched_idle = sched_idle_rq(rq); 6999 bool task_sleep = flags & DEQUEUE_SLEEP; 7000 bool task_delayed = flags & DEQUEUE_DELAYED; 7001 bool task_throttled = flags & DEQUEUE_THROTTLE; 7002 struct task_struct *p = NULL; 7003 int h_nr_idle = 0; 7004 int h_nr_queued = 0; 7005 int h_nr_runnable = 0; 7006 struct cfs_rq *cfs_rq; 7007 u64 slice = 0; 7008 7009 if (entity_is_task(se)) { 7010 p = task_of(se); 7011 h_nr_queued = 1; 7012 h_nr_idle = task_has_idle_policy(p); 7013 if (task_sleep || task_delayed || !se->sched_delayed) 7014 h_nr_runnable = 1; 7015 } 7016 7017 for_each_sched_entity(se) { 7018 cfs_rq = cfs_rq_of(se); 7019 7020 if (!dequeue_entity(cfs_rq, se, flags)) { 7021 if (p && &p->se == se) 7022 return -1; 7023 7024 slice = cfs_rq_min_slice(cfs_rq); 7025 break; 7026 } 7027 7028 cfs_rq->h_nr_runnable -= h_nr_runnable; 7029 cfs_rq->h_nr_queued -= h_nr_queued; 7030 cfs_rq->h_nr_idle -= h_nr_idle; 7031 7032 if (cfs_rq_is_idle(cfs_rq)) 7033 h_nr_idle = h_nr_queued; 7034 7035 if (throttled_hierarchy(cfs_rq) && task_throttled) 7036 record_throttle_clock(cfs_rq); 7037 7038 /* Don't dequeue parent if it has other entities besides us */ 7039 if (cfs_rq->load.weight) { 7040 slice = cfs_rq_min_slice(cfs_rq); 7041 7042 /* Avoid re-evaluating load for this entity: */ 7043 se = parent_entity(se); 7044 /* 7045 * Bias pick_next to pick a task from this cfs_rq, as 7046 * p is sleeping when it is within its sched_slice. 7047 */ 7048 if (task_sleep && se) 7049 set_next_buddy(se); 7050 break; 7051 } 7052 flags |= DEQUEUE_SLEEP; 7053 flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL); 7054 } 7055 7056 for_each_sched_entity(se) { 7057 cfs_rq = cfs_rq_of(se); 7058 7059 update_load_avg(cfs_rq, se, UPDATE_TG); 7060 se_update_runnable(se); 7061 update_cfs_group(se); 7062 7063 se->slice = slice; 7064 if (se != cfs_rq->curr) 7065 min_vruntime_cb_propagate(&se->run_node, NULL); 7066 slice = cfs_rq_min_slice(cfs_rq); 7067 7068 cfs_rq->h_nr_runnable -= h_nr_runnable; 7069 cfs_rq->h_nr_queued -= h_nr_queued; 7070 cfs_rq->h_nr_idle -= h_nr_idle; 7071 7072 if (cfs_rq_is_idle(cfs_rq)) 7073 h_nr_idle = h_nr_queued; 7074 7075 if (throttled_hierarchy(cfs_rq) && task_throttled) 7076 record_throttle_clock(cfs_rq); 7077 } 7078 7079 sub_nr_running(rq, h_nr_queued); 7080 7081 /* balance early to pull high priority tasks */ 7082 if (unlikely(!was_sched_idle && sched_idle_rq(rq))) 7083 rq->next_balance = jiffies; 7084 7085 if (p && task_delayed) { 7086 WARN_ON_ONCE(!task_sleep); 7087 WARN_ON_ONCE(p->on_rq != 1); 7088 7089 /* Fix-up what dequeue_task_fair() skipped */ 7090 hrtick_update(rq); 7091 7092 /* 7093 * Fix-up what block_task() skipped. 7094 * 7095 * Must be last, @p might not be valid after this. 7096 */ 7097 __block_task(rq, p); 7098 } 7099 7100 return 1; 7101 } 7102 7103 /* 7104 * The dequeue_task method is called before nr_running is 7105 * decreased. We remove the task from the rbtree and 7106 * update the fair scheduling stats: 7107 */ 7108 static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 7109 { 7110 if (task_is_throttled(p)) { 7111 dequeue_throttled_task(p, flags); 7112 return true; 7113 } 7114 7115 if (!p->se.sched_delayed) 7116 util_est_dequeue(&rq->cfs, p); 7117 7118 util_est_update(&rq->cfs, p, flags & DEQUEUE_SLEEP); 7119 if (dequeue_entities(rq, &p->se, flags) < 0) 7120 return false; 7121 7122 /* 7123 * Must not reference @p after dequeue_entities(DEQUEUE_DELAYED). 7124 */ 7125 7126 hrtick_update(rq); 7127 return true; 7128 } 7129 7130 static inline unsigned int cfs_h_nr_delayed(struct rq *rq) 7131 { 7132 return (rq->cfs.h_nr_queued - rq->cfs.h_nr_runnable); 7133 } 7134 7135 /* Working cpumask for: sched_balance_rq(), sched_balance_newidle(). */ 7136 static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); 7137 static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask); 7138 static DEFINE_PER_CPU(cpumask_var_t, should_we_balance_tmpmask); 7139 7140 #ifdef CONFIG_NO_HZ_COMMON 7141 7142 static struct { 7143 cpumask_var_t idle_cpus_mask; 7144 int has_blocked_load; /* Idle CPUS has blocked load */ 7145 int needs_update; /* Newly idle CPUs need their next_balance collated */ 7146 unsigned long next_balance; /* in jiffy units */ 7147 unsigned long next_blocked; /* Next update of blocked load in jiffies */ 7148 } nohz ____cacheline_aligned; 7149 7150 #endif /* CONFIG_NO_HZ_COMMON */ 7151 7152 static unsigned long cpu_load(struct rq *rq) 7153 { 7154 return cfs_rq_load_avg(&rq->cfs); 7155 } 7156 7157 /* 7158 * cpu_load_without - compute CPU load without any contributions from *p 7159 * @cpu: the CPU which load is requested 7160 * @p: the task which load should be discounted 7161 * 7162 * The load of a CPU is defined by the load of tasks currently enqueued on that 7163 * CPU as well as tasks which are currently sleeping after an execution on that 7164 * CPU. 7165 * 7166 * This method returns the load of the specified CPU by discounting the load of 7167 * the specified task, whenever the task is currently contributing to the CPU 7168 * load. 7169 */ 7170 static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p) 7171 { 7172 struct cfs_rq *cfs_rq; 7173 unsigned int load; 7174 7175 /* Task has no contribution or is new */ 7176 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 7177 return cpu_load(rq); 7178 7179 cfs_rq = &rq->cfs; 7180 load = READ_ONCE(cfs_rq->avg.load_avg); 7181 7182 /* Discount task's util from CPU's util */ 7183 lsub_positive(&load, task_h_load(p)); 7184 7185 return load; 7186 } 7187 7188 static unsigned long cpu_runnable(struct rq *rq) 7189 { 7190 return cfs_rq_runnable_avg(&rq->cfs); 7191 } 7192 7193 static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p) 7194 { 7195 struct cfs_rq *cfs_rq; 7196 unsigned int runnable; 7197 7198 /* Task has no contribution or is new */ 7199 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 7200 return cpu_runnable(rq); 7201 7202 cfs_rq = &rq->cfs; 7203 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 7204 7205 /* Discount task's runnable from CPU's runnable */ 7206 lsub_positive(&runnable, p->se.avg.runnable_avg); 7207 7208 return runnable; 7209 } 7210 7211 static unsigned long capacity_of(int cpu) 7212 { 7213 return cpu_rq(cpu)->cpu_capacity; 7214 } 7215 7216 static void record_wakee(struct task_struct *p) 7217 { 7218 /* 7219 * Only decay a single time; tasks that have less then 1 wakeup per 7220 * jiffy will not have built up many flips. 7221 */ 7222 if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 7223 current->wakee_flips >>= 1; 7224 current->wakee_flip_decay_ts = jiffies; 7225 } 7226 7227 if (current->last_wakee != p) { 7228 current->last_wakee = p; 7229 current->wakee_flips++; 7230 } 7231 } 7232 7233 /* 7234 * Detect M:N waker/wakee relationships via a switching-frequency heuristic. 7235 * 7236 * A waker of many should wake a different task than the one last awakened 7237 * at a frequency roughly N times higher than one of its wakees. 7238 * 7239 * In order to determine whether we should let the load spread vs consolidating 7240 * to shared cache, we look for a minimum 'flip' frequency of llc_size in one 7241 * partner, and a factor of lls_size higher frequency in the other. 7242 * 7243 * With both conditions met, we can be relatively sure that the relationship is 7244 * non-monogamous, with partner count exceeding socket size. 7245 * 7246 * Waker/wakee being client/server, worker/dispatcher, interrupt source or 7247 * whatever is irrelevant, spread criteria is apparent partner count exceeds 7248 * socket size. 7249 */ 7250 static int wake_wide(struct task_struct *p) 7251 { 7252 unsigned int master = current->wakee_flips; 7253 unsigned int slave = p->wakee_flips; 7254 int factor = __this_cpu_read(sd_llc_size); 7255 7256 if (master < slave) 7257 swap(master, slave); 7258 if (slave < factor || master < slave * factor) 7259 return 0; 7260 return 1; 7261 } 7262 7263 /* 7264 * The purpose of wake_affine() is to quickly determine on which CPU we can run 7265 * soonest. For the purpose of speed we only consider the waking and previous 7266 * CPU. 7267 * 7268 * wake_affine_idle() - only considers 'now', it check if the waking CPU is 7269 * cache-affine and is (or will be) idle. 7270 * 7271 * wake_affine_weight() - considers the weight to reflect the average 7272 * scheduling latency of the CPUs. This seems to work 7273 * for the overloaded case. 7274 */ 7275 static int 7276 wake_affine_idle(int this_cpu, int prev_cpu, int sync) 7277 { 7278 /* 7279 * If this_cpu is idle, it implies the wakeup is from interrupt 7280 * context. Only allow the move if cache is shared. Otherwise an 7281 * interrupt intensive workload could force all tasks onto one 7282 * node depending on the IO topology or IRQ affinity settings. 7283 * 7284 * If the prev_cpu is idle and cache affine then avoid a migration. 7285 * There is no guarantee that the cache hot data from an interrupt 7286 * is more important than cache hot data on the prev_cpu and from 7287 * a cpufreq perspective, it's better to have higher utilisation 7288 * on one CPU. 7289 */ 7290 if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 7291 return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 7292 7293 if (sync) { 7294 struct rq *rq = cpu_rq(this_cpu); 7295 7296 if ((rq->nr_running - cfs_h_nr_delayed(rq)) == 1) 7297 return this_cpu; 7298 } 7299 7300 if (available_idle_cpu(prev_cpu)) 7301 return prev_cpu; 7302 7303 return nr_cpumask_bits; 7304 } 7305 7306 static int 7307 wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 7308 int this_cpu, int prev_cpu, int sync) 7309 { 7310 s64 this_eff_load, prev_eff_load; 7311 unsigned long task_load; 7312 7313 this_eff_load = cpu_load(cpu_rq(this_cpu)); 7314 7315 if (sync) { 7316 unsigned long current_load = task_h_load(current); 7317 7318 if (current_load > this_eff_load) 7319 return this_cpu; 7320 7321 this_eff_load -= current_load; 7322 } 7323 7324 task_load = task_h_load(p); 7325 7326 this_eff_load += task_load; 7327 if (sched_feat(WA_BIAS)) 7328 this_eff_load *= 100; 7329 this_eff_load *= capacity_of(prev_cpu); 7330 7331 prev_eff_load = cpu_load(cpu_rq(prev_cpu)); 7332 prev_eff_load -= task_load; 7333 if (sched_feat(WA_BIAS)) 7334 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 7335 prev_eff_load *= capacity_of(this_cpu); 7336 7337 /* 7338 * If sync, adjust the weight of prev_eff_load such that if 7339 * prev_eff == this_eff that select_idle_sibling() will consider 7340 * stacking the wakee on top of the waker if no other CPU is 7341 * idle. 7342 */ 7343 if (sync) 7344 prev_eff_load += 1; 7345 7346 return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 7347 } 7348 7349 static int wake_affine(struct sched_domain *sd, struct task_struct *p, 7350 int this_cpu, int prev_cpu, int sync) 7351 { 7352 int target = nr_cpumask_bits; 7353 7354 if (sched_feat(WA_IDLE)) 7355 target = wake_affine_idle(this_cpu, prev_cpu, sync); 7356 7357 if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 7358 target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync); 7359 7360 schedstat_inc(p->stats.nr_wakeups_affine_attempts); 7361 if (target != this_cpu) 7362 return prev_cpu; 7363 7364 schedstat_inc(sd->ttwu_move_affine); 7365 schedstat_inc(p->stats.nr_wakeups_affine); 7366 return target; 7367 } 7368 7369 static struct sched_group * 7370 sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu); 7371 7372 /* 7373 * sched_balance_find_dst_group_cpu - find the idlest CPU among the CPUs in the group. 7374 */ 7375 static int 7376 sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 7377 { 7378 unsigned long load, min_load = ULONG_MAX; 7379 unsigned int min_exit_latency = UINT_MAX; 7380 u64 latest_idle_timestamp = 0; 7381 int least_loaded_cpu = this_cpu; 7382 int shallowest_idle_cpu = -1; 7383 int i; 7384 7385 /* Check if we have any choice: */ 7386 if (group->group_weight == 1) 7387 return cpumask_first(sched_group_span(group)); 7388 7389 /* Traverse only the allowed CPUs */ 7390 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 7391 struct rq *rq = cpu_rq(i); 7392 7393 if (!sched_core_cookie_match(rq, p)) 7394 continue; 7395 7396 if (sched_idle_cpu(i)) 7397 return i; 7398 7399 if (available_idle_cpu(i)) { 7400 struct cpuidle_state *idle = idle_get_state(rq); 7401 if (idle && idle->exit_latency < min_exit_latency) { 7402 /* 7403 * We give priority to a CPU whose idle state 7404 * has the smallest exit latency irrespective 7405 * of any idle timestamp. 7406 */ 7407 min_exit_latency = idle->exit_latency; 7408 latest_idle_timestamp = rq->idle_stamp; 7409 shallowest_idle_cpu = i; 7410 } else if ((!idle || idle->exit_latency == min_exit_latency) && 7411 rq->idle_stamp > latest_idle_timestamp) { 7412 /* 7413 * If equal or no active idle state, then 7414 * the most recently idled CPU might have 7415 * a warmer cache. 7416 */ 7417 latest_idle_timestamp = rq->idle_stamp; 7418 shallowest_idle_cpu = i; 7419 } 7420 } else if (shallowest_idle_cpu == -1) { 7421 load = cpu_load(cpu_rq(i)); 7422 if (load < min_load) { 7423 min_load = load; 7424 least_loaded_cpu = i; 7425 } 7426 } 7427 } 7428 7429 return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 7430 } 7431 7432 static inline int sched_balance_find_dst_cpu(struct sched_domain *sd, struct task_struct *p, 7433 int cpu, int prev_cpu, int sd_flag) 7434 { 7435 int new_cpu = cpu; 7436 7437 if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr)) 7438 return prev_cpu; 7439 7440 /* 7441 * We need task's util for cpu_util_without, sync it up to 7442 * prev_cpu's last_update_time. 7443 */ 7444 if (!(sd_flag & SD_BALANCE_FORK)) 7445 sync_entity_load_avg(&p->se); 7446 7447 while (sd) { 7448 struct sched_group *group; 7449 struct sched_domain *tmp; 7450 int weight; 7451 7452 if (!(sd->flags & sd_flag)) { 7453 sd = sd->child; 7454 continue; 7455 } 7456 7457 group = sched_balance_find_dst_group(sd, p, cpu); 7458 if (!group) { 7459 sd = sd->child; 7460 continue; 7461 } 7462 7463 new_cpu = sched_balance_find_dst_group_cpu(group, p, cpu); 7464 if (new_cpu == cpu) { 7465 /* Now try balancing at a lower domain level of 'cpu': */ 7466 sd = sd->child; 7467 continue; 7468 } 7469 7470 /* Now try balancing at a lower domain level of 'new_cpu': */ 7471 cpu = new_cpu; 7472 weight = sd->span_weight; 7473 sd = NULL; 7474 for_each_domain(cpu, tmp) { 7475 if (weight <= tmp->span_weight) 7476 break; 7477 if (tmp->flags & sd_flag) 7478 sd = tmp; 7479 } 7480 } 7481 7482 return new_cpu; 7483 } 7484 7485 static inline int __select_idle_cpu(int cpu, struct task_struct *p) 7486 { 7487 if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) && 7488 sched_cpu_cookie_match(cpu_rq(cpu), p)) 7489 return cpu; 7490 7491 return -1; 7492 } 7493 7494 #ifdef CONFIG_SCHED_SMT 7495 DEFINE_STATIC_KEY_FALSE(sched_smt_present); 7496 EXPORT_SYMBOL_GPL(sched_smt_present); 7497 7498 static inline void set_idle_cores(int cpu, int val) 7499 { 7500 struct sched_domain_shared *sds; 7501 7502 sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu)); 7503 if (sds) 7504 WRITE_ONCE(sds->has_idle_cores, val); 7505 } 7506 7507 static inline bool test_idle_cores(int cpu) 7508 { 7509 struct sched_domain_shared *sds; 7510 7511 sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu)); 7512 if (sds) 7513 return READ_ONCE(sds->has_idle_cores); 7514 7515 return false; 7516 } 7517 7518 /* 7519 * Scans the local SMT mask to see if the entire core is idle, and records this 7520 * information in sd_llc_shared->has_idle_cores. 7521 * 7522 * Since SMT siblings share all cache levels, inspecting this limited remote 7523 * state should be fairly cheap. 7524 */ 7525 void __update_idle_core(struct rq *rq) 7526 { 7527 int core = cpu_of(rq); 7528 int cpu; 7529 7530 rcu_read_lock(); 7531 if (test_idle_cores(core)) 7532 goto unlock; 7533 7534 for_each_cpu(cpu, cpu_smt_mask(core)) { 7535 if (cpu == core) 7536 continue; 7537 7538 if (!available_idle_cpu(cpu)) 7539 goto unlock; 7540 } 7541 7542 set_idle_cores(core, 1); 7543 unlock: 7544 rcu_read_unlock(); 7545 } 7546 7547 /* 7548 * Scan the entire LLC domain for idle cores; this dynamically switches off if 7549 * there are no idle cores left in the system; tracked through 7550 * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. 7551 */ 7552 static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7553 { 7554 bool idle = true; 7555 int cpu; 7556 7557 for_each_cpu(cpu, cpu_smt_mask(core)) { 7558 if (!available_idle_cpu(cpu)) { 7559 idle = false; 7560 if (*idle_cpu == -1) { 7561 if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, cpus)) { 7562 *idle_cpu = cpu; 7563 break; 7564 } 7565 continue; 7566 } 7567 break; 7568 } 7569 if (*idle_cpu == -1 && cpumask_test_cpu(cpu, cpus)) 7570 *idle_cpu = cpu; 7571 } 7572 7573 if (idle) 7574 return core; 7575 7576 cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); 7577 return -1; 7578 } 7579 7580 /* 7581 * Scan the local SMT mask for idle CPUs. 7582 */ 7583 static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 7584 { 7585 int cpu; 7586 7587 for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) { 7588 if (cpu == target) 7589 continue; 7590 /* 7591 * Check if the CPU is in the LLC scheduling domain of @target. 7592 * Due to isolcpus, there is no guarantee that all the siblings are in the domain. 7593 */ 7594 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 7595 continue; 7596 if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 7597 return cpu; 7598 } 7599 7600 return -1; 7601 } 7602 7603 #else /* !CONFIG_SCHED_SMT: */ 7604 7605 static inline void set_idle_cores(int cpu, int val) 7606 { 7607 } 7608 7609 static inline bool test_idle_cores(int cpu) 7610 { 7611 return false; 7612 } 7613 7614 static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7615 { 7616 return __select_idle_cpu(core, p); 7617 } 7618 7619 static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 7620 { 7621 return -1; 7622 } 7623 7624 #endif /* !CONFIG_SCHED_SMT */ 7625 7626 /* 7627 * Scan the LLC domain for idle CPUs; this is dynamically regulated by 7628 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the 7629 * average idle time for this rq (as found in rq->avg_idle). 7630 */ 7631 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target) 7632 { 7633 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7634 int i, cpu, idle_cpu = -1, nr = INT_MAX; 7635 struct sched_domain_shared *sd_share; 7636 7637 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7638 7639 if (sched_feat(SIS_UTIL)) { 7640 sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, target)); 7641 if (sd_share) { 7642 /* because !--nr is the condition to stop scan */ 7643 nr = READ_ONCE(sd_share->nr_idle_scan) + 1; 7644 /* overloaded LLC is unlikely to have idle cpu/core */ 7645 if (nr == 1) 7646 return -1; 7647 } 7648 } 7649 7650 if (static_branch_unlikely(&sched_cluster_active)) { 7651 struct sched_group *sg = sd->groups; 7652 7653 if (sg->flags & SD_CLUSTER) { 7654 for_each_cpu_wrap(cpu, sched_group_span(sg), target + 1) { 7655 if (!cpumask_test_cpu(cpu, cpus)) 7656 continue; 7657 7658 if (has_idle_core) { 7659 i = select_idle_core(p, cpu, cpus, &idle_cpu); 7660 if ((unsigned int)i < nr_cpumask_bits) 7661 return i; 7662 } else { 7663 if (--nr <= 0) 7664 return -1; 7665 idle_cpu = __select_idle_cpu(cpu, p); 7666 if ((unsigned int)idle_cpu < nr_cpumask_bits) 7667 return idle_cpu; 7668 } 7669 } 7670 cpumask_andnot(cpus, cpus, sched_group_span(sg)); 7671 } 7672 } 7673 7674 for_each_cpu_wrap(cpu, cpus, target + 1) { 7675 if (has_idle_core) { 7676 i = select_idle_core(p, cpu, cpus, &idle_cpu); 7677 if ((unsigned int)i < nr_cpumask_bits) 7678 return i; 7679 7680 } else { 7681 if (--nr <= 0) 7682 return -1; 7683 idle_cpu = __select_idle_cpu(cpu, p); 7684 if ((unsigned int)idle_cpu < nr_cpumask_bits) 7685 break; 7686 } 7687 } 7688 7689 if (has_idle_core) 7690 set_idle_cores(target, false); 7691 7692 return idle_cpu; 7693 } 7694 7695 /* 7696 * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which 7697 * the task fits. If no CPU is big enough, but there are idle ones, try to 7698 * maximize capacity. 7699 */ 7700 static int 7701 select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) 7702 { 7703 unsigned long task_util, util_min, util_max, best_cap = 0; 7704 int fits, best_fits = 0; 7705 int cpu, best_cpu = -1; 7706 struct cpumask *cpus; 7707 7708 cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7709 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7710 7711 task_util = task_util_est(p); 7712 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7713 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7714 7715 for_each_cpu_wrap(cpu, cpus, target) { 7716 unsigned long cpu_cap = capacity_of(cpu); 7717 7718 if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) 7719 continue; 7720 7721 fits = util_fits_cpu(task_util, util_min, util_max, cpu); 7722 7723 /* This CPU fits with all requirements */ 7724 if (fits > 0) 7725 return cpu; 7726 /* 7727 * Only the min performance hint (i.e. uclamp_min) doesn't fit. 7728 * Look for the CPU with best capacity. 7729 */ 7730 else if (fits < 0) 7731 cpu_cap = get_actual_cpu_capacity(cpu); 7732 7733 /* 7734 * First, select CPU which fits better (-1 being better than 0). 7735 * Then, select the one with best capacity at same level. 7736 */ 7737 if ((fits < best_fits) || 7738 ((fits == best_fits) && (cpu_cap > best_cap))) { 7739 best_cap = cpu_cap; 7740 best_cpu = cpu; 7741 best_fits = fits; 7742 } 7743 } 7744 7745 return best_cpu; 7746 } 7747 7748 static inline bool asym_fits_cpu(unsigned long util, 7749 unsigned long util_min, 7750 unsigned long util_max, 7751 int cpu) 7752 { 7753 if (sched_asym_cpucap_active()) 7754 /* 7755 * Return true only if the cpu fully fits the task requirements 7756 * which include the utilization and the performance hints. 7757 */ 7758 return (util_fits_cpu(util, util_min, util_max, cpu) > 0); 7759 7760 return true; 7761 } 7762 7763 /* 7764 * Try and locate an idle core/thread in the LLC cache domain. 7765 */ 7766 static int select_idle_sibling(struct task_struct *p, int prev, int target) 7767 { 7768 bool has_idle_core = false; 7769 struct sched_domain *sd; 7770 unsigned long task_util, util_min, util_max; 7771 int i, recent_used_cpu, prev_aff = -1; 7772 7773 /* 7774 * On asymmetric system, update task utilization because we will check 7775 * that the task fits with CPU's capacity. 7776 */ 7777 if (sched_asym_cpucap_active()) { 7778 sync_entity_load_avg(&p->se); 7779 task_util = task_util_est(p); 7780 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7781 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7782 } 7783 7784 /* 7785 * per-cpu select_rq_mask usage 7786 */ 7787 lockdep_assert_irqs_disabled(); 7788 7789 if ((available_idle_cpu(target) || sched_idle_cpu(target)) && 7790 asym_fits_cpu(task_util, util_min, util_max, target)) 7791 return target; 7792 7793 /* 7794 * If the previous CPU is cache affine and idle, don't be stupid: 7795 */ 7796 if (prev != target && cpus_share_cache(prev, target) && 7797 (available_idle_cpu(prev) || sched_idle_cpu(prev)) && 7798 asym_fits_cpu(task_util, util_min, util_max, prev)) { 7799 7800 if (!static_branch_unlikely(&sched_cluster_active) || 7801 cpus_share_resources(prev, target)) 7802 return prev; 7803 7804 prev_aff = prev; 7805 } 7806 7807 /* 7808 * Allow a per-cpu kthread to stack with the wakee if the 7809 * kworker thread and the tasks previous CPUs are the same. 7810 * The assumption is that the wakee queued work for the 7811 * per-cpu kthread that is now complete and the wakeup is 7812 * essentially a sync wakeup. An obvious example of this 7813 * pattern is IO completions. 7814 */ 7815 if (is_per_cpu_kthread(current) && 7816 in_task() && 7817 prev == smp_processor_id() && 7818 this_rq()->nr_running <= 1 && 7819 asym_fits_cpu(task_util, util_min, util_max, prev)) { 7820 return prev; 7821 } 7822 7823 /* Check a recently used CPU as a potential idle candidate: */ 7824 recent_used_cpu = p->recent_used_cpu; 7825 p->recent_used_cpu = prev; 7826 if (recent_used_cpu != prev && 7827 recent_used_cpu != target && 7828 cpus_share_cache(recent_used_cpu, target) && 7829 (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) && 7830 cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) && 7831 asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { 7832 7833 if (!static_branch_unlikely(&sched_cluster_active) || 7834 cpus_share_resources(recent_used_cpu, target)) 7835 return recent_used_cpu; 7836 7837 } else { 7838 recent_used_cpu = -1; 7839 } 7840 7841 /* 7842 * For asymmetric CPU capacity systems, our domain of interest is 7843 * sd_asym_cpucapacity rather than sd_llc. 7844 */ 7845 if (sched_asym_cpucap_active()) { 7846 sd = rcu_dereference_all(per_cpu(sd_asym_cpucapacity, target)); 7847 /* 7848 * On an asymmetric CPU capacity system where an exclusive 7849 * cpuset defines a symmetric island (i.e. one unique 7850 * capacity_orig value through the cpuset), the key will be set 7851 * but the CPUs within that cpuset will not have a domain with 7852 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric 7853 * capacity path. 7854 */ 7855 if (sd) { 7856 i = select_idle_capacity(p, sd, target); 7857 return ((unsigned)i < nr_cpumask_bits) ? i : target; 7858 } 7859 } 7860 7861 sd = rcu_dereference_all(per_cpu(sd_llc, target)); 7862 if (!sd) 7863 return target; 7864 7865 if (sched_smt_active()) { 7866 has_idle_core = test_idle_cores(target); 7867 7868 if (!has_idle_core && cpus_share_cache(prev, target)) { 7869 i = select_idle_smt(p, sd, prev); 7870 if ((unsigned int)i < nr_cpumask_bits) 7871 return i; 7872 } 7873 } 7874 7875 i = select_idle_cpu(p, sd, has_idle_core, target); 7876 if ((unsigned)i < nr_cpumask_bits) 7877 return i; 7878 7879 /* 7880 * For cluster machines which have lower sharing cache like L2 or 7881 * LLC Tag, we tend to find an idle CPU in the target's cluster 7882 * first. But prev_cpu or recent_used_cpu may also be a good candidate, 7883 * use them if possible when no idle CPU found in select_idle_cpu(). 7884 */ 7885 if ((unsigned int)prev_aff < nr_cpumask_bits) 7886 return prev_aff; 7887 if ((unsigned int)recent_used_cpu < nr_cpumask_bits) 7888 return recent_used_cpu; 7889 7890 return target; 7891 } 7892 7893 /** 7894 * cpu_util() - Estimates the amount of CPU capacity used by CFS tasks. 7895 * @cpu: the CPU to get the utilization for 7896 * @p: task for which the CPU utilization should be predicted or NULL 7897 * @dst_cpu: CPU @p migrates to, -1 if @p moves from @cpu or @p == NULL 7898 * @boost: 1 to enable boosting, otherwise 0 7899 * 7900 * The unit of the return value must be the same as the one of CPU capacity 7901 * so that CPU utilization can be compared with CPU capacity. 7902 * 7903 * CPU utilization is the sum of running time of runnable tasks plus the 7904 * recent utilization of currently non-runnable tasks on that CPU. 7905 * It represents the amount of CPU capacity currently used by CFS tasks in 7906 * the range [0..max CPU capacity] with max CPU capacity being the CPU 7907 * capacity at f_max. 7908 * 7909 * The estimated CPU utilization is defined as the maximum between CPU 7910 * utilization and sum of the estimated utilization of the currently 7911 * runnable tasks on that CPU. It preserves a utilization "snapshot" of 7912 * previously-executed tasks, which helps better deduce how busy a CPU will 7913 * be when a long-sleeping task wakes up. The contribution to CPU utilization 7914 * of such a task would be significantly decayed at this point of time. 7915 * 7916 * Boosted CPU utilization is defined as max(CPU runnable, CPU utilization). 7917 * CPU contention for CFS tasks can be detected by CPU runnable > CPU 7918 * utilization. Boosting is implemented in cpu_util() so that internal 7919 * users (e.g. EAS) can use it next to external users (e.g. schedutil), 7920 * latter via cpu_util_cfs_boost(). 7921 * 7922 * CPU utilization can be higher than the current CPU capacity 7923 * (f_curr/f_max * max CPU capacity) or even the max CPU capacity because 7924 * of rounding errors as well as task migrations or wakeups of new tasks. 7925 * CPU utilization has to be capped to fit into the [0..max CPU capacity] 7926 * range. Otherwise a group of CPUs (CPU0 util = 121% + CPU1 util = 80%) 7927 * could be seen as over-utilized even though CPU1 has 20% of spare CPU 7928 * capacity. CPU utilization is allowed to overshoot current CPU capacity 7929 * though since this is useful for predicting the CPU capacity required 7930 * after task migrations (scheduler-driven DVFS). 7931 * 7932 * Return: (Boosted) (estimated) utilization for the specified CPU. 7933 */ 7934 static unsigned long 7935 cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost) 7936 { 7937 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; 7938 unsigned long util = READ_ONCE(cfs_rq->avg.util_avg); 7939 unsigned long runnable; 7940 7941 if (boost) { 7942 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 7943 util = max(util, runnable); 7944 } 7945 7946 /* 7947 * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its 7948 * contribution. If @p migrates from another CPU to @cpu add its 7949 * contribution. In all the other cases @cpu is not impacted by the 7950 * migration so its util_avg is already correct. 7951 */ 7952 if (p && task_cpu(p) == cpu && dst_cpu != cpu) 7953 lsub_positive(&util, task_util(p)); 7954 else if (p && task_cpu(p) != cpu && dst_cpu == cpu) 7955 util += task_util(p); 7956 7957 if (sched_feat(UTIL_EST)) { 7958 unsigned long util_est; 7959 7960 util_est = READ_ONCE(cfs_rq->avg.util_est); 7961 7962 /* 7963 * During wake-up @p isn't enqueued yet and doesn't contribute 7964 * to any cpu_rq(cpu)->cfs.avg.util_est. 7965 * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p 7966 * has been enqueued. 7967 * 7968 * During exec (@dst_cpu = -1) @p is enqueued and does 7969 * contribute to cpu_rq(cpu)->cfs.util_est. 7970 * Remove it to "simulate" cpu_util without @p's contribution. 7971 * 7972 * Despite the task_on_rq_queued(@p) check there is still a 7973 * small window for a possible race when an exec 7974 * select_task_rq_fair() races with LB's detach_task(). 7975 * 7976 * detach_task() 7977 * deactivate_task() 7978 * p->on_rq = TASK_ON_RQ_MIGRATING; 7979 * -------------------------------- A 7980 * dequeue_task() \ 7981 * dequeue_task_fair() + Race Time 7982 * util_est_dequeue() / 7983 * -------------------------------- B 7984 * 7985 * The additional check "current == p" is required to further 7986 * reduce the race window. 7987 */ 7988 if (dst_cpu == cpu) 7989 util_est += _task_util_est(p); 7990 else if (p && unlikely(task_on_rq_queued(p) || current == p)) 7991 lsub_positive(&util_est, _task_util_est(p)); 7992 7993 util = max(util, util_est); 7994 } 7995 7996 return min(util, arch_scale_cpu_capacity(cpu)); 7997 } 7998 7999 unsigned long cpu_util_cfs(int cpu) 8000 { 8001 return cpu_util(cpu, NULL, -1, 0); 8002 } 8003 8004 unsigned long cpu_util_cfs_boost(int cpu) 8005 { 8006 return cpu_util(cpu, NULL, -1, 1); 8007 } 8008 8009 /* 8010 * cpu_util_without: compute cpu utilization without any contributions from *p 8011 * @cpu: the CPU which utilization is requested 8012 * @p: the task which utilization should be discounted 8013 * 8014 * The utilization of a CPU is defined by the utilization of tasks currently 8015 * enqueued on that CPU as well as tasks which are currently sleeping after an 8016 * execution on that CPU. 8017 * 8018 * This method returns the utilization of the specified CPU by discounting the 8019 * utilization of the specified task, whenever the task is currently 8020 * contributing to the CPU utilization. 8021 */ 8022 static unsigned long cpu_util_without(int cpu, struct task_struct *p) 8023 { 8024 /* Task has no contribution or is new */ 8025 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 8026 p = NULL; 8027 8028 return cpu_util(cpu, p, -1, 0); 8029 } 8030 8031 /* 8032 * This function computes an effective utilization for the given CPU, to be 8033 * used for frequency selection given the linear relation: f = u * f_max. 8034 * 8035 * The scheduler tracks the following metrics: 8036 * 8037 * cpu_util_{cfs,rt,dl,irq}() 8038 * cpu_bw_dl() 8039 * 8040 * Where the cfs,rt and dl util numbers are tracked with the same metric and 8041 * synchronized windows and are thus directly comparable. 8042 * 8043 * The cfs,rt,dl utilization are the running times measured with rq->clock_task 8044 * which excludes things like IRQ and steal-time. These latter are then accrued 8045 * in the IRQ utilization. 8046 * 8047 * The DL bandwidth number OTOH is not a measured metric but a value computed 8048 * based on the task model parameters and gives the minimal utilization 8049 * required to meet deadlines. 8050 */ 8051 unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, 8052 unsigned long *min, 8053 unsigned long *max) 8054 { 8055 unsigned long util, irq, scale; 8056 struct rq *rq = cpu_rq(cpu); 8057 8058 scale = arch_scale_cpu_capacity(cpu); 8059 8060 /* 8061 * Early check to see if IRQ/steal time saturates the CPU, can be 8062 * because of inaccuracies in how we track these -- see 8063 * update_irq_load_avg(). 8064 */ 8065 irq = cpu_util_irq(rq); 8066 if (unlikely(irq >= scale)) { 8067 if (min) 8068 *min = scale; 8069 if (max) 8070 *max = scale; 8071 return scale; 8072 } 8073 8074 if (min) { 8075 /* 8076 * The minimum utilization returns the highest level between: 8077 * - the computed DL bandwidth needed with the IRQ pressure which 8078 * steals time to the deadline task. 8079 * - The minimum performance requirement for CFS and/or RT. 8080 */ 8081 *min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN)); 8082 8083 /* 8084 * When an RT task is runnable and uclamp is not used, we must 8085 * ensure that the task will run at maximum compute capacity. 8086 */ 8087 if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt)) 8088 *min = max(*min, scale); 8089 } 8090 8091 /* 8092 * Because the time spend on RT/DL tasks is visible as 'lost' time to 8093 * CFS tasks and we use the same metric to track the effective 8094 * utilization (PELT windows are synchronized) we can directly add them 8095 * to obtain the CPU's actual utilization. 8096 */ 8097 util = util_cfs + cpu_util_rt(rq); 8098 util += cpu_util_dl(rq); 8099 8100 /* 8101 * The maximum hint is a soft bandwidth requirement, which can be lower 8102 * than the actual utilization because of uclamp_max requirements. 8103 */ 8104 if (max) 8105 *max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX)); 8106 8107 if (util >= scale) 8108 return scale; 8109 8110 /* 8111 * There is still idle time; further improve the number by using the 8112 * IRQ metric. Because IRQ/steal time is hidden from the task clock we 8113 * need to scale the task numbers: 8114 * 8115 * max - irq 8116 * U' = irq + --------- * U 8117 * max 8118 */ 8119 util = scale_irq_capacity(util, irq, scale); 8120 util += irq; 8121 8122 return min(scale, util); 8123 } 8124 8125 unsigned long sched_cpu_util(int cpu) 8126 { 8127 return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL); 8128 } 8129 8130 /* 8131 * energy_env - Utilization landscape for energy estimation. 8132 * @task_busy_time: Utilization contribution by the task for which we test the 8133 * placement. Given by eenv_task_busy_time(). 8134 * @pd_busy_time: Utilization of the whole perf domain without the task 8135 * contribution. Given by eenv_pd_busy_time(). 8136 * @cpu_cap: Maximum CPU capacity for the perf domain. 8137 * @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap). 8138 */ 8139 struct energy_env { 8140 unsigned long task_busy_time; 8141 unsigned long pd_busy_time; 8142 unsigned long cpu_cap; 8143 unsigned long pd_cap; 8144 }; 8145 8146 /* 8147 * Compute the task busy time for compute_energy(). This time cannot be 8148 * injected directly into effective_cpu_util() because of the IRQ scaling. 8149 * The latter only makes sense with the most recent CPUs where the task has 8150 * run. 8151 */ 8152 static inline void eenv_task_busy_time(struct energy_env *eenv, 8153 struct task_struct *p, int prev_cpu) 8154 { 8155 unsigned long busy_time, max_cap = arch_scale_cpu_capacity(prev_cpu); 8156 unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu)); 8157 8158 if (unlikely(irq >= max_cap)) 8159 busy_time = max_cap; 8160 else 8161 busy_time = scale_irq_capacity(task_util_est(p), irq, max_cap); 8162 8163 eenv->task_busy_time = busy_time; 8164 } 8165 8166 /* 8167 * Compute the perf_domain (PD) busy time for compute_energy(). Based on the 8168 * utilization for each @pd_cpus, it however doesn't take into account 8169 * clamping since the ratio (utilization / cpu_capacity) is already enough to 8170 * scale the EM reported power consumption at the (eventually clamped) 8171 * cpu_capacity. 8172 * 8173 * The contribution of the task @p for which we want to estimate the 8174 * energy cost is removed (by cpu_util()) and must be calculated 8175 * separately (see eenv_task_busy_time). This ensures: 8176 * 8177 * - A stable PD utilization, no matter which CPU of that PD we want to place 8178 * the task on. 8179 * 8180 * - A fair comparison between CPUs as the task contribution (task_util()) 8181 * will always be the same no matter which CPU utilization we rely on 8182 * (util_avg or util_est). 8183 * 8184 * Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't 8185 * exceed @eenv->pd_cap. 8186 */ 8187 static inline void eenv_pd_busy_time(struct energy_env *eenv, 8188 struct cpumask *pd_cpus, 8189 struct task_struct *p) 8190 { 8191 unsigned long busy_time = 0; 8192 int cpu; 8193 8194 for_each_cpu(cpu, pd_cpus) { 8195 unsigned long util = cpu_util(cpu, p, -1, 0); 8196 8197 busy_time += effective_cpu_util(cpu, util, NULL, NULL); 8198 } 8199 8200 eenv->pd_busy_time = min(eenv->pd_cap, busy_time); 8201 } 8202 8203 /* 8204 * Compute the maximum utilization for compute_energy() when the task @p 8205 * is placed on the cpu @dst_cpu. 8206 * 8207 * Returns the maximum utilization among @eenv->cpus. This utilization can't 8208 * exceed @eenv->cpu_cap. 8209 */ 8210 static inline unsigned long 8211 eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus, 8212 struct task_struct *p, int dst_cpu) 8213 { 8214 unsigned long max_util = 0; 8215 int cpu; 8216 8217 for_each_cpu(cpu, pd_cpus) { 8218 struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL; 8219 unsigned long util = cpu_util(cpu, p, dst_cpu, 1); 8220 unsigned long eff_util, min, max; 8221 8222 /* 8223 * Performance domain frequency: utilization clamping 8224 * must be considered since it affects the selection 8225 * of the performance domain frequency. 8226 * NOTE: in case RT tasks are running, by default the min 8227 * utilization can be max OPP. 8228 */ 8229 eff_util = effective_cpu_util(cpu, util, &min, &max); 8230 8231 /* Task's uclamp can modify min and max value */ 8232 if (tsk && uclamp_is_used()) { 8233 min = max(min, uclamp_eff_value(p, UCLAMP_MIN)); 8234 8235 /* 8236 * If there is no active max uclamp constraint, 8237 * directly use task's one, otherwise keep max. 8238 */ 8239 if (uclamp_rq_is_idle(cpu_rq(cpu))) 8240 max = uclamp_eff_value(p, UCLAMP_MAX); 8241 else 8242 max = max(max, uclamp_eff_value(p, UCLAMP_MAX)); 8243 } 8244 8245 eff_util = sugov_effective_cpu_perf(cpu, eff_util, min, max); 8246 max_util = max(max_util, eff_util); 8247 } 8248 8249 return min(max_util, eenv->cpu_cap); 8250 } 8251 8252 /* 8253 * compute_energy(): Use the Energy Model to estimate the energy that @pd would 8254 * consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task 8255 * contribution is ignored. 8256 */ 8257 static inline unsigned long 8258 compute_energy(struct energy_env *eenv, struct perf_domain *pd, 8259 struct cpumask *pd_cpus, struct task_struct *p, int dst_cpu) 8260 { 8261 unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu); 8262 unsigned long busy_time = eenv->pd_busy_time; 8263 unsigned long energy; 8264 8265 if (dst_cpu >= 0) 8266 busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time); 8267 8268 energy = em_cpu_energy(pd->em_pd, max_util, busy_time, eenv->cpu_cap); 8269 8270 trace_sched_compute_energy_tp(p, dst_cpu, energy, max_util, busy_time); 8271 8272 return energy; 8273 } 8274 8275 /* 8276 * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the 8277 * waking task. find_energy_efficient_cpu() looks for the CPU with maximum 8278 * spare capacity in each performance domain and uses it as a potential 8279 * candidate to execute the task. Then, it uses the Energy Model to figure 8280 * out which of the CPU candidates is the most energy-efficient. 8281 * 8282 * The rationale for this heuristic is as follows. In a performance domain, 8283 * all the most energy efficient CPU candidates (according to the Energy 8284 * Model) are those for which we'll request a low frequency. When there are 8285 * several CPUs for which the frequency request will be the same, we don't 8286 * have enough data to break the tie between them, because the Energy Model 8287 * only includes active power costs. With this model, if we assume that 8288 * frequency requests follow utilization (e.g. using schedutil), the CPU with 8289 * the maximum spare capacity in a performance domain is guaranteed to be among 8290 * the best candidates of the performance domain. 8291 * 8292 * In practice, it could be preferable from an energy standpoint to pack 8293 * small tasks on a CPU in order to let other CPUs go in deeper idle states, 8294 * but that could also hurt our chances to go cluster idle, and we have no 8295 * ways to tell with the current Energy Model if this is actually a good 8296 * idea or not. So, find_energy_efficient_cpu() basically favors 8297 * cluster-packing, and spreading inside a cluster. That should at least be 8298 * a good thing for latency, and this is consistent with the idea that most 8299 * of the energy savings of EAS come from the asymmetry of the system, and 8300 * not so much from breaking the tie between identical CPUs. That's also the 8301 * reason why EAS is enabled in the topology code only for systems where 8302 * SD_ASYM_CPUCAPACITY is set. 8303 * 8304 * NOTE: Forkees are not accepted in the energy-aware wake-up path because 8305 * they don't have any useful utilization data yet and it's not possible to 8306 * forecast their impact on energy consumption. Consequently, they will be 8307 * placed by sched_balance_find_dst_cpu() on the least loaded CPU, which might turn out 8308 * to be energy-inefficient in some use-cases. The alternative would be to 8309 * bias new tasks towards specific types of CPUs first, or to try to infer 8310 * their util_avg from the parent task, but those heuristics could hurt 8311 * other use-cases too. So, until someone finds a better way to solve this, 8312 * let's keep things simple by re-using the existing slow path. 8313 */ 8314 static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 8315 { 8316 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 8317 unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; 8318 unsigned long p_util_min = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MIN) : 0; 8319 unsigned long p_util_max = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MAX) : 1024; 8320 struct root_domain *rd = this_rq()->rd; 8321 int cpu, best_energy_cpu, target = -1; 8322 int prev_fits = -1, best_fits = -1; 8323 unsigned long best_actual_cap = 0; 8324 unsigned long prev_actual_cap = 0; 8325 struct sched_domain *sd; 8326 struct perf_domain *pd; 8327 struct energy_env eenv; 8328 8329 rcu_read_lock(); 8330 pd = rcu_dereference_all(rd->pd); 8331 if (!pd) 8332 goto unlock; 8333 8334 /* 8335 * Energy-aware wake-up happens on the lowest sched_domain starting 8336 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu. 8337 */ 8338 sd = rcu_dereference_all(*this_cpu_ptr(&sd_asym_cpucapacity)); 8339 while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 8340 sd = sd->parent; 8341 if (!sd) 8342 goto unlock; 8343 8344 target = prev_cpu; 8345 8346 sync_entity_load_avg(&p->se); 8347 if (!task_util_est(p) && p_util_min == 0) 8348 goto unlock; 8349 8350 eenv_task_busy_time(&eenv, p, prev_cpu); 8351 8352 for (; pd; pd = pd->next) { 8353 unsigned long util_min = p_util_min, util_max = p_util_max; 8354 unsigned long cpu_cap, cpu_actual_cap, util; 8355 long prev_spare_cap = -1, max_spare_cap = -1; 8356 unsigned long rq_util_min, rq_util_max; 8357 unsigned long cur_delta, base_energy; 8358 int max_spare_cap_cpu = -1; 8359 int fits, max_fits = -1; 8360 8361 if (!cpumask_and(cpus, perf_domain_span(pd), cpu_online_mask)) 8362 continue; 8363 8364 /* Account external pressure for the energy estimation */ 8365 cpu = cpumask_first(cpus); 8366 cpu_actual_cap = get_actual_cpu_capacity(cpu); 8367 8368 eenv.cpu_cap = cpu_actual_cap; 8369 eenv.pd_cap = 0; 8370 8371 for_each_cpu(cpu, cpus) { 8372 struct rq *rq = cpu_rq(cpu); 8373 8374 eenv.pd_cap += cpu_actual_cap; 8375 8376 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 8377 continue; 8378 8379 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 8380 continue; 8381 8382 util = cpu_util(cpu, p, cpu, 0); 8383 cpu_cap = capacity_of(cpu); 8384 8385 /* 8386 * Skip CPUs that cannot satisfy the capacity request. 8387 * IOW, placing the task there would make the CPU 8388 * overutilized. Take uclamp into account to see how 8389 * much capacity we can get out of the CPU; this is 8390 * aligned with sched_cpu_util(). 8391 */ 8392 if (uclamp_is_used() && !uclamp_rq_is_idle(rq)) { 8393 /* 8394 * Open code uclamp_rq_util_with() except for 8395 * the clamp() part. I.e.: apply max aggregation 8396 * only. util_fits_cpu() logic requires to 8397 * operate on non clamped util but must use the 8398 * max-aggregated uclamp_{min, max}. 8399 */ 8400 rq_util_min = uclamp_rq_get(rq, UCLAMP_MIN); 8401 rq_util_max = uclamp_rq_get(rq, UCLAMP_MAX); 8402 8403 util_min = max(rq_util_min, p_util_min); 8404 util_max = max(rq_util_max, p_util_max); 8405 } 8406 8407 fits = util_fits_cpu(util, util_min, util_max, cpu); 8408 if (!fits) 8409 continue; 8410 8411 lsub_positive(&cpu_cap, util); 8412 8413 if (cpu == prev_cpu) { 8414 /* Always use prev_cpu as a candidate. */ 8415 prev_spare_cap = cpu_cap; 8416 prev_fits = fits; 8417 } else if ((fits > max_fits) || 8418 ((fits == max_fits) && ((long)cpu_cap > max_spare_cap))) { 8419 /* 8420 * Find the CPU with the maximum spare capacity 8421 * among the remaining CPUs in the performance 8422 * domain. 8423 */ 8424 max_spare_cap = cpu_cap; 8425 max_spare_cap_cpu = cpu; 8426 max_fits = fits; 8427 } 8428 } 8429 8430 if (max_spare_cap_cpu < 0 && prev_spare_cap < 0) 8431 continue; 8432 8433 eenv_pd_busy_time(&eenv, cpus, p); 8434 /* Compute the 'base' energy of the pd, without @p */ 8435 base_energy = compute_energy(&eenv, pd, cpus, p, -1); 8436 8437 /* Evaluate the energy impact of using prev_cpu. */ 8438 if (prev_spare_cap > -1) { 8439 prev_delta = compute_energy(&eenv, pd, cpus, p, 8440 prev_cpu); 8441 /* CPU utilization has changed */ 8442 if (prev_delta < base_energy) 8443 goto unlock; 8444 prev_delta -= base_energy; 8445 prev_actual_cap = cpu_actual_cap; 8446 best_delta = min(best_delta, prev_delta); 8447 } 8448 8449 /* Evaluate the energy impact of using max_spare_cap_cpu. */ 8450 if (max_spare_cap_cpu >= 0 && max_spare_cap > prev_spare_cap) { 8451 /* Current best energy cpu fits better */ 8452 if (max_fits < best_fits) 8453 continue; 8454 8455 /* 8456 * Both don't fit performance hint (i.e. uclamp_min) 8457 * but best energy cpu has better capacity. 8458 */ 8459 if ((max_fits < 0) && 8460 (cpu_actual_cap <= best_actual_cap)) 8461 continue; 8462 8463 cur_delta = compute_energy(&eenv, pd, cpus, p, 8464 max_spare_cap_cpu); 8465 /* CPU utilization has changed */ 8466 if (cur_delta < base_energy) 8467 goto unlock; 8468 cur_delta -= base_energy; 8469 8470 /* 8471 * Both fit for the task but best energy cpu has lower 8472 * energy impact. 8473 */ 8474 if ((max_fits > 0) && (best_fits > 0) && 8475 (cur_delta >= best_delta)) 8476 continue; 8477 8478 best_delta = cur_delta; 8479 best_energy_cpu = max_spare_cap_cpu; 8480 best_fits = max_fits; 8481 best_actual_cap = cpu_actual_cap; 8482 } 8483 } 8484 rcu_read_unlock(); 8485 8486 if ((best_fits > prev_fits) || 8487 ((best_fits > 0) && (best_delta < prev_delta)) || 8488 ((best_fits < 0) && (best_actual_cap > prev_actual_cap))) 8489 target = best_energy_cpu; 8490 8491 return target; 8492 8493 unlock: 8494 rcu_read_unlock(); 8495 8496 return target; 8497 } 8498 8499 /* 8500 * select_task_rq_fair: Select target runqueue for the waking task in domains 8501 * that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE, 8502 * SD_BALANCE_FORK, or SD_BALANCE_EXEC. 8503 * 8504 * Balances load by selecting the idlest CPU in the idlest group, or under 8505 * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set. 8506 * 8507 * Returns the target CPU number. 8508 */ 8509 static int 8510 select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) 8511 { 8512 int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 8513 struct sched_domain *tmp, *sd = NULL; 8514 int cpu = smp_processor_id(); 8515 int new_cpu = prev_cpu; 8516 int want_affine = 0; 8517 /* SD_flags and WF_flags share the first nibble */ 8518 int sd_flag = wake_flags & 0xF; 8519 8520 /* 8521 * required for stable ->cpus_allowed 8522 */ 8523 lockdep_assert_held(&p->pi_lock); 8524 if (wake_flags & WF_TTWU) { 8525 record_wakee(p); 8526 8527 if ((wake_flags & WF_CURRENT_CPU) && 8528 cpumask_test_cpu(cpu, p->cpus_ptr)) 8529 return cpu; 8530 8531 if (!is_rd_overutilized(this_rq()->rd)) { 8532 new_cpu = find_energy_efficient_cpu(p, prev_cpu); 8533 if (new_cpu >= 0) 8534 return new_cpu; 8535 new_cpu = prev_cpu; 8536 } 8537 8538 want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr); 8539 } 8540 8541 rcu_read_lock(); 8542 for_each_domain(cpu, tmp) { 8543 /* 8544 * If both 'cpu' and 'prev_cpu' are part of this domain, 8545 * cpu is a valid SD_WAKE_AFFINE target. 8546 */ 8547 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && 8548 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { 8549 if (cpu != prev_cpu) 8550 new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync); 8551 8552 sd = NULL; /* Prefer wake_affine over balance flags */ 8553 break; 8554 } 8555 8556 /* 8557 * Usually only true for WF_EXEC and WF_FORK, as sched_domains 8558 * usually do not have SD_BALANCE_WAKE set. That means wakeup 8559 * will usually go to the fast path. 8560 */ 8561 if (tmp->flags & sd_flag) 8562 sd = tmp; 8563 else if (!want_affine) 8564 break; 8565 } 8566 8567 if (unlikely(sd)) { 8568 /* Slow path */ 8569 new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag); 8570 } else if (wake_flags & WF_TTWU) { /* XXX always ? */ 8571 /* Fast path */ 8572 new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); 8573 } 8574 rcu_read_unlock(); 8575 8576 return new_cpu; 8577 } 8578 8579 /* 8580 * Called immediately before a task is migrated to a new CPU; task_cpu(p) and 8581 * cfs_rq_of(p) references at time of call are still valid and identify the 8582 * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held. 8583 */ 8584 static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) 8585 { 8586 struct sched_entity *se = &p->se; 8587 8588 if (!task_on_rq_migrating(p)) { 8589 remove_entity_load_avg(se); 8590 8591 /* 8592 * Here, the task's PELT values have been updated according to 8593 * the current rq's clock. But if that clock hasn't been 8594 * updated in a while, a substantial idle time will be missed, 8595 * leading to an inflation after wake-up on the new rq. 8596 * 8597 * Estimate the missing time from the cfs_rq last_update_time 8598 * and update sched_avg to improve the PELT continuity after 8599 * migration. 8600 */ 8601 migrate_se_pelt_lag(se); 8602 } 8603 8604 /* Tell new CPU we are migrated */ 8605 se->avg.last_update_time = 0; 8606 8607 update_scan_period(p, new_cpu); 8608 } 8609 8610 static void task_dead_fair(struct task_struct *p) 8611 { 8612 struct sched_entity *se = &p->se; 8613 8614 if (se->sched_delayed) { 8615 struct rq_flags rf; 8616 struct rq *rq; 8617 8618 rq = task_rq_lock(p, &rf); 8619 if (se->sched_delayed) { 8620 update_rq_clock(rq); 8621 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 8622 } 8623 task_rq_unlock(rq, p, &rf); 8624 } 8625 8626 remove_entity_load_avg(se); 8627 } 8628 8629 /* 8630 * Set the max capacity the task is allowed to run at for misfit detection. 8631 */ 8632 static void set_task_max_allowed_capacity(struct task_struct *p) 8633 { 8634 struct asym_cap_data *entry; 8635 8636 if (!sched_asym_cpucap_active()) 8637 return; 8638 8639 rcu_read_lock(); 8640 list_for_each_entry_rcu(entry, &asym_cap_list, link) { 8641 cpumask_t *cpumask; 8642 8643 cpumask = cpu_capacity_span(entry); 8644 if (!cpumask_intersects(p->cpus_ptr, cpumask)) 8645 continue; 8646 8647 p->max_allowed_capacity = entry->capacity; 8648 break; 8649 } 8650 rcu_read_unlock(); 8651 } 8652 8653 static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context *ctx) 8654 { 8655 set_cpus_allowed_common(p, ctx); 8656 set_task_max_allowed_capacity(p); 8657 } 8658 8659 static void set_next_buddy(struct sched_entity *se) 8660 { 8661 for_each_sched_entity(se) { 8662 if (WARN_ON_ONCE(!se->on_rq)) 8663 return; 8664 if (se_is_idle(se)) 8665 return; 8666 cfs_rq_of(se)->next = se; 8667 } 8668 } 8669 8670 enum preempt_wakeup_action { 8671 PREEMPT_WAKEUP_NONE, /* No preemption. */ 8672 PREEMPT_WAKEUP_SHORT, /* Ignore slice protection. */ 8673 PREEMPT_WAKEUP_PICK, /* Let __pick_eevdf() decide. */ 8674 PREEMPT_WAKEUP_RESCHED, /* Force reschedule. */ 8675 }; 8676 8677 static inline bool 8678 set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags, 8679 struct sched_entity *pse, struct sched_entity *se) 8680 { 8681 /* 8682 * Keep existing buddy if the deadline is sooner than pse. 8683 * The older buddy may be cache cold and completely unrelated 8684 * to the current wakeup but that is unpredictable where as 8685 * obeying the deadline is more in line with EEVDF objectives. 8686 */ 8687 if (cfs_rq->next && entity_before(cfs_rq->next, pse)) 8688 return false; 8689 8690 set_next_buddy(pse); 8691 return true; 8692 } 8693 8694 /* 8695 * WF_SYNC|WF_TTWU indicates the waker expects to sleep but it is not 8696 * strictly enforced because the hint is either misunderstood or 8697 * multiple tasks must be woken up. 8698 */ 8699 static inline enum preempt_wakeup_action 8700 preempt_sync(struct rq *rq, int wake_flags, 8701 struct sched_entity *pse, struct sched_entity *se) 8702 { 8703 u64 threshold, delta; 8704 8705 /* 8706 * WF_SYNC without WF_TTWU is not expected so warn if it happens even 8707 * though it is likely harmless. 8708 */ 8709 WARN_ON_ONCE(!(wake_flags & WF_TTWU)); 8710 8711 threshold = sysctl_sched_migration_cost; 8712 delta = rq_clock_task(rq) - se->exec_start; 8713 if ((s64)delta < 0) 8714 delta = 0; 8715 8716 /* 8717 * WF_RQ_SELECTED implies the tasks are stacking on a CPU when they 8718 * could run on other CPUs. Reduce the threshold before preemption is 8719 * allowed to an arbitrary lower value as it is more likely (but not 8720 * guaranteed) the waker requires the wakee to finish. 8721 */ 8722 if (wake_flags & WF_RQ_SELECTED) 8723 threshold >>= 2; 8724 8725 /* 8726 * As WF_SYNC is not strictly obeyed, allow some runtime for batch 8727 * wakeups to be issued. 8728 */ 8729 if (entity_before(pse, se) && delta >= threshold) 8730 return PREEMPT_WAKEUP_RESCHED; 8731 8732 return PREEMPT_WAKEUP_NONE; 8733 } 8734 8735 /* 8736 * Preempt the current task with a newly woken task if needed: 8737 */ 8738 static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_flags) 8739 { 8740 enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK; 8741 struct task_struct *donor = rq->donor; 8742 struct sched_entity *se = &donor->se, *pse = &p->se; 8743 struct cfs_rq *cfs_rq = task_cfs_rq(donor); 8744 int cse_is_idle, pse_is_idle; 8745 8746 /* 8747 * XXX Getting preempted by higher class, try and find idle CPU? 8748 */ 8749 if (p->sched_class != &fair_sched_class) 8750 return; 8751 8752 if (unlikely(se == pse)) 8753 return; 8754 8755 /* 8756 * This is possible from callers such as attach_tasks(), in which we 8757 * unconditionally wakeup_preempt() after an enqueue (which may have 8758 * lead to a throttle). This both saves work and prevents false 8759 * next-buddy nomination below. 8760 */ 8761 if (task_is_throttled(p)) 8762 return; 8763 8764 /* 8765 * We can come here with TIF_NEED_RESCHED already set from new task 8766 * wake up path. 8767 * 8768 * Note: this also catches the edge-case of curr being in a throttled 8769 * group (e.g. via set_curr_task), since update_curr() (in the 8770 * enqueue of curr) will have resulted in resched being set. This 8771 * prevents us from potentially nominating it as a false LAST_BUDDY 8772 * below. 8773 */ 8774 if (test_tsk_need_resched(rq->curr)) 8775 return; 8776 8777 if (!sched_feat(WAKEUP_PREEMPTION)) 8778 return; 8779 8780 find_matching_se(&se, &pse); 8781 WARN_ON_ONCE(!pse); 8782 8783 cse_is_idle = se_is_idle(se); 8784 pse_is_idle = se_is_idle(pse); 8785 8786 /* 8787 * Preempt an idle entity in favor of a non-idle entity (and don't preempt 8788 * in the inverse case). 8789 */ 8790 if (cse_is_idle && !pse_is_idle) { 8791 /* 8792 * When non-idle entity preempt an idle entity, 8793 * don't give idle entity slice protection. 8794 */ 8795 preempt_action = PREEMPT_WAKEUP_SHORT; 8796 goto preempt; 8797 } 8798 8799 if (cse_is_idle != pse_is_idle) 8800 return; 8801 8802 /* 8803 * BATCH and IDLE tasks do not preempt others. 8804 */ 8805 if (unlikely(!normal_policy(p->policy))) 8806 return; 8807 8808 cfs_rq = cfs_rq_of(se); 8809 update_curr(cfs_rq); 8810 /* 8811 * If @p has a shorter slice than current and @p is eligible, override 8812 * current's slice protection in order to allow preemption. 8813 */ 8814 if (sched_feat(PREEMPT_SHORT) && (pse->slice < se->slice)) { 8815 preempt_action = PREEMPT_WAKEUP_SHORT; 8816 goto pick; 8817 } 8818 8819 /* 8820 * Ignore wakee preemption on WF_FORK as it is less likely that 8821 * there is shared data as exec often follow fork. Do not 8822 * preempt for tasks that are sched_delayed as it would violate 8823 * EEVDF to forcibly queue an ineligible task. 8824 */ 8825 if ((wake_flags & WF_FORK) || pse->sched_delayed) 8826 return; 8827 8828 /* Prefer picking wakee soon if appropriate. */ 8829 if (sched_feat(NEXT_BUDDY) && 8830 set_preempt_buddy(cfs_rq, wake_flags, pse, se)) { 8831 8832 /* 8833 * Decide whether to obey WF_SYNC hint for a new buddy. Old 8834 * buddies are ignored as they may not be relevant to the 8835 * waker and less likely to be cache hot. 8836 */ 8837 if (wake_flags & WF_SYNC) 8838 preempt_action = preempt_sync(rq, wake_flags, pse, se); 8839 } 8840 8841 switch (preempt_action) { 8842 case PREEMPT_WAKEUP_NONE: 8843 return; 8844 case PREEMPT_WAKEUP_RESCHED: 8845 goto preempt; 8846 case PREEMPT_WAKEUP_SHORT: 8847 fallthrough; 8848 case PREEMPT_WAKEUP_PICK: 8849 break; 8850 } 8851 8852 pick: 8853 /* 8854 * If @p has become the most eligible task, force preemption. 8855 */ 8856 if (__pick_eevdf(cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT) == pse) 8857 goto preempt; 8858 8859 if (sched_feat(RUN_TO_PARITY)) 8860 update_protect_slice(cfs_rq, se); 8861 8862 return; 8863 8864 preempt: 8865 if (preempt_action == PREEMPT_WAKEUP_SHORT) 8866 cancel_protect_slice(se); 8867 8868 resched_curr_lazy(rq); 8869 } 8870 8871 static struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) 8872 { 8873 struct sched_entity *se; 8874 struct cfs_rq *cfs_rq; 8875 struct task_struct *p; 8876 bool throttled; 8877 8878 again: 8879 cfs_rq = &rq->cfs; 8880 if (!cfs_rq->nr_queued) 8881 return NULL; 8882 8883 throttled = false; 8884 8885 do { 8886 /* Might not have done put_prev_entity() */ 8887 if (cfs_rq->curr && cfs_rq->curr->on_rq) 8888 update_curr(cfs_rq); 8889 8890 throttled |= check_cfs_rq_runtime(cfs_rq); 8891 8892 se = pick_next_entity(rq, cfs_rq); 8893 if (!se) 8894 goto again; 8895 cfs_rq = group_cfs_rq(se); 8896 } while (cfs_rq); 8897 8898 p = task_of(se); 8899 if (unlikely(throttled)) 8900 task_throttle_setup_work(p); 8901 return p; 8902 } 8903 8904 static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); 8905 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); 8906 8907 struct task_struct * 8908 pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 8909 __must_hold(__rq_lockp(rq)) 8910 { 8911 struct sched_entity *se; 8912 struct task_struct *p; 8913 int new_tasks; 8914 8915 again: 8916 p = pick_task_fair(rq, rf); 8917 if (!p) 8918 goto idle; 8919 se = &p->se; 8920 8921 #ifdef CONFIG_FAIR_GROUP_SCHED 8922 if (prev->sched_class != &fair_sched_class) 8923 goto simple; 8924 8925 __put_prev_set_next_dl_server(rq, prev, p); 8926 8927 /* 8928 * Because of the set_next_buddy() in dequeue_task_fair() it is rather 8929 * likely that a next task is from the same cgroup as the current. 8930 * 8931 * Therefore attempt to avoid putting and setting the entire cgroup 8932 * hierarchy, only change the part that actually changes. 8933 * 8934 * Since we haven't yet done put_prev_entity and if the selected task 8935 * is a different task than we started out with, try and touch the 8936 * least amount of cfs_rqs. 8937 */ 8938 if (prev != p) { 8939 struct sched_entity *pse = &prev->se; 8940 struct cfs_rq *cfs_rq; 8941 8942 while (!(cfs_rq = is_same_group(se, pse))) { 8943 int se_depth = se->depth; 8944 int pse_depth = pse->depth; 8945 8946 if (se_depth <= pse_depth) { 8947 put_prev_entity(cfs_rq_of(pse), pse); 8948 pse = parent_entity(pse); 8949 } 8950 if (se_depth >= pse_depth) { 8951 set_next_entity(cfs_rq_of(se), se); 8952 se = parent_entity(se); 8953 } 8954 } 8955 8956 put_prev_entity(cfs_rq, pse); 8957 set_next_entity(cfs_rq, se); 8958 8959 __set_next_task_fair(rq, p, true); 8960 } 8961 8962 return p; 8963 8964 simple: 8965 #endif /* CONFIG_FAIR_GROUP_SCHED */ 8966 put_prev_set_next_task(rq, prev, p); 8967 return p; 8968 8969 idle: 8970 if (rf) { 8971 new_tasks = sched_balance_newidle(rq, rf); 8972 8973 /* 8974 * Because sched_balance_newidle() releases (and re-acquires) 8975 * rq->lock, it is possible for any higher priority task to 8976 * appear. In that case we must re-start the pick_next_entity() 8977 * loop. 8978 */ 8979 if (new_tasks < 0) 8980 return RETRY_TASK; 8981 8982 if (new_tasks > 0) 8983 goto again; 8984 } 8985 8986 return NULL; 8987 } 8988 8989 static struct task_struct * 8990 fair_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf) 8991 { 8992 return pick_task_fair(dl_se->rq, rf); 8993 } 8994 8995 void fair_server_init(struct rq *rq) 8996 { 8997 struct sched_dl_entity *dl_se = &rq->fair_server; 8998 8999 init_dl_entity(dl_se); 9000 9001 dl_server_init(dl_se, rq, fair_server_pick_task); 9002 } 9003 9004 /* 9005 * Account for a descheduled task: 9006 */ 9007 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next) 9008 { 9009 struct sched_entity *se = &prev->se; 9010 struct cfs_rq *cfs_rq; 9011 9012 for_each_sched_entity(se) { 9013 cfs_rq = cfs_rq_of(se); 9014 put_prev_entity(cfs_rq, se); 9015 } 9016 } 9017 9018 /* 9019 * sched_yield() is very simple 9020 */ 9021 static void yield_task_fair(struct rq *rq) 9022 { 9023 struct task_struct *curr = rq->donor; 9024 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 9025 struct sched_entity *se = &curr->se; 9026 9027 /* 9028 * Are we the only task in the tree? 9029 */ 9030 if (unlikely(rq->nr_running == 1)) 9031 return; 9032 9033 clear_buddies(cfs_rq, se); 9034 9035 update_rq_clock(rq); 9036 /* 9037 * Update run-time statistics of the 'current'. 9038 */ 9039 update_curr(cfs_rq); 9040 /* 9041 * Tell update_rq_clock() that we've just updated, 9042 * so we don't do microscopic update in schedule() 9043 * and double the fastpath cost. 9044 */ 9045 rq_clock_skip_update(rq); 9046 9047 /* 9048 * Forfeit the remaining vruntime, only if the entity is eligible. This 9049 * condition is necessary because in core scheduling we prefer to run 9050 * ineligible tasks rather than force idling. If this happens we may 9051 * end up in a loop where the core scheduler picks the yielding task, 9052 * which yields immediately again; without the condition the vruntime 9053 * ends up quickly running away. 9054 */ 9055 if (entity_eligible(cfs_rq, se)) { 9056 se->vruntime = se->deadline; 9057 se->deadline += calc_delta_fair(se->slice, se); 9058 } 9059 } 9060 9061 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) 9062 { 9063 struct sched_entity *se = &p->se; 9064 9065 /* !se->on_rq also covers throttled task */ 9066 if (!se->on_rq) 9067 return false; 9068 9069 /* Tell the scheduler that we'd really like se to run next. */ 9070 set_next_buddy(se); 9071 9072 yield_task_fair(rq); 9073 9074 return true; 9075 } 9076 9077 /************************************************** 9078 * Fair scheduling class load-balancing methods. 9079 * 9080 * BASICS 9081 * 9082 * The purpose of load-balancing is to achieve the same basic fairness the 9083 * per-CPU scheduler provides, namely provide a proportional amount of compute 9084 * time to each task. This is expressed in the following equation: 9085 * 9086 * W_i,n/P_i == W_j,n/P_j for all i,j (1) 9087 * 9088 * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight 9089 * W_i,0 is defined as: 9090 * 9091 * W_i,0 = \Sum_j w_i,j (2) 9092 * 9093 * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight 9094 * is derived from the nice value as per sched_prio_to_weight[]. 9095 * 9096 * The weight average is an exponential decay average of the instantaneous 9097 * weight: 9098 * 9099 * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) 9100 * 9101 * C_i is the compute capacity of CPU i, typically it is the 9102 * fraction of 'recent' time available for SCHED_OTHER task execution. But it 9103 * can also include other factors [XXX]. 9104 * 9105 * To achieve this balance we define a measure of imbalance which follows 9106 * directly from (1): 9107 * 9108 * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) 9109 * 9110 * We them move tasks around to minimize the imbalance. In the continuous 9111 * function space it is obvious this converges, in the discrete case we get 9112 * a few fun cases generally called infeasible weight scenarios. 9113 * 9114 * [XXX expand on: 9115 * - infeasible weights; 9116 * - local vs global optima in the discrete case. ] 9117 * 9118 * 9119 * SCHED DOMAINS 9120 * 9121 * In order to solve the imbalance equation (4), and avoid the obvious O(n^2) 9122 * for all i,j solution, we create a tree of CPUs that follows the hardware 9123 * topology where each level pairs two lower groups (or better). This results 9124 * in O(log n) layers. Furthermore we reduce the number of CPUs going up the 9125 * tree to only the first of the previous level and we decrease the frequency 9126 * of load-balance at each level inversely proportional to the number of CPUs in 9127 * the groups. 9128 * 9129 * This yields: 9130 * 9131 * log_2 n 1 n 9132 * \Sum { --- * --- * 2^i } = O(n) (5) 9133 * i = 0 2^i 2^i 9134 * `- size of each group 9135 * | | `- number of CPUs doing load-balance 9136 * | `- freq 9137 * `- sum over all levels 9138 * 9139 * Coupled with a limit on how many tasks we can migrate every balance pass, 9140 * this makes (5) the runtime complexity of the balancer. 9141 * 9142 * An important property here is that each CPU is still (indirectly) connected 9143 * to every other CPU in at most O(log n) steps: 9144 * 9145 * The adjacency matrix of the resulting graph is given by: 9146 * 9147 * log_2 n 9148 * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) 9149 * k = 0 9150 * 9151 * And you'll find that: 9152 * 9153 * A^(log_2 n)_i,j != 0 for all i,j (7) 9154 * 9155 * Showing there's indeed a path between every CPU in at most O(log n) steps. 9156 * The task movement gives a factor of O(m), giving a convergence complexity 9157 * of: 9158 * 9159 * O(nm log n), n := nr_cpus, m := nr_tasks (8) 9160 * 9161 * 9162 * WORK CONSERVING 9163 * 9164 * In order to avoid CPUs going idle while there's still work to do, new idle 9165 * balancing is more aggressive and has the newly idle CPU iterate up the domain 9166 * tree itself instead of relying on other CPUs to bring it work. 9167 * 9168 * This adds some complexity to both (5) and (8) but it reduces the total idle 9169 * time. 9170 * 9171 * [XXX more?] 9172 * 9173 * 9174 * CGROUPS 9175 * 9176 * Cgroups make a horror show out of (2), instead of a simple sum we get: 9177 * 9178 * s_k,i 9179 * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) 9180 * S_k 9181 * 9182 * Where 9183 * 9184 * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) 9185 * 9186 * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i. 9187 * 9188 * The big problem is S_k, its a global sum needed to compute a local (W_i) 9189 * property. 9190 * 9191 * [XXX write more on how we solve this.. _after_ merging pjt's patches that 9192 * rewrite all of this once again.] 9193 */ 9194 9195 static unsigned long __read_mostly max_load_balance_interval = HZ/10; 9196 9197 enum fbq_type { regular, remote, all }; 9198 9199 /* 9200 * 'group_type' describes the group of CPUs at the moment of load balancing. 9201 * 9202 * The enum is ordered by pulling priority, with the group with lowest priority 9203 * first so the group_type can simply be compared when selecting the busiest 9204 * group. See update_sd_pick_busiest(). 9205 */ 9206 enum group_type { 9207 /* The group has spare capacity that can be used to run more tasks. */ 9208 group_has_spare = 0, 9209 /* 9210 * The group is fully used and the tasks don't compete for more CPU 9211 * cycles. Nevertheless, some tasks might wait before running. 9212 */ 9213 group_fully_busy, 9214 /* 9215 * One task doesn't fit with CPU's capacity and must be migrated to a 9216 * more powerful CPU. 9217 */ 9218 group_misfit_task, 9219 /* 9220 * Balance SMT group that's fully busy. Can benefit from migration 9221 * a task on SMT with busy sibling to another CPU on idle core. 9222 */ 9223 group_smt_balance, 9224 /* 9225 * SD_ASYM_PACKING only: One local CPU with higher capacity is available, 9226 * and the task should be migrated to it instead of running on the 9227 * current CPU. 9228 */ 9229 group_asym_packing, 9230 /* 9231 * The tasks' affinity constraints previously prevented the scheduler 9232 * from balancing the load across the system. 9233 */ 9234 group_imbalanced, 9235 /* 9236 * The CPU is overloaded and can't provide expected CPU cycles to all 9237 * tasks. 9238 */ 9239 group_overloaded 9240 }; 9241 9242 enum migration_type { 9243 migrate_load = 0, 9244 migrate_util, 9245 migrate_task, 9246 migrate_misfit 9247 }; 9248 9249 #define LBF_ALL_PINNED 0x01 9250 #define LBF_NEED_BREAK 0x02 9251 #define LBF_DST_PINNED 0x04 9252 #define LBF_SOME_PINNED 0x08 9253 #define LBF_ACTIVE_LB 0x10 9254 9255 struct lb_env { 9256 struct sched_domain *sd; 9257 9258 struct rq *src_rq; 9259 int src_cpu; 9260 9261 int dst_cpu; 9262 struct rq *dst_rq; 9263 9264 struct cpumask *dst_grpmask; 9265 int new_dst_cpu; 9266 enum cpu_idle_type idle; 9267 long imbalance; 9268 /* The set of CPUs under consideration for load-balancing */ 9269 struct cpumask *cpus; 9270 9271 unsigned int flags; 9272 9273 unsigned int loop; 9274 unsigned int loop_break; 9275 unsigned int loop_max; 9276 9277 enum fbq_type fbq_type; 9278 enum migration_type migration_type; 9279 struct list_head tasks; 9280 }; 9281 9282 /* 9283 * Is this task likely cache-hot: 9284 */ 9285 static int task_hot(struct task_struct *p, struct lb_env *env) 9286 { 9287 s64 delta; 9288 9289 lockdep_assert_rq_held(env->src_rq); 9290 9291 if (p->sched_class != &fair_sched_class) 9292 return 0; 9293 9294 if (unlikely(task_has_idle_policy(p))) 9295 return 0; 9296 9297 /* SMT siblings share cache */ 9298 if (env->sd->flags & SD_SHARE_CPUCAPACITY) 9299 return 0; 9300 9301 /* 9302 * Buddy candidates are cache hot: 9303 */ 9304 if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 9305 (&p->se == cfs_rq_of(&p->se)->next)) 9306 return 1; 9307 9308 if (sysctl_sched_migration_cost == -1) 9309 return 1; 9310 9311 /* 9312 * Don't migrate task if the task's cookie does not match 9313 * with the destination CPU's core cookie. 9314 */ 9315 if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p)) 9316 return 1; 9317 9318 if (sysctl_sched_migration_cost == 0) 9319 return 0; 9320 9321 delta = rq_clock_task(env->src_rq) - p->se.exec_start; 9322 9323 return delta < (s64)sysctl_sched_migration_cost; 9324 } 9325 9326 #ifdef CONFIG_NUMA_BALANCING 9327 /* 9328 * Returns a positive value, if task migration degrades locality. 9329 * Returns 0, if task migration is not affected by locality. 9330 * Returns a negative value, if task migration improves locality i.e migration preferred. 9331 */ 9332 static long migrate_degrades_locality(struct task_struct *p, struct lb_env *env) 9333 { 9334 struct numa_group *numa_group = rcu_dereference_all(p->numa_group); 9335 unsigned long src_weight, dst_weight; 9336 int src_nid, dst_nid, dist; 9337 9338 if (!static_branch_likely(&sched_numa_balancing)) 9339 return 0; 9340 9341 if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) 9342 return 0; 9343 9344 src_nid = cpu_to_node(env->src_cpu); 9345 dst_nid = cpu_to_node(env->dst_cpu); 9346 9347 if (src_nid == dst_nid) 9348 return 0; 9349 9350 /* Migrating away from the preferred node is always bad. */ 9351 if (src_nid == p->numa_preferred_nid) { 9352 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) 9353 return 1; 9354 else 9355 return 0; 9356 } 9357 9358 /* Encourage migration to the preferred node. */ 9359 if (dst_nid == p->numa_preferred_nid) 9360 return -1; 9361 9362 /* Leaving a core idle is often worse than degrading locality. */ 9363 if (env->idle == CPU_IDLE) 9364 return 0; 9365 9366 dist = node_distance(src_nid, dst_nid); 9367 if (numa_group) { 9368 src_weight = group_weight(p, src_nid, dist); 9369 dst_weight = group_weight(p, dst_nid, dist); 9370 } else { 9371 src_weight = task_weight(p, src_nid, dist); 9372 dst_weight = task_weight(p, dst_nid, dist); 9373 } 9374 9375 return src_weight - dst_weight; 9376 } 9377 9378 #else /* !CONFIG_NUMA_BALANCING: */ 9379 static inline long migrate_degrades_locality(struct task_struct *p, 9380 struct lb_env *env) 9381 { 9382 return 0; 9383 } 9384 #endif /* !CONFIG_NUMA_BALANCING */ 9385 9386 /* 9387 * Check whether the task is ineligible on the destination cpu 9388 * 9389 * When the PLACE_LAG scheduling feature is enabled and 9390 * dst_cfs_rq->nr_queued is greater than 1, if the task 9391 * is ineligible, it will also be ineligible when 9392 * it is migrated to the destination cpu. 9393 */ 9394 static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu) 9395 { 9396 struct cfs_rq *dst_cfs_rq; 9397 9398 #ifdef CONFIG_FAIR_GROUP_SCHED 9399 dst_cfs_rq = task_group(p)->cfs_rq[dest_cpu]; 9400 #else 9401 dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; 9402 #endif 9403 if (sched_feat(PLACE_LAG) && dst_cfs_rq->nr_queued && 9404 !entity_eligible(task_cfs_rq(p), &p->se)) 9405 return 1; 9406 9407 return 0; 9408 } 9409 9410 /* 9411 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? 9412 */ 9413 static 9414 int can_migrate_task(struct task_struct *p, struct lb_env *env) 9415 { 9416 long degrades, hot; 9417 9418 lockdep_assert_rq_held(env->src_rq); 9419 if (p->sched_task_hot) 9420 p->sched_task_hot = 0; 9421 9422 /* 9423 * We do not migrate tasks that are: 9424 * 1) delayed dequeued unless we migrate load, or 9425 * 2) target cfs_rq is in throttled hierarchy, or 9426 * 3) cannot be migrated to this CPU due to cpus_ptr, or 9427 * 4) running (obviously), or 9428 * 5) are cache-hot on their current CPU, or 9429 * 6) are blocked on mutexes (if SCHED_PROXY_EXEC is enabled) 9430 */ 9431 if ((p->se.sched_delayed) && (env->migration_type != migrate_load)) 9432 return 0; 9433 9434 if (lb_throttled_hierarchy(p, env->dst_cpu)) 9435 return 0; 9436 9437 /* 9438 * We want to prioritize the migration of eligible tasks. 9439 * For ineligible tasks we soft-limit them and only allow 9440 * them to migrate when nr_balance_failed is non-zero to 9441 * avoid load-balancing trying very hard to balance the load. 9442 */ 9443 if (!env->sd->nr_balance_failed && 9444 task_is_ineligible_on_dst_cpu(p, env->dst_cpu)) 9445 return 0; 9446 9447 /* Disregard percpu kthreads; they are where they need to be. */ 9448 if (kthread_is_per_cpu(p)) 9449 return 0; 9450 9451 if (task_is_blocked(p)) 9452 return 0; 9453 9454 if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { 9455 int cpu; 9456 9457 schedstat_inc(p->stats.nr_failed_migrations_affine); 9458 9459 env->flags |= LBF_SOME_PINNED; 9460 9461 /* 9462 * Remember if this task can be migrated to any other CPU in 9463 * our sched_group. We may want to revisit it if we couldn't 9464 * meet load balance goals by pulling other tasks on src_cpu. 9465 * 9466 * Avoid computing new_dst_cpu 9467 * - for NEWLY_IDLE 9468 * - if we have already computed one in current iteration 9469 * - if it's an active balance 9470 */ 9471 if (env->idle == CPU_NEWLY_IDLE || 9472 env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB)) 9473 return 0; 9474 9475 /* Prevent to re-select dst_cpu via env's CPUs: */ 9476 cpu = cpumask_first_and_and(env->dst_grpmask, env->cpus, p->cpus_ptr); 9477 9478 if (cpu < nr_cpu_ids) { 9479 env->flags |= LBF_DST_PINNED; 9480 env->new_dst_cpu = cpu; 9481 } 9482 9483 return 0; 9484 } 9485 9486 /* Record that we found at least one task that could run on dst_cpu */ 9487 env->flags &= ~LBF_ALL_PINNED; 9488 9489 if (task_on_cpu(env->src_rq, p) || 9490 task_current_donor(env->src_rq, p)) { 9491 schedstat_inc(p->stats.nr_failed_migrations_running); 9492 return 0; 9493 } 9494 9495 /* 9496 * Aggressive migration if: 9497 * 1) active balance 9498 * 2) destination numa is preferred 9499 * 3) task is cache cold, or 9500 * 4) too many balance attempts have failed. 9501 */ 9502 if (env->flags & LBF_ACTIVE_LB) 9503 return 1; 9504 9505 degrades = migrate_degrades_locality(p, env); 9506 if (!degrades) 9507 hot = task_hot(p, env); 9508 else 9509 hot = degrades > 0; 9510 9511 if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) { 9512 if (hot) 9513 p->sched_task_hot = 1; 9514 return 1; 9515 } 9516 9517 schedstat_inc(p->stats.nr_failed_migrations_hot); 9518 return 0; 9519 } 9520 9521 /* 9522 * detach_task() -- detach the task for the migration specified in env 9523 */ 9524 static void detach_task(struct task_struct *p, struct lb_env *env) 9525 { 9526 lockdep_assert_rq_held(env->src_rq); 9527 9528 if (p->sched_task_hot) { 9529 p->sched_task_hot = 0; 9530 schedstat_inc(env->sd->lb_hot_gained[env->idle]); 9531 schedstat_inc(p->stats.nr_forced_migrations); 9532 } 9533 9534 WARN_ON(task_current(env->src_rq, p)); 9535 WARN_ON(task_current_donor(env->src_rq, p)); 9536 9537 deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 9538 set_task_cpu(p, env->dst_cpu); 9539 } 9540 9541 /* 9542 * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as 9543 * part of active balancing operations within "domain". 9544 * 9545 * Returns a task if successful and NULL otherwise. 9546 */ 9547 static struct task_struct *detach_one_task(struct lb_env *env) 9548 { 9549 struct task_struct *p; 9550 9551 lockdep_assert_rq_held(env->src_rq); 9552 9553 list_for_each_entry_reverse(p, 9554 &env->src_rq->cfs_tasks, se.group_node) { 9555 if (!can_migrate_task(p, env)) 9556 continue; 9557 9558 detach_task(p, env); 9559 9560 /* 9561 * Right now, this is only the second place where 9562 * lb_gained[env->idle] is updated (other is detach_tasks) 9563 * so we can safely collect stats here rather than 9564 * inside detach_tasks(). 9565 */ 9566 schedstat_inc(env->sd->lb_gained[env->idle]); 9567 return p; 9568 } 9569 return NULL; 9570 } 9571 9572 /* 9573 * detach_tasks() -- tries to detach up to imbalance load/util/tasks from 9574 * busiest_rq, as part of a balancing operation within domain "sd". 9575 * 9576 * Returns number of detached tasks if successful and 0 otherwise. 9577 */ 9578 static int detach_tasks(struct lb_env *env) 9579 { 9580 struct list_head *tasks = &env->src_rq->cfs_tasks; 9581 unsigned long util, load; 9582 struct task_struct *p; 9583 int detached = 0; 9584 9585 lockdep_assert_rq_held(env->src_rq); 9586 9587 /* 9588 * Source run queue has been emptied by another CPU, clear 9589 * LBF_ALL_PINNED flag as we will not test any task. 9590 */ 9591 if (env->src_rq->nr_running <= 1) { 9592 env->flags &= ~LBF_ALL_PINNED; 9593 return 0; 9594 } 9595 9596 if (env->imbalance <= 0) 9597 return 0; 9598 9599 while (!list_empty(tasks)) { 9600 /* 9601 * We don't want to steal all, otherwise we may be treated likewise, 9602 * which could at worst lead to a livelock crash. 9603 */ 9604 if (env->idle && env->src_rq->nr_running <= 1) 9605 break; 9606 9607 env->loop++; 9608 /* We've more or less seen every task there is, call it quits */ 9609 if (env->loop > env->loop_max) 9610 break; 9611 9612 /* take a breather every nr_migrate tasks */ 9613 if (env->loop > env->loop_break) { 9614 env->loop_break += SCHED_NR_MIGRATE_BREAK; 9615 env->flags |= LBF_NEED_BREAK; 9616 break; 9617 } 9618 9619 p = list_last_entry(tasks, struct task_struct, se.group_node); 9620 9621 if (!can_migrate_task(p, env)) 9622 goto next; 9623 9624 switch (env->migration_type) { 9625 case migrate_load: 9626 /* 9627 * Depending of the number of CPUs and tasks and the 9628 * cgroup hierarchy, task_h_load() can return a null 9629 * value. Make sure that env->imbalance decreases 9630 * otherwise detach_tasks() will stop only after 9631 * detaching up to loop_max tasks. 9632 */ 9633 load = max_t(unsigned long, task_h_load(p), 1); 9634 9635 if (sched_feat(LB_MIN) && 9636 load < 16 && !env->sd->nr_balance_failed) 9637 goto next; 9638 9639 /* 9640 * Make sure that we don't migrate too much load. 9641 * Nevertheless, let relax the constraint if 9642 * scheduler fails to find a good waiting task to 9643 * migrate. 9644 */ 9645 if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance) 9646 goto next; 9647 9648 env->imbalance -= load; 9649 break; 9650 9651 case migrate_util: 9652 util = task_util_est(p); 9653 9654 if (shr_bound(util, env->sd->nr_balance_failed) > env->imbalance) 9655 goto next; 9656 9657 env->imbalance -= util; 9658 break; 9659 9660 case migrate_task: 9661 env->imbalance--; 9662 break; 9663 9664 case migrate_misfit: 9665 /* This is not a misfit task */ 9666 if (task_fits_cpu(p, env->src_cpu)) 9667 goto next; 9668 9669 env->imbalance = 0; 9670 break; 9671 } 9672 9673 detach_task(p, env); 9674 list_add(&p->se.group_node, &env->tasks); 9675 9676 detached++; 9677 9678 #ifdef CONFIG_PREEMPTION 9679 /* 9680 * NEWIDLE balancing is a source of latency, so preemptible 9681 * kernels will stop after the first task is detached to minimize 9682 * the critical section. 9683 */ 9684 if (env->idle == CPU_NEWLY_IDLE) 9685 break; 9686 #endif 9687 9688 /* 9689 * We only want to steal up to the prescribed amount of 9690 * load/util/tasks. 9691 */ 9692 if (env->imbalance <= 0) 9693 break; 9694 9695 continue; 9696 next: 9697 if (p->sched_task_hot) 9698 schedstat_inc(p->stats.nr_failed_migrations_hot); 9699 9700 list_move(&p->se.group_node, tasks); 9701 } 9702 9703 /* 9704 * Right now, this is one of only two places we collect this stat 9705 * so we can safely collect detach_one_task() stats here rather 9706 * than inside detach_one_task(). 9707 */ 9708 schedstat_add(env->sd->lb_gained[env->idle], detached); 9709 9710 return detached; 9711 } 9712 9713 /* 9714 * attach_task() -- attach the task detached by detach_task() to its new rq. 9715 */ 9716 static void attach_task(struct rq *rq, struct task_struct *p) 9717 { 9718 lockdep_assert_rq_held(rq); 9719 9720 WARN_ON_ONCE(task_rq(p) != rq); 9721 activate_task(rq, p, ENQUEUE_NOCLOCK); 9722 wakeup_preempt(rq, p, 0); 9723 } 9724 9725 /* 9726 * attach_one_task() -- attaches the task returned from detach_one_task() to 9727 * its new rq. 9728 */ 9729 static void attach_one_task(struct rq *rq, struct task_struct *p) 9730 { 9731 struct rq_flags rf; 9732 9733 rq_lock(rq, &rf); 9734 update_rq_clock(rq); 9735 attach_task(rq, p); 9736 rq_unlock(rq, &rf); 9737 } 9738 9739 /* 9740 * attach_tasks() -- attaches all tasks detached by detach_tasks() to their 9741 * new rq. 9742 */ 9743 static void attach_tasks(struct lb_env *env) 9744 { 9745 struct list_head *tasks = &env->tasks; 9746 struct task_struct *p; 9747 struct rq_flags rf; 9748 9749 rq_lock(env->dst_rq, &rf); 9750 update_rq_clock(env->dst_rq); 9751 9752 while (!list_empty(tasks)) { 9753 p = list_first_entry(tasks, struct task_struct, se.group_node); 9754 list_del_init(&p->se.group_node); 9755 9756 attach_task(env->dst_rq, p); 9757 } 9758 9759 rq_unlock(env->dst_rq, &rf); 9760 } 9761 9762 #ifdef CONFIG_NO_HZ_COMMON 9763 static inline bool cfs_rq_has_blocked_load(struct cfs_rq *cfs_rq) 9764 { 9765 if (cfs_rq->avg.load_avg) 9766 return true; 9767 9768 if (cfs_rq->avg.util_avg) 9769 return true; 9770 9771 return false; 9772 } 9773 9774 static inline bool others_have_blocked(struct rq *rq) 9775 { 9776 if (cpu_util_rt(rq)) 9777 return true; 9778 9779 if (cpu_util_dl(rq)) 9780 return true; 9781 9782 if (hw_load_avg(rq)) 9783 return true; 9784 9785 if (cpu_util_irq(rq)) 9786 return true; 9787 9788 return false; 9789 } 9790 9791 static inline void update_blocked_load_tick(struct rq *rq) 9792 { 9793 WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies); 9794 } 9795 9796 static inline void update_has_blocked_load_status(struct rq *rq, bool has_blocked_load) 9797 { 9798 if (!has_blocked_load) 9799 rq->has_blocked_load = 0; 9800 } 9801 #else /* !CONFIG_NO_HZ_COMMON: */ 9802 static inline bool cfs_rq_has_blocked_load(struct cfs_rq *cfs_rq) { return false; } 9803 static inline bool others_have_blocked(struct rq *rq) { return false; } 9804 static inline void update_blocked_load_tick(struct rq *rq) {} 9805 static inline void update_has_blocked_load_status(struct rq *rq, bool has_blocked_load) {} 9806 #endif /* !CONFIG_NO_HZ_COMMON */ 9807 9808 static bool __update_blocked_others(struct rq *rq, bool *done) 9809 { 9810 bool updated; 9811 9812 /* 9813 * update_load_avg() can call cpufreq_update_util(). Make sure that RT, 9814 * DL and IRQ signals have been updated before updating CFS. 9815 */ 9816 updated = update_other_load_avgs(rq); 9817 9818 if (others_have_blocked(rq)) 9819 *done = false; 9820 9821 return updated; 9822 } 9823 9824 #ifdef CONFIG_FAIR_GROUP_SCHED 9825 9826 static bool __update_blocked_fair(struct rq *rq, bool *done) 9827 { 9828 struct cfs_rq *cfs_rq, *pos; 9829 bool decayed = false; 9830 int cpu = cpu_of(rq); 9831 9832 /* 9833 * Iterates the task_group tree in a bottom up fashion, see 9834 * list_add_leaf_cfs_rq() for details. 9835 */ 9836 for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 9837 struct sched_entity *se; 9838 9839 if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { 9840 update_tg_load_avg(cfs_rq); 9841 9842 if (cfs_rq->nr_queued == 0) 9843 update_idle_cfs_rq_clock_pelt(cfs_rq); 9844 9845 if (cfs_rq == &rq->cfs) 9846 decayed = true; 9847 } 9848 9849 /* Propagate pending load changes to the parent, if any: */ 9850 se = cfs_rq->tg->se[cpu]; 9851 if (se && !skip_blocked_update(se)) 9852 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 9853 9854 /* 9855 * There can be a lot of idle CPU cgroups. Don't let fully 9856 * decayed cfs_rqs linger on the list. 9857 */ 9858 if (cfs_rq_is_decayed(cfs_rq)) 9859 list_del_leaf_cfs_rq(cfs_rq); 9860 9861 /* Don't need periodic decay once load/util_avg are null */ 9862 if (cfs_rq_has_blocked_load(cfs_rq)) 9863 *done = false; 9864 } 9865 9866 return decayed; 9867 } 9868 9869 /* 9870 * Compute the hierarchical load factor for cfs_rq and all its ascendants. 9871 * This needs to be done in a top-down fashion because the load of a child 9872 * group is a fraction of its parents load. 9873 */ 9874 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) 9875 { 9876 struct rq *rq = rq_of(cfs_rq); 9877 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 9878 unsigned long now = jiffies; 9879 unsigned long load; 9880 9881 if (cfs_rq->last_h_load_update == now) 9882 return; 9883 9884 WRITE_ONCE(cfs_rq->h_load_next, NULL); 9885 for_each_sched_entity(se) { 9886 cfs_rq = cfs_rq_of(se); 9887 WRITE_ONCE(cfs_rq->h_load_next, se); 9888 if (cfs_rq->last_h_load_update == now) 9889 break; 9890 } 9891 9892 if (!se) { 9893 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); 9894 cfs_rq->last_h_load_update = now; 9895 } 9896 9897 while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { 9898 load = cfs_rq->h_load; 9899 load = div64_ul(load * se->avg.load_avg, 9900 cfs_rq_load_avg(cfs_rq) + 1); 9901 cfs_rq = group_cfs_rq(se); 9902 cfs_rq->h_load = load; 9903 cfs_rq->last_h_load_update = now; 9904 } 9905 } 9906 9907 static unsigned long task_h_load(struct task_struct *p) 9908 { 9909 struct cfs_rq *cfs_rq = task_cfs_rq(p); 9910 9911 update_cfs_rq_h_load(cfs_rq); 9912 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 9913 cfs_rq_load_avg(cfs_rq) + 1); 9914 } 9915 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 9916 static bool __update_blocked_fair(struct rq *rq, bool *done) 9917 { 9918 struct cfs_rq *cfs_rq = &rq->cfs; 9919 bool decayed; 9920 9921 decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq); 9922 if (cfs_rq_has_blocked_load(cfs_rq)) 9923 *done = false; 9924 9925 return decayed; 9926 } 9927 9928 static unsigned long task_h_load(struct task_struct *p) 9929 { 9930 return p->se.avg.load_avg; 9931 } 9932 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 9933 9934 static void __sched_balance_update_blocked_averages(struct rq *rq) 9935 { 9936 bool decayed = false, done = true; 9937 9938 update_blocked_load_tick(rq); 9939 9940 decayed |= __update_blocked_others(rq, &done); 9941 decayed |= __update_blocked_fair(rq, &done); 9942 9943 update_has_blocked_load_status(rq, !done); 9944 if (decayed) 9945 cpufreq_update_util(rq, 0); 9946 } 9947 9948 static void sched_balance_update_blocked_averages(int cpu) 9949 { 9950 struct rq *rq = cpu_rq(cpu); 9951 9952 guard(rq_lock_irqsave)(rq); 9953 update_rq_clock(rq); 9954 __sched_balance_update_blocked_averages(rq); 9955 } 9956 9957 /********** Helpers for sched_balance_find_src_group ************************/ 9958 9959 /* 9960 * sg_lb_stats - stats of a sched_group required for load-balancing: 9961 */ 9962 struct sg_lb_stats { 9963 unsigned long avg_load; /* Avg load over the CPUs of the group */ 9964 unsigned long group_load; /* Total load over the CPUs of the group */ 9965 unsigned long group_capacity; /* Capacity over the CPUs of the group */ 9966 unsigned long group_util; /* Total utilization over the CPUs of the group */ 9967 unsigned long group_runnable; /* Total runnable time over the CPUs of the group */ 9968 unsigned int sum_nr_running; /* Nr of all tasks running in the group */ 9969 unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */ 9970 unsigned int idle_cpus; /* Nr of idle CPUs in the group */ 9971 unsigned int group_weight; 9972 enum group_type group_type; 9973 unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */ 9974 unsigned int group_smt_balance; /* Task on busy SMT be moved */ 9975 unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ 9976 #ifdef CONFIG_NUMA_BALANCING 9977 unsigned int nr_numa_running; 9978 unsigned int nr_preferred_running; 9979 #endif 9980 }; 9981 9982 /* 9983 * sd_lb_stats - stats of a sched_domain required for load-balancing: 9984 */ 9985 struct sd_lb_stats { 9986 struct sched_group *busiest; /* Busiest group in this sd */ 9987 struct sched_group *local; /* Local group in this sd */ 9988 unsigned long total_load; /* Total load of all groups in sd */ 9989 unsigned long total_capacity; /* Total capacity of all groups in sd */ 9990 unsigned long avg_load; /* Average load across all groups in sd */ 9991 unsigned int prefer_sibling; /* Tasks should go to sibling first */ 9992 9993 struct sg_lb_stats busiest_stat; /* Statistics of the busiest group */ 9994 struct sg_lb_stats local_stat; /* Statistics of the local group */ 9995 }; 9996 9997 static inline void init_sd_lb_stats(struct sd_lb_stats *sds) 9998 { 9999 /* 10000 * Skimp on the clearing to avoid duplicate work. We can avoid clearing 10001 * local_stat because update_sg_lb_stats() does a full clear/assignment. 10002 * We must however set busiest_stat::group_type and 10003 * busiest_stat::idle_cpus to the worst busiest group because 10004 * update_sd_pick_busiest() reads these before assignment. 10005 */ 10006 *sds = (struct sd_lb_stats){ 10007 .busiest = NULL, 10008 .local = NULL, 10009 .total_load = 0UL, 10010 .total_capacity = 0UL, 10011 .busiest_stat = { 10012 .idle_cpus = UINT_MAX, 10013 .group_type = group_has_spare, 10014 }, 10015 }; 10016 } 10017 10018 static unsigned long scale_rt_capacity(int cpu) 10019 { 10020 unsigned long max = get_actual_cpu_capacity(cpu); 10021 struct rq *rq = cpu_rq(cpu); 10022 unsigned long used, free; 10023 unsigned long irq; 10024 10025 irq = cpu_util_irq(rq); 10026 10027 if (unlikely(irq >= max)) 10028 return 1; 10029 10030 /* 10031 * avg_rt.util_avg and avg_dl.util_avg track binary signals 10032 * (running and not running) with weights 0 and 1024 respectively. 10033 */ 10034 used = cpu_util_rt(rq); 10035 used += cpu_util_dl(rq); 10036 10037 if (unlikely(used >= max)) 10038 return 1; 10039 10040 free = max - used; 10041 10042 return scale_irq_capacity(free, irq, max); 10043 } 10044 10045 static void update_cpu_capacity(struct sched_domain *sd, int cpu) 10046 { 10047 unsigned long capacity = scale_rt_capacity(cpu); 10048 struct sched_group *sdg = sd->groups; 10049 10050 if (!capacity) 10051 capacity = 1; 10052 10053 cpu_rq(cpu)->cpu_capacity = capacity; 10054 trace_sched_cpu_capacity_tp(cpu_rq(cpu)); 10055 10056 sdg->sgc->capacity = capacity; 10057 sdg->sgc->min_capacity = capacity; 10058 sdg->sgc->max_capacity = capacity; 10059 } 10060 10061 void update_group_capacity(struct sched_domain *sd, int cpu) 10062 { 10063 struct sched_domain *child = sd->child; 10064 struct sched_group *group, *sdg = sd->groups; 10065 unsigned long capacity, min_capacity, max_capacity; 10066 unsigned long interval; 10067 10068 interval = msecs_to_jiffies(sd->balance_interval); 10069 interval = clamp(interval, 1UL, max_load_balance_interval); 10070 sdg->sgc->next_update = jiffies + interval; 10071 10072 if (!child) { 10073 update_cpu_capacity(sd, cpu); 10074 return; 10075 } 10076 10077 capacity = 0; 10078 min_capacity = ULONG_MAX; 10079 max_capacity = 0; 10080 10081 if (child->flags & SD_NUMA) { 10082 /* 10083 * SD_NUMA domains cannot assume that child groups 10084 * span the current group. 10085 */ 10086 10087 for_each_cpu(cpu, sched_group_span(sdg)) { 10088 unsigned long cpu_cap = capacity_of(cpu); 10089 10090 capacity += cpu_cap; 10091 min_capacity = min(cpu_cap, min_capacity); 10092 max_capacity = max(cpu_cap, max_capacity); 10093 } 10094 } else { 10095 /* 10096 * !SD_NUMA domains can assume that child groups 10097 * span the current group. 10098 */ 10099 10100 group = child->groups; 10101 do { 10102 struct sched_group_capacity *sgc = group->sgc; 10103 10104 capacity += sgc->capacity; 10105 min_capacity = min(sgc->min_capacity, min_capacity); 10106 max_capacity = max(sgc->max_capacity, max_capacity); 10107 group = group->next; 10108 } while (group != child->groups); 10109 } 10110 10111 sdg->sgc->capacity = capacity; 10112 sdg->sgc->min_capacity = min_capacity; 10113 sdg->sgc->max_capacity = max_capacity; 10114 } 10115 10116 /* 10117 * Check whether the capacity of the rq has been noticeably reduced by side 10118 * activity. The imbalance_pct is used for the threshold. 10119 * Return true is the capacity is reduced 10120 */ 10121 static inline int 10122 check_cpu_capacity(struct rq *rq, struct sched_domain *sd) 10123 { 10124 return ((rq->cpu_capacity * sd->imbalance_pct) < 10125 (arch_scale_cpu_capacity(cpu_of(rq)) * 100)); 10126 } 10127 10128 /* Check if the rq has a misfit task */ 10129 static inline bool check_misfit_status(struct rq *rq) 10130 { 10131 return rq->misfit_task_load; 10132 } 10133 10134 /* 10135 * Group imbalance indicates (and tries to solve) the problem where balancing 10136 * groups is inadequate due to ->cpus_ptr constraints. 10137 * 10138 * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a 10139 * cpumask covering 1 CPU of the first group and 3 CPUs of the second group. 10140 * Something like: 10141 * 10142 * { 0 1 2 3 } { 4 5 6 7 } 10143 * * * * * 10144 * 10145 * If we were to balance group-wise we'd place two tasks in the first group and 10146 * two tasks in the second group. Clearly this is undesired as it will overload 10147 * cpu 3 and leave one of the CPUs in the second group unused. 10148 * 10149 * The current solution to this issue is detecting the skew in the first group 10150 * by noticing the lower domain failed to reach balance and had difficulty 10151 * moving tasks due to affinity constraints. 10152 * 10153 * When this is so detected; this group becomes a candidate for busiest; see 10154 * update_sd_pick_busiest(). And calculate_imbalance() and 10155 * sched_balance_find_src_group() avoid some of the usual balance conditions to allow it 10156 * to create an effective group imbalance. 10157 * 10158 * This is a somewhat tricky proposition since the next run might not find the 10159 * group imbalance and decide the groups need to be balanced again. A most 10160 * subtle and fragile situation. 10161 */ 10162 10163 static inline int sg_imbalanced(struct sched_group *group) 10164 { 10165 return group->sgc->imbalance; 10166 } 10167 10168 /* 10169 * group_has_capacity returns true if the group has spare capacity that could 10170 * be used by some tasks. 10171 * We consider that a group has spare capacity if the number of task is 10172 * smaller than the number of CPUs or if the utilization is lower than the 10173 * available capacity for CFS tasks. 10174 * For the latter, we use a threshold to stabilize the state, to take into 10175 * account the variance of the tasks' load and to return true if the available 10176 * capacity in meaningful for the load balancer. 10177 * As an example, an available capacity of 1% can appear but it doesn't make 10178 * any benefit for the load balance. 10179 */ 10180 static inline bool 10181 group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 10182 { 10183 if (sgs->sum_nr_running < sgs->group_weight) 10184 return true; 10185 10186 if ((sgs->group_capacity * imbalance_pct) < 10187 (sgs->group_runnable * 100)) 10188 return false; 10189 10190 if ((sgs->group_capacity * 100) > 10191 (sgs->group_util * imbalance_pct)) 10192 return true; 10193 10194 return false; 10195 } 10196 10197 /* 10198 * group_is_overloaded returns true if the group has more tasks than it can 10199 * handle. 10200 * group_is_overloaded is not equals to !group_has_capacity because a group 10201 * with the exact right number of tasks, has no more spare capacity but is not 10202 * overloaded so both group_has_capacity and group_is_overloaded return 10203 * false. 10204 */ 10205 static inline bool 10206 group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 10207 { 10208 if (sgs->sum_nr_running <= sgs->group_weight) 10209 return false; 10210 10211 if ((sgs->group_capacity * 100) < 10212 (sgs->group_util * imbalance_pct)) 10213 return true; 10214 10215 if ((sgs->group_capacity * imbalance_pct) < 10216 (sgs->group_runnable * 100)) 10217 return true; 10218 10219 return false; 10220 } 10221 10222 static inline enum 10223 group_type group_classify(unsigned int imbalance_pct, 10224 struct sched_group *group, 10225 struct sg_lb_stats *sgs) 10226 { 10227 if (group_is_overloaded(imbalance_pct, sgs)) 10228 return group_overloaded; 10229 10230 if (sg_imbalanced(group)) 10231 return group_imbalanced; 10232 10233 if (sgs->group_asym_packing) 10234 return group_asym_packing; 10235 10236 if (sgs->group_smt_balance) 10237 return group_smt_balance; 10238 10239 if (sgs->group_misfit_task_load) 10240 return group_misfit_task; 10241 10242 if (!group_has_capacity(imbalance_pct, sgs)) 10243 return group_fully_busy; 10244 10245 return group_has_spare; 10246 } 10247 10248 /** 10249 * sched_use_asym_prio - Check whether asym_packing priority must be used 10250 * @sd: The scheduling domain of the load balancing 10251 * @cpu: A CPU 10252 * 10253 * Always use CPU priority when balancing load between SMT siblings. When 10254 * balancing load between cores, it is not sufficient that @cpu is idle. Only 10255 * use CPU priority if the whole core is idle. 10256 * 10257 * Returns: True if the priority of @cpu must be followed. False otherwise. 10258 */ 10259 static bool sched_use_asym_prio(struct sched_domain *sd, int cpu) 10260 { 10261 if (!(sd->flags & SD_ASYM_PACKING)) 10262 return false; 10263 10264 if (!sched_smt_active()) 10265 return true; 10266 10267 return sd->flags & SD_SHARE_CPUCAPACITY || is_core_idle(cpu); 10268 } 10269 10270 static inline bool sched_asym(struct sched_domain *sd, int dst_cpu, int src_cpu) 10271 { 10272 /* 10273 * First check if @dst_cpu can do asym_packing load balance. Only do it 10274 * if it has higher priority than @src_cpu. 10275 */ 10276 return sched_use_asym_prio(sd, dst_cpu) && 10277 sched_asym_prefer(dst_cpu, src_cpu); 10278 } 10279 10280 /** 10281 * sched_group_asym - Check if the destination CPU can do asym_packing balance 10282 * @env: The load balancing environment 10283 * @sgs: Load-balancing statistics of the candidate busiest group 10284 * @group: The candidate busiest group 10285 * 10286 * @env::dst_cpu can do asym_packing if it has higher priority than the 10287 * preferred CPU of @group. 10288 * 10289 * Return: true if @env::dst_cpu can do with asym_packing load balance. False 10290 * otherwise. 10291 */ 10292 static inline bool 10293 sched_group_asym(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group *group) 10294 { 10295 /* 10296 * CPU priorities do not make sense for SMT cores with more than one 10297 * busy sibling. 10298 */ 10299 if ((group->flags & SD_SHARE_CPUCAPACITY) && 10300 (sgs->group_weight - sgs->idle_cpus != 1)) 10301 return false; 10302 10303 return sched_asym(env->sd, env->dst_cpu, READ_ONCE(group->asym_prefer_cpu)); 10304 } 10305 10306 /* One group has more than one SMT CPU while the other group does not */ 10307 static inline bool smt_vs_nonsmt_groups(struct sched_group *sg1, 10308 struct sched_group *sg2) 10309 { 10310 if (!sg1 || !sg2) 10311 return false; 10312 10313 return (sg1->flags & SD_SHARE_CPUCAPACITY) != 10314 (sg2->flags & SD_SHARE_CPUCAPACITY); 10315 } 10316 10317 static inline bool smt_balance(struct lb_env *env, struct sg_lb_stats *sgs, 10318 struct sched_group *group) 10319 { 10320 if (!env->idle) 10321 return false; 10322 10323 /* 10324 * For SMT source group, it is better to move a task 10325 * to a CPU that doesn't have multiple tasks sharing its CPU capacity. 10326 * Note that if a group has a single SMT, SD_SHARE_CPUCAPACITY 10327 * will not be on. 10328 */ 10329 if (group->flags & SD_SHARE_CPUCAPACITY && 10330 sgs->sum_h_nr_running > 1) 10331 return true; 10332 10333 return false; 10334 } 10335 10336 static inline long sibling_imbalance(struct lb_env *env, 10337 struct sd_lb_stats *sds, 10338 struct sg_lb_stats *busiest, 10339 struct sg_lb_stats *local) 10340 { 10341 int ncores_busiest, ncores_local; 10342 long imbalance; 10343 10344 if (!env->idle || !busiest->sum_nr_running) 10345 return 0; 10346 10347 ncores_busiest = sds->busiest->cores; 10348 ncores_local = sds->local->cores; 10349 10350 if (ncores_busiest == ncores_local) { 10351 imbalance = busiest->sum_nr_running; 10352 lsub_positive(&imbalance, local->sum_nr_running); 10353 return imbalance; 10354 } 10355 10356 /* Balance such that nr_running/ncores ratio are same on both groups */ 10357 imbalance = ncores_local * busiest->sum_nr_running; 10358 lsub_positive(&imbalance, ncores_busiest * local->sum_nr_running); 10359 /* Normalize imbalance and do rounding on normalization */ 10360 imbalance = 2 * imbalance + ncores_local + ncores_busiest; 10361 imbalance /= ncores_local + ncores_busiest; 10362 10363 /* Take advantage of resource in an empty sched group */ 10364 if (imbalance <= 1 && local->sum_nr_running == 0 && 10365 busiest->sum_nr_running > 1) 10366 imbalance = 2; 10367 10368 return imbalance; 10369 } 10370 10371 static inline bool 10372 sched_reduced_capacity(struct rq *rq, struct sched_domain *sd) 10373 { 10374 /* 10375 * When there is more than 1 task, the group_overloaded case already 10376 * takes care of cpu with reduced capacity 10377 */ 10378 if (rq->cfs.h_nr_runnable != 1) 10379 return false; 10380 10381 return check_cpu_capacity(rq, sd); 10382 } 10383 10384 /** 10385 * update_sg_lb_stats - Update sched_group's statistics for load balancing. 10386 * @env: The load balancing environment. 10387 * @sds: Load-balancing data with statistics of the local group. 10388 * @group: sched_group whose statistics are to be updated. 10389 * @sgs: variable to hold the statistics for this group. 10390 * @sg_overloaded: sched_group is overloaded 10391 * @sg_overutilized: sched_group is overutilized 10392 */ 10393 static inline void update_sg_lb_stats(struct lb_env *env, 10394 struct sd_lb_stats *sds, 10395 struct sched_group *group, 10396 struct sg_lb_stats *sgs, 10397 bool *sg_overloaded, 10398 bool *sg_overutilized) 10399 { 10400 int i, nr_running, local_group, sd_flags = env->sd->flags; 10401 bool balancing_at_rd = !env->sd->parent; 10402 10403 memset(sgs, 0, sizeof(*sgs)); 10404 10405 local_group = group == sds->local; 10406 10407 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 10408 struct rq *rq = cpu_rq(i); 10409 unsigned long load = cpu_load(rq); 10410 10411 sgs->group_load += load; 10412 sgs->group_util += cpu_util_cfs(i); 10413 sgs->group_runnable += cpu_runnable(rq); 10414 sgs->sum_h_nr_running += rq->cfs.h_nr_runnable; 10415 10416 nr_running = rq->nr_running; 10417 sgs->sum_nr_running += nr_running; 10418 10419 if (cpu_overutilized(i)) 10420 *sg_overutilized = 1; 10421 10422 /* 10423 * No need to call idle_cpu() if nr_running is not 0 10424 */ 10425 if (!nr_running && idle_cpu(i)) { 10426 sgs->idle_cpus++; 10427 /* Idle cpu can't have misfit task */ 10428 continue; 10429 } 10430 10431 /* Overload indicator is only updated at root domain */ 10432 if (balancing_at_rd && nr_running > 1) 10433 *sg_overloaded = 1; 10434 10435 #ifdef CONFIG_NUMA_BALANCING 10436 /* Only fbq_classify_group() uses this to classify NUMA groups */ 10437 if (sd_flags & SD_NUMA) { 10438 sgs->nr_numa_running += rq->nr_numa_running; 10439 sgs->nr_preferred_running += rq->nr_preferred_running; 10440 } 10441 #endif 10442 if (local_group) 10443 continue; 10444 10445 if (sd_flags & SD_ASYM_CPUCAPACITY) { 10446 /* Check for a misfit task on the cpu */ 10447 if (sgs->group_misfit_task_load < rq->misfit_task_load) { 10448 sgs->group_misfit_task_load = rq->misfit_task_load; 10449 *sg_overloaded = 1; 10450 } 10451 } else if (env->idle && sched_reduced_capacity(rq, env->sd)) { 10452 /* Check for a task running on a CPU with reduced capacity */ 10453 if (sgs->group_misfit_task_load < load) 10454 sgs->group_misfit_task_load = load; 10455 } 10456 } 10457 10458 sgs->group_capacity = group->sgc->capacity; 10459 10460 sgs->group_weight = group->group_weight; 10461 10462 /* Check if dst CPU is idle and preferred to this group */ 10463 if (!local_group && env->idle && sgs->sum_h_nr_running && 10464 sched_group_asym(env, sgs, group)) 10465 sgs->group_asym_packing = 1; 10466 10467 /* Check for loaded SMT group to be balanced to dst CPU */ 10468 if (!local_group && smt_balance(env, sgs, group)) 10469 sgs->group_smt_balance = 1; 10470 10471 sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 10472 10473 /* Computing avg_load makes sense only when group is overloaded */ 10474 if (sgs->group_type == group_overloaded) 10475 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 10476 sgs->group_capacity; 10477 } 10478 10479 /** 10480 * update_sd_pick_busiest - return 1 on busiest group 10481 * @env: The load balancing environment. 10482 * @sds: sched_domain statistics 10483 * @sg: sched_group candidate to be checked for being the busiest 10484 * @sgs: sched_group statistics 10485 * 10486 * Determine if @sg is a busier group than the previously selected 10487 * busiest group. 10488 * 10489 * Return: %true if @sg is a busier group than the previously selected 10490 * busiest group. %false otherwise. 10491 */ 10492 static bool update_sd_pick_busiest(struct lb_env *env, 10493 struct sd_lb_stats *sds, 10494 struct sched_group *sg, 10495 struct sg_lb_stats *sgs) 10496 { 10497 struct sg_lb_stats *busiest = &sds->busiest_stat; 10498 10499 /* Make sure that there is at least one task to pull */ 10500 if (!sgs->sum_h_nr_running) 10501 return false; 10502 10503 /* 10504 * Don't try to pull misfit tasks we can't help. 10505 * We can use max_capacity here as reduction in capacity on some 10506 * CPUs in the group should either be possible to resolve 10507 * internally or be covered by avg_load imbalance (eventually). 10508 */ 10509 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 10510 (sgs->group_type == group_misfit_task) && 10511 (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || 10512 sds->local_stat.group_type != group_has_spare)) 10513 return false; 10514 10515 if (sgs->group_type > busiest->group_type) 10516 return true; 10517 10518 if (sgs->group_type < busiest->group_type) 10519 return false; 10520 10521 /* 10522 * The candidate and the current busiest group are the same type of 10523 * group. Let check which one is the busiest according to the type. 10524 */ 10525 10526 switch (sgs->group_type) { 10527 case group_overloaded: 10528 /* Select the overloaded group with highest avg_load. */ 10529 return sgs->avg_load > busiest->avg_load; 10530 10531 case group_imbalanced: 10532 /* 10533 * Select the 1st imbalanced group as we don't have any way to 10534 * choose one more than another. 10535 */ 10536 return false; 10537 10538 case group_asym_packing: 10539 /* Prefer to move from lowest priority CPU's work */ 10540 return sched_asym_prefer(READ_ONCE(sds->busiest->asym_prefer_cpu), 10541 READ_ONCE(sg->asym_prefer_cpu)); 10542 10543 case group_misfit_task: 10544 /* 10545 * If we have more than one misfit sg go with the biggest 10546 * misfit. 10547 */ 10548 return sgs->group_misfit_task_load > busiest->group_misfit_task_load; 10549 10550 case group_smt_balance: 10551 /* 10552 * Check if we have spare CPUs on either SMT group to 10553 * choose has spare or fully busy handling. 10554 */ 10555 if (sgs->idle_cpus != 0 || busiest->idle_cpus != 0) 10556 goto has_spare; 10557 10558 fallthrough; 10559 10560 case group_fully_busy: 10561 /* 10562 * Select the fully busy group with highest avg_load. In 10563 * theory, there is no need to pull task from such kind of 10564 * group because tasks have all compute capacity that they need 10565 * but we can still improve the overall throughput by reducing 10566 * contention when accessing shared HW resources. 10567 * 10568 * XXX for now avg_load is not computed and always 0 so we 10569 * select the 1st one, except if @sg is composed of SMT 10570 * siblings. 10571 */ 10572 10573 if (sgs->avg_load < busiest->avg_load) 10574 return false; 10575 10576 if (sgs->avg_load == busiest->avg_load) { 10577 /* 10578 * SMT sched groups need more help than non-SMT groups. 10579 * If @sg happens to also be SMT, either choice is good. 10580 */ 10581 if (sds->busiest->flags & SD_SHARE_CPUCAPACITY) 10582 return false; 10583 } 10584 10585 break; 10586 10587 case group_has_spare: 10588 /* 10589 * Do not pick sg with SMT CPUs over sg with pure CPUs, 10590 * as we do not want to pull task off SMT core with one task 10591 * and make the core idle. 10592 */ 10593 if (smt_vs_nonsmt_groups(sds->busiest, sg)) { 10594 if (sg->flags & SD_SHARE_CPUCAPACITY && sgs->sum_h_nr_running <= 1) 10595 return false; 10596 else 10597 return true; 10598 } 10599 has_spare: 10600 10601 /* 10602 * Select not overloaded group with lowest number of idle CPUs 10603 * and highest number of running tasks. We could also compare 10604 * the spare capacity which is more stable but it can end up 10605 * that the group has less spare capacity but finally more idle 10606 * CPUs which means less opportunity to pull tasks. 10607 */ 10608 if (sgs->idle_cpus > busiest->idle_cpus) 10609 return false; 10610 else if ((sgs->idle_cpus == busiest->idle_cpus) && 10611 (sgs->sum_nr_running <= busiest->sum_nr_running)) 10612 return false; 10613 10614 break; 10615 } 10616 10617 /* 10618 * Candidate sg has no more than one task per CPU and has higher 10619 * per-CPU capacity. Migrating tasks to less capable CPUs may harm 10620 * throughput. Maximize throughput, power/energy consequences are not 10621 * considered. 10622 */ 10623 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 10624 (sgs->group_type <= group_fully_busy) && 10625 (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) 10626 return false; 10627 10628 return true; 10629 } 10630 10631 #ifdef CONFIG_NUMA_BALANCING 10632 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 10633 { 10634 if (sgs->sum_h_nr_running > sgs->nr_numa_running) 10635 return regular; 10636 if (sgs->sum_h_nr_running > sgs->nr_preferred_running) 10637 return remote; 10638 return all; 10639 } 10640 10641 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 10642 { 10643 if (rq->nr_running > rq->nr_numa_running) 10644 return regular; 10645 if (rq->nr_running > rq->nr_preferred_running) 10646 return remote; 10647 return all; 10648 } 10649 #else /* !CONFIG_NUMA_BALANCING: */ 10650 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 10651 { 10652 return all; 10653 } 10654 10655 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 10656 { 10657 return regular; 10658 } 10659 #endif /* !CONFIG_NUMA_BALANCING */ 10660 10661 10662 struct sg_lb_stats; 10663 10664 /* 10665 * task_running_on_cpu - return 1 if @p is running on @cpu. 10666 */ 10667 10668 static unsigned int task_running_on_cpu(int cpu, struct task_struct *p) 10669 { 10670 /* Task has no contribution or is new */ 10671 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 10672 return 0; 10673 10674 if (task_on_rq_queued(p)) 10675 return 1; 10676 10677 return 0; 10678 } 10679 10680 /** 10681 * idle_cpu_without - would a given CPU be idle without p ? 10682 * @cpu: the processor on which idleness is tested. 10683 * @p: task which should be ignored. 10684 * 10685 * Return: 1 if the CPU would be idle. 0 otherwise. 10686 */ 10687 static int idle_cpu_without(int cpu, struct task_struct *p) 10688 { 10689 struct rq *rq = cpu_rq(cpu); 10690 10691 if (rq->curr != rq->idle && rq->curr != p) 10692 return 0; 10693 10694 /* 10695 * rq->nr_running can't be used but an updated version without the 10696 * impact of p on cpu must be used instead. The updated nr_running 10697 * be computed and tested before calling idle_cpu_without(). 10698 */ 10699 10700 if (rq->ttwu_pending) 10701 return 0; 10702 10703 return 1; 10704 } 10705 10706 /* 10707 * update_sg_wakeup_stats - Update sched_group's statistics for wakeup. 10708 * @sd: The sched_domain level to look for idlest group. 10709 * @group: sched_group whose statistics are to be updated. 10710 * @sgs: variable to hold the statistics for this group. 10711 * @p: The task for which we look for the idlest group/CPU. 10712 */ 10713 static inline void update_sg_wakeup_stats(struct sched_domain *sd, 10714 struct sched_group *group, 10715 struct sg_lb_stats *sgs, 10716 struct task_struct *p) 10717 { 10718 int i, nr_running; 10719 10720 memset(sgs, 0, sizeof(*sgs)); 10721 10722 /* Assume that task can't fit any CPU of the group */ 10723 if (sd->flags & SD_ASYM_CPUCAPACITY) 10724 sgs->group_misfit_task_load = 1; 10725 10726 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 10727 struct rq *rq = cpu_rq(i); 10728 unsigned int local; 10729 10730 sgs->group_load += cpu_load_without(rq, p); 10731 sgs->group_util += cpu_util_without(i, p); 10732 sgs->group_runnable += cpu_runnable_without(rq, p); 10733 local = task_running_on_cpu(i, p); 10734 sgs->sum_h_nr_running += rq->cfs.h_nr_runnable - local; 10735 10736 nr_running = rq->nr_running - local; 10737 sgs->sum_nr_running += nr_running; 10738 10739 /* 10740 * No need to call idle_cpu_without() if nr_running is not 0 10741 */ 10742 if (!nr_running && idle_cpu_without(i, p)) 10743 sgs->idle_cpus++; 10744 10745 /* Check if task fits in the CPU */ 10746 if (sd->flags & SD_ASYM_CPUCAPACITY && 10747 sgs->group_misfit_task_load && 10748 task_fits_cpu(p, i)) 10749 sgs->group_misfit_task_load = 0; 10750 10751 } 10752 10753 sgs->group_capacity = group->sgc->capacity; 10754 10755 sgs->group_weight = group->group_weight; 10756 10757 sgs->group_type = group_classify(sd->imbalance_pct, group, sgs); 10758 10759 /* 10760 * Computing avg_load makes sense only when group is fully busy or 10761 * overloaded 10762 */ 10763 if (sgs->group_type == group_fully_busy || 10764 sgs->group_type == group_overloaded) 10765 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 10766 sgs->group_capacity; 10767 } 10768 10769 static bool update_pick_idlest(struct sched_group *idlest, 10770 struct sg_lb_stats *idlest_sgs, 10771 struct sched_group *group, 10772 struct sg_lb_stats *sgs) 10773 { 10774 if (sgs->group_type < idlest_sgs->group_type) 10775 return true; 10776 10777 if (sgs->group_type > idlest_sgs->group_type) 10778 return false; 10779 10780 /* 10781 * The candidate and the current idlest group are the same type of 10782 * group. Let check which one is the idlest according to the type. 10783 */ 10784 10785 switch (sgs->group_type) { 10786 case group_overloaded: 10787 case group_fully_busy: 10788 /* Select the group with lowest avg_load. */ 10789 if (idlest_sgs->avg_load <= sgs->avg_load) 10790 return false; 10791 break; 10792 10793 case group_imbalanced: 10794 case group_asym_packing: 10795 case group_smt_balance: 10796 /* Those types are not used in the slow wakeup path */ 10797 return false; 10798 10799 case group_misfit_task: 10800 /* Select group with the highest max capacity */ 10801 if (idlest->sgc->max_capacity >= group->sgc->max_capacity) 10802 return false; 10803 break; 10804 10805 case group_has_spare: 10806 /* Select group with most idle CPUs */ 10807 if (idlest_sgs->idle_cpus > sgs->idle_cpus) 10808 return false; 10809 10810 /* Select group with lowest group_util */ 10811 if (idlest_sgs->idle_cpus == sgs->idle_cpus && 10812 idlest_sgs->group_util <= sgs->group_util) 10813 return false; 10814 10815 break; 10816 } 10817 10818 return true; 10819 } 10820 10821 /* 10822 * sched_balance_find_dst_group() finds and returns the least busy CPU group within the 10823 * domain. 10824 * 10825 * Assumes p is allowed on at least one CPU in sd. 10826 */ 10827 static struct sched_group * 10828 sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) 10829 { 10830 struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups; 10831 struct sg_lb_stats local_sgs, tmp_sgs; 10832 struct sg_lb_stats *sgs; 10833 unsigned long imbalance; 10834 struct sg_lb_stats idlest_sgs = { 10835 .avg_load = UINT_MAX, 10836 .group_type = group_overloaded, 10837 }; 10838 10839 do { 10840 int local_group; 10841 10842 /* Skip over this group if it has no CPUs allowed */ 10843 if (!cpumask_intersects(sched_group_span(group), 10844 p->cpus_ptr)) 10845 continue; 10846 10847 /* Skip over this group if no cookie matched */ 10848 if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group)) 10849 continue; 10850 10851 local_group = cpumask_test_cpu(this_cpu, 10852 sched_group_span(group)); 10853 10854 if (local_group) { 10855 sgs = &local_sgs; 10856 local = group; 10857 } else { 10858 sgs = &tmp_sgs; 10859 } 10860 10861 update_sg_wakeup_stats(sd, group, sgs, p); 10862 10863 if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) { 10864 idlest = group; 10865 idlest_sgs = *sgs; 10866 } 10867 10868 } while (group = group->next, group != sd->groups); 10869 10870 10871 /* There is no idlest group to push tasks to */ 10872 if (!idlest) 10873 return NULL; 10874 10875 /* The local group has been skipped because of CPU affinity */ 10876 if (!local) 10877 return idlest; 10878 10879 /* 10880 * If the local group is idler than the selected idlest group 10881 * don't try and push the task. 10882 */ 10883 if (local_sgs.group_type < idlest_sgs.group_type) 10884 return NULL; 10885 10886 /* 10887 * If the local group is busier than the selected idlest group 10888 * try and push the task. 10889 */ 10890 if (local_sgs.group_type > idlest_sgs.group_type) 10891 return idlest; 10892 10893 switch (local_sgs.group_type) { 10894 case group_overloaded: 10895 case group_fully_busy: 10896 10897 /* Calculate allowed imbalance based on load */ 10898 imbalance = scale_load_down(NICE_0_LOAD) * 10899 (sd->imbalance_pct-100) / 100; 10900 10901 /* 10902 * When comparing groups across NUMA domains, it's possible for 10903 * the local domain to be very lightly loaded relative to the 10904 * remote domains but "imbalance" skews the comparison making 10905 * remote CPUs look much more favourable. When considering 10906 * cross-domain, add imbalance to the load on the remote node 10907 * and consider staying local. 10908 */ 10909 10910 if ((sd->flags & SD_NUMA) && 10911 ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load)) 10912 return NULL; 10913 10914 /* 10915 * If the local group is less loaded than the selected 10916 * idlest group don't try and push any tasks. 10917 */ 10918 if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance)) 10919 return NULL; 10920 10921 if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load) 10922 return NULL; 10923 break; 10924 10925 case group_imbalanced: 10926 case group_asym_packing: 10927 case group_smt_balance: 10928 /* Those type are not used in the slow wakeup path */ 10929 return NULL; 10930 10931 case group_misfit_task: 10932 /* Select group with the highest max capacity */ 10933 if (local->sgc->max_capacity >= idlest->sgc->max_capacity) 10934 return NULL; 10935 break; 10936 10937 case group_has_spare: 10938 #ifdef CONFIG_NUMA 10939 if (sd->flags & SD_NUMA) { 10940 int imb_numa_nr = sd->imb_numa_nr; 10941 #ifdef CONFIG_NUMA_BALANCING 10942 int idlest_cpu; 10943 /* 10944 * If there is spare capacity at NUMA, try to select 10945 * the preferred node 10946 */ 10947 if (cpu_to_node(this_cpu) == p->numa_preferred_nid) 10948 return NULL; 10949 10950 idlest_cpu = cpumask_first(sched_group_span(idlest)); 10951 if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid) 10952 return idlest; 10953 #endif /* CONFIG_NUMA_BALANCING */ 10954 /* 10955 * Otherwise, keep the task close to the wakeup source 10956 * and improve locality if the number of running tasks 10957 * would remain below threshold where an imbalance is 10958 * allowed while accounting for the possibility the 10959 * task is pinned to a subset of CPUs. If there is a 10960 * real need of migration, periodic load balance will 10961 * take care of it. 10962 */ 10963 if (p->nr_cpus_allowed != NR_CPUS) { 10964 unsigned int w = cpumask_weight_and(p->cpus_ptr, 10965 sched_group_span(local)); 10966 imb_numa_nr = min(w, sd->imb_numa_nr); 10967 } 10968 10969 imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus); 10970 if (!adjust_numa_imbalance(imbalance, 10971 local_sgs.sum_nr_running + 1, 10972 imb_numa_nr)) { 10973 return NULL; 10974 } 10975 } 10976 #endif /* CONFIG_NUMA */ 10977 10978 /* 10979 * Select group with highest number of idle CPUs. We could also 10980 * compare the utilization which is more stable but it can end 10981 * up that the group has less spare capacity but finally more 10982 * idle CPUs which means more opportunity to run task. 10983 */ 10984 if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus) 10985 return NULL; 10986 break; 10987 } 10988 10989 return idlest; 10990 } 10991 10992 static void update_idle_cpu_scan(struct lb_env *env, 10993 unsigned long sum_util) 10994 { 10995 struct sched_domain_shared *sd_share; 10996 int llc_weight, pct; 10997 u64 x, y, tmp; 10998 /* 10999 * Update the number of CPUs to scan in LLC domain, which could 11000 * be used as a hint in select_idle_cpu(). The update of sd_share 11001 * could be expensive because it is within a shared cache line. 11002 * So the write of this hint only occurs during periodic load 11003 * balancing, rather than CPU_NEWLY_IDLE, because the latter 11004 * can fire way more frequently than the former. 11005 */ 11006 if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE) 11007 return; 11008 11009 llc_weight = per_cpu(sd_llc_size, env->dst_cpu); 11010 if (env->sd->span_weight != llc_weight) 11011 return; 11012 11013 sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, env->dst_cpu)); 11014 if (!sd_share) 11015 return; 11016 11017 /* 11018 * The number of CPUs to search drops as sum_util increases, when 11019 * sum_util hits 85% or above, the scan stops. 11020 * The reason to choose 85% as the threshold is because this is the 11021 * imbalance_pct(117) when a LLC sched group is overloaded. 11022 * 11023 * let y = SCHED_CAPACITY_SCALE - p * x^2 [1] 11024 * and y'= y / SCHED_CAPACITY_SCALE 11025 * 11026 * x is the ratio of sum_util compared to the CPU capacity: 11027 * x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE) 11028 * y' is the ratio of CPUs to be scanned in the LLC domain, 11029 * and the number of CPUs to scan is calculated by: 11030 * 11031 * nr_scan = llc_weight * y' [2] 11032 * 11033 * When x hits the threshold of overloaded, AKA, when 11034 * x = 100 / pct, y drops to 0. According to [1], 11035 * p should be SCHED_CAPACITY_SCALE * pct^2 / 10000 11036 * 11037 * Scale x by SCHED_CAPACITY_SCALE: 11038 * x' = sum_util / llc_weight; [3] 11039 * 11040 * and finally [1] becomes: 11041 * y = SCHED_CAPACITY_SCALE - 11042 * x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4] 11043 * 11044 */ 11045 /* equation [3] */ 11046 x = sum_util; 11047 do_div(x, llc_weight); 11048 11049 /* equation [4] */ 11050 pct = env->sd->imbalance_pct; 11051 tmp = x * x * pct * pct; 11052 do_div(tmp, 10000 * SCHED_CAPACITY_SCALE); 11053 tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE); 11054 y = SCHED_CAPACITY_SCALE - tmp; 11055 11056 /* equation [2] */ 11057 y *= llc_weight; 11058 do_div(y, SCHED_CAPACITY_SCALE); 11059 if ((int)y != sd_share->nr_idle_scan) 11060 WRITE_ONCE(sd_share->nr_idle_scan, (int)y); 11061 } 11062 11063 /** 11064 * update_sd_lb_stats - Update sched_domain's statistics for load balancing. 11065 * @env: The load balancing environment. 11066 * @sds: variable to hold the statistics for this sched_domain. 11067 */ 11068 11069 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) 11070 { 11071 struct sched_group *sg = env->sd->groups; 11072 struct sg_lb_stats *local = &sds->local_stat; 11073 struct sg_lb_stats tmp_sgs; 11074 unsigned long sum_util = 0; 11075 bool sg_overloaded = 0, sg_overutilized = 0; 11076 11077 do { 11078 struct sg_lb_stats *sgs = &tmp_sgs; 11079 int local_group; 11080 11081 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg)); 11082 if (local_group) { 11083 sds->local = sg; 11084 sgs = local; 11085 11086 if (env->idle != CPU_NEWLY_IDLE || 11087 time_after_eq(jiffies, sg->sgc->next_update)) 11088 update_group_capacity(env->sd, env->dst_cpu); 11089 } 11090 11091 update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded, &sg_overutilized); 11092 11093 if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) { 11094 sds->busiest = sg; 11095 sds->busiest_stat = *sgs; 11096 } 11097 11098 /* Now, start updating sd_lb_stats */ 11099 sds->total_load += sgs->group_load; 11100 sds->total_capacity += sgs->group_capacity; 11101 11102 sum_util += sgs->group_util; 11103 sg = sg->next; 11104 } while (sg != env->sd->groups); 11105 11106 /* 11107 * Indicate that the child domain of the busiest group prefers tasks 11108 * go to a child's sibling domains first. NB the flags of a sched group 11109 * are those of the child domain. 11110 */ 11111 if (sds->busiest) 11112 sds->prefer_sibling = !!(sds->busiest->flags & SD_PREFER_SIBLING); 11113 11114 11115 if (env->sd->flags & SD_NUMA) 11116 env->fbq_type = fbq_classify_group(&sds->busiest_stat); 11117 11118 if (!env->sd->parent) { 11119 /* update overload indicator if we are at root domain */ 11120 set_rd_overloaded(env->dst_rq->rd, sg_overloaded); 11121 11122 /* Update over-utilization (tipping point, U >= 0) indicator */ 11123 set_rd_overutilized(env->dst_rq->rd, sg_overutilized); 11124 } else if (sg_overutilized) { 11125 set_rd_overutilized(env->dst_rq->rd, sg_overutilized); 11126 } 11127 11128 update_idle_cpu_scan(env, sum_util); 11129 } 11130 11131 /** 11132 * calculate_imbalance - Calculate the amount of imbalance present within the 11133 * groups of a given sched_domain during load balance. 11134 * @env: load balance environment 11135 * @sds: statistics of the sched_domain whose imbalance is to be calculated. 11136 */ 11137 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) 11138 { 11139 struct sg_lb_stats *local, *busiest; 11140 11141 local = &sds->local_stat; 11142 busiest = &sds->busiest_stat; 11143 11144 if (busiest->group_type == group_misfit_task) { 11145 if (env->sd->flags & SD_ASYM_CPUCAPACITY) { 11146 /* Set imbalance to allow misfit tasks to be balanced. */ 11147 env->migration_type = migrate_misfit; 11148 env->imbalance = 1; 11149 } else { 11150 /* 11151 * Set load imbalance to allow moving task from cpu 11152 * with reduced capacity. 11153 */ 11154 env->migration_type = migrate_load; 11155 env->imbalance = busiest->group_misfit_task_load; 11156 } 11157 return; 11158 } 11159 11160 if (busiest->group_type == group_asym_packing) { 11161 /* 11162 * In case of asym capacity, we will try to migrate all load to 11163 * the preferred CPU. 11164 */ 11165 env->migration_type = migrate_task; 11166 env->imbalance = busiest->sum_h_nr_running; 11167 return; 11168 } 11169 11170 if (busiest->group_type == group_smt_balance) { 11171 /* Reduce number of tasks sharing CPU capacity */ 11172 env->migration_type = migrate_task; 11173 env->imbalance = 1; 11174 return; 11175 } 11176 11177 if (busiest->group_type == group_imbalanced) { 11178 /* 11179 * In the group_imb case we cannot rely on group-wide averages 11180 * to ensure CPU-load equilibrium, try to move any task to fix 11181 * the imbalance. The next load balance will take care of 11182 * balancing back the system. 11183 */ 11184 env->migration_type = migrate_task; 11185 env->imbalance = 1; 11186 return; 11187 } 11188 11189 /* 11190 * Try to use spare capacity of local group without overloading it or 11191 * emptying busiest. 11192 */ 11193 if (local->group_type == group_has_spare) { 11194 if ((busiest->group_type > group_fully_busy) && 11195 !(env->sd->flags & SD_SHARE_LLC)) { 11196 /* 11197 * If busiest is overloaded, try to fill spare 11198 * capacity. This might end up creating spare capacity 11199 * in busiest or busiest still being overloaded but 11200 * there is no simple way to directly compute the 11201 * amount of load to migrate in order to balance the 11202 * system. 11203 */ 11204 env->migration_type = migrate_util; 11205 env->imbalance = max(local->group_capacity, local->group_util) - 11206 local->group_util; 11207 11208 /* 11209 * In some cases, the group's utilization is max or even 11210 * higher than capacity because of migrations but the 11211 * local CPU is (newly) idle. There is at least one 11212 * waiting task in this overloaded busiest group. Let's 11213 * try to pull it. 11214 */ 11215 if (env->idle && env->imbalance == 0) { 11216 env->migration_type = migrate_task; 11217 env->imbalance = 1; 11218 } 11219 11220 return; 11221 } 11222 11223 if (busiest->group_weight == 1 || sds->prefer_sibling) { 11224 /* 11225 * When prefer sibling, evenly spread running tasks on 11226 * groups. 11227 */ 11228 env->migration_type = migrate_task; 11229 env->imbalance = sibling_imbalance(env, sds, busiest, local); 11230 } else { 11231 11232 /* 11233 * If there is no overload, we just want to even the number of 11234 * idle CPUs. 11235 */ 11236 env->migration_type = migrate_task; 11237 env->imbalance = max_t(long, 0, 11238 (local->idle_cpus - busiest->idle_cpus)); 11239 } 11240 11241 #ifdef CONFIG_NUMA 11242 /* Consider allowing a small imbalance between NUMA groups */ 11243 if (env->sd->flags & SD_NUMA) { 11244 env->imbalance = adjust_numa_imbalance(env->imbalance, 11245 local->sum_nr_running + 1, 11246 env->sd->imb_numa_nr); 11247 } 11248 #endif 11249 11250 /* Number of tasks to move to restore balance */ 11251 env->imbalance >>= 1; 11252 11253 return; 11254 } 11255 11256 /* 11257 * Local is fully busy but has to take more load to relieve the 11258 * busiest group 11259 */ 11260 if (local->group_type < group_overloaded) { 11261 /* 11262 * Local will become overloaded so the avg_load metrics are 11263 * finally needed. 11264 */ 11265 11266 local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 11267 local->group_capacity; 11268 11269 /* 11270 * If the local group is more loaded than the selected 11271 * busiest group don't try to pull any tasks. 11272 */ 11273 if (local->avg_load >= busiest->avg_load) { 11274 env->imbalance = 0; 11275 return; 11276 } 11277 11278 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 11279 sds->total_capacity; 11280 11281 /* 11282 * If the local group is more loaded than the average system 11283 * load, don't try to pull any tasks. 11284 */ 11285 if (local->avg_load >= sds->avg_load) { 11286 env->imbalance = 0; 11287 return; 11288 } 11289 11290 } 11291 11292 /* 11293 * Both group are or will become overloaded and we're trying to get all 11294 * the CPUs to the average_load, so we don't want to push ourselves 11295 * above the average load, nor do we wish to reduce the max loaded CPU 11296 * below the average load. At the same time, we also don't want to 11297 * reduce the group load below the group capacity. Thus we look for 11298 * the minimum possible imbalance. 11299 */ 11300 env->migration_type = migrate_load; 11301 env->imbalance = min( 11302 (busiest->avg_load - sds->avg_load) * busiest->group_capacity, 11303 (sds->avg_load - local->avg_load) * local->group_capacity 11304 ) / SCHED_CAPACITY_SCALE; 11305 } 11306 11307 /******* sched_balance_find_src_group() helpers end here *********************/ 11308 11309 /* 11310 * Decision matrix according to the local and busiest group type: 11311 * 11312 * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded 11313 * has_spare nr_idle balanced N/A N/A balanced balanced 11314 * fully_busy nr_idle nr_idle N/A N/A balanced balanced 11315 * misfit_task force N/A N/A N/A N/A N/A 11316 * asym_packing force force N/A N/A force force 11317 * imbalanced force force N/A N/A force force 11318 * overloaded force force N/A N/A force avg_load 11319 * 11320 * N/A : Not Applicable because already filtered while updating 11321 * statistics. 11322 * balanced : The system is balanced for these 2 groups. 11323 * force : Calculate the imbalance as load migration is probably needed. 11324 * avg_load : Only if imbalance is significant enough. 11325 * nr_idle : dst_cpu is not busy and the number of idle CPUs is quite 11326 * different in groups. 11327 */ 11328 11329 /** 11330 * sched_balance_find_src_group - Returns the busiest group within the sched_domain 11331 * if there is an imbalance. 11332 * @env: The load balancing environment. 11333 * 11334 * Also calculates the amount of runnable load which should be moved 11335 * to restore balance. 11336 * 11337 * Return: - The busiest group if imbalance exists. 11338 */ 11339 static struct sched_group *sched_balance_find_src_group(struct lb_env *env) 11340 { 11341 struct sg_lb_stats *local, *busiest; 11342 struct sd_lb_stats sds; 11343 11344 init_sd_lb_stats(&sds); 11345 11346 /* 11347 * Compute the various statistics relevant for load balancing at 11348 * this level. 11349 */ 11350 update_sd_lb_stats(env, &sds); 11351 11352 /* There is no busy sibling group to pull tasks from */ 11353 if (!sds.busiest) 11354 goto out_balanced; 11355 11356 busiest = &sds.busiest_stat; 11357 11358 /* Misfit tasks should be dealt with regardless of the avg load */ 11359 if (busiest->group_type == group_misfit_task) 11360 goto force_balance; 11361 11362 if (!is_rd_overutilized(env->dst_rq->rd) && 11363 rcu_dereference_all(env->dst_rq->rd->pd)) 11364 goto out_balanced; 11365 11366 /* ASYM feature bypasses nice load balance check */ 11367 if (busiest->group_type == group_asym_packing) 11368 goto force_balance; 11369 11370 /* 11371 * If the busiest group is imbalanced the below checks don't 11372 * work because they assume all things are equal, which typically 11373 * isn't true due to cpus_ptr constraints and the like. 11374 */ 11375 if (busiest->group_type == group_imbalanced) 11376 goto force_balance; 11377 11378 local = &sds.local_stat; 11379 /* 11380 * If the local group is busier than the selected busiest group 11381 * don't try and pull any tasks. 11382 */ 11383 if (local->group_type > busiest->group_type) 11384 goto out_balanced; 11385 11386 /* 11387 * When groups are overloaded, use the avg_load to ensure fairness 11388 * between tasks. 11389 */ 11390 if (local->group_type == group_overloaded) { 11391 /* 11392 * If the local group is more loaded than the selected 11393 * busiest group don't try to pull any tasks. 11394 */ 11395 if (local->avg_load >= busiest->avg_load) 11396 goto out_balanced; 11397 11398 /* XXX broken for overlapping NUMA groups */ 11399 sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) / 11400 sds.total_capacity; 11401 11402 /* 11403 * Don't pull any tasks if this group is already above the 11404 * domain average load. 11405 */ 11406 if (local->avg_load >= sds.avg_load) 11407 goto out_balanced; 11408 11409 /* 11410 * If the busiest group is more loaded, use imbalance_pct to be 11411 * conservative. 11412 */ 11413 if (100 * busiest->avg_load <= 11414 env->sd->imbalance_pct * local->avg_load) 11415 goto out_balanced; 11416 } 11417 11418 /* 11419 * Try to move all excess tasks to a sibling domain of the busiest 11420 * group's child domain. 11421 */ 11422 if (sds.prefer_sibling && local->group_type == group_has_spare && 11423 sibling_imbalance(env, &sds, busiest, local) > 1) 11424 goto force_balance; 11425 11426 if (busiest->group_type != group_overloaded) { 11427 if (!env->idle) { 11428 /* 11429 * If the busiest group is not overloaded (and as a 11430 * result the local one too) but this CPU is already 11431 * busy, let another idle CPU try to pull task. 11432 */ 11433 goto out_balanced; 11434 } 11435 11436 if (busiest->group_type == group_smt_balance && 11437 smt_vs_nonsmt_groups(sds.local, sds.busiest)) { 11438 /* Let non SMT CPU pull from SMT CPU sharing with sibling */ 11439 goto force_balance; 11440 } 11441 11442 if (busiest->group_weight > 1 && 11443 local->idle_cpus <= (busiest->idle_cpus + 1)) { 11444 /* 11445 * If the busiest group is not overloaded 11446 * and there is no imbalance between this and busiest 11447 * group wrt idle CPUs, it is balanced. The imbalance 11448 * becomes significant if the diff is greater than 1 11449 * otherwise we might end up to just move the imbalance 11450 * on another group. Of course this applies only if 11451 * there is more than 1 CPU per group. 11452 */ 11453 goto out_balanced; 11454 } 11455 11456 if (busiest->sum_h_nr_running == 1) { 11457 /* 11458 * busiest doesn't have any tasks waiting to run 11459 */ 11460 goto out_balanced; 11461 } 11462 } 11463 11464 force_balance: 11465 /* Looks like there is an imbalance. Compute it */ 11466 calculate_imbalance(env, &sds); 11467 return env->imbalance ? sds.busiest : NULL; 11468 11469 out_balanced: 11470 env->imbalance = 0; 11471 return NULL; 11472 } 11473 11474 /* 11475 * sched_balance_find_src_rq - find the busiest runqueue among the CPUs in the group. 11476 */ 11477 static struct rq *sched_balance_find_src_rq(struct lb_env *env, 11478 struct sched_group *group) 11479 { 11480 struct rq *busiest = NULL, *rq; 11481 unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; 11482 unsigned int busiest_nr = 0; 11483 int i; 11484 11485 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 11486 unsigned long capacity, load, util; 11487 unsigned int nr_running; 11488 enum fbq_type rt; 11489 11490 rq = cpu_rq(i); 11491 rt = fbq_classify_rq(rq); 11492 11493 /* 11494 * We classify groups/runqueues into three groups: 11495 * - regular: there are !numa tasks 11496 * - remote: there are numa tasks that run on the 'wrong' node 11497 * - all: there is no distinction 11498 * 11499 * In order to avoid migrating ideally placed numa tasks, 11500 * ignore those when there's better options. 11501 * 11502 * If we ignore the actual busiest queue to migrate another 11503 * task, the next balance pass can still reduce the busiest 11504 * queue by moving tasks around inside the node. 11505 * 11506 * If we cannot move enough load due to this classification 11507 * the next pass will adjust the group classification and 11508 * allow migration of more tasks. 11509 * 11510 * Both cases only affect the total convergence complexity. 11511 */ 11512 if (rt > env->fbq_type) 11513 continue; 11514 11515 nr_running = rq->cfs.h_nr_runnable; 11516 if (!nr_running) 11517 continue; 11518 11519 capacity = capacity_of(i); 11520 11521 /* 11522 * For ASYM_CPUCAPACITY domains, don't pick a CPU that could 11523 * eventually lead to active_balancing high->low capacity. 11524 * Higher per-CPU capacity is considered better than balancing 11525 * average load. 11526 */ 11527 if (env->sd->flags & SD_ASYM_CPUCAPACITY && 11528 !capacity_greater(capacity_of(env->dst_cpu), capacity) && 11529 nr_running == 1) 11530 continue; 11531 11532 /* 11533 * Make sure we only pull tasks from a CPU of lower priority 11534 * when balancing between SMT siblings. 11535 * 11536 * If balancing between cores, let lower priority CPUs help 11537 * SMT cores with more than one busy sibling. 11538 */ 11539 if (sched_asym(env->sd, i, env->dst_cpu) && nr_running == 1) 11540 continue; 11541 11542 switch (env->migration_type) { 11543 case migrate_load: 11544 /* 11545 * When comparing with load imbalance, use cpu_load() 11546 * which is not scaled with the CPU capacity. 11547 */ 11548 load = cpu_load(rq); 11549 11550 if (nr_running == 1 && load > env->imbalance && 11551 !check_cpu_capacity(rq, env->sd)) 11552 break; 11553 11554 /* 11555 * For the load comparisons with the other CPUs, 11556 * consider the cpu_load() scaled with the CPU 11557 * capacity, so that the load can be moved away 11558 * from the CPU that is potentially running at a 11559 * lower capacity. 11560 * 11561 * Thus we're looking for max(load_i / capacity_i), 11562 * crosswise multiplication to rid ourselves of the 11563 * division works out to: 11564 * load_i * capacity_j > load_j * capacity_i; 11565 * where j is our previous maximum. 11566 */ 11567 if (load * busiest_capacity > busiest_load * capacity) { 11568 busiest_load = load; 11569 busiest_capacity = capacity; 11570 busiest = rq; 11571 } 11572 break; 11573 11574 case migrate_util: 11575 util = cpu_util_cfs_boost(i); 11576 11577 /* 11578 * Don't try to pull utilization from a CPU with one 11579 * running task. Whatever its utilization, we will fail 11580 * detach the task. 11581 */ 11582 if (nr_running <= 1) 11583 continue; 11584 11585 if (busiest_util < util) { 11586 busiest_util = util; 11587 busiest = rq; 11588 } 11589 break; 11590 11591 case migrate_task: 11592 if (busiest_nr < nr_running) { 11593 busiest_nr = nr_running; 11594 busiest = rq; 11595 } 11596 break; 11597 11598 case migrate_misfit: 11599 /* 11600 * For ASYM_CPUCAPACITY domains with misfit tasks we 11601 * simply seek the "biggest" misfit task. 11602 */ 11603 if (rq->misfit_task_load > busiest_load) { 11604 busiest_load = rq->misfit_task_load; 11605 busiest = rq; 11606 } 11607 11608 break; 11609 11610 } 11611 } 11612 11613 return busiest; 11614 } 11615 11616 /* 11617 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but 11618 * so long as it is large enough. 11619 */ 11620 #define MAX_PINNED_INTERVAL 512 11621 11622 static inline bool 11623 asym_active_balance(struct lb_env *env) 11624 { 11625 /* 11626 * ASYM_PACKING needs to force migrate tasks from busy but lower 11627 * priority CPUs in order to pack all tasks in the highest priority 11628 * CPUs. When done between cores, do it only if the whole core if the 11629 * whole core is idle. 11630 * 11631 * If @env::src_cpu is an SMT core with busy siblings, let 11632 * the lower priority @env::dst_cpu help it. Do not follow 11633 * CPU priority. 11634 */ 11635 return env->idle && sched_use_asym_prio(env->sd, env->dst_cpu) && 11636 (sched_asym_prefer(env->dst_cpu, env->src_cpu) || 11637 !sched_use_asym_prio(env->sd, env->src_cpu)); 11638 } 11639 11640 static inline bool 11641 imbalanced_active_balance(struct lb_env *env) 11642 { 11643 struct sched_domain *sd = env->sd; 11644 11645 /* 11646 * The imbalanced case includes the case of pinned tasks preventing a fair 11647 * distribution of the load on the system but also the even distribution of the 11648 * threads on a system with spare capacity 11649 */ 11650 if ((env->migration_type == migrate_task) && 11651 (sd->nr_balance_failed > sd->cache_nice_tries+2)) 11652 return 1; 11653 11654 return 0; 11655 } 11656 11657 static int need_active_balance(struct lb_env *env) 11658 { 11659 struct sched_domain *sd = env->sd; 11660 11661 if (asym_active_balance(env)) 11662 return 1; 11663 11664 if (imbalanced_active_balance(env)) 11665 return 1; 11666 11667 /* 11668 * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. 11669 * It's worth migrating the task if the src_cpu's capacity is reduced 11670 * because of other sched_class or IRQs if more capacity stays 11671 * available on dst_cpu. 11672 */ 11673 if (env->idle && 11674 (env->src_rq->cfs.h_nr_runnable == 1)) { 11675 if ((check_cpu_capacity(env->src_rq, sd)) && 11676 (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) 11677 return 1; 11678 } 11679 11680 if (env->migration_type == migrate_misfit) 11681 return 1; 11682 11683 return 0; 11684 } 11685 11686 static int active_load_balance_cpu_stop(void *data); 11687 11688 static int should_we_balance(struct lb_env *env) 11689 { 11690 struct cpumask *swb_cpus = this_cpu_cpumask_var_ptr(should_we_balance_tmpmask); 11691 struct sched_group *sg = env->sd->groups; 11692 int cpu, idle_smt = -1; 11693 11694 /* 11695 * Ensure the balancing environment is consistent; can happen 11696 * when the softirq triggers 'during' hotplug. 11697 */ 11698 if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 11699 return 0; 11700 11701 /* 11702 * In the newly idle case, we will allow all the CPUs 11703 * to do the newly idle load balance. 11704 * 11705 * However, we bail out if we already have tasks or a wakeup pending, 11706 * to optimize wakeup latency. 11707 */ 11708 if (env->idle == CPU_NEWLY_IDLE) { 11709 if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending) 11710 return 0; 11711 return 1; 11712 } 11713 11714 cpumask_copy(swb_cpus, group_balance_mask(sg)); 11715 /* Try to find first idle CPU */ 11716 for_each_cpu_and(cpu, swb_cpus, env->cpus) { 11717 if (!idle_cpu(cpu)) 11718 continue; 11719 11720 /* 11721 * Don't balance to idle SMT in busy core right away when 11722 * balancing cores, but remember the first idle SMT CPU for 11723 * later consideration. Find CPU on an idle core first. 11724 */ 11725 if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) { 11726 if (idle_smt == -1) 11727 idle_smt = cpu; 11728 /* 11729 * If the core is not idle, and first SMT sibling which is 11730 * idle has been found, then its not needed to check other 11731 * SMT siblings for idleness: 11732 */ 11733 #ifdef CONFIG_SCHED_SMT 11734 cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu)); 11735 #endif 11736 continue; 11737 } 11738 11739 /* 11740 * Are we the first idle core in a non-SMT domain or higher, 11741 * or the first idle CPU in a SMT domain? 11742 */ 11743 return cpu == env->dst_cpu; 11744 } 11745 11746 /* Are we the first idle CPU with busy siblings? */ 11747 if (idle_smt != -1) 11748 return idle_smt == env->dst_cpu; 11749 11750 /* Are we the first CPU of this group ? */ 11751 return group_balance_cpu(sg) == env->dst_cpu; 11752 } 11753 11754 static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd, 11755 enum cpu_idle_type idle) 11756 { 11757 if (!schedstat_enabled()) 11758 return; 11759 11760 switch (env->migration_type) { 11761 case migrate_load: 11762 __schedstat_add(sd->lb_imbalance_load[idle], env->imbalance); 11763 break; 11764 case migrate_util: 11765 __schedstat_add(sd->lb_imbalance_util[idle], env->imbalance); 11766 break; 11767 case migrate_task: 11768 __schedstat_add(sd->lb_imbalance_task[idle], env->imbalance); 11769 break; 11770 case migrate_misfit: 11771 __schedstat_add(sd->lb_imbalance_misfit[idle], env->imbalance); 11772 break; 11773 } 11774 } 11775 11776 /* 11777 * This flag serializes load-balancing passes over large domains 11778 * (above the NODE topology level) - only one load-balancing instance 11779 * may run at a time, to reduce overhead on very large systems with 11780 * lots of CPUs and large NUMA distances. 11781 * 11782 * - Note that load-balancing passes triggered while another one 11783 * is executing are skipped and not re-tried. 11784 * 11785 * - Also note that this does not serialize rebalance_domains() 11786 * execution, as non-SD_SERIALIZE domains will still be 11787 * load-balanced in parallel. 11788 */ 11789 static atomic_t sched_balance_running = ATOMIC_INIT(0); 11790 11791 /* 11792 * Check this_cpu to ensure it is balanced within domain. Attempt to move 11793 * tasks if there is an imbalance. 11794 */ 11795 static int sched_balance_rq(int this_cpu, struct rq *this_rq, 11796 struct sched_domain *sd, enum cpu_idle_type idle, 11797 int *continue_balancing) 11798 { 11799 int ld_moved, cur_ld_moved, active_balance = 0; 11800 struct sched_domain *sd_parent = sd->parent; 11801 struct sched_group *group; 11802 struct rq *busiest; 11803 struct rq_flags rf; 11804 struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 11805 struct lb_env env = { 11806 .sd = sd, 11807 .dst_cpu = this_cpu, 11808 .dst_rq = this_rq, 11809 .dst_grpmask = group_balance_mask(sd->groups), 11810 .idle = idle, 11811 .loop_break = SCHED_NR_MIGRATE_BREAK, 11812 .cpus = cpus, 11813 .fbq_type = all, 11814 .tasks = LIST_HEAD_INIT(env.tasks), 11815 }; 11816 bool need_unlock = false; 11817 11818 cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); 11819 11820 schedstat_inc(sd->lb_count[idle]); 11821 11822 redo: 11823 if (!should_we_balance(&env)) { 11824 *continue_balancing = 0; 11825 goto out_balanced; 11826 } 11827 11828 if (!need_unlock && (sd->flags & SD_SERIALIZE)) { 11829 int zero = 0; 11830 if (!atomic_try_cmpxchg_acquire(&sched_balance_running, &zero, 1)) 11831 goto out_balanced; 11832 11833 need_unlock = true; 11834 } 11835 11836 group = sched_balance_find_src_group(&env); 11837 if (!group) { 11838 schedstat_inc(sd->lb_nobusyg[idle]); 11839 goto out_balanced; 11840 } 11841 11842 busiest = sched_balance_find_src_rq(&env, group); 11843 if (!busiest) { 11844 schedstat_inc(sd->lb_nobusyq[idle]); 11845 goto out_balanced; 11846 } 11847 11848 WARN_ON_ONCE(busiest == env.dst_rq); 11849 11850 update_lb_imbalance_stat(&env, sd, idle); 11851 11852 env.src_cpu = busiest->cpu; 11853 env.src_rq = busiest; 11854 11855 ld_moved = 0; 11856 /* Clear this flag as soon as we find a pullable task */ 11857 env.flags |= LBF_ALL_PINNED; 11858 if (busiest->nr_running > 1) { 11859 /* 11860 * Attempt to move tasks. If sched_balance_find_src_group has found 11861 * an imbalance but busiest->nr_running <= 1, the group is 11862 * still unbalanced. ld_moved simply stays zero, so it is 11863 * correctly treated as an imbalance. 11864 */ 11865 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 11866 11867 more_balance: 11868 rq_lock_irqsave(busiest, &rf); 11869 update_rq_clock(busiest); 11870 11871 /* 11872 * cur_ld_moved - load moved in current iteration 11873 * ld_moved - cumulative load moved across iterations 11874 */ 11875 cur_ld_moved = detach_tasks(&env); 11876 11877 /* 11878 * We've detached some tasks from busiest_rq. Every 11879 * task is masked "TASK_ON_RQ_MIGRATING", so we can safely 11880 * unlock busiest->lock, and we are able to be sure 11881 * that nobody can manipulate the tasks in parallel. 11882 * See task_rq_lock() family for the details. 11883 */ 11884 11885 rq_unlock(busiest, &rf); 11886 11887 if (cur_ld_moved) { 11888 attach_tasks(&env); 11889 ld_moved += cur_ld_moved; 11890 } 11891 11892 local_irq_restore(rf.flags); 11893 11894 if (env.flags & LBF_NEED_BREAK) { 11895 env.flags &= ~LBF_NEED_BREAK; 11896 goto more_balance; 11897 } 11898 11899 /* 11900 * Revisit (affine) tasks on src_cpu that couldn't be moved to 11901 * us and move them to an alternate dst_cpu in our sched_group 11902 * where they can run. The upper limit on how many times we 11903 * iterate on same src_cpu is dependent on number of CPUs in our 11904 * sched_group. 11905 * 11906 * This changes load balance semantics a bit on who can move 11907 * load to a given_cpu. In addition to the given_cpu itself 11908 * (or a ilb_cpu acting on its behalf where given_cpu is 11909 * nohz-idle), we now have balance_cpu in a position to move 11910 * load to given_cpu. In rare situations, this may cause 11911 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding 11912 * _independently_ and at _same_ time to move some load to 11913 * given_cpu) causing excess load to be moved to given_cpu. 11914 * This however should not happen so much in practice and 11915 * moreover subsequent load balance cycles should correct the 11916 * excess load moved. 11917 */ 11918 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { 11919 11920 /* Prevent to re-select dst_cpu via env's CPUs */ 11921 __cpumask_clear_cpu(env.dst_cpu, env.cpus); 11922 11923 env.dst_rq = cpu_rq(env.new_dst_cpu); 11924 env.dst_cpu = env.new_dst_cpu; 11925 env.flags &= ~LBF_DST_PINNED; 11926 env.loop = 0; 11927 env.loop_break = SCHED_NR_MIGRATE_BREAK; 11928 11929 /* 11930 * Go back to "more_balance" rather than "redo" since we 11931 * need to continue with same src_cpu. 11932 */ 11933 goto more_balance; 11934 } 11935 11936 /* 11937 * We failed to reach balance because of affinity. 11938 */ 11939 if (sd_parent) { 11940 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 11941 11942 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) 11943 *group_imbalance = 1; 11944 } 11945 11946 /* All tasks on this runqueue were pinned by CPU affinity */ 11947 if (unlikely(env.flags & LBF_ALL_PINNED)) { 11948 __cpumask_clear_cpu(cpu_of(busiest), cpus); 11949 /* 11950 * Attempting to continue load balancing at the current 11951 * sched_domain level only makes sense if there are 11952 * active CPUs remaining as possible busiest CPUs to 11953 * pull load from which are not contained within the 11954 * destination group that is receiving any migrated 11955 * load. 11956 */ 11957 if (!cpumask_subset(cpus, env.dst_grpmask)) { 11958 env.loop = 0; 11959 env.loop_break = SCHED_NR_MIGRATE_BREAK; 11960 goto redo; 11961 } 11962 goto out_all_pinned; 11963 } 11964 } 11965 11966 if (!ld_moved) { 11967 schedstat_inc(sd->lb_failed[idle]); 11968 /* 11969 * Increment the failure counter only on periodic balance. 11970 * We do not want newidle balance, which can be very 11971 * frequent, pollute the failure counter causing 11972 * excessive cache_hot migrations and active balances. 11973 * 11974 * Similarly for migration_misfit which is not related to 11975 * load/util migration, don't pollute nr_balance_failed. 11976 */ 11977 if (idle != CPU_NEWLY_IDLE && 11978 env.migration_type != migrate_misfit) 11979 sd->nr_balance_failed++; 11980 11981 if (need_active_balance(&env)) { 11982 unsigned long flags; 11983 11984 raw_spin_rq_lock_irqsave(busiest, flags); 11985 11986 /* 11987 * Don't kick the active_load_balance_cpu_stop, 11988 * if the curr task on busiest CPU can't be 11989 * moved to this_cpu: 11990 */ 11991 if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { 11992 raw_spin_rq_unlock_irqrestore(busiest, flags); 11993 goto out_one_pinned; 11994 } 11995 11996 /* Record that we found at least one task that could run on this_cpu */ 11997 env.flags &= ~LBF_ALL_PINNED; 11998 11999 /* 12000 * ->active_balance synchronizes accesses to 12001 * ->active_balance_work. Once set, it's cleared 12002 * only after active load balance is finished. 12003 */ 12004 if (!busiest->active_balance) { 12005 busiest->active_balance = 1; 12006 busiest->push_cpu = this_cpu; 12007 active_balance = 1; 12008 } 12009 12010 preempt_disable(); 12011 raw_spin_rq_unlock_irqrestore(busiest, flags); 12012 if (active_balance) { 12013 stop_one_cpu_nowait(cpu_of(busiest), 12014 active_load_balance_cpu_stop, busiest, 12015 &busiest->active_balance_work); 12016 } 12017 preempt_enable(); 12018 } 12019 } else { 12020 sd->nr_balance_failed = 0; 12021 } 12022 12023 if (likely(!active_balance) || need_active_balance(&env)) { 12024 /* We were unbalanced, so reset the balancing interval */ 12025 sd->balance_interval = sd->min_interval; 12026 } 12027 12028 goto out; 12029 12030 out_balanced: 12031 /* 12032 * We reach balance although we may have faced some affinity 12033 * constraints. Clear the imbalance flag only if other tasks got 12034 * a chance to move and fix the imbalance. 12035 */ 12036 if (sd_parent && !(env.flags & LBF_ALL_PINNED)) { 12037 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 12038 12039 if (*group_imbalance) 12040 *group_imbalance = 0; 12041 } 12042 12043 out_all_pinned: 12044 /* 12045 * We reach balance because all tasks are pinned at this level so 12046 * we can't migrate them. Let the imbalance flag set so parent level 12047 * can try to migrate them. 12048 */ 12049 schedstat_inc(sd->lb_balanced[idle]); 12050 12051 sd->nr_balance_failed = 0; 12052 12053 out_one_pinned: 12054 ld_moved = 0; 12055 12056 /* 12057 * sched_balance_newidle() disregards balance intervals, so we could 12058 * repeatedly reach this code, which would lead to balance_interval 12059 * skyrocketing in a short amount of time. Skip the balance_interval 12060 * increase logic to avoid that. 12061 * 12062 * Similarly misfit migration which is not necessarily an indication of 12063 * the system being busy and requires lb to backoff to let it settle 12064 * down. 12065 */ 12066 if (env.idle == CPU_NEWLY_IDLE || 12067 env.migration_type == migrate_misfit) 12068 goto out; 12069 12070 /* tune up the balancing interval */ 12071 if ((env.flags & LBF_ALL_PINNED && 12072 sd->balance_interval < MAX_PINNED_INTERVAL) || 12073 sd->balance_interval < sd->max_interval) 12074 sd->balance_interval *= 2; 12075 out: 12076 if (need_unlock) 12077 atomic_set_release(&sched_balance_running, 0); 12078 12079 return ld_moved; 12080 } 12081 12082 static inline unsigned long 12083 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) 12084 { 12085 unsigned long interval = sd->balance_interval; 12086 12087 if (cpu_busy) 12088 interval *= sd->busy_factor; 12089 12090 /* scale ms to jiffies */ 12091 interval = msecs_to_jiffies(interval); 12092 12093 /* 12094 * Reduce likelihood of busy balancing at higher domains racing with 12095 * balancing at lower domains by preventing their balancing periods 12096 * from being multiples of each other. 12097 */ 12098 if (cpu_busy) 12099 interval -= 1; 12100 12101 interval = clamp(interval, 1UL, max_load_balance_interval); 12102 12103 return interval; 12104 } 12105 12106 static inline void 12107 update_next_balance(struct sched_domain *sd, unsigned long *next_balance) 12108 { 12109 unsigned long interval, next; 12110 12111 /* used by idle balance, so cpu_busy = 0 */ 12112 interval = get_sd_balance_interval(sd, 0); 12113 next = sd->last_balance + interval; 12114 12115 if (time_after(*next_balance, next)) 12116 *next_balance = next; 12117 } 12118 12119 /* 12120 * active_load_balance_cpu_stop is run by the CPU stopper. It pushes 12121 * running tasks off the busiest CPU onto idle CPUs. It requires at 12122 * least 1 task to be running on each physical CPU where possible, and 12123 * avoids physical / logical imbalances. 12124 */ 12125 static int active_load_balance_cpu_stop(void *data) 12126 { 12127 struct rq *busiest_rq = data; 12128 int busiest_cpu = cpu_of(busiest_rq); 12129 int target_cpu = busiest_rq->push_cpu; 12130 struct rq *target_rq = cpu_rq(target_cpu); 12131 struct sched_domain *sd; 12132 struct task_struct *p = NULL; 12133 struct rq_flags rf; 12134 12135 rq_lock_irq(busiest_rq, &rf); 12136 /* 12137 * Between queueing the stop-work and running it is a hole in which 12138 * CPUs can become inactive. We should not move tasks from or to 12139 * inactive CPUs. 12140 */ 12141 if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu)) 12142 goto out_unlock; 12143 12144 /* Make sure the requested CPU hasn't gone down in the meantime: */ 12145 if (unlikely(busiest_cpu != smp_processor_id() || 12146 !busiest_rq->active_balance)) 12147 goto out_unlock; 12148 12149 /* Is there any task to move? */ 12150 if (busiest_rq->nr_running <= 1) 12151 goto out_unlock; 12152 12153 /* 12154 * This condition is "impossible", if it occurs 12155 * we need to fix it. Originally reported by 12156 * Bjorn Helgaas on a 128-CPU setup. 12157 */ 12158 WARN_ON_ONCE(busiest_rq == target_rq); 12159 12160 /* Search for an sd spanning us and the target CPU. */ 12161 rcu_read_lock(); 12162 for_each_domain(target_cpu, sd) { 12163 if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) 12164 break; 12165 } 12166 12167 if (likely(sd)) { 12168 struct lb_env env = { 12169 .sd = sd, 12170 .dst_cpu = target_cpu, 12171 .dst_rq = target_rq, 12172 .src_cpu = busiest_rq->cpu, 12173 .src_rq = busiest_rq, 12174 .idle = CPU_IDLE, 12175 .flags = LBF_ACTIVE_LB, 12176 }; 12177 12178 schedstat_inc(sd->alb_count); 12179 update_rq_clock(busiest_rq); 12180 12181 p = detach_one_task(&env); 12182 if (p) { 12183 schedstat_inc(sd->alb_pushed); 12184 /* Active balancing done, reset the failure counter. */ 12185 sd->nr_balance_failed = 0; 12186 } else { 12187 schedstat_inc(sd->alb_failed); 12188 } 12189 } 12190 rcu_read_unlock(); 12191 out_unlock: 12192 busiest_rq->active_balance = 0; 12193 rq_unlock(busiest_rq, &rf); 12194 12195 if (p) 12196 attach_one_task(target_rq, p); 12197 12198 local_irq_enable(); 12199 12200 return 0; 12201 } 12202 12203 /* 12204 * Scale the max sched_balance_rq interval with the number of CPUs in the system. 12205 * This trades load-balance latency on larger machines for less cross talk. 12206 */ 12207 void update_max_interval(void) 12208 { 12209 max_load_balance_interval = HZ*num_online_cpus()/10; 12210 } 12211 12212 static inline void update_newidle_stats(struct sched_domain *sd, unsigned int success) 12213 { 12214 sd->newidle_call++; 12215 sd->newidle_success += success; 12216 12217 if (sd->newidle_call >= 1024) { 12218 sd->newidle_ratio = sd->newidle_success; 12219 sd->newidle_call /= 2; 12220 sd->newidle_success /= 2; 12221 } 12222 } 12223 12224 static inline bool 12225 update_newidle_cost(struct sched_domain *sd, u64 cost, unsigned int success) 12226 { 12227 unsigned long next_decay = sd->last_decay_max_lb_cost + HZ; 12228 unsigned long now = jiffies; 12229 12230 if (cost) 12231 update_newidle_stats(sd, success); 12232 12233 if (cost > sd->max_newidle_lb_cost) { 12234 /* 12235 * Track max cost of a domain to make sure to not delay the 12236 * next wakeup on the CPU. 12237 */ 12238 sd->max_newidle_lb_cost = cost; 12239 sd->last_decay_max_lb_cost = now; 12240 12241 } else if (time_after(now, next_decay)) { 12242 /* 12243 * Decay the newidle max times by ~1% per second to ensure that 12244 * it is not outdated and the current max cost is actually 12245 * shorter. 12246 */ 12247 sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256; 12248 sd->last_decay_max_lb_cost = now; 12249 return true; 12250 } 12251 12252 return false; 12253 } 12254 12255 /* 12256 * It checks each scheduling domain to see if it is due to be balanced, 12257 * and initiates a balancing operation if so. 12258 * 12259 * Balancing parameters are set up in init_sched_domains. 12260 */ 12261 static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle) 12262 { 12263 int continue_balancing = 1; 12264 int cpu = rq->cpu; 12265 int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 12266 unsigned long interval; 12267 struct sched_domain *sd; 12268 /* Earliest time when we have to do rebalance again */ 12269 unsigned long next_balance = jiffies + 60*HZ; 12270 int update_next_balance = 0; 12271 int need_decay = 0; 12272 u64 max_cost = 0; 12273 12274 rcu_read_lock(); 12275 for_each_domain(cpu, sd) { 12276 /* 12277 * Decay the newidle max times here because this is a regular 12278 * visit to all the domains. 12279 */ 12280 need_decay = update_newidle_cost(sd, 0, 0); 12281 max_cost += sd->max_newidle_lb_cost; 12282 12283 /* 12284 * Stop the load balance at this level. There is another 12285 * CPU in our sched group which is doing load balancing more 12286 * actively. 12287 */ 12288 if (!continue_balancing) { 12289 if (need_decay) 12290 continue; 12291 break; 12292 } 12293 12294 interval = get_sd_balance_interval(sd, busy); 12295 if (time_after_eq(jiffies, sd->last_balance + interval)) { 12296 if (sched_balance_rq(cpu, rq, sd, idle, &continue_balancing)) { 12297 /* 12298 * The LBF_DST_PINNED logic could have changed 12299 * env->dst_cpu, so we can't know our idle 12300 * state even if we migrated tasks. Update it. 12301 */ 12302 idle = idle_cpu(cpu); 12303 busy = !idle && !sched_idle_cpu(cpu); 12304 } 12305 sd->last_balance = jiffies; 12306 interval = get_sd_balance_interval(sd, busy); 12307 } 12308 if (time_after(next_balance, sd->last_balance + interval)) { 12309 next_balance = sd->last_balance + interval; 12310 update_next_balance = 1; 12311 } 12312 } 12313 if (need_decay) { 12314 /* 12315 * Ensure the rq-wide value also decays but keep it at a 12316 * reasonable floor to avoid funnies with rq->avg_idle. 12317 */ 12318 rq->max_idle_balance_cost = 12319 max((u64)sysctl_sched_migration_cost, max_cost); 12320 } 12321 rcu_read_unlock(); 12322 12323 /* 12324 * next_balance will be updated only when there is a need. 12325 * When the cpu is attached to null domain for ex, it will not be 12326 * updated. 12327 */ 12328 if (likely(update_next_balance)) 12329 rq->next_balance = next_balance; 12330 12331 } 12332 12333 static inline int on_null_domain(struct rq *rq) 12334 { 12335 return unlikely(!rcu_dereference_sched(rq->sd)); 12336 } 12337 12338 #ifdef CONFIG_NO_HZ_COMMON 12339 /* 12340 * NOHZ idle load balancing (ILB) details: 12341 * 12342 * - When one of the busy CPUs notices that there may be an idle rebalancing 12343 * needed, they will kick the idle load balancer, which then does idle 12344 * load balancing for all the idle CPUs. 12345 */ 12346 static inline int find_new_ilb(void) 12347 { 12348 const struct cpumask *hk_mask; 12349 int ilb_cpu; 12350 12351 hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE); 12352 12353 for_each_cpu_and(ilb_cpu, nohz.idle_cpus_mask, hk_mask) { 12354 12355 if (ilb_cpu == smp_processor_id()) 12356 continue; 12357 12358 if (idle_cpu(ilb_cpu)) 12359 return ilb_cpu; 12360 } 12361 12362 return -1; 12363 } 12364 12365 /* 12366 * Kick a CPU to do the NOHZ balancing, if it is time for it, via a cross-CPU 12367 * SMP function call (IPI). 12368 * 12369 * We pick the first idle CPU in the HK_TYPE_KERNEL_NOISE housekeeping set 12370 * (if there is one). 12371 */ 12372 static void kick_ilb(unsigned int flags) 12373 { 12374 int ilb_cpu; 12375 12376 /* 12377 * Increase nohz.next_balance only when if full ilb is triggered but 12378 * not if we only update stats. 12379 */ 12380 if (flags & NOHZ_BALANCE_KICK) 12381 nohz.next_balance = jiffies+1; 12382 12383 ilb_cpu = find_new_ilb(); 12384 if (ilb_cpu < 0) 12385 return; 12386 12387 /* 12388 * Don't bother if no new NOHZ balance work items for ilb_cpu, 12389 * i.e. all bits in flags are already set in ilb_cpu. 12390 */ 12391 if ((atomic_read(nohz_flags(ilb_cpu)) & flags) == flags) 12392 return; 12393 12394 /* 12395 * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets 12396 * the first flag owns it; cleared by nohz_csd_func(). 12397 */ 12398 flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu)); 12399 if (flags & NOHZ_KICK_MASK) 12400 return; 12401 12402 /* 12403 * This way we generate an IPI on the target CPU which 12404 * is idle, and the softirq performing NOHZ idle load balancing 12405 * will be run before returning from the IPI. 12406 */ 12407 smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd); 12408 } 12409 12410 /* 12411 * Current decision point for kicking the idle load balancer in the presence 12412 * of idle CPUs in the system. 12413 */ 12414 static void nohz_balancer_kick(struct rq *rq) 12415 { 12416 unsigned long now = jiffies; 12417 struct sched_domain_shared *sds; 12418 struct sched_domain *sd; 12419 int nr_busy, i, cpu = rq->cpu; 12420 unsigned int flags = 0; 12421 12422 if (unlikely(rq->idle_balance)) 12423 return; 12424 12425 /* 12426 * We may be recently in ticked or tickless idle mode. At the first 12427 * busy tick after returning from idle, we will update the busy stats. 12428 */ 12429 nohz_balance_exit_idle(rq); 12430 12431 if (READ_ONCE(nohz.has_blocked_load) && 12432 time_after(now, READ_ONCE(nohz.next_blocked))) 12433 flags = NOHZ_STATS_KICK; 12434 12435 /* 12436 * Most of the time system is not 100% busy. i.e nohz.nr_cpus > 0 12437 * Skip the read if time is not due. 12438 * 12439 * If none are in tickless mode, there maybe a narrow window 12440 * (28 jiffies, HZ=1000) where flags maybe set and kick_ilb called. 12441 * But idle load balancing is not done as find_new_ilb fails. 12442 * That's very rare. So read nohz.nr_cpus only if time is due. 12443 */ 12444 if (time_before(now, nohz.next_balance)) 12445 goto out; 12446 12447 /* 12448 * None are in tickless mode and hence no need for NOHZ idle load 12449 * balancing 12450 */ 12451 if (unlikely(cpumask_empty(nohz.idle_cpus_mask))) 12452 return; 12453 12454 if (rq->nr_running >= 2) { 12455 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12456 goto out; 12457 } 12458 12459 rcu_read_lock(); 12460 12461 sd = rcu_dereference_all(rq->sd); 12462 if (sd) { 12463 /* 12464 * If there's a runnable CFS task and the current CPU has reduced 12465 * capacity, kick the ILB to see if there's a better CPU to run on: 12466 */ 12467 if (rq->cfs.h_nr_runnable >= 1 && check_cpu_capacity(rq, sd)) { 12468 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12469 goto unlock; 12470 } 12471 } 12472 12473 sd = rcu_dereference_all(per_cpu(sd_asym_packing, cpu)); 12474 if (sd) { 12475 /* 12476 * When ASYM_PACKING; see if there's a more preferred CPU 12477 * currently idle; in which case, kick the ILB to move tasks 12478 * around. 12479 * 12480 * When balancing between cores, all the SMT siblings of the 12481 * preferred CPU must be idle. 12482 */ 12483 for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { 12484 if (sched_asym(sd, i, cpu)) { 12485 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12486 goto unlock; 12487 } 12488 } 12489 } 12490 12491 sd = rcu_dereference_all(per_cpu(sd_asym_cpucapacity, cpu)); 12492 if (sd) { 12493 /* 12494 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU 12495 * to run the misfit task on. 12496 */ 12497 if (check_misfit_status(rq)) { 12498 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12499 goto unlock; 12500 } 12501 12502 /* 12503 * For asymmetric systems, we do not want to nicely balance 12504 * cache use, instead we want to embrace asymmetry and only 12505 * ensure tasks have enough CPU capacity. 12506 * 12507 * Skip the LLC logic because it's not relevant in that case. 12508 */ 12509 goto unlock; 12510 } 12511 12512 sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu)); 12513 if (sds) { 12514 /* 12515 * If there is an imbalance between LLC domains (IOW we could 12516 * increase the overall cache utilization), we need a less-loaded LLC 12517 * domain to pull some load from. Likewise, we may need to spread 12518 * load within the current LLC domain (e.g. packed SMT cores but 12519 * other CPUs are idle). We can't really know from here how busy 12520 * the others are - so just get a NOHZ balance going if it looks 12521 * like this LLC domain has tasks we could move. 12522 */ 12523 nr_busy = atomic_read(&sds->nr_busy_cpus); 12524 if (nr_busy > 1) { 12525 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12526 goto unlock; 12527 } 12528 } 12529 unlock: 12530 rcu_read_unlock(); 12531 out: 12532 if (READ_ONCE(nohz.needs_update)) 12533 flags |= NOHZ_NEXT_KICK; 12534 12535 if (flags) 12536 kick_ilb(flags); 12537 } 12538 12539 static void set_cpu_sd_state_busy(int cpu) 12540 { 12541 struct sched_domain *sd; 12542 12543 rcu_read_lock(); 12544 sd = rcu_dereference_all(per_cpu(sd_llc, cpu)); 12545 12546 if (!sd || !sd->nohz_idle) 12547 goto unlock; 12548 sd->nohz_idle = 0; 12549 12550 atomic_inc(&sd->shared->nr_busy_cpus); 12551 unlock: 12552 rcu_read_unlock(); 12553 } 12554 12555 void nohz_balance_exit_idle(struct rq *rq) 12556 { 12557 WARN_ON_ONCE(rq != this_rq()); 12558 12559 if (likely(!rq->nohz_tick_stopped)) 12560 return; 12561 12562 rq->nohz_tick_stopped = 0; 12563 cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask); 12564 12565 set_cpu_sd_state_busy(rq->cpu); 12566 } 12567 12568 static void set_cpu_sd_state_idle(int cpu) 12569 { 12570 struct sched_domain *sd; 12571 12572 rcu_read_lock(); 12573 sd = rcu_dereference_all(per_cpu(sd_llc, cpu)); 12574 12575 if (!sd || sd->nohz_idle) 12576 goto unlock; 12577 sd->nohz_idle = 1; 12578 12579 atomic_dec(&sd->shared->nr_busy_cpus); 12580 unlock: 12581 rcu_read_unlock(); 12582 } 12583 12584 /* 12585 * This routine will record that the CPU is going idle with tick stopped. 12586 * This info will be used in performing idle load balancing in the future. 12587 */ 12588 void nohz_balance_enter_idle(int cpu) 12589 { 12590 struct rq *rq = cpu_rq(cpu); 12591 12592 WARN_ON_ONCE(cpu != smp_processor_id()); 12593 12594 /* If this CPU is going down, then nothing needs to be done: */ 12595 if (!cpu_active(cpu)) 12596 return; 12597 12598 /* 12599 * Can be set safely without rq->lock held 12600 * If a clear happens, it will have evaluated last additions because 12601 * rq->lock is held during the check and the clear 12602 */ 12603 rq->has_blocked_load = 1; 12604 12605 /* 12606 * The tick is still stopped but load could have been added in the 12607 * meantime. We set the nohz.has_blocked_load flag to trig a check of the 12608 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear 12609 * of nohz.has_blocked_load can only happen after checking the new load 12610 */ 12611 if (rq->nohz_tick_stopped) 12612 goto out; 12613 12614 /* If we're a completely isolated CPU, we don't play: */ 12615 if (on_null_domain(rq)) 12616 return; 12617 12618 rq->nohz_tick_stopped = 1; 12619 12620 cpumask_set_cpu(cpu, nohz.idle_cpus_mask); 12621 12622 /* 12623 * Ensures that if nohz_idle_balance() fails to observe our 12624 * @idle_cpus_mask store, it must observe the @has_blocked_load 12625 * and @needs_update stores. 12626 */ 12627 smp_mb__after_atomic(); 12628 12629 set_cpu_sd_state_idle(cpu); 12630 12631 WRITE_ONCE(nohz.needs_update, 1); 12632 out: 12633 /* 12634 * Each time a cpu enter idle, we assume that it has blocked load and 12635 * enable the periodic update of the load of idle CPUs 12636 */ 12637 WRITE_ONCE(nohz.has_blocked_load, 1); 12638 } 12639 12640 static bool update_nohz_stats(struct rq *rq) 12641 { 12642 unsigned int cpu = rq->cpu; 12643 12644 if (!rq->has_blocked_load) 12645 return false; 12646 12647 if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask)) 12648 return false; 12649 12650 if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick))) 12651 return true; 12652 12653 sched_balance_update_blocked_averages(cpu); 12654 12655 return rq->has_blocked_load; 12656 } 12657 12658 /* 12659 * Internal function that runs load balance for all idle CPUs. The load balance 12660 * can be a simple update of blocked load or a complete load balance with 12661 * tasks movement depending of flags. 12662 */ 12663 static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags) 12664 { 12665 /* Earliest time when we have to do rebalance again */ 12666 unsigned long now = jiffies; 12667 unsigned long next_balance = now + 60*HZ; 12668 bool has_blocked_load = false; 12669 int update_next_balance = 0; 12670 int this_cpu = this_rq->cpu; 12671 int balance_cpu; 12672 struct rq *rq; 12673 12674 WARN_ON_ONCE((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); 12675 12676 /* 12677 * We assume there will be no idle load after this update and clear 12678 * the has_blocked_load flag. If a cpu enters idle in the mean time, it will 12679 * set the has_blocked_load flag and trigger another update of idle load. 12680 * Because a cpu that becomes idle, is added to idle_cpus_mask before 12681 * setting the flag, we are sure to not clear the state and not 12682 * check the load of an idle cpu. 12683 * 12684 * Same applies to idle_cpus_mask vs needs_update. 12685 */ 12686 if (flags & NOHZ_STATS_KICK) 12687 WRITE_ONCE(nohz.has_blocked_load, 0); 12688 if (flags & NOHZ_NEXT_KICK) 12689 WRITE_ONCE(nohz.needs_update, 0); 12690 12691 /* 12692 * Ensures that if we miss the CPU, we must see the has_blocked_load 12693 * store from nohz_balance_enter_idle(). 12694 */ 12695 smp_mb(); 12696 12697 /* 12698 * Start with the next CPU after this_cpu so we will end with this_cpu and let a 12699 * chance for other idle cpu to pull load. 12700 */ 12701 for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) { 12702 if (!idle_cpu(balance_cpu)) 12703 continue; 12704 12705 /* 12706 * If this CPU gets work to do, stop the load balancing 12707 * work being done for other CPUs. Next load 12708 * balancing owner will pick it up. 12709 */ 12710 if (!idle_cpu(this_cpu) && need_resched()) { 12711 if (flags & NOHZ_STATS_KICK) 12712 has_blocked_load = true; 12713 if (flags & NOHZ_NEXT_KICK) 12714 WRITE_ONCE(nohz.needs_update, 1); 12715 goto abort; 12716 } 12717 12718 rq = cpu_rq(balance_cpu); 12719 12720 if (flags & NOHZ_STATS_KICK) 12721 has_blocked_load |= update_nohz_stats(rq); 12722 12723 /* 12724 * If time for next balance is due, 12725 * do the balance. 12726 */ 12727 if (time_after_eq(jiffies, rq->next_balance)) { 12728 struct rq_flags rf; 12729 12730 rq_lock_irqsave(rq, &rf); 12731 update_rq_clock(rq); 12732 rq_unlock_irqrestore(rq, &rf); 12733 12734 if (flags & NOHZ_BALANCE_KICK) 12735 sched_balance_domains(rq, CPU_IDLE); 12736 } 12737 12738 if (time_after(next_balance, rq->next_balance)) { 12739 next_balance = rq->next_balance; 12740 update_next_balance = 1; 12741 } 12742 } 12743 12744 /* 12745 * next_balance will be updated only when there is a need. 12746 * When the CPU is attached to null domain for ex, it will not be 12747 * updated. 12748 */ 12749 if (likely(update_next_balance)) 12750 nohz.next_balance = next_balance; 12751 12752 if (flags & NOHZ_STATS_KICK) 12753 WRITE_ONCE(nohz.next_blocked, 12754 now + msecs_to_jiffies(LOAD_AVG_PERIOD)); 12755 12756 abort: 12757 /* There is still blocked load, enable periodic update */ 12758 if (has_blocked_load) 12759 WRITE_ONCE(nohz.has_blocked_load, 1); 12760 } 12761 12762 /* 12763 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the 12764 * rebalancing for all the CPUs for whom scheduler ticks are stopped. 12765 */ 12766 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 12767 { 12768 unsigned int flags = this_rq->nohz_idle_balance; 12769 12770 if (!flags) 12771 return false; 12772 12773 this_rq->nohz_idle_balance = 0; 12774 12775 if (idle != CPU_IDLE) 12776 return false; 12777 12778 _nohz_idle_balance(this_rq, flags); 12779 12780 return true; 12781 } 12782 12783 /* 12784 * Check if we need to directly run the ILB for updating blocked load before 12785 * entering idle state. Here we run ILB directly without issuing IPIs. 12786 * 12787 * Note that when this function is called, the tick may not yet be stopped on 12788 * this CPU yet. nohz.idle_cpus_mask is updated only when tick is stopped and 12789 * cleared on the next busy tick. In other words, nohz.idle_cpus_mask updates 12790 * don't align with CPUs enter/exit idle to avoid bottlenecks due to high idle 12791 * entry/exit rate (usec). So it is possible that _nohz_idle_balance() is 12792 * called from this function on (this) CPU that's not yet in the mask. That's 12793 * OK because the goal of nohz_run_idle_balance() is to run ILB only for 12794 * updating the blocked load of already idle CPUs without waking up one of 12795 * those idle CPUs and outside the preempt disable / IRQ off phase of the local 12796 * cpu about to enter idle, because it can take a long time. 12797 */ 12798 void nohz_run_idle_balance(int cpu) 12799 { 12800 unsigned int flags; 12801 12802 flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu)); 12803 12804 /* 12805 * Update the blocked load only if no SCHED_SOFTIRQ is about to happen 12806 * (i.e. NOHZ_STATS_KICK set) and will do the same. 12807 */ 12808 if ((flags == NOHZ_NEWILB_KICK) && !need_resched()) 12809 _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK); 12810 } 12811 12812 static void nohz_newidle_balance(struct rq *this_rq) 12813 { 12814 int this_cpu = this_rq->cpu; 12815 12816 /* Will wake up very soon. No time for doing anything else*/ 12817 if (this_rq->avg_idle < sysctl_sched_migration_cost) 12818 return; 12819 12820 /* Don't need to update blocked load of idle CPUs*/ 12821 if (!READ_ONCE(nohz.has_blocked_load) || 12822 time_before(jiffies, READ_ONCE(nohz.next_blocked))) 12823 return; 12824 12825 /* 12826 * Set the need to trigger ILB in order to update blocked load 12827 * before entering idle state. 12828 */ 12829 atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu)); 12830 } 12831 12832 #else /* !CONFIG_NO_HZ_COMMON: */ 12833 static inline void nohz_balancer_kick(struct rq *rq) { } 12834 12835 static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 12836 { 12837 return false; 12838 } 12839 12840 static inline void nohz_newidle_balance(struct rq *this_rq) { } 12841 #endif /* !CONFIG_NO_HZ_COMMON */ 12842 12843 /* 12844 * sched_balance_newidle is called by schedule() if this_cpu is about to become 12845 * idle. Attempts to pull tasks from other CPUs. 12846 * 12847 * Returns: 12848 * < 0 - we released the lock and there are !fair tasks present 12849 * 0 - failed, no new tasks 12850 * > 0 - success, new (fair) tasks present 12851 */ 12852 static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) 12853 __must_hold(__rq_lockp(this_rq)) 12854 { 12855 unsigned long next_balance = jiffies + HZ; 12856 int this_cpu = this_rq->cpu; 12857 int continue_balancing = 1; 12858 u64 t0, t1, curr_cost = 0; 12859 struct sched_domain *sd; 12860 int pulled_task = 0; 12861 12862 update_misfit_status(NULL, this_rq); 12863 12864 /* 12865 * There is a task waiting to run. No need to search for one. 12866 * Return 0; the task will be enqueued when switching to idle. 12867 */ 12868 if (this_rq->ttwu_pending) 12869 return 0; 12870 12871 /* 12872 * We must set idle_stamp _before_ calling sched_balance_rq() 12873 * for CPU_NEWLY_IDLE, such that we measure the this duration 12874 * as idle time. 12875 */ 12876 this_rq->idle_stamp = rq_clock(this_rq); 12877 12878 /* 12879 * Do not pull tasks towards !active CPUs... 12880 */ 12881 if (!cpu_active(this_cpu)) 12882 return 0; 12883 12884 /* 12885 * This is OK, because current is on_cpu, which avoids it being picked 12886 * for load-balance and preemption/IRQs are still disabled avoiding 12887 * further scheduler activity on it and we're being very careful to 12888 * re-start the picking loop. 12889 */ 12890 rq_unpin_lock(this_rq, rf); 12891 12892 sd = rcu_dereference_sched_domain(this_rq->sd); 12893 if (!sd) 12894 goto out; 12895 12896 if (!get_rd_overloaded(this_rq->rd) || 12897 this_rq->avg_idle < sd->max_newidle_lb_cost) { 12898 12899 update_next_balance(sd, &next_balance); 12900 goto out; 12901 } 12902 12903 /* 12904 * Include sched_balance_update_blocked_averages() in the cost 12905 * calculation because it can be quite costly -- this ensures we skip 12906 * it when avg_idle gets to be very low. 12907 */ 12908 t0 = sched_clock_cpu(this_cpu); 12909 __sched_balance_update_blocked_averages(this_rq); 12910 12911 this_rq->next_class = &fair_sched_class; 12912 raw_spin_rq_unlock(this_rq); 12913 12914 for_each_domain(this_cpu, sd) { 12915 u64 domain_cost; 12916 12917 update_next_balance(sd, &next_balance); 12918 12919 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) 12920 break; 12921 12922 if (sd->flags & SD_BALANCE_NEWIDLE) { 12923 unsigned int weight = 1; 12924 12925 if (sched_feat(NI_RANDOM)) { 12926 /* 12927 * Throw a 1k sided dice; and only run 12928 * newidle_balance according to the success 12929 * rate. 12930 */ 12931 u32 d1k = sched_rng() % 1024; 12932 weight = 1 + sd->newidle_ratio; 12933 if (d1k > weight) { 12934 update_newidle_stats(sd, 0); 12935 continue; 12936 } 12937 weight = (1024 + weight/2) / weight; 12938 } 12939 12940 pulled_task = sched_balance_rq(this_cpu, this_rq, 12941 sd, CPU_NEWLY_IDLE, 12942 &continue_balancing); 12943 12944 t1 = sched_clock_cpu(this_cpu); 12945 domain_cost = t1 - t0; 12946 curr_cost += domain_cost; 12947 t0 = t1; 12948 12949 /* 12950 * Track max cost of a domain to make sure to not delay the 12951 * next wakeup on the CPU. 12952 */ 12953 update_newidle_cost(sd, domain_cost, weight * !!pulled_task); 12954 } 12955 12956 /* 12957 * Stop searching for tasks to pull if there are 12958 * now runnable tasks on this rq. 12959 */ 12960 if (pulled_task || !continue_balancing) 12961 break; 12962 } 12963 12964 raw_spin_rq_lock(this_rq); 12965 12966 if (curr_cost > this_rq->max_idle_balance_cost) 12967 this_rq->max_idle_balance_cost = curr_cost; 12968 12969 /* 12970 * While browsing the domains, we released the rq lock, a task could 12971 * have been enqueued in the meantime. Since we're not going idle, 12972 * pretend we pulled a task. 12973 */ 12974 if (this_rq->cfs.h_nr_queued && !pulled_task) 12975 pulled_task = 1; 12976 12977 /* If a higher prio class was modified, restart the pick */ 12978 if (sched_class_above(this_rq->next_class, &fair_sched_class)) 12979 pulled_task = -1; 12980 12981 out: 12982 /* Move the next balance forward */ 12983 if (time_after(this_rq->next_balance, next_balance)) 12984 this_rq->next_balance = next_balance; 12985 12986 if (pulled_task) 12987 this_rq->idle_stamp = 0; 12988 else 12989 nohz_newidle_balance(this_rq); 12990 12991 rq_repin_lock(this_rq, rf); 12992 12993 return pulled_task; 12994 } 12995 12996 /* 12997 * This softirq handler is triggered via SCHED_SOFTIRQ from two places: 12998 * 12999 * - directly from the local sched_tick() for periodic load balancing 13000 * 13001 * - indirectly from a remote sched_tick() for NOHZ idle balancing 13002 * through the SMP cross-call nohz_csd_func() 13003 */ 13004 static __latent_entropy void sched_balance_softirq(void) 13005 { 13006 struct rq *this_rq = this_rq(); 13007 enum cpu_idle_type idle = this_rq->idle_balance; 13008 /* 13009 * If this CPU has a pending NOHZ_BALANCE_KICK, then do the 13010 * balancing on behalf of the other idle CPUs whose ticks are 13011 * stopped. Do nohz_idle_balance *before* sched_balance_domains to 13012 * give the idle CPUs a chance to load balance. Else we may 13013 * load balance only within the local sched_domain hierarchy 13014 * and abort nohz_idle_balance altogether if we pull some load. 13015 */ 13016 if (nohz_idle_balance(this_rq, idle)) 13017 return; 13018 13019 /* normal load balance */ 13020 sched_balance_update_blocked_averages(this_rq->cpu); 13021 sched_balance_domains(this_rq, idle); 13022 } 13023 13024 /* 13025 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. 13026 */ 13027 void sched_balance_trigger(struct rq *rq) 13028 { 13029 /* 13030 * Don't need to rebalance while attached to NULL domain or 13031 * runqueue CPU is not active 13032 */ 13033 if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq)))) 13034 return; 13035 13036 if (time_after_eq(jiffies, rq->next_balance)) 13037 raise_softirq(SCHED_SOFTIRQ); 13038 13039 nohz_balancer_kick(rq); 13040 } 13041 13042 static void rq_online_fair(struct rq *rq) 13043 { 13044 update_sysctl(); 13045 13046 update_runtime_enabled(rq); 13047 } 13048 13049 static void rq_offline_fair(struct rq *rq) 13050 { 13051 update_sysctl(); 13052 13053 /* Ensure any throttled groups are reachable by pick_next_task */ 13054 unthrottle_offline_cfs_rqs(rq); 13055 13056 /* Ensure that we remove rq contribution to group share: */ 13057 clear_tg_offline_cfs_rqs(rq); 13058 } 13059 13060 #ifdef CONFIG_SCHED_CORE 13061 static inline bool 13062 __entity_slice_used(struct sched_entity *se, int min_nr_tasks) 13063 { 13064 u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime; 13065 u64 slice = se->slice; 13066 13067 return (rtime * min_nr_tasks > slice); 13068 } 13069 13070 #define MIN_NR_TASKS_DURING_FORCEIDLE 2 13071 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) 13072 { 13073 if (!sched_core_enabled(rq)) 13074 return; 13075 13076 /* 13077 * If runqueue has only one task which used up its slice and 13078 * if the sibling is forced idle, then trigger schedule to 13079 * give forced idle task a chance. 13080 * 13081 * sched_slice() considers only this active rq and it gets the 13082 * whole slice. But during force idle, we have siblings acting 13083 * like a single runqueue and hence we need to consider runnable 13084 * tasks on this CPU and the forced idle CPU. Ideally, we should 13085 * go through the forced idle rq, but that would be a perf hit. 13086 * We can assume that the forced idle CPU has at least 13087 * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check 13088 * if we need to give up the CPU. 13089 */ 13090 if (rq->core->core_forceidle_count && rq->cfs.nr_queued == 1 && 13091 __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) 13092 resched_curr(rq); 13093 } 13094 13095 /* 13096 * Consider any infeasible weight scenario. Take for instance two tasks, 13097 * each bound to their respective sibling, one with weight 1 and one with 13098 * weight 2. Then the lower weight task will run ahead of the higher weight 13099 * task without bound. 13100 * 13101 * This utterly destroys the concept of a shared time base. 13102 * 13103 * Remember; all this is about a proportionally fair scheduling, where each 13104 * tasks receives: 13105 * 13106 * w_i 13107 * dt_i = ---------- dt (1) 13108 * \Sum_j w_j 13109 * 13110 * which we do by tracking a virtual time, s_i: 13111 * 13112 * 1 13113 * s_i = --- d[t]_i (2) 13114 * w_i 13115 * 13116 * Where d[t] is a delta of discrete time, while dt is an infinitesimal. 13117 * The immediate corollary is that the ideal schedule S, where (2) to use 13118 * an infinitesimal delta, is: 13119 * 13120 * 1 13121 * S = ---------- dt (3) 13122 * \Sum_i w_i 13123 * 13124 * From which we can define the lag, or deviation from the ideal, as: 13125 * 13126 * lag(i) = S - s_i (4) 13127 * 13128 * And since the one and only purpose is to approximate S, we get that: 13129 * 13130 * \Sum_i w_i lag(i) := 0 (5) 13131 * 13132 * If this were not so, we no longer converge to S, and we can no longer 13133 * claim our scheduler has any of the properties we derive from S. This is 13134 * exactly what you did above, you broke it! 13135 * 13136 * 13137 * Let's continue for a while though; to see if there is anything useful to 13138 * be learned. We can combine (1)-(3) or (4)-(5) and express S in s_i: 13139 * 13140 * \Sum_i w_i s_i 13141 * S = -------------- (6) 13142 * \Sum_i w_i 13143 * 13144 * Which gives us a way to compute S, given our s_i. Now, if you've read 13145 * our code, you know that we do not in fact do this, the reason for this 13146 * is two-fold. Firstly, computing S in that way requires a 64bit division 13147 * for every time we'd use it (see 12), and secondly, this only describes 13148 * the steady-state, it doesn't handle dynamics. 13149 * 13150 * Anyway, in (6): s_i -> x + (s_i - x), to get: 13151 * 13152 * \Sum_i w_i (s_i - x) 13153 * S - x = -------------------- (7) 13154 * \Sum_i w_i 13155 * 13156 * Which shows that S and s_i transform alike (which makes perfect sense 13157 * given that S is basically the (weighted) average of s_i). 13158 * 13159 * So the thing to remember is that the above is strictly UP. It is 13160 * possible to generalize to multiple runqueues -- however it gets really 13161 * yuck when you have to add affinity support, as illustrated by our very 13162 * first counter-example. 13163 * 13164 * Luckily I think we can avoid needing a full multi-queue variant for 13165 * core-scheduling (or load-balancing). The crucial observation is that we 13166 * only actually need this comparison in the presence of forced-idle; only 13167 * then do we need to tell if the stalled rq has higher priority over the 13168 * other. 13169 * 13170 * [XXX assumes SMT2; better consider the more general case, I suspect 13171 * it'll work out because our comparison is always between 2 rqs and the 13172 * answer is only interesting if one of them is forced-idle] 13173 * 13174 * And (under assumption of SMT2) when there is forced-idle, there is only 13175 * a single queue, so everything works like normal. 13176 * 13177 * Let, for our runqueue 'k': 13178 * 13179 * T_k = \Sum_i w_i s_i 13180 * W_k = \Sum_i w_i ; for all i of k (8) 13181 * 13182 * Then we can write (6) like: 13183 * 13184 * T_k 13185 * S_k = --- (9) 13186 * W_k 13187 * 13188 * From which immediately follows that: 13189 * 13190 * T_k + T_l 13191 * S_k+l = --------- (10) 13192 * W_k + W_l 13193 * 13194 * On which we can define a combined lag: 13195 * 13196 * lag_k+l(i) := S_k+l - s_i (11) 13197 * 13198 * And that gives us the tools to compare tasks across a combined runqueue. 13199 * 13200 * 13201 * Combined this gives the following: 13202 * 13203 * a) when a runqueue enters force-idle, sync it against it's sibling rq(s) 13204 * using (7); this only requires storing single 'time'-stamps. 13205 * 13206 * b) when comparing tasks between 2 runqueues of which one is forced-idle, 13207 * compare the combined lag, per (11). 13208 * 13209 * Now, of course cgroups (I so hate them) make this more interesting in 13210 * that a) seems to suggest we need to iterate all cgroup on a CPU at such 13211 * boundaries, but I think we can avoid that. The force-idle is for the 13212 * whole CPU, all it's rqs. So we can mark it in the root and lazily 13213 * propagate downward on demand. 13214 */ 13215 13216 /* 13217 * So this sync is basically a relative reset of S to 0. 13218 * 13219 * So with 2 queues, when one goes idle, we drop them both to 0 and one 13220 * then increases due to not being idle, and the idle one builds up lag to 13221 * get re-elected. So far so simple, right? 13222 * 13223 * When there's 3, we can have the situation where 2 run and one is idle, 13224 * we sync to 0 and let the idle one build up lag to get re-election. Now 13225 * suppose another one also drops idle. At this point dropping all to 0 13226 * again would destroy the built-up lag from the queue that was already 13227 * idle, not good. 13228 * 13229 * So instead of syncing everything, we can: 13230 * 13231 * less := !((s64)(s_a - s_b) <= 0) 13232 * 13233 * (v_a - S_a) - (v_b - S_b) == v_a - v_b - S_a + S_b 13234 * == v_a - (v_b - S_a + S_b) 13235 * 13236 * IOW, we can recast the (lag) comparison to a one-sided difference. 13237 * So if then, instead of syncing the whole queue, sync the idle queue 13238 * against the active queue with S_a + S_b at the point where we sync. 13239 * 13240 * (XXX consider the implication of living in a cyclic group: N / 2^n N) 13241 * 13242 * This gives us means of syncing single queues against the active queue, 13243 * and for already idle queues to preserve their build-up lag. 13244 * 13245 * Of course, then we get the situation where there's 2 active and one 13246 * going idle, who do we pick to sync against? Theory would have us sync 13247 * against the combined S, but as we've already demonstrated, there is no 13248 * such thing in infeasible weight scenarios. 13249 * 13250 * One thing I've considered; and this is where that core_active rudiment 13251 * came from, is having active queues sync up between themselves after 13252 * every tick. This limits the observed divergence due to the work 13253 * conservancy. 13254 * 13255 * On top of that, we can improve upon things by employing (10) here. 13256 */ 13257 13258 /* 13259 * se_fi_update - Update the cfs_rq->zero_vruntime_fi in a CFS hierarchy if needed. 13260 */ 13261 static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq, 13262 bool forceidle) 13263 { 13264 for_each_sched_entity(se) { 13265 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13266 13267 if (forceidle) { 13268 if (cfs_rq->forceidle_seq == fi_seq) 13269 break; 13270 cfs_rq->forceidle_seq = fi_seq; 13271 } 13272 13273 cfs_rq->zero_vruntime_fi = cfs_rq->zero_vruntime; 13274 } 13275 } 13276 13277 void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi) 13278 { 13279 struct sched_entity *se = &p->se; 13280 13281 if (p->sched_class != &fair_sched_class) 13282 return; 13283 13284 se_fi_update(se, rq->core->core_forceidle_seq, in_fi); 13285 } 13286 13287 bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, 13288 bool in_fi) 13289 { 13290 struct rq *rq = task_rq(a); 13291 const struct sched_entity *sea = &a->se; 13292 const struct sched_entity *seb = &b->se; 13293 struct cfs_rq *cfs_rqa; 13294 struct cfs_rq *cfs_rqb; 13295 s64 delta; 13296 13297 WARN_ON_ONCE(task_rq(b)->core != rq->core); 13298 13299 #ifdef CONFIG_FAIR_GROUP_SCHED 13300 /* 13301 * Find an se in the hierarchy for tasks a and b, such that the se's 13302 * are immediate siblings. 13303 */ 13304 while (sea->cfs_rq->tg != seb->cfs_rq->tg) { 13305 int sea_depth = sea->depth; 13306 int seb_depth = seb->depth; 13307 13308 if (sea_depth >= seb_depth) 13309 sea = parent_entity(sea); 13310 if (sea_depth <= seb_depth) 13311 seb = parent_entity(seb); 13312 } 13313 13314 se_fi_update(sea, rq->core->core_forceidle_seq, in_fi); 13315 se_fi_update(seb, rq->core->core_forceidle_seq, in_fi); 13316 13317 cfs_rqa = sea->cfs_rq; 13318 cfs_rqb = seb->cfs_rq; 13319 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 13320 cfs_rqa = &task_rq(a)->cfs; 13321 cfs_rqb = &task_rq(b)->cfs; 13322 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 13323 13324 /* 13325 * Find delta after normalizing se's vruntime with its cfs_rq's 13326 * zero_vruntime_fi, which would have been updated in prior calls 13327 * to se_fi_update(). 13328 */ 13329 delta = vruntime_op(sea->vruntime, "-", seb->vruntime) + 13330 vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi); 13331 13332 return delta > 0; 13333 } 13334 13335 static int task_is_throttled_fair(struct task_struct *p, int cpu) 13336 { 13337 struct cfs_rq *cfs_rq; 13338 13339 #ifdef CONFIG_FAIR_GROUP_SCHED 13340 cfs_rq = task_group(p)->cfs_rq[cpu]; 13341 #else 13342 cfs_rq = &cpu_rq(cpu)->cfs; 13343 #endif 13344 return throttled_hierarchy(cfs_rq); 13345 } 13346 #else /* !CONFIG_SCHED_CORE: */ 13347 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} 13348 #endif /* !CONFIG_SCHED_CORE */ 13349 13350 /* 13351 * scheduler tick hitting a task of our scheduling class. 13352 * 13353 * NOTE: This function can be called remotely by the tick offload that 13354 * goes along full dynticks. Therefore no local assumption can be made 13355 * and everything must be accessed through the @rq and @curr passed in 13356 * parameters. 13357 */ 13358 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) 13359 { 13360 struct cfs_rq *cfs_rq; 13361 struct sched_entity *se = &curr->se; 13362 13363 for_each_sched_entity(se) { 13364 cfs_rq = cfs_rq_of(se); 13365 entity_tick(cfs_rq, se, queued); 13366 } 13367 13368 if (queued) { 13369 if (!need_resched()) 13370 hrtick_start_fair(rq, curr); 13371 return; 13372 } 13373 13374 if (static_branch_unlikely(&sched_numa_balancing)) 13375 task_tick_numa(rq, curr); 13376 13377 update_misfit_status(curr, rq); 13378 check_update_overutilized_status(task_rq(curr)); 13379 13380 task_tick_core(rq, curr); 13381 } 13382 13383 /* 13384 * called on fork with the child task as argument from the parent's context 13385 * - child not yet on the tasklist 13386 * - preemption disabled 13387 */ 13388 static void task_fork_fair(struct task_struct *p) 13389 { 13390 set_task_max_allowed_capacity(p); 13391 } 13392 13393 /* 13394 * Priority of the task has changed. Check to see if we preempt 13395 * the current task. 13396 */ 13397 static void 13398 prio_changed_fair(struct rq *rq, struct task_struct *p, u64 oldprio) 13399 { 13400 if (!task_on_rq_queued(p)) 13401 return; 13402 13403 if (p->prio == oldprio) 13404 return; 13405 13406 if (rq->cfs.nr_queued == 1) 13407 return; 13408 13409 /* 13410 * Reschedule if we are currently running on this runqueue and 13411 * our priority decreased, or if we are not currently running on 13412 * this runqueue and our priority is higher than the current's 13413 */ 13414 if (task_current_donor(rq, p)) { 13415 if (p->prio > oldprio) 13416 resched_curr(rq); 13417 } else { 13418 wakeup_preempt(rq, p, 0); 13419 } 13420 } 13421 13422 #ifdef CONFIG_FAIR_GROUP_SCHED 13423 /* 13424 * Propagate the changes of the sched_entity across the tg tree to make it 13425 * visible to the root 13426 */ 13427 static void propagate_entity_cfs_rq(struct sched_entity *se) 13428 { 13429 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13430 13431 /* 13432 * If a task gets attached to this cfs_rq and before being queued, 13433 * it gets migrated to another CPU due to reasons like affinity 13434 * change, make sure this cfs_rq stays on leaf cfs_rq list to have 13435 * that removed load decayed or it can cause faireness problem. 13436 */ 13437 if (!cfs_rq_pelt_clock_throttled(cfs_rq)) 13438 list_add_leaf_cfs_rq(cfs_rq); 13439 13440 /* Start to propagate at parent */ 13441 se = se->parent; 13442 13443 for_each_sched_entity(se) { 13444 cfs_rq = cfs_rq_of(se); 13445 13446 update_load_avg(cfs_rq, se, UPDATE_TG); 13447 13448 if (!cfs_rq_pelt_clock_throttled(cfs_rq)) 13449 list_add_leaf_cfs_rq(cfs_rq); 13450 } 13451 13452 assert_list_leaf_cfs_rq(rq_of(cfs_rq)); 13453 } 13454 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 13455 static void propagate_entity_cfs_rq(struct sched_entity *se) { } 13456 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 13457 13458 static void detach_entity_cfs_rq(struct sched_entity *se) 13459 { 13460 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13461 13462 /* 13463 * In case the task sched_avg hasn't been attached: 13464 * - A forked task which hasn't been woken up by wake_up_new_task(). 13465 * - A task which has been woken up by try_to_wake_up() but is 13466 * waiting for actually being woken up by sched_ttwu_pending(). 13467 */ 13468 if (!se->avg.last_update_time) 13469 return; 13470 13471 /* Catch up with the cfs_rq and remove our load when we leave */ 13472 update_load_avg(cfs_rq, se, 0); 13473 detach_entity_load_avg(cfs_rq, se); 13474 update_tg_load_avg(cfs_rq); 13475 propagate_entity_cfs_rq(se); 13476 } 13477 13478 static void attach_entity_cfs_rq(struct sched_entity *se) 13479 { 13480 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13481 13482 /* Synchronize entity with its cfs_rq */ 13483 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); 13484 attach_entity_load_avg(cfs_rq, se); 13485 update_tg_load_avg(cfs_rq); 13486 propagate_entity_cfs_rq(se); 13487 } 13488 13489 static void detach_task_cfs_rq(struct task_struct *p) 13490 { 13491 struct sched_entity *se = &p->se; 13492 13493 detach_entity_cfs_rq(se); 13494 } 13495 13496 static void attach_task_cfs_rq(struct task_struct *p) 13497 { 13498 struct sched_entity *se = &p->se; 13499 13500 attach_entity_cfs_rq(se); 13501 } 13502 13503 static void switching_from_fair(struct rq *rq, struct task_struct *p) 13504 { 13505 if (p->se.sched_delayed) 13506 dequeue_task(rq, p, DEQUEUE_SLEEP | DEQUEUE_DELAYED | DEQUEUE_NOCLOCK); 13507 } 13508 13509 static void switched_from_fair(struct rq *rq, struct task_struct *p) 13510 { 13511 detach_task_cfs_rq(p); 13512 } 13513 13514 static void switched_to_fair(struct rq *rq, struct task_struct *p) 13515 { 13516 WARN_ON_ONCE(p->se.sched_delayed); 13517 13518 attach_task_cfs_rq(p); 13519 13520 set_task_max_allowed_capacity(p); 13521 13522 if (task_on_rq_queued(p)) { 13523 /* 13524 * We were most likely switched from sched_rt, so 13525 * kick off the schedule if running, otherwise just see 13526 * if we can still preempt the current task. 13527 */ 13528 if (task_current_donor(rq, p)) 13529 resched_curr(rq); 13530 else 13531 wakeup_preempt(rq, p, 0); 13532 } 13533 } 13534 13535 static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 13536 { 13537 struct sched_entity *se = &p->se; 13538 13539 if (task_on_rq_queued(p)) { 13540 /* 13541 * Move the next running task to the front of the list, so our 13542 * cfs_tasks list becomes MRU one. 13543 */ 13544 list_move(&se->group_node, &rq->cfs_tasks); 13545 } 13546 if (!first) 13547 return; 13548 13549 WARN_ON_ONCE(se->sched_delayed); 13550 13551 if (hrtick_enabled_fair(rq)) 13552 hrtick_start_fair(rq, p); 13553 13554 update_misfit_status(p, rq); 13555 sched_fair_update_stop_tick(rq, p); 13556 } 13557 13558 /* 13559 * Account for a task changing its policy or group. 13560 * 13561 * This routine is mostly called to set cfs_rq->curr field when a task 13562 * migrates between groups/classes. 13563 */ 13564 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 13565 { 13566 struct sched_entity *se = &p->se; 13567 13568 for_each_sched_entity(se) { 13569 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13570 13571 set_next_entity(cfs_rq, se); 13572 /* ensure bandwidth has been allocated on our new cfs_rq */ 13573 account_cfs_rq_runtime(cfs_rq, 0); 13574 } 13575 13576 __set_next_task_fair(rq, p, first); 13577 } 13578 13579 void init_cfs_rq(struct cfs_rq *cfs_rq) 13580 { 13581 cfs_rq->tasks_timeline = RB_ROOT_CACHED; 13582 cfs_rq->zero_vruntime = (u64)(-(1LL << 20)); 13583 raw_spin_lock_init(&cfs_rq->removed.lock); 13584 } 13585 13586 #ifdef CONFIG_FAIR_GROUP_SCHED 13587 static void task_change_group_fair(struct task_struct *p) 13588 { 13589 /* 13590 * We couldn't detach or attach a forked task which 13591 * hasn't been woken up by wake_up_new_task(). 13592 */ 13593 if (READ_ONCE(p->__state) == TASK_NEW) 13594 return; 13595 13596 detach_task_cfs_rq(p); 13597 13598 /* Tell se's cfs_rq has been changed -- migrated */ 13599 p->se.avg.last_update_time = 0; 13600 set_task_rq(p, task_cpu(p)); 13601 attach_task_cfs_rq(p); 13602 } 13603 13604 void free_fair_sched_group(struct task_group *tg) 13605 { 13606 int i; 13607 13608 for_each_possible_cpu(i) { 13609 if (tg->cfs_rq) 13610 kfree(tg->cfs_rq[i]); 13611 if (tg->se) 13612 kfree(tg->se[i]); 13613 } 13614 13615 kfree(tg->cfs_rq); 13616 kfree(tg->se); 13617 } 13618 13619 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 13620 { 13621 struct sched_entity *se; 13622 struct cfs_rq *cfs_rq; 13623 int i; 13624 13625 tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL); 13626 if (!tg->cfs_rq) 13627 goto err; 13628 tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL); 13629 if (!tg->se) 13630 goto err; 13631 13632 tg->shares = NICE_0_LOAD; 13633 13634 init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent)); 13635 13636 for_each_possible_cpu(i) { 13637 cfs_rq = kzalloc_node(sizeof(struct cfs_rq), 13638 GFP_KERNEL, cpu_to_node(i)); 13639 if (!cfs_rq) 13640 goto err; 13641 13642 se = kzalloc_node(sizeof(struct sched_entity_stats), 13643 GFP_KERNEL, cpu_to_node(i)); 13644 if (!se) 13645 goto err_free_rq; 13646 13647 init_cfs_rq(cfs_rq); 13648 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); 13649 init_entity_runnable_average(se); 13650 } 13651 13652 return 1; 13653 13654 err_free_rq: 13655 kfree(cfs_rq); 13656 err: 13657 return 0; 13658 } 13659 13660 void online_fair_sched_group(struct task_group *tg) 13661 { 13662 struct sched_entity *se; 13663 struct rq_flags rf; 13664 struct rq *rq; 13665 int i; 13666 13667 for_each_possible_cpu(i) { 13668 rq = cpu_rq(i); 13669 se = tg->se[i]; 13670 rq_lock_irq(rq, &rf); 13671 update_rq_clock(rq); 13672 attach_entity_cfs_rq(se); 13673 sync_throttle(tg, i); 13674 rq_unlock_irq(rq, &rf); 13675 } 13676 } 13677 13678 void unregister_fair_sched_group(struct task_group *tg) 13679 { 13680 int cpu; 13681 13682 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); 13683 13684 for_each_possible_cpu(cpu) { 13685 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu]; 13686 struct sched_entity *se = tg->se[cpu]; 13687 struct rq *rq = cpu_rq(cpu); 13688 13689 if (se) { 13690 if (se->sched_delayed) { 13691 guard(rq_lock_irqsave)(rq); 13692 if (se->sched_delayed) { 13693 update_rq_clock(rq); 13694 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 13695 } 13696 list_del_leaf_cfs_rq(cfs_rq); 13697 } 13698 remove_entity_load_avg(se); 13699 } 13700 13701 /* 13702 * Only empty task groups can be destroyed; so we can speculatively 13703 * check on_list without danger of it being re-added. 13704 */ 13705 if (cfs_rq->on_list) { 13706 guard(rq_lock_irqsave)(rq); 13707 list_del_leaf_cfs_rq(cfs_rq); 13708 } 13709 } 13710 } 13711 13712 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, 13713 struct sched_entity *se, int cpu, 13714 struct sched_entity *parent) 13715 { 13716 struct rq *rq = cpu_rq(cpu); 13717 13718 cfs_rq->tg = tg; 13719 cfs_rq->rq = rq; 13720 init_cfs_rq_runtime(cfs_rq); 13721 13722 tg->cfs_rq[cpu] = cfs_rq; 13723 tg->se[cpu] = se; 13724 13725 /* se could be NULL for root_task_group */ 13726 if (!se) 13727 return; 13728 13729 if (!parent) { 13730 se->cfs_rq = &rq->cfs; 13731 se->depth = 0; 13732 } else { 13733 se->cfs_rq = parent->my_q; 13734 se->depth = parent->depth + 1; 13735 } 13736 13737 se->my_q = cfs_rq; 13738 /* guarantee group entities always have weight */ 13739 update_load_set(&se->load, NICE_0_LOAD); 13740 se->parent = parent; 13741 } 13742 13743 static DEFINE_MUTEX(shares_mutex); 13744 13745 static int __sched_group_set_shares(struct task_group *tg, unsigned long shares) 13746 { 13747 int i; 13748 13749 lockdep_assert_held(&shares_mutex); 13750 13751 /* 13752 * We can't change the weight of the root cgroup. 13753 */ 13754 if (!tg->se[0]) 13755 return -EINVAL; 13756 13757 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); 13758 13759 if (tg->shares == shares) 13760 return 0; 13761 13762 tg->shares = shares; 13763 for_each_possible_cpu(i) { 13764 struct rq *rq = cpu_rq(i); 13765 struct sched_entity *se = tg->se[i]; 13766 struct rq_flags rf; 13767 13768 /* Propagate contribution to hierarchy */ 13769 rq_lock_irqsave(rq, &rf); 13770 update_rq_clock(rq); 13771 for_each_sched_entity(se) { 13772 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 13773 update_cfs_group(se); 13774 } 13775 rq_unlock_irqrestore(rq, &rf); 13776 } 13777 13778 return 0; 13779 } 13780 13781 int sched_group_set_shares(struct task_group *tg, unsigned long shares) 13782 { 13783 int ret; 13784 13785 mutex_lock(&shares_mutex); 13786 if (tg_is_idle(tg)) 13787 ret = -EINVAL; 13788 else 13789 ret = __sched_group_set_shares(tg, shares); 13790 mutex_unlock(&shares_mutex); 13791 13792 return ret; 13793 } 13794 13795 int sched_group_set_idle(struct task_group *tg, long idle) 13796 { 13797 int i; 13798 13799 if (tg == &root_task_group) 13800 return -EINVAL; 13801 13802 if (idle < 0 || idle > 1) 13803 return -EINVAL; 13804 13805 mutex_lock(&shares_mutex); 13806 13807 if (tg->idle == idle) { 13808 mutex_unlock(&shares_mutex); 13809 return 0; 13810 } 13811 13812 tg->idle = idle; 13813 13814 for_each_possible_cpu(i) { 13815 struct rq *rq = cpu_rq(i); 13816 struct sched_entity *se = tg->se[i]; 13817 struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i]; 13818 bool was_idle = cfs_rq_is_idle(grp_cfs_rq); 13819 long idle_task_delta; 13820 struct rq_flags rf; 13821 13822 rq_lock_irqsave(rq, &rf); 13823 13824 grp_cfs_rq->idle = idle; 13825 if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq))) 13826 goto next_cpu; 13827 13828 idle_task_delta = grp_cfs_rq->h_nr_queued - 13829 grp_cfs_rq->h_nr_idle; 13830 if (!cfs_rq_is_idle(grp_cfs_rq)) 13831 idle_task_delta *= -1; 13832 13833 for_each_sched_entity(se) { 13834 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13835 13836 if (!se->on_rq) 13837 break; 13838 13839 cfs_rq->h_nr_idle += idle_task_delta; 13840 13841 /* Already accounted at parent level and above. */ 13842 if (cfs_rq_is_idle(cfs_rq)) 13843 break; 13844 } 13845 13846 next_cpu: 13847 rq_unlock_irqrestore(rq, &rf); 13848 } 13849 13850 /* Idle groups have minimum weight. */ 13851 if (tg_is_idle(tg)) 13852 __sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO)); 13853 else 13854 __sched_group_set_shares(tg, NICE_0_LOAD); 13855 13856 mutex_unlock(&shares_mutex); 13857 return 0; 13858 } 13859 13860 #endif /* CONFIG_FAIR_GROUP_SCHED */ 13861 13862 13863 static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) 13864 { 13865 struct sched_entity *se = &task->se; 13866 unsigned int rr_interval = 0; 13867 13868 /* 13869 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise 13870 * idle runqueue: 13871 */ 13872 if (rq->cfs.load.weight) 13873 rr_interval = NS_TO_JIFFIES(se->slice); 13874 13875 return rr_interval; 13876 } 13877 13878 /* 13879 * All the scheduling class methods: 13880 */ 13881 DEFINE_SCHED_CLASS(fair) = { 13882 .enqueue_task = enqueue_task_fair, 13883 .dequeue_task = dequeue_task_fair, 13884 .yield_task = yield_task_fair, 13885 .yield_to_task = yield_to_task_fair, 13886 13887 .wakeup_preempt = wakeup_preempt_fair, 13888 13889 .pick_task = pick_task_fair, 13890 .pick_next_task = pick_next_task_fair, 13891 .put_prev_task = put_prev_task_fair, 13892 .set_next_task = set_next_task_fair, 13893 13894 .select_task_rq = select_task_rq_fair, 13895 .migrate_task_rq = migrate_task_rq_fair, 13896 13897 .rq_online = rq_online_fair, 13898 .rq_offline = rq_offline_fair, 13899 13900 .task_dead = task_dead_fair, 13901 .set_cpus_allowed = set_cpus_allowed_fair, 13902 13903 .task_tick = task_tick_fair, 13904 .task_fork = task_fork_fair, 13905 13906 .reweight_task = reweight_task_fair, 13907 .prio_changed = prio_changed_fair, 13908 .switching_from = switching_from_fair, 13909 .switched_from = switched_from_fair, 13910 .switched_to = switched_to_fair, 13911 13912 .get_rr_interval = get_rr_interval_fair, 13913 13914 .update_curr = update_curr_fair, 13915 13916 #ifdef CONFIG_FAIR_GROUP_SCHED 13917 .task_change_group = task_change_group_fair, 13918 #endif 13919 13920 #ifdef CONFIG_SCHED_CORE 13921 .task_is_throttled = task_is_throttled_fair, 13922 #endif 13923 13924 #ifdef CONFIG_UCLAMP_TASK 13925 .uclamp_enabled = 1, 13926 #endif 13927 }; 13928 13929 void print_cfs_stats(struct seq_file *m, int cpu) 13930 { 13931 struct cfs_rq *cfs_rq, *pos; 13932 13933 rcu_read_lock(); 13934 for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos) 13935 print_cfs_rq(m, cpu, cfs_rq); 13936 rcu_read_unlock(); 13937 } 13938 13939 #ifdef CONFIG_NUMA_BALANCING 13940 void show_numa_stats(struct task_struct *p, struct seq_file *m) 13941 { 13942 int node; 13943 unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; 13944 struct numa_group *ng; 13945 13946 rcu_read_lock(); 13947 ng = rcu_dereference_all(p->numa_group); 13948 for_each_online_node(node) { 13949 if (p->numa_faults) { 13950 tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; 13951 tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; 13952 } 13953 if (ng) { 13954 gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)], 13955 gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 13956 } 13957 print_numa_stats(m, node, tsf, tpf, gsf, gpf); 13958 } 13959 rcu_read_unlock(); 13960 } 13961 #endif /* CONFIG_NUMA_BALANCING */ 13962 13963 __init void init_sched_fair_class(void) 13964 { 13965 int i; 13966 13967 for_each_possible_cpu(i) { 13968 zalloc_cpumask_var_node(&per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(i)); 13969 zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i)); 13970 zalloc_cpumask_var_node(&per_cpu(should_we_balance_tmpmask, i), 13971 GFP_KERNEL, cpu_to_node(i)); 13972 13973 #ifdef CONFIG_CFS_BANDWIDTH 13974 INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i)); 13975 INIT_LIST_HEAD(&cpu_rq(i)->cfsb_csd_list); 13976 #endif 13977 } 13978 13979 open_softirq(SCHED_SOFTIRQ, sched_balance_softirq); 13980 13981 #ifdef CONFIG_NO_HZ_COMMON 13982 nohz.next_balance = jiffies; 13983 nohz.next_blocked = jiffies; 13984 zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); 13985 #endif 13986 } 13987