1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) 4 * 5 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * 7 * Interactivity improvements by Mike Galbraith 8 * (C) 2007 Mike Galbraith <efault@gmx.de> 9 * 10 * Various enhancements by Dmitry Adamushko. 11 * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com> 12 * 13 * Group scheduling enhancements by Srivatsa Vaddagiri 14 * Copyright IBM Corporation, 2007 15 * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> 16 * 17 * Scaled math optimizations by Thomas Gleixner 18 * Copyright (C) 2007, Linutronix GmbH, Thomas Gleixner <tglx@kernel.org> 19 * 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 */ 23 #include <linux/energy_model.h> 24 #include <linux/mmap_lock.h> 25 #include <linux/jiffies.h> 26 #include <linux/math.h> 27 #include <linux/mm_api.h> 28 #include <linux/highmem.h> 29 #include <linux/hrtimer.h> 30 #include <linux/hrtimer_bases.h> 31 #include <linux/spinlock_api.h> 32 #include <linux/cpumask_api.h> 33 #include <linux/lockdep_api.h> 34 #include <linux/softirq.h> 35 #include <linux/refcount_api.h> 36 #include <linux/topology.h> 37 #include <linux/sched/clock.h> 38 #include <linux/sched/cond_resched.h> 39 #include <linux/sched/cputime.h> 40 #include <linux/sched/isolation.h> 41 #include <linux/sched/nohz.h> 42 #include <linux/sched/prio.h> 43 #include <linux/static_call.h> 44 45 #include <linux/cpuidle.h> 46 #include <linux/interrupt.h> 47 #include <linux/memory-tiers.h> 48 #include <linux/mempolicy.h> 49 #include <linux/mutex_api.h> 50 #include <linux/profile.h> 51 #include <linux/psi.h> 52 #include <linux/ratelimit.h> 53 #include <linux/task_work.h> 54 #include <linux/rbtree_augmented.h> 55 56 #include <asm/switch_to.h> 57 58 #include <uapi/linux/sched/types.h> 59 60 #include "sched.h" 61 #include "stats.h" 62 #include "autogroup.h" 63 64 /* 65 * The initial- and re-scaling of tunables is configurable 66 * 67 * Options are: 68 * 69 * SCHED_TUNABLESCALING_NONE - unscaled, always *1 70 * SCHED_TUNABLESCALING_LOG - scaled logarithmically, *1+ilog(ncpus) 71 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus 72 * 73 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) 74 */ 75 unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; 76 77 /* 78 * Default base time slice (request size r_i) for SCHED_NORMAL/SCHED_BATCH: 79 * 80 * Under EEVDF this is the request size used to compute the virtual 81 * deadline; see update_deadline(). 82 * 83 * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) 84 */ 85 unsigned int sysctl_sched_base_slice = 700000ULL; 86 static unsigned int normalized_sysctl_sched_base_slice = 700000ULL; 87 88 __read_mostly unsigned int sysctl_sched_migration_cost = 500000UL; 89 90 static int __init setup_sched_thermal_decay_shift(char *str) 91 { 92 pr_warn("Ignoring the deprecated sched_thermal_decay_shift= option\n"); 93 return 1; 94 } 95 __setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift); 96 97 /* 98 * For asym packing, by default the lower numbered CPU has higher priority. 99 */ 100 int __weak arch_asym_cpu_priority(int cpu) 101 { 102 return -cpu; 103 } 104 105 /* 106 * The margin used when comparing utilization with CPU capacity. 107 * 108 * (default: ~20%) 109 */ 110 #define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024) 111 112 /* 113 * The margin used when comparing CPU capacities. 114 * is 'cap1' noticeably greater than 'cap2' 115 * 116 * (default: ~5%) 117 */ 118 #define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078) 119 120 #ifdef CONFIG_CFS_BANDWIDTH 121 /* 122 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool 123 * each time a cfs_rq requests quota. 124 * 125 * Note: in the case that the slice exceeds the runtime remaining (either due 126 * to consumption or the quota being specified to be smaller than the slice) 127 * we will always only issue the remaining available time. 128 * 129 * (default: 5 msec, units: microseconds) 130 */ 131 static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; 132 #endif 133 134 #ifdef CONFIG_NUMA_BALANCING 135 /* Restrict the NUMA promotion throughput (MB/s) for each target node. */ 136 static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; 137 #endif 138 139 #ifdef CONFIG_SYSCTL 140 static const struct ctl_table sched_fair_sysctls[] = { 141 #ifdef CONFIG_CFS_BANDWIDTH 142 { 143 .procname = "sched_cfs_bandwidth_slice_us", 144 .data = &sysctl_sched_cfs_bandwidth_slice, 145 .maxlen = sizeof(unsigned int), 146 .mode = 0644, 147 .proc_handler = proc_dointvec_minmax, 148 .extra1 = SYSCTL_ONE, 149 }, 150 #endif 151 #ifdef CONFIG_NUMA_BALANCING 152 { 153 .procname = "numa_balancing_promote_rate_limit_MBps", 154 .data = &sysctl_numa_balancing_promote_rate_limit, 155 .maxlen = sizeof(unsigned int), 156 .mode = 0644, 157 .proc_handler = proc_dointvec_minmax, 158 .extra1 = SYSCTL_ZERO, 159 }, 160 #endif /* CONFIG_NUMA_BALANCING */ 161 }; 162 163 static int __init sched_fair_sysctl_init(void) 164 { 165 register_sysctl_init("kernel", sched_fair_sysctls); 166 return 0; 167 } 168 late_initcall(sched_fair_sysctl_init); 169 #endif /* CONFIG_SYSCTL */ 170 171 static inline void update_load_add(struct load_weight *lw, unsigned long inc) 172 { 173 lw->weight += inc; 174 lw->inv_weight = 0; 175 } 176 177 static inline void update_load_sub(struct load_weight *lw, unsigned long dec) 178 { 179 lw->weight -= dec; 180 lw->inv_weight = 0; 181 } 182 183 static inline void update_load_set(struct load_weight *lw, unsigned long w) 184 { 185 lw->weight = w; 186 lw->inv_weight = 0; 187 } 188 189 /* 190 * Increase the granularity value when there are more CPUs, 191 * because with more CPUs the 'effective latency' as visible 192 * to users decreases. But the relationship is not linear, 193 * so pick a second-best guess by going with the log2 of the 194 * number of CPUs. 195 * 196 * This idea comes from the SD scheduler of Con Kolivas: 197 */ 198 static unsigned int get_update_sysctl_factor(void) 199 { 200 unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); 201 unsigned int factor; 202 203 switch (sysctl_sched_tunable_scaling) { 204 case SCHED_TUNABLESCALING_NONE: 205 factor = 1; 206 break; 207 case SCHED_TUNABLESCALING_LINEAR: 208 factor = cpus; 209 break; 210 case SCHED_TUNABLESCALING_LOG: 211 default: 212 factor = 1 + ilog2(cpus); 213 break; 214 } 215 216 return factor; 217 } 218 219 static void update_sysctl(void) 220 { 221 unsigned int factor = get_update_sysctl_factor(); 222 223 #define SET_SYSCTL(name) \ 224 (sysctl_##name = (factor) * normalized_sysctl_##name) 225 SET_SYSCTL(sched_base_slice); 226 #undef SET_SYSCTL 227 } 228 229 void __init sched_init_granularity(void) 230 { 231 update_sysctl(); 232 } 233 234 #ifndef CONFIG_64BIT 235 #define WMULT_CONST (~0U) 236 #define WMULT_SHIFT 32 237 238 static void __update_inv_weight(struct load_weight *lw) 239 { 240 unsigned long w; 241 242 if (likely(lw->inv_weight)) 243 return; 244 245 w = scale_load_down(lw->weight); 246 247 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 248 lw->inv_weight = 1; 249 else if (unlikely(!w)) 250 lw->inv_weight = WMULT_CONST; 251 else 252 lw->inv_weight = WMULT_CONST / w; 253 } 254 255 /* 256 * delta_exec * weight / lw.weight 257 * OR 258 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT 259 * 260 * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case 261 * we're guaranteed shift stays positive because inv_weight is guaranteed to 262 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. 263 * 264 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus 265 * weight/lw.weight <= 1, and therefore our shift will also be positive. 266 */ 267 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 268 { 269 u64 fact = scale_load_down(weight); 270 u32 fact_hi = (u32)(fact >> 32); 271 int shift = WMULT_SHIFT; 272 int fs; 273 274 __update_inv_weight(lw); 275 276 if (unlikely(fact_hi)) { 277 fs = fls(fact_hi); 278 shift -= fs; 279 fact >>= fs; 280 } 281 282 fact = mul_u32_u32(fact, lw->inv_weight); 283 284 fact_hi = (u32)(fact >> 32); 285 if (fact_hi) { 286 fs = fls(fact_hi); 287 shift -= fs; 288 fact >>= fs; 289 } 290 291 return mul_u64_u32_shr(delta_exec, fact, shift); 292 } 293 #else 294 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 295 { 296 return (delta_exec * weight) / lw->weight; 297 } 298 #endif 299 300 /* 301 * delta /= w 302 */ 303 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) 304 { 305 if (se->h_load.weight != NICE_0_LOAD) 306 delta = __calc_delta(delta, NICE_0_LOAD, &se->h_load); 307 308 return delta; 309 } 310 311 const struct sched_class fair_sched_class; 312 313 /************************************************************** 314 * CFS operations on generic schedulable entities: 315 */ 316 317 #ifdef CONFIG_FAIR_GROUP_SCHED 318 319 /* Walk up scheduling entities hierarchy */ 320 #define for_each_sched_entity(se) \ 321 for (; se; se = se->parent) 322 323 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 324 { 325 struct rq *rq = rq_of(cfs_rq); 326 int cpu = cpu_of(rq); 327 328 if (cfs_rq->on_list) 329 return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list; 330 331 cfs_rq->on_list = 1; 332 333 /* 334 * Ensure we either appear before our parent (if already 335 * enqueued) or force our parent to appear after us when it is 336 * enqueued. The fact that we always enqueue bottom-up 337 * reduces this to two cases and a special case for the root 338 * cfs_rq. Furthermore, it also means that we will always reset 339 * tmp_alone_branch either when the branch is connected 340 * to a tree or when we reach the top of the tree 341 */ 342 if (cfs_rq->tg->parent && 343 tg_cfs_rq(cfs_rq->tg->parent, cpu)->on_list) { 344 /* 345 * If parent is already on the list, we add the child 346 * just before. Thanks to circular linked property of 347 * the list, this means to put the child at the tail 348 * of the list that starts by parent. 349 */ 350 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 351 &(tg_cfs_rq(cfs_rq->tg->parent, cpu)->leaf_cfs_rq_list)); 352 /* 353 * The branch is now connected to its tree so we can 354 * reset tmp_alone_branch to the beginning of the 355 * list. 356 */ 357 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 358 return true; 359 } 360 361 if (!cfs_rq->tg->parent) { 362 /* 363 * cfs rq without parent should be put 364 * at the tail of the list. 365 */ 366 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 367 &rq->leaf_cfs_rq_list); 368 /* 369 * We have reach the top of a tree so we can reset 370 * tmp_alone_branch to the beginning of the list. 371 */ 372 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 373 return true; 374 } 375 376 /* 377 * The parent has not already been added so we want to 378 * make sure that it will be put after us. 379 * tmp_alone_branch points to the begin of the branch 380 * where we will add parent. 381 */ 382 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); 383 /* 384 * update tmp_alone_branch to points to the new begin 385 * of the branch 386 */ 387 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 388 return false; 389 } 390 391 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 392 { 393 if (cfs_rq->on_list) { 394 struct rq *rq = rq_of(cfs_rq); 395 396 /* 397 * With cfs_rq being unthrottled/throttled during an enqueue, 398 * it can happen the tmp_alone_branch points to the leaf that 399 * we finally want to delete. In this case, tmp_alone_branch moves 400 * to the prev element but it will point to rq->leaf_cfs_rq_list 401 * at the end of the enqueue. 402 */ 403 if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) 404 rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev; 405 406 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 407 cfs_rq->on_list = 0; 408 } 409 } 410 411 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 412 { 413 WARN_ON_ONCE(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list); 414 } 415 416 /* Iterate through all leaf cfs_rq's on a runqueue */ 417 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 418 list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \ 419 leaf_cfs_rq_list) 420 421 /* Do the two (enqueued) entities belong to the same group ? */ 422 static inline struct cfs_rq * 423 is_same_group(struct sched_entity *se, struct sched_entity *pse) 424 { 425 if (se->cfs_rq == pse->cfs_rq) 426 return se->cfs_rq; 427 428 return NULL; 429 } 430 431 static inline struct sched_entity *parent_entity(const struct sched_entity *se) 432 { 433 return se->parent; 434 } 435 436 static int tg_is_idle(struct task_group *tg) 437 { 438 return tg->idle > 0; 439 } 440 441 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 442 { 443 return cfs_rq->idle > 0; 444 } 445 446 static int se_is_idle(struct sched_entity *se) 447 { 448 if (entity_is_task(se)) 449 return task_has_idle_policy(task_of(se)); 450 return cfs_rq_is_idle(group_cfs_rq(se)); 451 } 452 453 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 454 455 #define for_each_sched_entity(se) \ 456 for (; se; se = NULL) 457 458 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 459 { 460 return true; 461 } 462 463 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 464 { 465 } 466 467 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 468 { 469 } 470 471 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 472 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos) 473 474 static inline struct sched_entity *parent_entity(struct sched_entity *se) 475 { 476 return NULL; 477 } 478 479 static inline int tg_is_idle(struct task_group *tg) 480 { 481 return 0; 482 } 483 484 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 485 { 486 return 0; 487 } 488 489 static int se_is_idle(struct sched_entity *se) 490 { 491 return task_has_idle_policy(task_of(se)); 492 } 493 494 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 495 496 static __always_inline 497 bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); 498 499 /************************************************************** 500 * Scheduling class tree data structure manipulation methods: 501 */ 502 503 extern void __BUILD_BUG_vruntime_cmp(void); 504 505 /* Use __builtin_strcmp() because of __HAVE_ARCH_STRCMP: */ 506 507 #define vruntime_cmp(A, CMP_STR, B) ({ \ 508 int __res = 0; \ 509 \ 510 if (!__builtin_strcmp(CMP_STR, "<")) { \ 511 __res = ((s64)((A)-(B)) < 0); \ 512 } else if (!__builtin_strcmp(CMP_STR, "<=")) { \ 513 __res = ((s64)((A)-(B)) <= 0); \ 514 } else if (!__builtin_strcmp(CMP_STR, ">")) { \ 515 __res = ((s64)((A)-(B)) > 0); \ 516 } else if (!__builtin_strcmp(CMP_STR, ">=")) { \ 517 __res = ((s64)((A)-(B)) >= 0); \ 518 } else { \ 519 /* Unknown operator throws linker error: */ \ 520 __BUILD_BUG_vruntime_cmp(); \ 521 } \ 522 \ 523 __res; \ 524 }) 525 526 extern void __BUILD_BUG_vruntime_op(void); 527 528 #define vruntime_op(A, OP_STR, B) ({ \ 529 s64 __res = 0; \ 530 \ 531 if (!__builtin_strcmp(OP_STR, "-")) { \ 532 __res = (s64)((A)-(B)); \ 533 } else { \ 534 /* Unknown operator throws linker error: */ \ 535 __BUILD_BUG_vruntime_op(); \ 536 } \ 537 \ 538 __res; \ 539 }) 540 541 542 static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime) 543 { 544 if (vruntime_cmp(vruntime, ">", max_vruntime)) 545 max_vruntime = vruntime; 546 547 return max_vruntime; 548 } 549 550 static inline __maybe_unused u64 min_vruntime(u64 min_vruntime, u64 vruntime) 551 { 552 if (vruntime_cmp(vruntime, "<", min_vruntime)) 553 min_vruntime = vruntime; 554 555 return min_vruntime; 556 } 557 558 static inline bool entity_before(const struct sched_entity *a, 559 const struct sched_entity *b) 560 { 561 /* 562 * Tiebreak on vruntime seems unnecessary since it can 563 * hardly happen. 564 */ 565 return vruntime_cmp(a->deadline, "<", b->deadline); 566 } 567 568 /* 569 * Per avg_vruntime() below, cfs_rq::zero_vruntime is only slightly stale 570 * and this value should be no more than two lag bounds. Which puts it in the 571 * general order of: 572 * 573 * (slice + TICK_NSEC) << NICE_0_LOAD_SHIFT 574 * 575 * which is around 44 bits in size (on 64bit); that is 20 for 576 * NICE_0_LOAD_SHIFT, another 20 for NSEC_PER_MSEC and then a handful for 577 * however many msec the actual slice+tick ends up begin. 578 * 579 * (disregarding the actual divide-by-weight part makes for the worst case 580 * weight of 2, which nicely cancels vs the fuzz in zero_vruntime not actually 581 * being the zero-lag point). 582 */ 583 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) 584 { 585 return vruntime_op(se->vruntime, "-", cfs_rq->zero_vruntime); 586 } 587 588 #define __node_2_se(node) \ 589 rb_entry((node), struct sched_entity, run_node) 590 591 /* 592 * Compute virtual time from the per-task service numbers: 593 * 594 * Fair schedulers conserve lag: 595 * 596 * \Sum lag_i = 0 597 * 598 * Where lag_i is given by: 599 * 600 * lag_i = S - s_i = w_i * (V - v_i) 601 * 602 * Where S is the ideal service time and V is it's virtual time counterpart. 603 * Therefore: 604 * 605 * \Sum lag_i = 0 606 * \Sum w_i * (V - v_i) = 0 607 * \Sum (w_i * V - w_i * v_i) = 0 608 * 609 * From which we can solve an expression for V in v_i (which we have in 610 * se->vruntime): 611 * 612 * \Sum v_i * w_i \Sum v_i * w_i 613 * V = -------------- = -------------- 614 * \Sum w_i W 615 * 616 * Specifically, this is the weighted average of all entity virtual runtimes. 617 * 618 * [[ NOTE: this is only equal to the ideal scheduler under the condition 619 * that join/leave operations happen at lag_i = 0, otherwise the 620 * virtual time has non-contiguous motion equivalent to: 621 * 622 * V +-= lag_i / W 623 * 624 * Also see the comment in place_entity() that deals with this. ]] 625 * 626 * However, since v_i is u64, and the multiplication could easily overflow 627 * transform it into a relative form that uses smaller quantities: 628 * 629 * Substitute: v_i == (v_i - v0) + v0 630 * 631 * \Sum ((v_i - v0) + v0) * w_i \Sum (v_i - v0) * w_i 632 * V = ---------------------------- = --------------------- + v0 633 * W W 634 * 635 * Which we track using: 636 * 637 * v0 := cfs_rq->zero_vruntime 638 * \Sum (v_i - v0) * w_i := cfs_rq->sum_w_vruntime 639 * \Sum w_i := cfs_rq->sum_weight 640 * 641 * Since zero_vruntime closely tracks the per-task service, these 642 * deltas: (v_i - v0), will be in the order of the maximal (virtual) lag 643 * induced in the system due to quantisation. 644 */ 645 static inline unsigned long avg_vruntime_weight(struct cfs_rq *cfs_rq, unsigned long w) 646 { 647 #ifdef CONFIG_64BIT 648 if (cfs_rq->sum_shift) 649 w = max(2UL, w >> cfs_rq->sum_shift); 650 #endif 651 return w; 652 } 653 654 static inline void 655 __sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) 656 { 657 unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); 658 s64 w_vruntime, key = entity_key(cfs_rq, se); 659 660 w_vruntime = key * weight; 661 WARN_ON_ONCE((w_vruntime >> 63) != (w_vruntime >> 62)); 662 663 cfs_rq->sum_w_vruntime += w_vruntime; 664 cfs_rq->sum_weight += weight; 665 } 666 667 static void 668 sum_w_vruntime_add_paranoid(struct cfs_rq *cfs_rq, struct sched_entity *se) 669 { 670 unsigned long weight; 671 s64 key, tmp; 672 673 again: 674 weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); 675 key = entity_key(cfs_rq, se); 676 677 if (check_mul_overflow(key, weight, &key)) 678 goto overflow; 679 680 if (check_add_overflow(cfs_rq->sum_w_vruntime, key, &tmp)) 681 goto overflow; 682 683 cfs_rq->sum_w_vruntime = tmp; 684 cfs_rq->sum_weight += weight; 685 return; 686 687 overflow: 688 /* 689 * There's gotta be a limit -- if we're still failing at this point 690 * there's really nothing much to be done about things. 691 */ 692 BUG_ON(cfs_rq->sum_shift >= 10); 693 cfs_rq->sum_shift++; 694 695 /* 696 * Note: \Sum (k_i * (w_i >> 1)) != (\Sum (k_i * w_i)) >> 1 697 */ 698 cfs_rq->sum_w_vruntime = 0; 699 cfs_rq->sum_weight = 0; 700 701 for (struct rb_node *node = cfs_rq->tasks_timeline.rb_leftmost; 702 node; node = rb_next(node)) 703 __sum_w_vruntime_add(cfs_rq, __node_2_se(node)); 704 705 goto again; 706 } 707 708 static void 709 sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) 710 { 711 if (sched_feat(PARANOID_AVG)) 712 return sum_w_vruntime_add_paranoid(cfs_rq, se); 713 714 __sum_w_vruntime_add(cfs_rq, se); 715 } 716 717 static void 718 sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) 719 { 720 unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); 721 s64 key = entity_key(cfs_rq, se); 722 723 cfs_rq->sum_w_vruntime -= key * weight; 724 cfs_rq->sum_weight -= weight; 725 } 726 727 static inline 728 void update_zero_vruntime(struct cfs_rq *cfs_rq, s64 delta) 729 { 730 /* 731 * v' = v + d ==> sum_w_vruntime' = sum_w_vruntime - d*sum_weight 732 */ 733 cfs_rq->sum_w_vruntime -= cfs_rq->sum_weight * delta; 734 cfs_rq->zero_vruntime += delta; 735 } 736 737 /* 738 * Specifically: avg_vruntime() + 0 must result in entity_eligible() := true 739 * For this to be so, the result of this function must have a left bias. 740 * 741 * Called in: 742 * - place_entity() -- before enqueue 743 * - update_entity_lag() -- before dequeue 744 * - update_deadline() -- slice expiration 745 * 746 * This means it is one entry 'behind' but that puts it close enough to where 747 * the bound on entity_key() is at most two lag bounds. 748 */ 749 u64 avg_vruntime(struct cfs_rq *cfs_rq) 750 { 751 struct sched_entity *curr = cfs_rq->curr; 752 long weight = cfs_rq->sum_weight; 753 s64 delta = 0; 754 755 if (curr && !curr->on_rq) 756 curr = NULL; 757 758 if (weight) { 759 s64 runtime = cfs_rq->sum_w_vruntime; 760 761 if (curr) { 762 unsigned long w = avg_vruntime_weight(cfs_rq, curr->h_load.weight); 763 764 runtime += entity_key(cfs_rq, curr) * w; 765 weight += w; 766 } 767 768 /* sign flips effective floor / ceiling */ 769 if (runtime < 0) 770 runtime -= (weight - 1); 771 772 delta = div64_long(runtime, weight); 773 } else if (curr) { 774 /* 775 * When there is but one element, it is the average. 776 */ 777 delta = curr->vruntime - cfs_rq->zero_vruntime; 778 } 779 780 update_zero_vruntime(cfs_rq, delta); 781 782 return cfs_rq->zero_vruntime; 783 } 784 785 /* 786 * \Sum (v_i - v0)*w_i 787 * V = ------------------- + v0 788 * \Sum w_i 789 * 790 * Let W = \Sum w_i, and move v_j such that 'v_j == V', thus: 791 * 792 * V = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0 793 * 794 * v_j = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0 795 * 796 * v_j = 1/W * (v_j - v0)*w_j + 1/W * \Sum_i!=j (v_i - v0)*w_i + v0 797 * 798 * v_j - 1/W * (v_j - v0)*w_j = 1/W * \Sum_i!=j (v_i - v0)*w_i + v0 799 * 800 * v_j*W - (v_j - v0)*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W 801 * 802 * v_j*(W - w_j) + v0*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W 803 * 804 * v_j*(W - w_j) = \Sum_i!=j (v_i - v0)*w_i + v0*(W - w_j) 805 * 806 * \Sum_i!=j (v_i - v0)*w_i 807 * v_j = ------------------------ + v0 808 * W - w_j 809 * 810 * When v_j happens to be curr, then '\Sum_i!=j (v_i - v0)*w_i' 811 * is cfs_rq->sum_w_runtime, and 'W - w_j' is cfs_rq->sum_weight, since curr 812 * is not included in the sum. 813 */ 814 static u64 ineligible_vruntime(struct cfs_rq *cfs_rq) 815 { 816 struct sched_entity *curr = cfs_rq->curr; 817 long weight = cfs_rq->sum_weight; 818 s64 delta = 0; 819 820 if (curr && !curr->on_rq) 821 curr = NULL; 822 823 if (weight) { 824 s64 runtime = cfs_rq->sum_w_vruntime; 825 826 /* 827 * Do not add @curr to obtain the effective '- w_j' terms. 828 */ 829 830 /* sign flips effective floor / ceiling */ 831 if (runtime < 0) 832 runtime -= (weight - 1); 833 834 delta = div64_long(runtime, weight); 835 } 836 837 return cfs_rq->zero_vruntime + delta + 1; 838 } 839 840 static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); 841 842 /* 843 * lag_i = S - s_i = w_i * (V - v_i) 844 * 845 * However, since V is approximated by the weighted average of all entities it 846 * is possible -- by addition/removal/reweight to the tree -- to move V around 847 * and end up with a larger lag than we started with. 848 * 849 * Limit this to either double the slice length with a minimum of TICK_NSEC 850 * since that is the timing granularity. 851 * 852 * EEVDF gives the following limit for a steady state system: 853 * 854 * -r_max < lag < max(r_max, q) 855 */ 856 static s64 entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 avruntime) 857 { 858 u64 max_slice = cfs_rq_max_slice(cfs_rq) + TICK_NSEC; 859 s64 vlag, limit; 860 861 vlag = avruntime - se->vruntime; 862 limit = calc_delta_fair(max_slice, se); 863 864 return clamp(vlag, -limit, limit); 865 } 866 867 /* 868 * Delayed dequeue aims to reduce the negative lag of a dequeued task. While 869 * updating the lag of an entity, check that negative lag didn't increase 870 * during the delayed dequeue period which would be unfair. 871 * Similarly, check that the entity didn't gain positive lag when DELAY_ZERO 872 * is set. 873 * 874 * Return true if the vlag has been modified. Specifically: 875 * 876 * se->vlag != avg_vruntime() - se->vruntime 877 * 878 * This can be due to clamping in entity_lag() or clamping due to 879 * sched_delayed. Either way, when vlag is modified and the entity is 880 * retained, the tree needs to be adjusted. 881 */ 882 static __always_inline 883 bool update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) 884 { 885 u64 avruntime = avg_vruntime(cfs_rq); 886 s64 vlag = entity_lag(cfs_rq, se, avruntime); 887 888 if (se->sched_delayed) { 889 /* previous vlag < 0 otherwise se would not be delayed */ 890 vlag = max(vlag, se->vlag); 891 if (sched_feat(DELAY_ZERO)) 892 vlag = min(vlag, 0); 893 } 894 se->vlag = vlag; 895 896 return avruntime - vlag != se->vruntime; 897 } 898 899 /* 900 * Entity is eligible once it received less service than it ought to have, 901 * eg. lag >= 0. 902 * 903 * lag_i = S - s_i = w_i*(V - v_i) 904 * 905 * lag_i >= 0 -> V >= v_i 906 * 907 * \Sum (v_i - v0)*w_i 908 * V = ------------------- + v0 909 * \Sum w_i 910 * 911 * lag_i >= 0 -> \Sum (v_i - v0)*w_i >= (v_i - v0)*(\Sum w_i) 912 * 913 * Note: using 'avg_vruntime() > se->vruntime' is inaccurate due 914 * to the loss in precision caused by the division. 915 */ 916 static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) 917 { 918 struct sched_entity *curr = cfs_rq->curr; 919 s64 key, avg = cfs_rq->sum_w_vruntime; 920 long load = cfs_rq->sum_weight; 921 922 if (curr && curr->on_rq) { 923 unsigned long weight = avg_vruntime_weight(cfs_rq, curr->h_load.weight); 924 925 avg += entity_key(cfs_rq, curr) * weight; 926 load += weight; 927 } 928 929 key = vruntime_op(vruntime, "-", cfs_rq->zero_vruntime); 930 931 /* 932 * The worst case term for @key includes 'NSEC_TICK * NICE_0_LOAD' 933 * and @load obviously includes NICE_0_LOAD. NSEC_TICK is around 24 934 * bits, while NICE_0_LOAD is 20 on 64bit and 10 otherwise. 935 * 936 * This gives that on 64bit the product will be at least 64bit which 937 * overflows s64, while on 32bit it will only be 44bits and should fit 938 * comfortably. 939 */ 940 #ifdef CONFIG_64BIT 941 #ifdef CONFIG_ARCH_SUPPORTS_INT128 942 /* This often results in simpler code than __builtin_mul_overflow(). */ 943 return avg >= (__int128)key * load; 944 #else 945 s64 rhs; 946 /* 947 * On overflow, the sign of key tells us the correct answer: a large 948 * positive key means vruntime >> V, so not eligible; a large negative 949 * key means vruntime << V, so eligible. 950 */ 951 if (check_mul_overflow(key, load, &rhs)) 952 return key <= 0; 953 954 return avg >= rhs; 955 #endif 956 #else /* 32bit */ 957 return avg >= key * load; 958 #endif 959 } 960 961 int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) 962 { 963 return vruntime_eligible(cfs_rq, se->vruntime); 964 } 965 966 static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) 967 { 968 struct sched_entity *root = __pick_root_entity(cfs_rq); 969 struct sched_entity *curr = cfs_rq->curr; 970 u64 min_slice = ~0ULL; 971 972 if (curr && curr->on_rq) 973 min_slice = curr->slice; 974 975 if (root) 976 min_slice = min(min_slice, root->min_slice); 977 978 return min_slice; 979 } 980 981 static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq) 982 { 983 struct sched_entity *root = __pick_root_entity(cfs_rq); 984 struct sched_entity *curr = cfs_rq->curr; 985 u64 max_slice = 0ULL; 986 987 if (curr && curr->on_rq) 988 max_slice = curr->slice; 989 990 if (root) 991 max_slice = max(max_slice, root->max_slice); 992 993 return max_slice; 994 } 995 996 static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) 997 { 998 return entity_before(__node_2_se(a), __node_2_se(b)); 999 } 1000 1001 /* min() for wrapping vruntimes */ 1002 static inline u64 __min_vruntime(u64 a, u64 b) 1003 { 1004 return vruntime_cmp(a, "<", b) ? a : b; 1005 } 1006 1007 RB_DECLARE_CALLBACKS(static, min_vruntime_cb, 1008 struct sched_entity, run_node, 1009 RB_AUG(vruntime, min_vruntime, __min_vruntime), 1010 RB_AUG(slice, min_slice, min), 1011 RB_AUG(slice, max_slice, max)); 1012 1013 /* 1014 * Enqueue an entity into the rb-tree: 1015 */ 1016 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 1017 { 1018 WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); 1019 WARN_ON_ONCE(!entity_is_task(se)); 1020 1021 sum_w_vruntime_add(cfs_rq, se); 1022 se->min_vruntime = se->vruntime; 1023 se->min_slice = se->slice; 1024 se->max_slice = se->slice; 1025 1026 rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 1027 __entity_less, &min_vruntime_cb); 1028 } 1029 1030 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 1031 { 1032 WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); 1033 WARN_ON_ONCE(!entity_is_task(se)); 1034 1035 rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 1036 &min_vruntime_cb); 1037 sum_w_vruntime_sub(cfs_rq, se); 1038 } 1039 1040 struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq) 1041 { 1042 struct rb_node *root = cfs_rq->tasks_timeline.rb_root.rb_node; 1043 1044 if (!root) 1045 return NULL; 1046 1047 return __node_2_se(root); 1048 } 1049 1050 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 1051 { 1052 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 1053 1054 if (!left) 1055 return NULL; 1056 1057 return __node_2_se(left); 1058 } 1059 1060 /* 1061 * Set the vruntime up to which an entity can run before looking 1062 * for another entity to pick. 1063 * In case of run to parity, we use the shortest slice of the enqueued 1064 * entities to set the protected period. 1065 * When run to parity is disabled, we give a minimum quantum to the running 1066 * entity to ensure progress. 1067 */ 1068 static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 1069 { 1070 u64 slice = normalized_sysctl_sched_base_slice; 1071 u64 vprot = se->deadline; 1072 1073 if (sched_feat(RUN_TO_PARITY)) 1074 slice = cfs_rq_min_slice(cfs_rq); 1075 1076 slice = min(slice, se->slice); 1077 1078 /* If there are shorter slices than se's one */ 1079 if (slice != se->slice) { 1080 vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); 1081 if (sched_feat(PREEMPT_SHORT)) 1082 vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq)); 1083 } 1084 1085 se->vprot = vprot; 1086 } 1087 1088 static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 1089 { 1090 u64 vruntime = min_vruntime(se->vruntime, avg_vruntime(cfs_rq)); 1091 u64 slice = cfs_rq_min_slice(cfs_rq); 1092 u64 vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se)); 1093 1094 if (sched_feat(PREEMPT_SHORT) && slice != se->slice) 1095 vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq)); 1096 1097 se->vprot = vprot; 1098 } 1099 1100 static inline bool protect_slice(struct sched_entity *se) 1101 { 1102 return vruntime_cmp(se->vruntime, "<", se->vprot); 1103 } 1104 1105 static inline void cancel_protect_slice(struct sched_entity *se) 1106 { 1107 if (protect_slice(se)) 1108 se->vprot = se->vruntime; 1109 } 1110 1111 /* 1112 * Earliest Eligible Virtual Deadline First 1113 * 1114 * In order to provide latency guarantees for different request sizes 1115 * EEVDF selects the best runnable task from two criteria: 1116 * 1117 * 1) the task must be eligible (must be owed service) 1118 * 1119 * 2) from those tasks that meet 1), we select the one 1120 * with the earliest virtual deadline. 1121 * 1122 * We can do this in O(log n) time due to an augmented RB-tree. The 1123 * tree keeps the entries sorted on deadline, but also functions as a 1124 * heap based on the vruntime by keeping: 1125 * 1126 * se->min_vruntime = min(se->vruntime, se->{left,right}->min_vruntime) 1127 * 1128 * Which allows tree pruning through eligibility. 1129 */ 1130 static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, bool protect) 1131 { 1132 struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; 1133 struct sched_entity *se = __pick_first_entity(cfs_rq); 1134 struct sched_entity *curr = cfs_rq->curr; 1135 struct sched_entity *best = NULL; 1136 1137 /* 1138 * We can safely skip eligibility check if there is only one entity 1139 * in this cfs_rq, saving some cycles. 1140 */ 1141 if (cfs_rq->h_nr_queued == 1) 1142 return curr && curr->on_rq ? curr : se; 1143 1144 /* 1145 * Picking the ->next buddy will affect latency but not fairness. 1146 */ 1147 if (sched_feat(PICK_BUDDY) && protect && 1148 cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) { 1149 /* ->next will never be delayed */ 1150 WARN_ON_ONCE(cfs_rq->next->sched_delayed); 1151 return cfs_rq->next; 1152 } 1153 1154 if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr))) 1155 curr = NULL; 1156 1157 if (curr && protect && protect_slice(curr)) 1158 return curr; 1159 1160 /* Pick the leftmost entity if it's eligible */ 1161 if (se && entity_eligible(cfs_rq, se)) { 1162 best = se; 1163 goto found; 1164 } 1165 1166 /* Heap search for the EEVD entity */ 1167 while (node) { 1168 struct rb_node *left = node->rb_left; 1169 1170 /* 1171 * Eligible entities in left subtree are always better 1172 * choices, since they have earlier deadlines. 1173 */ 1174 if (left && vruntime_eligible(cfs_rq, 1175 __node_2_se(left)->min_vruntime)) { 1176 node = left; 1177 continue; 1178 } 1179 1180 se = __node_2_se(node); 1181 1182 /* 1183 * The left subtree either is empty or has no eligible 1184 * entity, so check the current node since it is the one 1185 * with earliest deadline that might be eligible. 1186 */ 1187 if (entity_eligible(cfs_rq, se)) { 1188 best = se; 1189 break; 1190 } 1191 1192 node = node->rb_right; 1193 } 1194 found: 1195 if (!best || (curr && entity_before(curr, best))) 1196 best = curr; 1197 1198 return best; 1199 } 1200 1201 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) 1202 { 1203 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); 1204 1205 if (!last) 1206 return NULL; 1207 1208 return __node_2_se(last); 1209 } 1210 1211 /************************************************************** 1212 * Scheduling class statistics methods: 1213 */ 1214 int sched_update_scaling(void) 1215 { 1216 unsigned int factor = get_update_sysctl_factor(); 1217 1218 #define WRT_SYSCTL(name) \ 1219 (normalized_sysctl_##name = sysctl_##name / (factor)) 1220 WRT_SYSCTL(sched_base_slice); 1221 #undef WRT_SYSCTL 1222 1223 return 0; 1224 } 1225 1226 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se); 1227 1228 /* 1229 * XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i 1230 * this is probably good enough. 1231 */ 1232 static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) 1233 { 1234 if (vruntime_cmp(se->vruntime, "<", se->deadline)) 1235 return false; 1236 1237 /* 1238 * For EEVDF the virtual time slope is determined by w_i (iow. 1239 * nice) while the request time r_i is determined by 1240 * sysctl_sched_base_slice. 1241 */ 1242 if (!se->custom_slice) 1243 se->slice = sysctl_sched_base_slice; 1244 1245 /* 1246 * EEVDF: vd_i = ve_i + r_i / w_i 1247 */ 1248 se->deadline = se->vruntime + calc_delta_fair(se->slice, se); 1249 avg_vruntime(cfs_rq); 1250 1251 /* 1252 * The task has consumed its request, reschedule. 1253 */ 1254 return true; 1255 } 1256 1257 #include "pelt.h" 1258 1259 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); 1260 static unsigned long task_h_load(struct task_struct *p); 1261 static unsigned long capacity_of(int cpu); 1262 1263 /* Give new sched_entity start runnable values to heavy its load in infant time */ 1264 void init_entity_runnable_average(struct sched_entity *se) 1265 { 1266 struct sched_avg *sa = &se->avg; 1267 1268 memset(sa, 0, sizeof(*sa)); 1269 1270 /* 1271 * Tasks are initialized with full load to be seen as heavy tasks until 1272 * they get a chance to stabilize to their real load level. 1273 * Group entities are initialized with zero load to reflect the fact that 1274 * nothing has been attached to the task group yet. 1275 */ 1276 if (entity_is_task(se)) 1277 sa->load_avg = scale_load_down(se->load.weight); 1278 1279 /* when this task is enqueued, it will contribute to its cfs_rq's load_avg */ 1280 } 1281 1282 /* 1283 * With new tasks being created, their initial util_avgs are extrapolated 1284 * based on the cfs_rq's current util_avg: 1285 * 1286 * util_avg = cfs_rq->avg.util_avg / (cfs_rq->avg.load_avg + 1) 1287 * * se_weight(se) 1288 * 1289 * However, in many cases, the above util_avg does not give a desired 1290 * value. Moreover, the sum of the util_avgs may be divergent, such 1291 * as when the series is a harmonic series. 1292 * 1293 * To solve this problem, we also cap the util_avg of successive tasks to 1294 * only 1/2 of the left utilization budget: 1295 * 1296 * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n 1297 * 1298 * where n denotes the nth task and cpu_scale the CPU capacity. 1299 * 1300 * For example, for a CPU with 1024 of capacity, a simplest series from 1301 * the beginning would be like: 1302 * 1303 * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... 1304 * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... 1305 * 1306 * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) 1307 * if util_avg > util_avg_cap. 1308 */ 1309 void post_init_entity_util_avg(struct task_struct *p) 1310 { 1311 struct sched_entity *se = &p->se; 1312 struct cfs_rq *cfs_rq = cfs_rq_of(se); 1313 struct sched_avg *sa = &se->avg; 1314 long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq))); 1315 long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; 1316 1317 if (p->sched_class != &fair_sched_class) { 1318 /* 1319 * For !fair tasks do: 1320 * 1321 update_cfs_rq_load_avg(now, cfs_rq); 1322 attach_entity_load_avg(cfs_rq, se); 1323 switched_from_fair(rq, p); 1324 * 1325 * such that the next switched_to_fair() has the 1326 * expected state. 1327 */ 1328 se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq); 1329 return; 1330 } 1331 1332 if (cap > 0) { 1333 if (cfs_rq->avg.util_avg != 0) { 1334 sa->util_avg = cfs_rq->avg.util_avg * se_weight(se); 1335 sa->util_avg /= (cfs_rq->avg.load_avg + 1); 1336 1337 if (sa->util_avg > cap) 1338 sa->util_avg = cap; 1339 } else { 1340 sa->util_avg = cap; 1341 } 1342 } 1343 1344 sa->runnable_avg = sa->util_avg; 1345 } 1346 1347 static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec); 1348 1349 static s64 update_se(struct rq *rq, struct sched_entity *se) 1350 { 1351 u64 now = rq_clock_task(rq); 1352 s64 delta_exec; 1353 1354 delta_exec = now - se->exec_start; 1355 if (unlikely(delta_exec <= 0)) 1356 return delta_exec; 1357 1358 se->exec_start = now; 1359 if (entity_is_task(se)) { 1360 struct task_struct *running = rq->curr; 1361 /* 1362 * If se is a task, we account the time against the running 1363 * task, as w/ proxy-exec they may not be the same. 1364 */ 1365 running->se.exec_start = now; 1366 running->se.sum_exec_runtime += delta_exec; 1367 1368 trace_sched_stat_runtime(running, delta_exec); 1369 account_group_exec_runtime(running, delta_exec); 1370 account_mm_sched(rq, running, delta_exec); 1371 1372 cgroup_account_cputime(running, delta_exec); 1373 } else { 1374 /* If not task, account the time against donor se */ 1375 se->sum_exec_runtime += delta_exec; 1376 } 1377 1378 if (schedstat_enabled()) { 1379 struct sched_statistics *stats; 1380 1381 stats = __schedstats_from_se(se); 1382 __schedstat_set(stats->exec_max, 1383 max(delta_exec, stats->exec_max)); 1384 } 1385 1386 return delta_exec; 1387 } 1388 1389 #ifdef CONFIG_SCHED_CACHE 1390 1391 /* 1392 * XXX numbers come from a place the sun don't shine -- probably wants to be SD 1393 * tunable or so. 1394 */ 1395 #define EPOCH_PERIOD (HZ / 100) /* 10 ms */ 1396 #define EPOCH_LLC_AFFINITY_TIMEOUT 5 /* 50 ms */ 1397 __read_mostly unsigned int llc_aggr_tolerance = 1; 1398 __read_mostly unsigned int llc_epoch_period = EPOCH_PERIOD; 1399 __read_mostly unsigned int llc_epoch_affinity_timeout = EPOCH_LLC_AFFINITY_TIMEOUT; 1400 __read_mostly unsigned int llc_imb_pct = 20; 1401 __read_mostly unsigned int llc_overaggr_pct = 50; 1402 1403 static int llc_id(int cpu) 1404 { 1405 if (cpu < 0) 1406 return -1; 1407 1408 return per_cpu(sd_llc_id, cpu); 1409 } 1410 1411 static inline int get_sched_cache_scale(int mul) 1412 { 1413 unsigned int tol = READ_ONCE(llc_aggr_tolerance); 1414 1415 if (!tol) 1416 return 0; 1417 1418 if (tol >= 100) 1419 return INT_MAX; 1420 1421 return (1 + (tol - 1) * mul); 1422 } 1423 1424 static bool exceed_llc_capacity(struct sched_cache_group *grp, int cpu) 1425 { 1426 #ifdef CONFIG_NUMA_BALANCING 1427 unsigned long llc, footprint; 1428 struct sched_domain *sd; 1429 int scale; 1430 1431 guard(rcu)(); 1432 1433 sd = rcu_dereference_sched_domain(cpu_rq(cpu)->sd); 1434 if (!sd) 1435 return true; 1436 1437 if (static_branch_likely(&sched_numa_balancing)) { 1438 /* 1439 * TBD: RDT exclusive LLC ways reserved should be 1440 * excluded. 1441 */ 1442 llc = sd->llc_bytes; 1443 footprint = READ_ONCE(grp->footprint); 1444 1445 /* 1446 * Scale the LLC size by 256*llc_aggr_tolerance 1447 * and compare it to the task's footprint. 1448 * 1449 * Suppose the L3 size is 32MB. If the 1450 * llc_aggr_tolerance is 1: 1451 * When the footprint is larger than 32MB, the 1452 * process is regarded as exceeding the LLC 1453 * capacity. If the llc_aggr_tolerance is 99: 1454 * When the footprint is larger than 784GB, the 1455 * process is regarded as exceeding the LLC 1456 * capacity: 1457 * 784GB = (1 + (99 - 1) * 256) * 32MB 1458 * If the llc_aggr_tolerance is 100: 1459 * ignore the footprint and do the aggregation 1460 * anyway. 1461 */ 1462 scale = get_sched_cache_scale(256); 1463 if (scale == INT_MAX) 1464 return false; 1465 1466 return ((llc * (u64)scale) < (footprint * PAGE_SIZE)); 1467 } 1468 #endif 1469 return false; 1470 } 1471 1472 /* Are the running threads (in SMT siblings) below ~80% of the LLC's CPUs? */ 1473 static inline bool fits_llc_nr(u64 nr_threads, unsigned int nr_cpus) 1474 { 1475 return nr_threads * 5 < (u64)nr_cpus * 4; 1476 } 1477 1478 static bool invalid_llc_nr(struct sched_cache_group *grp, struct task_struct *p, 1479 int cpu) 1480 { 1481 int scale; 1482 1483 if (get_nr_threads(p) <= 1) 1484 return true; 1485 1486 /* 1487 * Scale the number of 'cores' in a LLC by llc_aggr_tolerance 1488 * and compare it to the task's active threads. 1489 */ 1490 scale = get_sched_cache_scale(1); 1491 if (scale == INT_MAX) 1492 return false; 1493 1494 return !fits_llc_nr(READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads, 1495 scale * per_cpu(sd_llc_size, cpu)); 1496 } 1497 1498 /* 1499 * A task counts in nr_pref_llc_running while it is queued on its preferred 1500 * LLC (pref_llc_queued) and runnable (!sched_delayed), keeping the counter in 1501 * the runnable domain so alb_break_llc() can compare it with h_nr_runnable. 1502 */ 1503 static bool task_pref_llc_runnable(struct task_struct *p) 1504 { 1505 return p->pref_llc_queued && !p->se.sched_delayed; 1506 } 1507 1508 static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) 1509 { 1510 if (task_pref_llc_runnable(p)) 1511 rq->nr_pref_llc_running++; 1512 } 1513 1514 static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) 1515 { 1516 if (task_pref_llc_runnable(p)) 1517 rq->nr_pref_llc_running--; 1518 } 1519 1520 static void account_llc_enqueue(struct rq *rq, struct task_struct *p) 1521 { 1522 int pref_llc, pref_llc_queued; 1523 struct sched_domain *sd; 1524 1525 pref_llc = p->preferred_llc; 1526 if (pref_llc < 0) 1527 return; 1528 1529 pref_llc_queued = (pref_llc == task_llc(p)); 1530 rq->nr_llc_running++; 1531 1532 /* 1533 * Record whether p is enqueued on its preferred 1534 * LLC, in order to pair with account_llc_dequeue() 1535 * to maintain a consistent nr_pref_llc_running per 1536 * runqueue. 1537 * This is necessary because a race condition exists: 1538 * after a task is enqueued on a runqueue, task_llc(p) 1539 * may change due to CPU hotplug. Therefore, checking 1540 * task_llc(p) to determine whether the task is being 1541 * dequeued from its preferred LLC is unreliable and 1542 * can cause inconsistent values - checking the 1543 * p->pref_llc_queued in account_llc_dequeue() would 1544 * be reliable. 1545 */ 1546 p->pref_llc_queued = pref_llc_queued; 1547 1548 /* Skipped while delayed; clear_delayed() adds it back on wake. */ 1549 pref_llc_running_inc(rq, p); 1550 1551 sd = rcu_dereference_all(rq->sd); 1552 if (sd && (unsigned int)pref_llc < sd->llc_max) 1553 sd->llc_counts[pref_llc]++; 1554 } 1555 1556 static void account_llc_dequeue(struct rq *rq, struct task_struct *p) 1557 { 1558 struct sched_domain *sd; 1559 int pref_llc; 1560 1561 pref_llc = p->preferred_llc; 1562 if (pref_llc < 0) 1563 return; 1564 1565 rq->nr_llc_running--; 1566 if (p->pref_llc_queued) { 1567 /* 1568 * Skipped if still delayed (set_delayed() already removed it); 1569 * clearing pref_llc_queued below also stops clear_delayed() 1570 * from re-adding it. 1571 */ 1572 pref_llc_running_dec(rq, p); 1573 /* 1574 * Update the status in case 1575 * other logic might query 1576 * this. 1577 */ 1578 p->pref_llc_queued = 0; 1579 } 1580 1581 sd = rcu_dereference_all(rq->sd); 1582 if (sd && (unsigned int)pref_llc < sd->llc_max) { 1583 /* 1584 * There is a race condition between dequeue 1585 * and CPU hotplug. After a task has been enqueued 1586 * on CPUx, a CPU hotplug event occurs, and all online 1587 * CPUs (including CPUx) rebuild their sched_domains 1588 * and reset statistics to zero(including sd->llc_counts). 1589 * This can cause temporary undercount and we have to 1590 * check for such underflow in sd->llc_counts. 1591 * 1592 * This undercount is temporary and accurate accounting 1593 * will resume once the rq has a chance to be idle. 1594 */ 1595 if (sd->llc_counts[pref_llc]) 1596 sd->llc_counts[pref_llc]--; 1597 } 1598 } 1599 1600 int mm_init_sched(struct mm_struct *mm, 1601 struct sched_cache_time __percpu *_pcpu_sched) 1602 { 1603 struct sched_cache_group *grp; 1604 unsigned long epoch = 0; 1605 int i; 1606 1607 grp = kzalloc_obj(*grp); 1608 if (!grp) { 1609 free_percpu(_pcpu_sched); 1610 mm->sched_cache_grp = NULL; 1611 return -ENOMEM; 1612 } 1613 1614 for_each_possible_cpu(i) { 1615 struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i); 1616 struct rq *rq = cpu_rq(i); 1617 1618 pcpu_sched->runtime = 0; 1619 /* a slightly stale cpu epoch is acceptible */ 1620 pcpu_sched->epoch = rq->cpu_epoch; 1621 epoch = rq->cpu_epoch; 1622 } 1623 1624 raw_spin_lock_init(&grp->lock); 1625 grp->epoch = epoch; 1626 grp->cpu = -1; 1627 grp->next_scan = jiffies; 1628 grp->nr_running_avg = 0; 1629 grp->footprint = 0; 1630 refcount_set(&grp->refcnt, 1); 1631 /* 1632 * The update to grp->pcpu_sched should not be reordered 1633 * before initialization to grp's other fields, in case 1634 * the readers may get invalid mm_sched_epoch, etc. 1635 */ 1636 smp_store_release(&grp->pcpu_sched, _pcpu_sched); 1637 /* 1638 * Publish the group last. Not every reader qualifies it by 1639 * grp->pcpu_sched - can_migrate_llc_task() only checks that the 1640 * pointer is non-NULL before reading grp->footprint and 1641 * grp->nr_running_avg - so a reachable group must already be 1642 * fully initialized. 1643 */ 1644 smp_store_release(&mm->sched_cache_grp, grp); 1645 return 0; 1646 } 1647 1648 static void sched_cache_group_free_rcu(struct rcu_head *rcu) 1649 { 1650 struct sched_cache_group *grp = 1651 container_of(rcu, struct sched_cache_group, rcu); 1652 1653 free_percpu(grp->pcpu_sched); 1654 kfree(grp); 1655 } 1656 1657 static void sched_cache_group_put(struct sched_cache_group *grp) 1658 { 1659 if (!grp || !refcount_dec_and_test(&grp->refcnt)) 1660 return; 1661 1662 call_rcu(&grp->rcu, sched_cache_group_free_rcu); 1663 } 1664 1665 DEFINE_FREE(sched_cache_group_put, struct sched_cache_group *, 1666 sched_cache_group_put(_T)); 1667 1668 #define rcu_deref_sched_cache_grp(tsk) \ 1669 rcu_dereference_check((tsk)->sched_cache_grp, (tsk) == current) 1670 1671 static struct sched_cache_group *sched_cache_replace_grp(struct task_struct *p, 1672 struct sched_cache_group *new) 1673 { 1674 struct sched_cache_group *old; 1675 1676 old = rcu_deref_sched_cache_grp(p); 1677 rcu_assign_pointer(p->sched_cache_grp, new); 1678 1679 return old; 1680 } 1681 1682 struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp) 1683 { 1684 /* 1685 * refcount_inc_not_zero() is the acquire primitive for lockless 1686 * (RCU) lookups; plain refcount_inc() would scribble the count if 1687 * it already reached zero. Return NULL in that case. 1688 */ 1689 if (grp && !refcount_inc_not_zero(&grp->refcnt)) 1690 grp = NULL; 1691 1692 return grp; 1693 } 1694 1695 struct sched_cache_group *task_cache_group_get(struct task_struct *p) 1696 { 1697 guard(rcu)(); 1698 return sched_cache_group_get(rcu_dereference(p->sched_cache_grp)); 1699 } 1700 1701 void sched_cache_fork(struct task_struct *p) 1702 { 1703 /* 1704 * The child takes its own reference on the mm's cache group, separate 1705 * from the reference held by the mm. @p is not yet visible to readers, 1706 * so a plain initializing store is enough. 1707 */ 1708 RCU_INIT_POINTER(p->sched_cache_grp, 1709 sched_cache_group_get(p->mm->sched_cache_grp)); 1710 } 1711 1712 void sched_cache_fork_cleanup(struct task_struct *p) 1713 { 1714 /* 1715 * A fork that fails after sched_cache_fork() never reaches exit_mm(), 1716 * so drop the reference here. @p never became visible, so there are no 1717 * concurrent readers and the reference we hold keeps the group alive. 1718 */ 1719 sched_cache_group_put(rcu_access_pointer(p->sched_cache_grp)); 1720 RCU_INIT_POINTER(p->sched_cache_grp, NULL); 1721 } 1722 1723 void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm) 1724 { 1725 struct sched_cache_group *old; 1726 1727 /* 1728 * Acquire the new reference before publishing the pointer, then drop 1729 * the old one. @p is current and the only writer of its own pointer. 1730 */ 1731 old = sched_cache_replace_grp(p, sched_cache_group_get(mm->sched_cache_grp)); 1732 sched_cache_group_put(old); 1733 } 1734 1735 void sched_cache_exit_mm(struct task_struct *p) 1736 { 1737 struct sched_cache_group *grp = sched_cache_replace_grp(p, NULL); 1738 1739 #ifdef CONFIG_NUMA_BALANCING 1740 /* 1741 * Subtract this task's footprint from the group before dropping the 1742 * reference, so the group footprint converges as its threads exit. 1743 * Unlocked for performance; clamp to avoid underflow. 1744 */ 1745 if (grp && p->total_numa_faults) { 1746 unsigned long fp = READ_ONCE(grp->footprint); 1747 unsigned long sub = min(fp, p->total_numa_faults); 1748 1749 WRITE_ONCE(grp->footprint, fp - sub); 1750 } 1751 #endif 1752 sched_cache_group_put(grp); 1753 } 1754 1755 void mm_destroy_sched(struct mm_struct *mm) 1756 { 1757 sched_cache_group_put(mm->sched_cache_grp); 1758 mm->sched_cache_grp = NULL; 1759 } 1760 1761 /* because why would C be fully specified */ 1762 static __always_inline void __shr_u64(u64 *val, unsigned int n) 1763 { 1764 if (n >= 64) { 1765 *val = 0; 1766 return; 1767 } 1768 *val >>= n; 1769 } 1770 1771 static inline void __update_mm_sched(struct rq *rq, 1772 struct sched_cache_time *pcpu_sched) 1773 { 1774 lockdep_assert_held(&rq->cpu_epoch_lock); 1775 1776 unsigned int period = max(READ_ONCE(llc_epoch_period), 1U); 1777 unsigned long n, now = jiffies; 1778 long delta = now - rq->cpu_epoch_next; 1779 1780 if (delta > 0) { 1781 n = (delta + period - 1) / period; 1782 rq->cpu_epoch += n; 1783 rq->cpu_epoch_next += n * period; 1784 __shr_u64(&rq->cpu_runtime, n); 1785 } 1786 1787 n = rq->cpu_epoch - pcpu_sched->epoch; 1788 if (n) { 1789 pcpu_sched->epoch += n; 1790 __shr_u64(&pcpu_sched->runtime, n); 1791 } 1792 } 1793 1794 static unsigned long fraction_mm_sched(struct rq *rq, 1795 struct sched_cache_time *pcpu_sched) 1796 { 1797 guard(raw_spinlock_irqsave)(&rq->cpu_epoch_lock); 1798 1799 __update_mm_sched(rq, pcpu_sched); 1800 1801 /* 1802 * Runtime is a geometric series (r=0.5) and as such will sum to twice 1803 * the accumulation period, this means the multiplcation here should 1804 * not overflow. 1805 */ 1806 return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1); 1807 } 1808 1809 static int get_pref_llc(struct task_struct *p, struct sched_cache_group *grp) 1810 { 1811 int mm_sched_llc = -1, mm_sched_cpu; 1812 1813 if (!grp) 1814 return -1; 1815 1816 mm_sched_cpu = READ_ONCE(grp->cpu); 1817 if (mm_sched_cpu != -1) { 1818 mm_sched_llc = llc_id(mm_sched_cpu); 1819 1820 #ifdef CONFIG_NUMA_BALANCING 1821 /* 1822 * Don't assign preferred LLC if it 1823 * conflicts with NUMA balancing. 1824 * This can happen when sched_setnuma() gets 1825 * called, however it is not much of an issue 1826 * because we expect account_mm_sched() to get 1827 * called fairly regularly -- at a higher rate 1828 * than sched_setnuma() at least -- and thus the 1829 * conflict only exists for a short period of time. 1830 */ 1831 if (static_branch_likely(&sched_numa_balancing) && 1832 p->numa_preferred_nid >= 0 && 1833 cpu_to_node(mm_sched_cpu) != p->numa_preferred_nid) 1834 mm_sched_llc = -1; 1835 #endif 1836 } 1837 1838 return mm_sched_llc; 1839 } 1840 1841 static unsigned int task_running_on_cpu(int cpu, struct task_struct *p); 1842 1843 static inline 1844 void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) 1845 { 1846 struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp); 1847 struct sched_cache_time *pcpu_sched; 1848 int mm_sched_llc = -1; 1849 unsigned long epoch; 1850 1851 if (!sched_cache_enabled()) 1852 return; 1853 1854 if (p->sched_class != &fair_sched_class) 1855 return; 1856 /* 1857 * init_task, kthreads and user thread created 1858 * by user_mode_thread() don't have a cache group. 1859 * In theory a kernel thread does not have any valid 1860 * cache group, because sched_cache_fork() is not 1861 * invoked for a kernel thread - !grp should gate the 1862 * kernel thread. Use the PF_KTHREAD check explicitly 1863 * here for safety reasons, to guard against future 1864 * modifications and to pair with task_tick_cache(). 1865 */ 1866 if (p->flags & PF_KTHREAD || !grp || !grp->pcpu_sched) 1867 return; 1868 1869 pcpu_sched = per_cpu_ptr(grp->pcpu_sched, cpu_of(rq)); 1870 1871 scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) { 1872 __update_mm_sched(rq, pcpu_sched); 1873 pcpu_sched->runtime += delta_exec; 1874 rq->cpu_runtime += delta_exec; 1875 epoch = rq->cpu_epoch; 1876 } 1877 1878 /* 1879 * If this process hasn't hit task_cache_work() for a while invalidate 1880 * its preferred state. 1881 */ 1882 if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout || 1883 invalid_llc_nr(grp, p, cpu_of(rq)) || 1884 exceed_llc_capacity(grp, cpu_of(rq))) { 1885 if (READ_ONCE(grp->cpu) != -1) 1886 WRITE_ONCE(grp->cpu, -1); 1887 } 1888 1889 mm_sched_llc = get_pref_llc(p, grp); 1890 1891 /* task not on rq accounted later in account_entity_enqueue() */ 1892 if (task_running_on_cpu(rq->cpu, p) && 1893 READ_ONCE(p->preferred_llc) != mm_sched_llc) { 1894 account_llc_dequeue(rq, p); 1895 WRITE_ONCE(p->preferred_llc, mm_sched_llc); 1896 account_llc_enqueue(rq, p); 1897 } 1898 } 1899 1900 static void task_tick_cache(struct rq *rq, struct task_struct *p) 1901 { 1902 struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp); 1903 struct callback_head *work = &p->cache_work; 1904 unsigned long epoch; 1905 1906 if (!sched_cache_enabled()) 1907 return; 1908 1909 if (!grp || p->flags & PF_KTHREAD || 1910 !grp->pcpu_sched) 1911 return; 1912 1913 epoch = rq->cpu_epoch; 1914 /* avoid moving backwards */ 1915 if (time_after_eq(grp->epoch, epoch)) 1916 return; 1917 1918 guard(raw_spinlock)(&grp->lock); 1919 1920 if (work->next == work) { 1921 task_work_add(p, work, TWA_RESUME); 1922 WRITE_ONCE(grp->epoch, epoch); 1923 } 1924 } 1925 1926 static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p, 1927 struct sched_cache_group *grp) 1928 { 1929 #ifdef CONFIG_NUMA_BALANCING 1930 int cpu, curr_cpu, nid, pref_nid; 1931 1932 if (!static_branch_likely(&sched_numa_balancing)) 1933 goto out; 1934 1935 cpu = READ_ONCE(grp->cpu); 1936 if (cpu != -1) 1937 nid = cpu_to_node(cpu); 1938 curr_cpu = task_cpu(p); 1939 1940 /* 1941 * Scanning in the preferred NUMA node is ideal. However, the NUMA 1942 * preferred node is per-task rather than per-process. It is possible 1943 * for different threads of the process to have distinct preferred 1944 * nodes; consequently, the process-wide preferred LLC may bounce 1945 * between different nodes. As a workaround, maintain the scan 1946 * CPU mask to also cover the process's current preferred LLC and the 1947 * current running node to mitigate the bouncing risk. 1948 * TBD: numa_group should be considered during task aggregation. 1949 */ 1950 pref_nid = p->numa_preferred_nid; 1951 /* honor the task's preferred node */ 1952 if (pref_nid == NUMA_NO_NODE) 1953 goto out; 1954 1955 cpumask_or(cpus, cpus, cpumask_of_node(pref_nid)); 1956 1957 /* honor the task's preferred LLC CPU */ 1958 if (cpu != -1 && !cpumask_test_cpu(cpu, cpus) && nid != NUMA_NO_NODE) 1959 cpumask_or(cpus, cpus, cpumask_of_node(nid)); 1960 1961 /* make sure the task's current running node is included */ 1962 if (!cpumask_test_cpu(curr_cpu, cpus)) 1963 cpumask_or(cpus, cpus, cpumask_of_node(cpu_to_node(curr_cpu))); 1964 1965 return; 1966 1967 out: 1968 #endif 1969 cpumask_copy(cpus, cpu_online_mask); 1970 } 1971 1972 static inline void update_avg_scale(u64 *avg, u64 sample) 1973 { 1974 int factor = per_cpu(sd_llc_size, raw_smp_processor_id()); 1975 s64 diff = sample - *avg; 1976 u32 divisor; 1977 1978 /* 1979 * Scale the divisor based on the number of CPUs contained 1980 * in the LLC. This scaling ensures smaller LLC domains use 1981 * a smaller divisor to achieve more precise sensitivity to 1982 * changes in nr_running, while larger LLC domains are capped 1983 * at a maximum divisor of 8 which is the default smoothing 1984 * factor of EWMA in update_avg(). 1985 */ 1986 divisor = clamp_t(u32, (factor >> 2), 2, 8); 1987 *avg += div64_s64(diff, divisor); 1988 } 1989 1990 static void task_cache_work(struct callback_head *work) 1991 { 1992 struct sched_cache_group *grp __free(sched_cache_group_put) = NULL; 1993 cpumask_var_t cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL; 1994 int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu; 1995 unsigned long next_scan, now = jiffies; 1996 struct task_struct *p = current, *cur; 1997 unsigned long curr_m_a_occ = 0; 1998 unsigned long m_a_occ = 0; 1999 2000 WARN_ON_ONCE(work != &p->cache_work); 2001 2002 work->next = work; 2003 2004 if (p->flags & PF_EXITING) 2005 return; 2006 2007 /* 2008 * A reference makes sure grp is not released by others. The rcu 2009 * lock can not be held till after zalloc_cpumask_var() below, 2010 * because the latter might sleep. 2011 */ 2012 grp = task_cache_group_get(p); 2013 if (!grp) 2014 return; 2015 2016 next_scan = READ_ONCE(grp->next_scan); 2017 if (time_before(now, next_scan)) 2018 return; 2019 2020 /* only 1 thread is allowed to scan */ 2021 if (!try_cmpxchg(&grp->next_scan, &next_scan, 2022 now + max_t(unsigned long, 2023 READ_ONCE(llc_epoch_period), 1))) 2024 return; 2025 2026 curr_cpu = task_cpu(p); 2027 if (invalid_llc_nr(grp, p, curr_cpu) || 2028 exceed_llc_capacity(grp, curr_cpu)) { 2029 if (READ_ONCE(grp->cpu) != -1) 2030 WRITE_ONCE(grp->cpu, -1); 2031 2032 return; 2033 } 2034 2035 if (!zalloc_cpumask_var(&cpus, GFP_KERNEL)) 2036 return; 2037 2038 scoped_guard (cpus_read_lock) { 2039 guard(rcu)(); 2040 2041 get_scan_cpumasks(cpus, p, grp); 2042 2043 for_each_cpu(cpu, cpus) { 2044 /* XXX sched_cluster_active */ 2045 struct sched_domain *sd = rcu_dereference_all(per_cpu(sd_llc, cpu)); 2046 unsigned long occ, m_occ = 0, a_occ = 0; 2047 int m_cpu = -1, i; 2048 2049 if (!sd) 2050 continue; 2051 2052 for_each_cpu(i, sched_domain_span(sd)) { 2053 occ = fraction_mm_sched(cpu_rq(i), 2054 per_cpu_ptr(grp->pcpu_sched, i)); 2055 a_occ += occ; 2056 if (occ > m_occ) { 2057 m_occ = occ; 2058 m_cpu = i; 2059 } 2060 2061 /* 2062 * rcu_access_pointer() is used because the 2063 * pointer is only compared, never dereferenced. 2064 */ 2065 cur = rcu_dereference_all(cpu_rq(i)->curr); 2066 if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) && 2067 rcu_access_pointer(cur->sched_cache_grp) == grp) 2068 nr_running++; 2069 } 2070 2071 /* 2072 * Compare the accumulated occupancy of each LLC. The 2073 * reason for using accumulated occupancy rather than average 2074 * per CPU occupancy is that it works better in asymmetric LLC 2075 * scenarios. 2076 * For example, if there are 2 threads in a 4CPU LLC and 3 2077 * threads in an 8CPU LLC, it might be better to choose the one 2078 * with 3 threads. However, this would not be the case if the 2079 * occupancy is divided by the number of CPUs in an LLC (i.e., 2080 * if average per CPU occupancy is used). 2081 * Besides, NUMA balancing fault statistics behave similarly: 2082 * the total number of faults per node is compared rather than 2083 * the average number of faults per CPU. This strategy is also 2084 * followed here. 2085 */ 2086 if (a_occ > m_a_occ) { 2087 m_a_occ = a_occ; 2088 m_a_cpu = m_cpu; 2089 } 2090 2091 if (llc_id(cpu) == llc_id(READ_ONCE(grp->cpu))) 2092 curr_m_a_occ = a_occ; 2093 2094 cpumask_andnot(cpus, cpus, sched_domain_span(sd)); 2095 } 2096 } 2097 2098 if (m_a_occ > (2 * curr_m_a_occ)) { 2099 /* 2100 * Avoid switching sched_cache_grp->cpu too fast. 2101 * The reason to choose 2X is because: 2102 * 1. It is better to keep the preferred LLC stable, 2103 * rather than changing it frequently and cause migrations 2104 * 2. 2X means the new preferred LLC has at least 1 more 2105 * busy CPU than the old one(200% vs 100%, eg) 2106 * 3. 2X is chosen based on test results, as it delivers 2107 * the optimal performance gain so far. 2108 */ 2109 WRITE_ONCE(grp->cpu, m_a_cpu); 2110 } 2111 2112 update_avg_scale(&grp->nr_running_avg, nr_running); 2113 } 2114 2115 void init_sched_mm(struct task_struct *p) 2116 { 2117 struct callback_head *work = &p->cache_work; 2118 2119 init_task_work(work, task_cache_work); 2120 work->next = work; 2121 /* 2122 * dup_task_struct() copies the parent's task_struct, including its 2123 * sched_cache_grp, for which the child holds no reference. Clear it 2124 * here - before copy_mm() runs - so the child never carries a 2125 * borrowed pointer that the fork error path would put. 2126 */ 2127 RCU_INIT_POINTER(p->sched_cache_grp, NULL); 2128 /* 2129 * Reset new task's preference to avoid 2130 * polluting account_llc_enqueue(). 2131 */ 2132 p->preferred_llc = -1; 2133 p->pref_llc_queued = 0; 2134 } 2135 2136 #else /* CONFIG_SCHED_CACHE */ 2137 2138 static inline void account_mm_sched(struct rq *rq, struct task_struct *p, 2139 s64 delta_exec) { } 2140 2141 void init_sched_mm(struct task_struct *p) { } 2142 2143 static void task_tick_cache(struct rq *rq, struct task_struct *p) { } 2144 2145 static inline int get_pref_llc(struct task_struct *p, 2146 struct sched_cache_group *grp) 2147 { 2148 return -1; 2149 } 2150 2151 static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {} 2152 2153 static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {} 2154 2155 static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) {} 2156 2157 static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) {} 2158 2159 #endif /* CONFIG_SCHED_CACHE */ 2160 2161 /* 2162 * Used by other classes to account runtime. 2163 */ 2164 s64 update_curr_common(struct rq *rq) 2165 { 2166 return update_se(rq, &rq->donor->se); 2167 } 2168 2169 /* 2170 * Update the current task's runtime statistics. 2171 */ 2172 static void update_curr(struct cfs_rq *cfs_rq) 2173 { 2174 /* 2175 * Note: cfs_rq->curr corresponds to the task picked to 2176 * run (ie: rq->donor.se) which due to proxy-exec may 2177 * not necessarily be the actual task running 2178 * (rq->curr.se). This is easy to confuse! 2179 */ 2180 struct sched_entity *curr = cfs_rq->h_curr; 2181 struct rq *rq = rq_of(cfs_rq); 2182 s64 delta_exec; 2183 bool resched; 2184 2185 if (unlikely(!curr)) 2186 return; 2187 2188 delta_exec = update_se(rq, curr); 2189 if (unlikely(delta_exec <= 0)) 2190 return; 2191 2192 account_cfs_rq_runtime(cfs_rq, delta_exec); 2193 2194 if (!entity_is_task(curr)) 2195 return; 2196 2197 cfs_rq = &rq->cfs; 2198 2199 curr->vruntime += calc_delta_fair(delta_exec, curr); 2200 resched = update_deadline(cfs_rq, curr); 2201 2202 /* 2203 * If the fair_server is active, we need to account for the 2204 * fair_server time whether or not the task is running on 2205 * behalf of fair_server or not: 2206 * - If the task is running on behalf of fair_server, we need 2207 * to limit its time based on the assigned runtime. 2208 * - Fair task that runs outside of fair_server should account 2209 * against fair_server such that it can account for this time 2210 * and possibly avoid running this period. 2211 */ 2212 dl_server_update(&rq->fair_server, delta_exec); 2213 2214 if (cfs_rq->h_nr_queued == 1) 2215 return; 2216 2217 if (resched || !protect_slice(curr)) { 2218 resched_curr_lazy(rq); 2219 clear_buddies(cfs_rq, curr); 2220 } 2221 } 2222 2223 static void update_curr_fair(struct rq *rq) 2224 { 2225 struct sched_entity *se = &rq->donor->se; 2226 2227 for_each_sched_entity(se) 2228 update_curr(cfs_rq_of(se)); 2229 } 2230 2231 static inline void 2232 update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 2233 { 2234 struct sched_statistics *stats; 2235 struct task_struct *p = NULL; 2236 2237 if (!schedstat_enabled()) 2238 return; 2239 2240 stats = __schedstats_from_se(se); 2241 2242 if (entity_is_task(se)) 2243 p = task_of(se); 2244 2245 __update_stats_wait_start(rq_of(cfs_rq), p, stats); 2246 } 2247 2248 static inline void 2249 update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 2250 { 2251 struct sched_statistics *stats; 2252 struct task_struct *p = NULL; 2253 2254 if (!schedstat_enabled()) 2255 return; 2256 2257 stats = __schedstats_from_se(se); 2258 2259 /* 2260 * When the sched_schedstat changes from 0 to 1, some sched se 2261 * maybe already in the runqueue, the se->statistics.wait_start 2262 * will be 0.So it will let the delta wrong. We need to avoid this 2263 * scenario. 2264 */ 2265 if (unlikely(!schedstat_val(stats->wait_start))) 2266 return; 2267 2268 if (entity_is_task(se)) 2269 p = task_of(se); 2270 2271 __update_stats_wait_end(rq_of(cfs_rq), p, stats); 2272 } 2273 2274 static inline void 2275 update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 2276 { 2277 struct sched_statistics *stats; 2278 struct task_struct *tsk = NULL; 2279 2280 if (!schedstat_enabled()) 2281 return; 2282 2283 stats = __schedstats_from_se(se); 2284 2285 if (entity_is_task(se)) 2286 tsk = task_of(se); 2287 2288 __update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats); 2289 } 2290 2291 /* 2292 * Task is being enqueued - update stats: 2293 */ 2294 static inline void 2295 update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 2296 { 2297 if (!schedstat_enabled()) 2298 return; 2299 2300 /* 2301 * Are we enqueueing a waiting task? (for current tasks 2302 * a dequeue/enqueue event is a NOP) 2303 */ 2304 if (se != cfs_rq->h_curr) 2305 update_stats_wait_start_fair(cfs_rq, se); 2306 2307 if (flags & ENQUEUE_WAKEUP) 2308 update_stats_enqueue_sleeper_fair(cfs_rq, se); 2309 } 2310 2311 static inline void 2312 update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 2313 { 2314 2315 if (!schedstat_enabled()) 2316 return; 2317 2318 /* 2319 * Mark the end of the wait period if dequeueing a 2320 * waiting task: 2321 */ 2322 if (se != cfs_rq->h_curr) 2323 update_stats_wait_end_fair(cfs_rq, se); 2324 2325 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { 2326 struct task_struct *tsk = task_of(se); 2327 unsigned int state; 2328 2329 /* XXX racy against TTWU */ 2330 state = READ_ONCE(tsk->__state); 2331 if (state & TASK_INTERRUPTIBLE) 2332 __schedstat_set(tsk->stats.sleep_start, 2333 rq_clock(rq_of(cfs_rq))); 2334 if (state & TASK_UNINTERRUPTIBLE) 2335 __schedstat_set(tsk->stats.block_start, 2336 rq_clock(rq_of(cfs_rq))); 2337 } 2338 } 2339 2340 /* 2341 * We are picking a new current task - update its stats: 2342 */ 2343 static inline void 2344 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 2345 { 2346 /* 2347 * We are starting a new run period: 2348 */ 2349 se->exec_start = rq_clock_task(rq_of(cfs_rq)); 2350 } 2351 2352 /* Check sched_smt_active before calling this to avoid overheads in fastpaths */ 2353 static inline bool is_core_idle(int cpu) 2354 { 2355 int sibling; 2356 2357 for_each_cpu(sibling, cpu_smt_mask(cpu)) { 2358 if (cpu == sibling) 2359 continue; 2360 2361 if (!idle_cpu(sibling)) 2362 return false; 2363 } 2364 2365 return true; 2366 } 2367 2368 #ifdef CONFIG_NUMA 2369 #define NUMA_IMBALANCE_MIN 2 2370 2371 static inline long 2372 adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr) 2373 { 2374 /* 2375 * Allow a NUMA imbalance if busy CPUs is less than the maximum 2376 * threshold. Above this threshold, individual tasks may be contending 2377 * for both memory bandwidth and any shared HT resources. This is an 2378 * approximation as the number of running tasks may not be related to 2379 * the number of busy CPUs due to sched_setaffinity. 2380 */ 2381 if (dst_running > imb_numa_nr) 2382 return imbalance; 2383 2384 /* 2385 * Allow a small imbalance based on a simple pair of communicating 2386 * tasks that remain local when the destination is lightly loaded. 2387 */ 2388 if (imbalance <= NUMA_IMBALANCE_MIN) 2389 return 0; 2390 2391 return imbalance; 2392 } 2393 #endif /* CONFIG_NUMA */ 2394 2395 #ifdef CONFIG_NUMA_BALANCING 2396 /* 2397 * Approximate time to scan a full NUMA task in ms. The task scan period is 2398 * calculated based on the tasks virtual memory size and 2399 * numa_balancing_scan_size. 2400 */ 2401 unsigned int sysctl_numa_balancing_scan_period_min = 1000; 2402 unsigned int sysctl_numa_balancing_scan_period_max = 60000; 2403 2404 /* Portion of address space to scan in MB */ 2405 unsigned int sysctl_numa_balancing_scan_size = 256; 2406 2407 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ 2408 unsigned int sysctl_numa_balancing_scan_delay = 1000; 2409 2410 /* The page with hint page fault latency < threshold in ms is considered hot */ 2411 unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC; 2412 2413 struct numa_group { 2414 refcount_t refcount; 2415 2416 spinlock_t lock; /* nr_tasks, tasks */ 2417 int nr_tasks; 2418 pid_t gid; 2419 int active_nodes; 2420 2421 struct rcu_head rcu; 2422 unsigned long total_faults; 2423 unsigned long max_faults_cpu; 2424 /* 2425 * faults[] array is split into two regions: faults_mem and faults_cpu. 2426 * 2427 * Faults_cpu is used to decide whether memory should move 2428 * towards the CPU. As a consequence, these stats are weighted 2429 * more by CPU use than by memory faults. 2430 */ 2431 unsigned long faults[]; 2432 }; 2433 2434 /* 2435 * For functions that can be called in multiple contexts that permit reading 2436 * ->numa_group (see struct task_struct for locking rules). 2437 */ 2438 static struct numa_group *deref_task_numa_group(struct task_struct *p) 2439 { 2440 return rcu_dereference_check(p->numa_group, p == current || 2441 (lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu))); 2442 } 2443 2444 static struct numa_group *deref_curr_numa_group(struct task_struct *p) 2445 { 2446 return rcu_dereference_protected(p->numa_group, p == current); 2447 } 2448 2449 static inline unsigned long group_faults_priv(struct numa_group *ng); 2450 static inline unsigned long group_faults_shared(struct numa_group *ng); 2451 2452 static unsigned int task_nr_scan_windows(struct task_struct *p) 2453 { 2454 unsigned long rss = 0; 2455 unsigned long nr_scan_pages; 2456 2457 /* 2458 * Calculations based on RSS as non-present and empty pages are skipped 2459 * by the PTE scanner and NUMA hinting faults should be trapped based 2460 * on resident pages 2461 */ 2462 nr_scan_pages = MB_TO_PAGES(sysctl_numa_balancing_scan_size); 2463 rss = get_mm_rss(p->mm); 2464 if (!rss) 2465 rss = nr_scan_pages; 2466 2467 rss = round_up(rss, nr_scan_pages); 2468 return rss / nr_scan_pages; 2469 } 2470 2471 /* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ 2472 #define MAX_SCAN_WINDOW 2560 2473 2474 static unsigned int task_scan_min(struct task_struct *p) 2475 { 2476 unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); 2477 unsigned int scan, floor; 2478 unsigned int windows = 1; 2479 2480 if (scan_size < MAX_SCAN_WINDOW) 2481 windows = MAX_SCAN_WINDOW / scan_size; 2482 floor = 1000 / windows; 2483 2484 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); 2485 return max_t(unsigned int, floor, scan); 2486 } 2487 2488 static unsigned int task_scan_start(struct task_struct *p) 2489 { 2490 unsigned long smin = task_scan_min(p); 2491 unsigned long period = smin; 2492 struct numa_group *ng; 2493 2494 /* Scale the maximum scan period with the amount of shared memory. */ 2495 rcu_read_lock(); 2496 ng = rcu_dereference_all(p->numa_group); 2497 if (ng) { 2498 unsigned long shared = group_faults_shared(ng); 2499 unsigned long private = group_faults_priv(ng); 2500 2501 period *= refcount_read(&ng->refcount); 2502 period *= shared + 1; 2503 period /= private + shared + 1; 2504 } 2505 rcu_read_unlock(); 2506 2507 return max(smin, period); 2508 } 2509 2510 static unsigned int task_scan_max(struct task_struct *p) 2511 { 2512 unsigned long smin = task_scan_min(p); 2513 unsigned long smax; 2514 struct numa_group *ng; 2515 2516 /* Watch for min being lower than max due to floor calculations */ 2517 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); 2518 2519 /* Scale the maximum scan period with the amount of shared memory. */ 2520 ng = deref_curr_numa_group(p); 2521 if (ng) { 2522 unsigned long shared = group_faults_shared(ng); 2523 unsigned long private = group_faults_priv(ng); 2524 unsigned long period = smax; 2525 2526 period *= refcount_read(&ng->refcount); 2527 period *= shared + 1; 2528 period /= private + shared + 1; 2529 2530 smax = max(smax, period); 2531 } 2532 2533 return max(smin, smax); 2534 } 2535 2536 static void account_numa_enqueue(struct rq *rq, struct task_struct *p) 2537 { 2538 rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE); 2539 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 2540 } 2541 2542 static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 2543 { 2544 rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE); 2545 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 2546 } 2547 2548 /* Shared or private faults. */ 2549 #define NR_NUMA_HINT_FAULT_TYPES 2 2550 2551 /* Memory and CPU locality */ 2552 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) 2553 2554 /* Averaged statistics, and temporary buffers. */ 2555 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) 2556 2557 pid_t task_numa_group_id(struct task_struct *p) 2558 { 2559 struct numa_group *ng; 2560 pid_t gid = 0; 2561 2562 rcu_read_lock(); 2563 ng = rcu_dereference_all(p->numa_group); 2564 if (ng) 2565 gid = ng->gid; 2566 rcu_read_unlock(); 2567 2568 return gid; 2569 } 2570 2571 /* 2572 * The averaged statistics, shared & private, memory & CPU, 2573 * occupy the first half of the array. The second half of the 2574 * array is for current counters, which are averaged into the 2575 * first set by task_numa_placement. 2576 */ 2577 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) 2578 { 2579 return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 2580 } 2581 2582 static inline unsigned long task_faults(struct task_struct *p, int nid) 2583 { 2584 if (!p->numa_faults) 2585 return 0; 2586 2587 return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + 2588 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; 2589 } 2590 2591 static inline unsigned long group_faults(struct task_struct *p, int nid) 2592 { 2593 struct numa_group *ng = deref_task_numa_group(p); 2594 2595 if (!ng) 2596 return 0; 2597 2598 return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 2599 ng->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 2600 } 2601 2602 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 2603 { 2604 return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] + 2605 group->faults[task_faults_idx(NUMA_CPU, nid, 1)]; 2606 } 2607 2608 static inline unsigned long group_faults_priv(struct numa_group *ng) 2609 { 2610 unsigned long faults = 0; 2611 int node; 2612 2613 for_each_online_node(node) { 2614 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 2615 } 2616 2617 return faults; 2618 } 2619 2620 static inline unsigned long group_faults_shared(struct numa_group *ng) 2621 { 2622 unsigned long faults = 0; 2623 int node; 2624 2625 for_each_online_node(node) { 2626 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 2627 } 2628 2629 return faults; 2630 } 2631 2632 /* 2633 * A node triggering more than 1/3 as many NUMA faults as the maximum is 2634 * considered part of a numa group's pseudo-interleaving set. Migrations 2635 * between these nodes are slowed down, to allow things to settle down. 2636 */ 2637 #define ACTIVE_NODE_FRACTION 3 2638 2639 static bool numa_is_active_node(int nid, struct numa_group *ng) 2640 { 2641 return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; 2642 } 2643 2644 /* Handle placement on systems where not all nodes are directly connected. */ 2645 static unsigned long score_nearby_nodes(struct task_struct *p, int nid, 2646 int lim_dist, bool task) 2647 { 2648 unsigned long score = 0; 2649 int node, max_dist; 2650 2651 /* 2652 * All nodes are directly connected, and the same distance 2653 * from each other. No need for fancy placement algorithms. 2654 */ 2655 if (sched_numa_topology_type == NUMA_DIRECT) 2656 return 0; 2657 2658 /* sched_max_numa_distance may be changed in parallel. */ 2659 max_dist = READ_ONCE(sched_max_numa_distance); 2660 /* 2661 * This code is called for each node, introducing N^2 complexity, 2662 * which should be OK given the number of nodes rarely exceeds 8. 2663 */ 2664 for_each_online_node(node) { 2665 unsigned long faults; 2666 int dist = node_distance(nid, node); 2667 2668 /* 2669 * The furthest away nodes in the system are not interesting 2670 * for placement; nid was already counted. 2671 */ 2672 if (dist >= max_dist || node == nid) 2673 continue; 2674 2675 /* 2676 * On systems with a backplane NUMA topology, compare groups 2677 * of nodes, and move tasks towards the group with the most 2678 * memory accesses. When comparing two nodes at distance 2679 * "hoplimit", only nodes closer by than "hoplimit" are part 2680 * of each group. Skip other nodes. 2681 */ 2682 if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist) 2683 continue; 2684 2685 /* Add up the faults from nearby nodes. */ 2686 if (task) 2687 faults = task_faults(p, node); 2688 else 2689 faults = group_faults(p, node); 2690 2691 /* 2692 * On systems with a glueless mesh NUMA topology, there are 2693 * no fixed "groups of nodes". Instead, nodes that are not 2694 * directly connected bounce traffic through intermediate 2695 * nodes; a numa_group can occupy any set of nodes. 2696 * The further away a node is, the less the faults count. 2697 * This seems to result in good task placement. 2698 */ 2699 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 2700 faults *= (max_dist - dist); 2701 faults /= (max_dist - LOCAL_DISTANCE); 2702 } 2703 2704 score += faults; 2705 } 2706 2707 return score; 2708 } 2709 2710 /* 2711 * These return the fraction of accesses done by a particular task, or 2712 * task group, on a particular numa node. The group weight is given a 2713 * larger multiplier, in order to group tasks together that are almost 2714 * evenly spread out between numa nodes. 2715 */ 2716 static inline unsigned long task_weight(struct task_struct *p, int nid, 2717 int dist) 2718 { 2719 unsigned long faults, total_faults; 2720 2721 if (!p->numa_faults) 2722 return 0; 2723 2724 total_faults = p->total_numa_faults; 2725 2726 if (!total_faults) 2727 return 0; 2728 2729 faults = task_faults(p, nid); 2730 faults += score_nearby_nodes(p, nid, dist, true); 2731 2732 return 1000 * faults / total_faults; 2733 } 2734 2735 static inline unsigned long group_weight(struct task_struct *p, int nid, 2736 int dist) 2737 { 2738 struct numa_group *ng = deref_task_numa_group(p); 2739 unsigned long faults, total_faults; 2740 2741 if (!ng) 2742 return 0; 2743 2744 total_faults = ng->total_faults; 2745 2746 if (!total_faults) 2747 return 0; 2748 2749 faults = group_faults(p, nid); 2750 faults += score_nearby_nodes(p, nid, dist, false); 2751 2752 return 1000 * faults / total_faults; 2753 } 2754 2755 /* 2756 * If memory tiering mode is enabled, cpupid of slow memory page is 2757 * used to record scan time instead of CPU and PID. When tiering mode 2758 * is disabled at run time, the scan time (in cpupid) will be 2759 * interpreted as CPU and PID. So CPU needs to be checked to avoid to 2760 * access out of array bound. 2761 */ 2762 static inline bool cpupid_valid(int cpupid) 2763 { 2764 return cpupid_to_cpu(cpupid) < nr_cpu_ids; 2765 } 2766 2767 /* 2768 * For memory tiering mode, if there are enough free pages (more than 2769 * enough watermark defined here) in fast memory node, to take full 2770 * advantage of fast memory capacity, all recently accessed slow 2771 * memory pages will be migrated to fast memory node without 2772 * considering hot threshold. 2773 */ 2774 static bool pgdat_free_space_enough(struct pglist_data *pgdat) 2775 { 2776 int z; 2777 unsigned long enough_wmark; 2778 2779 enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT, 2780 pgdat->node_present_pages >> 4); 2781 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 2782 struct zone *zone = pgdat->node_zones + z; 2783 2784 if (!populated_zone(zone)) 2785 continue; 2786 2787 if (zone_watermark_ok(zone, 0, 2788 promo_wmark_pages(zone) + enough_wmark, 2789 ZONE_MOVABLE, 0)) 2790 return true; 2791 } 2792 return false; 2793 } 2794 2795 /* 2796 * For memory tiering mode, when page tables are scanned, the scan 2797 * time will be recorded in struct page in addition to make page 2798 * PROT_NONE for slow memory page. So when the page is accessed, in 2799 * hint page fault handler, the hint page fault latency is calculated 2800 * via, 2801 * 2802 * hint page fault latency = hint page fault time - scan time 2803 * 2804 * The smaller the hint page fault latency, the higher the possibility 2805 * for the page to be hot. 2806 */ 2807 static int numa_hint_fault_latency(struct folio *folio) 2808 { 2809 int last_time, time; 2810 2811 time = jiffies_to_msecs(jiffies); 2812 last_time = folio_xchg_access_time(folio, time); 2813 2814 return (time - last_time) & PAGE_ACCESS_TIME_MASK; 2815 } 2816 2817 /* 2818 * For memory tiering mode, too high promotion/demotion throughput may 2819 * hurt application latency. So we provide a mechanism to rate limit 2820 * the number of pages that are tried to be promoted. 2821 */ 2822 static bool numa_promotion_rate_limit(struct pglist_data *pgdat, 2823 unsigned long rate_limit, int nr) 2824 { 2825 unsigned long nr_cand; 2826 unsigned int now, start; 2827 2828 now = jiffies_to_msecs(jiffies); 2829 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr); 2830 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 2831 start = pgdat->nbp_rl_start; 2832 if (now - start > MSEC_PER_SEC && 2833 cmpxchg(&pgdat->nbp_rl_start, start, now) == start) 2834 pgdat->nbp_rl_nr_cand = nr_cand; 2835 if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit) 2836 return true; 2837 return false; 2838 } 2839 2840 #define NUMA_MIGRATION_ADJUST_STEPS 16 2841 2842 static void numa_promotion_adjust_threshold(struct pglist_data *pgdat, 2843 unsigned long rate_limit, 2844 unsigned int ref_th) 2845 { 2846 unsigned int now, start, th_period, unit_th, th; 2847 unsigned long nr_cand, ref_cand, diff_cand; 2848 2849 now = jiffies_to_msecs(jiffies); 2850 th_period = sysctl_numa_balancing_scan_period_max; 2851 start = pgdat->nbp_th_start; 2852 if (now - start > th_period && 2853 cmpxchg(&pgdat->nbp_th_start, start, now) == start) { 2854 ref_cand = rate_limit * 2855 sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC; 2856 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 2857 diff_cand = nr_cand - pgdat->nbp_th_nr_cand; 2858 unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS; 2859 th = pgdat->nbp_threshold ? : ref_th; 2860 if (diff_cand > ref_cand * 11 / 10) 2861 th = max(th - unit_th, unit_th); 2862 else if (diff_cand < ref_cand * 9 / 10) 2863 th = min(th + unit_th, ref_th * 2); 2864 pgdat->nbp_th_nr_cand = nr_cand; 2865 pgdat->nbp_threshold = th; 2866 } 2867 } 2868 2869 bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio, 2870 int src_nid, int dst_cpu) 2871 { 2872 struct numa_group *ng = deref_curr_numa_group(p); 2873 int dst_nid = cpu_to_node(dst_cpu); 2874 int last_cpupid, this_cpupid; 2875 2876 /* 2877 * Cannot migrate to memoryless nodes. 2878 */ 2879 if (!node_state(dst_nid, N_MEMORY)) 2880 return false; 2881 2882 /* 2883 * The pages in slow memory node should be migrated according 2884 * to hot/cold instead of private/shared. 2885 */ 2886 if (folio_use_access_time(folio)) { 2887 struct pglist_data *pgdat; 2888 unsigned long rate_limit; 2889 unsigned int latency, th, def_th; 2890 long nr = folio_nr_pages(folio); 2891 2892 pgdat = NODE_DATA(dst_nid); 2893 if (pgdat_free_space_enough(pgdat)) { 2894 /* workload changed, reset hot threshold */ 2895 pgdat->nbp_threshold = 0; 2896 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE_NRL, nr); 2897 return true; 2898 } 2899 2900 def_th = sysctl_numa_balancing_hot_threshold; 2901 rate_limit = MB_TO_PAGES(sysctl_numa_balancing_promote_rate_limit); 2902 numa_promotion_adjust_threshold(pgdat, rate_limit, def_th); 2903 2904 th = pgdat->nbp_threshold ? : def_th; 2905 latency = numa_hint_fault_latency(folio); 2906 if (latency >= th) 2907 return false; 2908 2909 return !numa_promotion_rate_limit(pgdat, rate_limit, nr); 2910 } 2911 2912 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); 2913 last_cpupid = folio_xchg_last_cpupid(folio, this_cpupid); 2914 2915 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 2916 !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid)) 2917 return false; 2918 2919 /* 2920 * Allow first faults or private faults to migrate immediately early in 2921 * the lifetime of a task. The magic number 4 is based on waiting for 2922 * two full passes of the "multi-stage node selection" test that is 2923 * executed below. 2924 */ 2925 if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) && 2926 (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid))) 2927 return true; 2928 2929 /* 2930 * Multi-stage node selection is used in conjunction with a periodic 2931 * migration fault to build a temporal task<->page relation. By using 2932 * a two-stage filter we remove short/unlikely relations. 2933 * 2934 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate 2935 * a task's usage of a particular page (n_p) per total usage of this 2936 * page (n_t) (in a given time-span) to a probability. 2937 * 2938 * Our periodic faults will sample this probability and getting the 2939 * same result twice in a row, given these samples are fully 2940 * independent, is then given by P(n)^2, provided our sample period 2941 * is sufficiently short compared to the usage pattern. 2942 * 2943 * This quadric squishes small probabilities, making it less likely we 2944 * act on an unlikely task<->page relation. 2945 */ 2946 if (!cpupid_pid_unset(last_cpupid) && 2947 cpupid_to_nid(last_cpupid) != dst_nid) 2948 return false; 2949 2950 /* Always allow migrate on private faults */ 2951 if (cpupid_match_pid(p, last_cpupid)) 2952 return true; 2953 2954 /* A shared fault, but p->numa_group has not been set up yet. */ 2955 if (!ng) 2956 return true; 2957 2958 /* 2959 * Destination node is much more heavily used than the source 2960 * node? Allow migration. 2961 */ 2962 if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * 2963 ACTIVE_NODE_FRACTION) 2964 return true; 2965 2966 /* 2967 * Distribute memory according to CPU & memory use on each node, 2968 * with 3/4 hysteresis to avoid unnecessary memory migrations: 2969 * 2970 * faults_cpu(dst) 3 faults_cpu(src) 2971 * --------------- * - > --------------- 2972 * faults_mem(dst) 4 faults_mem(src) 2973 */ 2974 return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > 2975 group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; 2976 } 2977 2978 /* 2979 * 'numa_type' describes the node at the moment of load balancing. 2980 */ 2981 enum numa_type { 2982 /* The node has spare capacity that can be used to run more tasks. */ 2983 node_has_spare = 0, 2984 /* 2985 * The node is fully used and the tasks don't compete for more CPU 2986 * cycles. Nevertheless, some tasks might wait before running. 2987 */ 2988 node_fully_busy, 2989 /* 2990 * The node is overloaded and can't provide expected CPU cycles to all 2991 * tasks. 2992 */ 2993 node_overloaded 2994 }; 2995 2996 /* Cached statistics for all CPUs within a node */ 2997 struct numa_stats { 2998 unsigned long load; 2999 unsigned long runnable; 3000 unsigned long util; 3001 /* Total compute capacity of CPUs on a node */ 3002 unsigned long compute_capacity; 3003 unsigned int nr_running; 3004 unsigned int weight; 3005 enum numa_type node_type; 3006 int idle_cpu; 3007 }; 3008 3009 struct task_numa_env { 3010 struct task_struct *p; 3011 3012 int src_cpu, src_nid; 3013 int dst_cpu, dst_nid; 3014 int imb_numa_nr; 3015 3016 struct numa_stats src_stats, dst_stats; 3017 3018 int imbalance_pct; 3019 int dist; 3020 3021 struct task_struct *best_task; 3022 long best_imp; 3023 int best_cpu; 3024 }; 3025 3026 static unsigned long cpu_load(struct rq *rq); 3027 static unsigned long cpu_runnable(struct rq *rq); 3028 3029 static inline enum 3030 numa_type numa_classify(unsigned int imbalance_pct, 3031 struct numa_stats *ns) 3032 { 3033 if ((ns->nr_running > ns->weight) && 3034 (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) || 3035 ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100)))) 3036 return node_overloaded; 3037 3038 if ((ns->nr_running < ns->weight) || 3039 (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) && 3040 ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100)))) 3041 return node_has_spare; 3042 3043 return node_fully_busy; 3044 } 3045 3046 /* Forward declarations of select_idle_sibling helpers */ 3047 static inline bool test_idle_cores(int cpu); 3048 static inline int numa_idle_core(int idle_core, int cpu) 3049 { 3050 if (!sched_smt_active() || 3051 idle_core >= 0 || !test_idle_cores(cpu)) 3052 return idle_core; 3053 3054 /* 3055 * Prefer cores instead of packing HT siblings 3056 * and triggering future load balancing. 3057 */ 3058 if (is_core_idle(cpu)) 3059 idle_core = cpu; 3060 3061 return idle_core; 3062 } 3063 3064 /* 3065 * Gather all necessary information to make NUMA balancing placement 3066 * decisions that are compatible with standard load balancer. This 3067 * borrows code and logic from update_sg_lb_stats but sharing a 3068 * common implementation is impractical. 3069 */ 3070 static void update_numa_stats(struct task_numa_env *env, 3071 struct numa_stats *ns, int nid, 3072 bool find_idle) 3073 { 3074 int cpu, idle_core = -1; 3075 3076 memset(ns, 0, sizeof(*ns)); 3077 ns->idle_cpu = -1; 3078 3079 rcu_read_lock(); 3080 for_each_cpu(cpu, cpumask_of_node(nid)) { 3081 struct rq *rq = cpu_rq(cpu); 3082 3083 ns->load += cpu_load(rq); 3084 ns->runnable += cpu_runnable(rq); 3085 ns->util += cpu_util_cfs(cpu); 3086 ns->nr_running += rq->cfs.h_nr_runnable; 3087 ns->compute_capacity += capacity_of(cpu); 3088 3089 if (find_idle && idle_core < 0 && !rq->nr_running && idle_cpu(cpu)) { 3090 if (READ_ONCE(rq->numa_migrate_on) || 3091 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) 3092 continue; 3093 3094 if (ns->idle_cpu == -1) 3095 ns->idle_cpu = cpu; 3096 3097 idle_core = numa_idle_core(idle_core, cpu); 3098 } 3099 } 3100 rcu_read_unlock(); 3101 3102 ns->weight = cpumask_weight(cpumask_of_node(nid)); 3103 3104 ns->node_type = numa_classify(env->imbalance_pct, ns); 3105 3106 if (idle_core >= 0) 3107 ns->idle_cpu = idle_core; 3108 } 3109 3110 static void task_numa_assign(struct task_numa_env *env, 3111 struct task_struct *p, long imp) 3112 { 3113 struct rq *rq = cpu_rq(env->dst_cpu); 3114 3115 /* Check if run-queue part of active NUMA balance. */ 3116 if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) { 3117 int cpu; 3118 int start = env->dst_cpu; 3119 3120 /* Find alternative idle CPU. */ 3121 for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) { 3122 if (cpu == env->best_cpu || !idle_cpu(cpu) || 3123 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) { 3124 continue; 3125 } 3126 3127 env->dst_cpu = cpu; 3128 rq = cpu_rq(env->dst_cpu); 3129 if (!xchg(&rq->numa_migrate_on, 1)) 3130 goto assign; 3131 } 3132 3133 /* Failed to find an alternative idle CPU */ 3134 return; 3135 } 3136 3137 assign: 3138 /* 3139 * Clear previous best_cpu/rq numa-migrate flag, since task now 3140 * found a better CPU to move/swap. 3141 */ 3142 if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) { 3143 rq = cpu_rq(env->best_cpu); 3144 WRITE_ONCE(rq->numa_migrate_on, 0); 3145 } 3146 3147 if (env->best_task) 3148 put_task_struct(env->best_task); 3149 if (p) 3150 get_task_struct(p); 3151 3152 env->best_task = p; 3153 env->best_imp = imp; 3154 env->best_cpu = env->dst_cpu; 3155 } 3156 3157 static bool load_too_imbalanced(long src_load, long dst_load, 3158 struct task_numa_env *env) 3159 { 3160 long imb, old_imb; 3161 long orig_src_load, orig_dst_load; 3162 long src_capacity, dst_capacity; 3163 3164 /* 3165 * The load is corrected for the CPU capacity available on each node. 3166 * 3167 * src_load dst_load 3168 * ------------ vs --------- 3169 * src_capacity dst_capacity 3170 */ 3171 src_capacity = env->src_stats.compute_capacity; 3172 dst_capacity = env->dst_stats.compute_capacity; 3173 3174 imb = abs(dst_load * src_capacity - src_load * dst_capacity); 3175 3176 orig_src_load = env->src_stats.load; 3177 orig_dst_load = env->dst_stats.load; 3178 3179 old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity); 3180 3181 /* Would this change make things worse? */ 3182 return (imb > old_imb); 3183 } 3184 3185 /* 3186 * Maximum NUMA importance can be 1998 (2*999); 3187 * SMALLIMP @ 30 would be close to 1998/64. 3188 * Used to deter task migration. 3189 */ 3190 #define SMALLIMP 30 3191 3192 /* 3193 * This checks if the overall compute and NUMA accesses of the system would 3194 * be improved if the source tasks was migrated to the target dst_cpu taking 3195 * into account that it might be best if task running on the dst_cpu should 3196 * be exchanged with the source task 3197 */ 3198 static bool task_numa_compare(struct task_numa_env *env, 3199 long taskimp, long groupimp, bool maymove) 3200 { 3201 struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p); 3202 struct rq *dst_rq = cpu_rq(env->dst_cpu); 3203 long imp = p_ng ? groupimp : taskimp; 3204 struct task_struct *cur; 3205 long src_load, dst_load; 3206 int dist = env->dist; 3207 long moveimp = imp; 3208 long load; 3209 bool stopsearch = false; 3210 3211 if (READ_ONCE(dst_rq->numa_migrate_on)) 3212 return false; 3213 3214 rcu_read_lock(); 3215 cur = rcu_dereference_all(dst_rq->curr); 3216 if (cur && ((cur->flags & (PF_EXITING | PF_KTHREAD)) || 3217 !cur->mm)) 3218 cur = NULL; 3219 3220 /* 3221 * Because we have preemption enabled we can get migrated around and 3222 * end try selecting ourselves (current == env->p) as a swap candidate. 3223 */ 3224 if (cur == env->p) { 3225 stopsearch = true; 3226 goto unlock; 3227 } 3228 3229 if (!cur) { 3230 if (maymove && moveimp >= env->best_imp) 3231 goto assign; 3232 else 3233 goto unlock; 3234 } 3235 3236 /* Skip this swap candidate if cannot move to the source cpu. */ 3237 if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr)) 3238 goto unlock; 3239 3240 /* 3241 * Skip this swap candidate if it is not moving to its preferred 3242 * node and the best task is. 3243 */ 3244 if (env->best_task && 3245 env->best_task->numa_preferred_nid == env->src_nid && 3246 cur->numa_preferred_nid != env->src_nid) { 3247 goto unlock; 3248 } 3249 3250 /* 3251 * "imp" is the fault differential for the source task between the 3252 * source and destination node. Calculate the total differential for 3253 * the source task and potential destination task. The more negative 3254 * the value is, the more remote accesses that would be expected to 3255 * be incurred if the tasks were swapped. 3256 * 3257 * If dst and source tasks are in the same NUMA group, or not 3258 * in any group then look only at task weights. 3259 */ 3260 cur_ng = rcu_dereference_all(cur->numa_group); 3261 if (cur_ng == p_ng) { 3262 /* 3263 * Do not swap within a group or between tasks that have 3264 * no group if there is spare capacity. Swapping does 3265 * not address the load imbalance and helps one task at 3266 * the cost of punishing another. 3267 */ 3268 if (env->dst_stats.node_type == node_has_spare) 3269 goto unlock; 3270 3271 imp = taskimp + task_weight(cur, env->src_nid, dist) - 3272 task_weight(cur, env->dst_nid, dist); 3273 /* 3274 * Add some hysteresis to prevent swapping the 3275 * tasks within a group over tiny differences. 3276 */ 3277 if (cur_ng) 3278 imp -= imp / 16; 3279 } else { 3280 /* 3281 * Compare the group weights. If a task is all by itself 3282 * (not part of a group), use the task weight instead. 3283 */ 3284 if (cur_ng && p_ng) 3285 imp += group_weight(cur, env->src_nid, dist) - 3286 group_weight(cur, env->dst_nid, dist); 3287 else 3288 imp += task_weight(cur, env->src_nid, dist) - 3289 task_weight(cur, env->dst_nid, dist); 3290 } 3291 3292 /* Discourage picking a task already on its preferred node */ 3293 if (cur->numa_preferred_nid == env->dst_nid) 3294 imp -= imp / 16; 3295 3296 /* 3297 * Encourage picking a task that moves to its preferred node. 3298 * This potentially makes imp larger than it's maximum of 3299 * 1998 (see SMALLIMP and task_weight for why) but in this 3300 * case, it does not matter. 3301 */ 3302 if (cur->numa_preferred_nid == env->src_nid) 3303 imp += imp / 8; 3304 3305 if (maymove && moveimp > imp && moveimp > env->best_imp) { 3306 imp = moveimp; 3307 cur = NULL; 3308 goto assign; 3309 } 3310 3311 /* 3312 * Prefer swapping with a task moving to its preferred node over a 3313 * task that is not. 3314 */ 3315 if (env->best_task && cur->numa_preferred_nid == env->src_nid && 3316 env->best_task->numa_preferred_nid != env->src_nid) { 3317 goto assign; 3318 } 3319 3320 /* 3321 * If the NUMA importance is less than SMALLIMP, 3322 * task migration might only result in ping pong 3323 * of tasks and also hurt performance due to cache 3324 * misses. 3325 */ 3326 if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2) 3327 goto unlock; 3328 3329 /* 3330 * In the overloaded case, try and keep the load balanced. 3331 */ 3332 load = task_h_load(env->p) - task_h_load(cur); 3333 if (!load) 3334 goto assign; 3335 3336 dst_load = env->dst_stats.load + load; 3337 src_load = env->src_stats.load - load; 3338 3339 if (load_too_imbalanced(src_load, dst_load, env)) 3340 goto unlock; 3341 3342 assign: 3343 /* Evaluate an idle CPU for a task numa move. */ 3344 if (!cur) { 3345 int cpu = env->dst_stats.idle_cpu; 3346 3347 /* Nothing cached so current CPU went idle since the search. */ 3348 if (cpu < 0) 3349 cpu = env->dst_cpu; 3350 3351 /* 3352 * If the CPU is no longer truly idle and the previous best CPU 3353 * is, keep using it. 3354 */ 3355 if (!idle_cpu(cpu) && env->best_cpu >= 0 && 3356 idle_cpu(env->best_cpu)) { 3357 cpu = env->best_cpu; 3358 } 3359 3360 env->dst_cpu = cpu; 3361 } 3362 3363 task_numa_assign(env, cur, imp); 3364 3365 /* 3366 * If a move to idle is allowed because there is capacity or load 3367 * balance improves then stop the search. While a better swap 3368 * candidate may exist, a search is not free. 3369 */ 3370 if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu)) 3371 stopsearch = true; 3372 3373 /* 3374 * If a swap candidate must be identified and the current best task 3375 * moves its preferred node then stop the search. 3376 */ 3377 if (!maymove && env->best_task && 3378 env->best_task->numa_preferred_nid == env->src_nid) { 3379 stopsearch = true; 3380 } 3381 unlock: 3382 rcu_read_unlock(); 3383 3384 return stopsearch; 3385 } 3386 3387 static void task_numa_find_cpu(struct task_numa_env *env, 3388 long taskimp, long groupimp) 3389 { 3390 bool maymove = false; 3391 int cpu; 3392 3393 /* 3394 * If dst node has spare capacity, then check if there is an 3395 * imbalance that would be overruled by the load balancer. 3396 */ 3397 if (env->dst_stats.node_type == node_has_spare) { 3398 unsigned int imbalance; 3399 int src_running, dst_running; 3400 3401 /* 3402 * Would movement cause an imbalance? Note that if src has 3403 * more running tasks that the imbalance is ignored as the 3404 * move improves the imbalance from the perspective of the 3405 * CPU load balancer. 3406 * */ 3407 src_running = env->src_stats.nr_running - 1; 3408 dst_running = env->dst_stats.nr_running + 1; 3409 imbalance = max(0, dst_running - src_running); 3410 imbalance = adjust_numa_imbalance(imbalance, dst_running, 3411 env->imb_numa_nr); 3412 3413 /* Use idle CPU if there is no imbalance */ 3414 if (!imbalance) { 3415 maymove = true; 3416 if (env->dst_stats.idle_cpu >= 0) { 3417 env->dst_cpu = env->dst_stats.idle_cpu; 3418 task_numa_assign(env, NULL, 0); 3419 return; 3420 } 3421 } 3422 } else { 3423 long src_load, dst_load, load; 3424 /* 3425 * If the improvement from just moving env->p direction is better 3426 * than swapping tasks around, check if a move is possible. 3427 */ 3428 load = task_h_load(env->p); 3429 dst_load = env->dst_stats.load + load; 3430 src_load = env->src_stats.load - load; 3431 maymove = !load_too_imbalanced(src_load, dst_load, env); 3432 } 3433 3434 /* Skip CPUs if the source task cannot migrate */ 3435 for_each_cpu_and(cpu, cpumask_of_node(env->dst_nid), env->p->cpus_ptr) { 3436 env->dst_cpu = cpu; 3437 if (task_numa_compare(env, taskimp, groupimp, maymove)) 3438 break; 3439 } 3440 } 3441 3442 static int task_numa_migrate(struct task_struct *p) 3443 { 3444 struct task_numa_env env = { 3445 .p = p, 3446 3447 .src_cpu = task_cpu(p), 3448 .src_nid = task_node(p), 3449 3450 .imbalance_pct = 112, 3451 3452 .best_task = NULL, 3453 .best_imp = 0, 3454 .best_cpu = -1, 3455 }; 3456 unsigned long taskweight, groupweight; 3457 struct sched_domain *sd; 3458 long taskimp, groupimp; 3459 struct numa_group *ng; 3460 struct rq *best_rq; 3461 int nid, ret, dist; 3462 3463 /* 3464 * Pick the lowest SD_NUMA domain, as that would have the smallest 3465 * imbalance and would be the first to start moving tasks about. 3466 * 3467 * And we want to avoid any moving of tasks about, as that would create 3468 * random movement of tasks -- counter the numa conditions we're trying 3469 * to satisfy here. 3470 */ 3471 rcu_read_lock(); 3472 sd = rcu_dereference_all(per_cpu(sd_numa, env.src_cpu)); 3473 if (sd) { 3474 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; 3475 env.imb_numa_nr = sd->imb_numa_nr; 3476 } 3477 rcu_read_unlock(); 3478 3479 /* 3480 * Cpusets can break the scheduler domain tree into smaller 3481 * balance domains, some of which do not cross NUMA boundaries. 3482 * Tasks that are "trapped" in such domains cannot be migrated 3483 * elsewhere, so there is no point in (re)trying. 3484 */ 3485 if (unlikely(!sd)) { 3486 sched_setnuma(p, task_node(p)); 3487 return -EINVAL; 3488 } 3489 3490 env.dst_nid = p->numa_preferred_nid; 3491 dist = env.dist = node_distance(env.src_nid, env.dst_nid); 3492 taskweight = task_weight(p, env.src_nid, dist); 3493 groupweight = group_weight(p, env.src_nid, dist); 3494 update_numa_stats(&env, &env.src_stats, env.src_nid, false); 3495 taskimp = task_weight(p, env.dst_nid, dist) - taskweight; 3496 groupimp = group_weight(p, env.dst_nid, dist) - groupweight; 3497 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 3498 3499 /* Try to find a spot on the preferred nid. */ 3500 task_numa_find_cpu(&env, taskimp, groupimp); 3501 3502 /* 3503 * Look at other nodes in these cases: 3504 * - there is no space available on the preferred_nid 3505 * - the task is part of a numa_group that is interleaved across 3506 * multiple NUMA nodes; in order to better consolidate the group, 3507 * we need to check other locations. 3508 */ 3509 ng = deref_curr_numa_group(p); 3510 if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) { 3511 for_each_node_state(nid, N_CPU) { 3512 if (nid == env.src_nid || nid == p->numa_preferred_nid) 3513 continue; 3514 3515 dist = node_distance(env.src_nid, env.dst_nid); 3516 if (sched_numa_topology_type == NUMA_BACKPLANE && 3517 dist != env.dist) { 3518 taskweight = task_weight(p, env.src_nid, dist); 3519 groupweight = group_weight(p, env.src_nid, dist); 3520 } 3521 3522 /* Only consider nodes where both task and groups benefit */ 3523 taskimp = task_weight(p, nid, dist) - taskweight; 3524 groupimp = group_weight(p, nid, dist) - groupweight; 3525 if (taskimp < 0 && groupimp < 0) 3526 continue; 3527 3528 env.dist = dist; 3529 env.dst_nid = nid; 3530 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 3531 task_numa_find_cpu(&env, taskimp, groupimp); 3532 } 3533 } 3534 3535 /* 3536 * If the task is part of a workload that spans multiple NUMA nodes, 3537 * and is migrating into one of the workload's active nodes, remember 3538 * this node as the task's preferred numa node, so the workload can 3539 * settle down. 3540 * A task that migrated to a second choice node will be better off 3541 * trying for a better one later. Do not set the preferred node here. 3542 */ 3543 if (ng) { 3544 if (env.best_cpu == -1) 3545 nid = env.src_nid; 3546 else 3547 nid = cpu_to_node(env.best_cpu); 3548 3549 if (nid != p->numa_preferred_nid) 3550 sched_setnuma(p, nid); 3551 } 3552 3553 /* No better CPU than the current one was found. */ 3554 if (env.best_cpu == -1) { 3555 trace_sched_stick_numa(p, env.src_cpu, NULL, -1); 3556 return -EAGAIN; 3557 } 3558 3559 best_rq = cpu_rq(env.best_cpu); 3560 if (env.best_task == NULL) { 3561 ret = migrate_task_to(p, env.best_cpu); 3562 WRITE_ONCE(best_rq->numa_migrate_on, 0); 3563 if (ret != 0) 3564 trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu); 3565 return ret; 3566 } 3567 3568 ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu); 3569 WRITE_ONCE(best_rq->numa_migrate_on, 0); 3570 3571 if (ret != 0) 3572 trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu); 3573 put_task_struct(env.best_task); 3574 return ret; 3575 } 3576 3577 /* Attempt to migrate a task to a CPU on the preferred node. */ 3578 static void numa_migrate_preferred(struct task_struct *p) 3579 { 3580 unsigned long interval = HZ; 3581 3582 /* This task has no NUMA fault statistics yet */ 3583 if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults)) 3584 return; 3585 3586 /* Periodically retry migrating the task to the preferred node */ 3587 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); 3588 p->numa_migrate_retry = jiffies + interval; 3589 3590 /* Success if task is already running on preferred CPU */ 3591 if (task_node(p) == p->numa_preferred_nid) 3592 return; 3593 3594 /* Otherwise, try migrate to a CPU on the preferred node */ 3595 task_numa_migrate(p); 3596 } 3597 3598 /* 3599 * Find out how many nodes the workload is actively running on. Do this by 3600 * tracking the nodes from which NUMA hinting faults are triggered. This can 3601 * be different from the set of nodes where the workload's memory is currently 3602 * located. 3603 */ 3604 static void numa_group_count_active_nodes(struct numa_group *numa_group) 3605 { 3606 unsigned long faults, max_faults = 0; 3607 int nid, active_nodes = 0; 3608 3609 for_each_node_state(nid, N_CPU) { 3610 faults = group_faults_cpu(numa_group, nid); 3611 if (faults > max_faults) 3612 max_faults = faults; 3613 } 3614 3615 for_each_node_state(nid, N_CPU) { 3616 faults = group_faults_cpu(numa_group, nid); 3617 if (faults * ACTIVE_NODE_FRACTION > max_faults) 3618 active_nodes++; 3619 } 3620 3621 numa_group->max_faults_cpu = max_faults; 3622 numa_group->active_nodes = active_nodes; 3623 } 3624 3625 /* 3626 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS 3627 * increments. The more local the fault statistics are, the higher the scan 3628 * period will be for the next scan window. If local/(local+remote) ratio is 3629 * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) 3630 * the scan period will decrease. Aim for 70% local accesses. 3631 */ 3632 #define NUMA_PERIOD_SLOTS 10 3633 #define NUMA_PERIOD_THRESHOLD 7 3634 3635 /* 3636 * Increase the scan period (slow down scanning) if the majority of 3637 * our memory is already on our local node, or if the majority of 3638 * the page accesses are shared with other processes. 3639 * Otherwise, decrease the scan period. 3640 */ 3641 static void update_task_scan_period(struct task_struct *p, 3642 unsigned long shared, unsigned long private) 3643 { 3644 unsigned int period_slot; 3645 int lr_ratio, ps_ratio; 3646 int diff; 3647 3648 unsigned long remote = p->numa_faults_locality[0]; 3649 unsigned long local = p->numa_faults_locality[1]; 3650 3651 /* 3652 * If there were no record hinting faults then either the task is 3653 * completely idle or all activity is in areas that are not of interest 3654 * to automatic numa balancing. Related to that, if there were failed 3655 * migration then it implies we are migrating too quickly or the local 3656 * node is overloaded. In either case, scan slower 3657 */ 3658 if (local + shared == 0 || p->numa_faults_locality[2]) { 3659 p->numa_scan_period = min(p->numa_scan_period_max, 3660 p->numa_scan_period << 1); 3661 3662 p->mm->numa_next_scan = jiffies + 3663 msecs_to_jiffies(p->numa_scan_period); 3664 3665 goto out; 3666 } 3667 3668 /* 3669 * Prepare to scale scan period relative to the current period. 3670 * == NUMA_PERIOD_THRESHOLD scan period stays the same 3671 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) 3672 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) 3673 */ 3674 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); 3675 lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); 3676 ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); 3677 3678 if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { 3679 /* 3680 * Most memory accesses are local. There is no need to 3681 * do fast NUMA scanning, since memory is already local. 3682 */ 3683 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; 3684 if (!slot) 3685 slot = 1; 3686 diff = slot * period_slot; 3687 } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { 3688 /* 3689 * Most memory accesses are shared with other tasks. 3690 * There is no point in continuing fast NUMA scanning, 3691 * since other tasks may just move the memory elsewhere. 3692 */ 3693 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; 3694 if (!slot) 3695 slot = 1; 3696 diff = slot * period_slot; 3697 } else { 3698 /* 3699 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, 3700 * yet they are not on the local NUMA node. Speed up 3701 * NUMA scanning to get the memory moved over. 3702 */ 3703 int ratio = max(lr_ratio, ps_ratio); 3704 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; 3705 } 3706 3707 p->numa_scan_period = clamp(p->numa_scan_period + diff, 3708 task_scan_min(p), task_scan_max(p)); 3709 3710 out: 3711 memset(p->numa_faults_locality, 0, 3712 sizeof(p->numa_faults_locality)); 3713 } 3714 3715 /* 3716 * Get the fraction of time the task has been running since the last 3717 * NUMA placement cycle. The scheduler keeps similar statistics, but 3718 * decays those on a 32ms period, which is orders of magnitude off 3719 * from the dozens-of-seconds NUMA balancing period. Use the scheduler 3720 * stats only if the task is so new there are no NUMA statistics yet. 3721 */ 3722 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) 3723 { 3724 u64 runtime, delta, now; 3725 /* Use the start of this time slice to avoid calculations. */ 3726 now = p->se.exec_start; 3727 runtime = p->se.sum_exec_runtime; 3728 3729 if (p->last_task_numa_placement) { 3730 delta = runtime - p->last_sum_exec_runtime; 3731 *period = now - p->last_task_numa_placement; 3732 3733 /* Avoid time going backwards, prevent potential divide error: */ 3734 if (unlikely((s64)*period < 0)) 3735 *period = 0; 3736 } else { 3737 delta = p->se.avg.load_sum; 3738 *period = LOAD_AVG_MAX; 3739 } 3740 3741 p->last_sum_exec_runtime = runtime; 3742 p->last_task_numa_placement = now; 3743 3744 return delta; 3745 } 3746 3747 /* 3748 * Determine the preferred nid for a task in a numa_group. This needs to 3749 * be done in a way that produces consistent results with group_weight, 3750 * otherwise workloads might not converge. 3751 */ 3752 static int preferred_group_nid(struct task_struct *p, int nid) 3753 { 3754 nodemask_t nodes; 3755 int dist; 3756 3757 /* Direct connections between all NUMA nodes. */ 3758 if (sched_numa_topology_type == NUMA_DIRECT) 3759 return nid; 3760 3761 /* 3762 * On a system with glueless mesh NUMA topology, group_weight 3763 * scores nodes according to the number of NUMA hinting faults on 3764 * both the node itself, and on nearby nodes. 3765 */ 3766 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 3767 unsigned long score, max_score = 0; 3768 int node, max_node = nid; 3769 3770 dist = sched_max_numa_distance; 3771 3772 for_each_node_state(node, N_CPU) { 3773 score = group_weight(p, node, dist); 3774 if (score > max_score) { 3775 max_score = score; 3776 max_node = node; 3777 } 3778 } 3779 return max_node; 3780 } 3781 3782 /* 3783 * Finding the preferred nid in a system with NUMA backplane 3784 * interconnect topology is more involved. The goal is to locate 3785 * tasks from numa_groups near each other in the system, and 3786 * untangle workloads from different sides of the system. This requires 3787 * searching down the hierarchy of node groups, recursively searching 3788 * inside the highest scoring group of nodes. The nodemask tricks 3789 * keep the complexity of the search down. 3790 */ 3791 nodes = node_states[N_CPU]; 3792 for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { 3793 unsigned long max_faults = 0; 3794 nodemask_t max_group = NODE_MASK_NONE; 3795 int a, b; 3796 3797 /* Are there nodes at this distance from each other? */ 3798 if (!find_numa_distance(dist)) 3799 continue; 3800 3801 for_each_node_mask(a, nodes) { 3802 unsigned long faults = 0; 3803 nodemask_t this_group; 3804 nodes_clear(this_group); 3805 3806 /* Sum group's NUMA faults; includes a==b case. */ 3807 for_each_node_mask(b, nodes) { 3808 if (node_distance(a, b) < dist) { 3809 faults += group_faults(p, b); 3810 node_set(b, this_group); 3811 node_clear(b, nodes); 3812 } 3813 } 3814 3815 /* Remember the top group. */ 3816 if (faults > max_faults) { 3817 max_faults = faults; 3818 max_group = this_group; 3819 /* 3820 * subtle: at the smallest distance there is 3821 * just one node left in each "group", the 3822 * winner is the preferred nid. 3823 */ 3824 nid = a; 3825 } 3826 } 3827 /* Next round, evaluate the nodes within max_group. */ 3828 if (!max_faults) 3829 break; 3830 nodes = max_group; 3831 } 3832 return nid; 3833 } 3834 3835 static void task_numa_placement(struct task_struct *p) 3836 __context_unsafe(/* conditional locking */) 3837 { 3838 struct sched_cache_group __maybe_unused *grp; 3839 int seq, nid, max_nid = NUMA_NO_NODE; 3840 unsigned long max_faults = 0; 3841 unsigned long fault_types[2] = { 0, 0 }; 3842 unsigned long total_faults; 3843 u64 runtime, period; 3844 spinlock_t *group_lock = NULL; 3845 long __maybe_unused new_fp; 3846 struct numa_group *ng; 3847 3848 /* 3849 * The p->mm->numa_scan_seq field gets updated without 3850 * exclusive access. Use READ_ONCE() here to ensure 3851 * that the field is read in a single access: 3852 */ 3853 seq = READ_ONCE(p->mm->numa_scan_seq); 3854 if (p->numa_scan_seq == seq) 3855 return; 3856 p->numa_scan_seq = seq; 3857 p->numa_scan_period_max = task_scan_max(p); 3858 3859 total_faults = p->numa_faults_locality[0] + 3860 p->numa_faults_locality[1]; 3861 runtime = numa_get_avg_runtime(p, &period); 3862 3863 /* If the task is part of a group prevent parallel updates to group stats */ 3864 ng = deref_curr_numa_group(p); 3865 if (ng) { 3866 group_lock = &ng->lock; 3867 spin_lock_irq(group_lock); 3868 } 3869 3870 /* Find the node with the highest number of faults */ 3871 for_each_online_node(nid) { 3872 /* Keep track of the offsets in numa_faults array */ 3873 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; 3874 unsigned long faults = 0, group_faults = 0; 3875 int priv; 3876 3877 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { 3878 long diff, f_diff, f_weight; 3879 3880 mem_idx = task_faults_idx(NUMA_MEM, nid, priv); 3881 membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); 3882 cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); 3883 cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); 3884 3885 /* Decay existing window, copy faults since last scan */ 3886 diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; 3887 fault_types[priv] += p->numa_faults[membuf_idx]; 3888 p->numa_faults[membuf_idx] = 0; 3889 3890 /* 3891 * Normalize the faults_from, so all tasks in a group 3892 * count according to CPU use, instead of by the raw 3893 * number of faults. Tasks with little runtime have 3894 * little over-all impact on throughput, and thus their 3895 * faults are less important. 3896 */ 3897 f_weight = div64_u64(runtime << 16, period + 1); 3898 f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / 3899 (total_faults + 1); 3900 f_diff = f_weight - p->numa_faults[cpu_idx] / 2; 3901 p->numa_faults[cpubuf_idx] = 0; 3902 3903 p->numa_faults[mem_idx] += diff; 3904 p->numa_faults[cpu_idx] += f_diff; 3905 faults += p->numa_faults[mem_idx]; 3906 p->total_numa_faults += diff; 3907 if (ng) { 3908 /* 3909 * safe because we can only change our own group 3910 * 3911 * mem_idx represents the offset for a given 3912 * nid and priv in a specific region because it 3913 * is at the beginning of the numa_faults array. 3914 */ 3915 ng->faults[mem_idx] += diff; 3916 ng->faults[cpu_idx] += f_diff; 3917 ng->total_faults += diff; 3918 group_faults += ng->faults[mem_idx]; 3919 } 3920 #ifdef CONFIG_SCHED_CACHE 3921 /* 3922 * Per task p->numa_faults[mem_idx] converges, 3923 * so the accumulation of each task's faults 3924 * converges too - Given the number of threads, 3925 * it cannot overflow an unsigned long. 3926 * Racy with concurrent updates from other threads 3927 * sharing this mm. Acceptable since footprint is a 3928 * heuristic and occasional lost updates are tolerable. 3929 * 3930 * If a task exits, its corresponding footprint must 3931 * be subtracted from p->sched_cache_grp->footprint, 3932 * otherwise the footprint will not converge: the 3933 * exiting thread's footprint remains unchanged/undecayed. 3934 * See exit_mm(). 3935 * 3936 * Lost updates and unsynchronized subtraction 3937 * in exit_mm() can cause footprint + diff to 3938 * go negative. Clamp to zero to prevent the 3939 * unsigned footprint from wrapping. 3940 */ 3941 scoped_guard(rcu) { 3942 grp = rcu_dereference(p->sched_cache_grp); 3943 3944 if (grp) { 3945 new_fp = (long)READ_ONCE(grp->footprint) + diff; 3946 WRITE_ONCE(grp->footprint, max(new_fp, 0L)); 3947 } 3948 } 3949 #endif 3950 } 3951 3952 if (!ng) { 3953 if (faults > max_faults) { 3954 max_faults = faults; 3955 max_nid = nid; 3956 } 3957 } else if (group_faults > max_faults) { 3958 max_faults = group_faults; 3959 max_nid = nid; 3960 } 3961 } 3962 3963 /* Cannot migrate task to CPU-less node */ 3964 max_nid = numa_nearest_node(max_nid, N_CPU); 3965 3966 if (ng) { 3967 numa_group_count_active_nodes(ng); 3968 spin_unlock_irq(group_lock); 3969 max_nid = preferred_group_nid(p, max_nid); 3970 } 3971 3972 if (max_faults) { 3973 /* Set the new preferred node */ 3974 if (max_nid != p->numa_preferred_nid) 3975 sched_setnuma(p, max_nid); 3976 } 3977 3978 update_task_scan_period(p, fault_types[0], fault_types[1]); 3979 } 3980 3981 static inline int get_numa_group(struct numa_group *grp) 3982 { 3983 return refcount_inc_not_zero(&grp->refcount); 3984 } 3985 3986 static inline void put_numa_group(struct numa_group *grp) 3987 { 3988 if (refcount_dec_and_test(&grp->refcount)) 3989 kfree_rcu(grp, rcu); 3990 } 3991 3992 static void task_numa_group(struct task_struct *p, int cpupid, int flags, 3993 int *priv) 3994 { 3995 struct numa_group *grp, *my_grp; 3996 struct task_struct *tsk; 3997 bool join = false; 3998 int cpu = cpupid_to_cpu(cpupid); 3999 int i; 4000 4001 if (unlikely(!deref_curr_numa_group(p))) { 4002 unsigned int size = sizeof(struct numa_group) + 4003 NR_NUMA_HINT_FAULT_STATS * 4004 nr_node_ids * sizeof(unsigned long); 4005 4006 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 4007 if (!grp) 4008 return; 4009 4010 refcount_set(&grp->refcount, 1); 4011 grp->active_nodes = 1; 4012 grp->max_faults_cpu = 0; 4013 spin_lock_init(&grp->lock); 4014 grp->gid = p->pid; 4015 4016 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 4017 grp->faults[i] = p->numa_faults[i]; 4018 4019 grp->total_faults = p->total_numa_faults; 4020 4021 grp->nr_tasks++; 4022 rcu_assign_pointer(p->numa_group, grp); 4023 } 4024 4025 rcu_read_lock(); 4026 tsk = READ_ONCE(cpu_rq(cpu)->curr); 4027 4028 if (!cpupid_match_pid(tsk, cpupid)) 4029 goto no_join; 4030 4031 grp = rcu_dereference_all(tsk->numa_group); 4032 if (!grp) 4033 goto no_join; 4034 4035 my_grp = deref_curr_numa_group(p); 4036 if (grp == my_grp) 4037 goto no_join; 4038 4039 /* 4040 * Only join the other group if its bigger; if we're the bigger group, 4041 * the other task will join us. 4042 */ 4043 if (my_grp->nr_tasks > grp->nr_tasks) 4044 goto no_join; 4045 4046 /* 4047 * Tie-break on the grp address. 4048 */ 4049 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) 4050 goto no_join; 4051 4052 /* Always join threads in the same process. */ 4053 if (tsk->mm == current->mm) 4054 join = true; 4055 4056 /* Simple filter to avoid false positives due to PID collisions */ 4057 if (flags & TNF_SHARED) 4058 join = true; 4059 4060 /* Update priv based on whether false sharing was detected */ 4061 *priv = !join; 4062 4063 if (join && !get_numa_group(grp)) 4064 goto no_join; 4065 4066 rcu_read_unlock(); 4067 4068 if (!join) 4069 return; 4070 4071 WARN_ON_ONCE(irqs_disabled()); 4072 double_lock_irq(&my_grp->lock, &grp->lock); 4073 4074 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { 4075 my_grp->faults[i] -= p->numa_faults[i]; 4076 grp->faults[i] += p->numa_faults[i]; 4077 } 4078 my_grp->total_faults -= p->total_numa_faults; 4079 grp->total_faults += p->total_numa_faults; 4080 4081 my_grp->nr_tasks--; 4082 grp->nr_tasks++; 4083 4084 spin_unlock(&my_grp->lock); 4085 spin_unlock_irq(&grp->lock); 4086 4087 rcu_assign_pointer(p->numa_group, grp); 4088 4089 put_numa_group(my_grp); 4090 return; 4091 4092 no_join: 4093 rcu_read_unlock(); 4094 return; 4095 } 4096 4097 /* 4098 * Get rid of NUMA statistics associated with a task (either current or dead). 4099 * If @final is set, the task is dead and has reached refcount zero, so we can 4100 * safely free all relevant data structures. Otherwise, there might be 4101 * concurrent reads from places like load balancing and procfs, and we should 4102 * reset the data back to default state without freeing ->numa_faults. 4103 */ 4104 void task_numa_free(struct task_struct *p, bool final) 4105 { 4106 /* safe: p either is current or is being freed by current */ 4107 struct numa_group *grp = rcu_dereference_raw(p->numa_group); 4108 unsigned long *numa_faults = p->numa_faults; 4109 unsigned long flags; 4110 int i; 4111 4112 if (!numa_faults) 4113 return; 4114 4115 if (grp) { 4116 spin_lock_irqsave(&grp->lock, flags); 4117 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 4118 grp->faults[i] -= p->numa_faults[i]; 4119 grp->total_faults -= p->total_numa_faults; 4120 4121 grp->nr_tasks--; 4122 spin_unlock_irqrestore(&grp->lock, flags); 4123 RCU_INIT_POINTER(p->numa_group, NULL); 4124 put_numa_group(grp); 4125 } 4126 4127 if (final) { 4128 p->numa_faults = NULL; 4129 kfree(numa_faults); 4130 } else { 4131 p->total_numa_faults = 0; 4132 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 4133 numa_faults[i] = 0; 4134 } 4135 } 4136 4137 /* 4138 * Got a PROT_NONE fault for a page on @node. 4139 */ 4140 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) 4141 { 4142 struct task_struct *p = current; 4143 bool migrated = flags & TNF_MIGRATED; 4144 int cpu_node = task_node(current); 4145 int local = !!(flags & TNF_FAULT_LOCAL); 4146 struct numa_group *ng; 4147 int priv; 4148 4149 if (!static_branch_likely(&sched_numa_balancing)) 4150 return; 4151 4152 /* for example, ksmd faulting in a user's mm */ 4153 if (!p->mm) 4154 return; 4155 4156 /* 4157 * NUMA faults statistics are unnecessary for the slow memory 4158 * node for memory tiering mode. 4159 */ 4160 if (!node_is_toptier(mem_node) && 4161 (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING || 4162 !cpupid_valid(last_cpupid))) 4163 return; 4164 4165 /* Allocate buffer to track faults on a per-node basis */ 4166 if (unlikely(!p->numa_faults)) { 4167 int size = sizeof(*p->numa_faults) * 4168 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; 4169 4170 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); 4171 if (!p->numa_faults) 4172 return; 4173 4174 p->total_numa_faults = 0; 4175 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 4176 } 4177 4178 /* 4179 * First accesses are treated as private, otherwise consider accesses 4180 * to be private if the accessing pid has not changed 4181 */ 4182 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { 4183 priv = 1; 4184 } else { 4185 priv = cpupid_match_pid(p, last_cpupid); 4186 if (!priv && !(flags & TNF_NO_GROUP)) 4187 task_numa_group(p, last_cpupid, flags, &priv); 4188 } 4189 4190 /* 4191 * If a workload spans multiple NUMA nodes, a shared fault that 4192 * occurs wholly within the set of nodes that the workload is 4193 * actively using should be counted as local. This allows the 4194 * scan rate to slow down when a workload has settled down. 4195 */ 4196 ng = deref_curr_numa_group(p); 4197 if (!priv && !local && ng && ng->active_nodes > 1 && 4198 numa_is_active_node(cpu_node, ng) && 4199 numa_is_active_node(mem_node, ng)) 4200 local = 1; 4201 4202 /* 4203 * Retry to migrate task to preferred node periodically, in case it 4204 * previously failed, or the scheduler moved us. 4205 */ 4206 if (time_after(jiffies, p->numa_migrate_retry)) { 4207 task_numa_placement(p); 4208 numa_migrate_preferred(p); 4209 } 4210 4211 if (migrated) 4212 p->numa_pages_migrated += pages; 4213 if (flags & TNF_MIGRATE_FAIL) 4214 p->numa_faults_locality[2] += pages; 4215 4216 p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; 4217 p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; 4218 p->numa_faults_locality[local] += pages; 4219 } 4220 4221 static void reset_ptenuma_scan(struct task_struct *p) 4222 { 4223 /* 4224 * We only did a read acquisition of the mmap sem, so 4225 * p->mm->numa_scan_seq is written to without exclusive access 4226 * and the update is not guaranteed to be atomic. That's not 4227 * much of an issue though, since this is just used for 4228 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not 4229 * expensive, to avoid any form of compiler optimizations: 4230 */ 4231 WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 4232 p->mm->numa_scan_offset = 0; 4233 } 4234 4235 static bool vma_is_accessed(struct mm_struct *mm, struct vm_area_struct *vma) 4236 { 4237 unsigned long pids; 4238 /* 4239 * Allow unconditional access first two times, so that all the (pages) 4240 * of VMAs get prot_none fault introduced irrespective of accesses. 4241 * This is also done to avoid any side effect of task scanning 4242 * amplifying the unfairness of disjoint set of VMAs' access. 4243 */ 4244 if ((READ_ONCE(current->mm->numa_scan_seq) - vma->numab_state->start_scan_seq) < 2) 4245 return true; 4246 4247 pids = vma->numab_state->pids_active[0] | vma->numab_state->pids_active[1]; 4248 if (test_bit(hash_32(current->pid, ilog2(BITS_PER_LONG)), &pids)) 4249 return true; 4250 4251 /* 4252 * Complete a scan that has already started regardless of PID access, or 4253 * some VMAs may never be scanned in multi-threaded applications: 4254 */ 4255 if (mm->numa_scan_offset > vma->vm_start) { 4256 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_IGNORE_PID); 4257 return true; 4258 } 4259 4260 /* 4261 * This vma has not been accessed for a while, and if the number 4262 * the threads in the same process is low, which means no other 4263 * threads can help scan this vma, force a vma scan. 4264 */ 4265 if (READ_ONCE(mm->numa_scan_seq) > 4266 (vma->numab_state->prev_scan_seq + get_nr_threads(current))) 4267 return true; 4268 4269 return false; 4270 } 4271 4272 #define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay) 4273 4274 /* 4275 * The expensive part of numa migration is done from task_work context. 4276 * Triggered from task_tick_numa(). 4277 */ 4278 static void task_numa_work(struct callback_head *work) 4279 { 4280 unsigned long migrate, next_scan, now = jiffies; 4281 struct task_struct *p = current; 4282 struct mm_struct *mm = p->mm; 4283 u64 runtime = p->se.sum_exec_runtime; 4284 struct vm_area_struct *vma; 4285 unsigned long start, end; 4286 unsigned long nr_pte_updates = 0; 4287 long pages, virtpages; 4288 struct vma_iterator vmi; 4289 bool vma_pids_skipped; 4290 bool vma_pids_forced = false; 4291 4292 WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work)); 4293 4294 work->next = work; 4295 /* 4296 * Who cares about NUMA placement when they're dying. 4297 * 4298 * NOTE: make sure not to dereference p->mm before this check, 4299 * exit_task_work() happens _after_ exit_mm() so we could be called 4300 * without p->mm even though we still had it when we enqueued this 4301 * work. 4302 */ 4303 if (p->flags & PF_EXITING) 4304 return; 4305 4306 /* 4307 * Memory is pinned to only one NUMA node via cpuset.mems, naturally 4308 * no page can be migrated. 4309 */ 4310 if (cpusets_enabled() && nodes_weight(cpuset_current_mems_allowed) == 1) { 4311 trace_sched_skip_cpuset_numa(current, &cpuset_current_mems_allowed); 4312 return; 4313 } 4314 4315 if (!mm->numa_next_scan) { 4316 mm->numa_next_scan = now + 4317 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 4318 } 4319 4320 /* 4321 * Enforce maximal scan/migration frequency.. 4322 */ 4323 migrate = mm->numa_next_scan; 4324 if (time_before(now, migrate)) 4325 return; 4326 4327 if (p->numa_scan_period == 0) { 4328 p->numa_scan_period_max = task_scan_max(p); 4329 p->numa_scan_period = task_scan_start(p); 4330 } 4331 4332 next_scan = now + msecs_to_jiffies(p->numa_scan_period); 4333 if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan)) 4334 return; 4335 4336 /* 4337 * Delay this task enough that another task of this mm will likely win 4338 * the next time around. 4339 */ 4340 p->node_stamp += 2 * TICK_NSEC; 4341 4342 pages = sysctl_numa_balancing_scan_size; 4343 pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 4344 virtpages = pages * 8; /* Scan up to this much virtual space */ 4345 if (!pages) 4346 return; 4347 4348 4349 if (!mmap_read_trylock(mm)) 4350 return; 4351 4352 /* 4353 * VMAs are skipped if the current PID has not trapped a fault within 4354 * the VMA recently. Allow scanning to be forced if there is no 4355 * suitable VMA remaining. 4356 */ 4357 vma_pids_skipped = false; 4358 4359 retry_pids: 4360 start = mm->numa_scan_offset; 4361 vma_iter_init(&vmi, mm, start); 4362 vma = vma_next(&vmi); 4363 if (!vma) { 4364 reset_ptenuma_scan(p); 4365 start = 0; 4366 vma_iter_set(&vmi, start); 4367 vma = vma_next(&vmi); 4368 } 4369 4370 for (; vma; vma = vma_next(&vmi)) { 4371 if (!vma_migratable(vma) || !vma_policy_mof(vma) || 4372 vma_is_hugetlb(vma) || !vma_is_mm_managed(vma)) { 4373 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_UNSUITABLE); 4374 continue; 4375 } 4376 4377 /* 4378 * Shared library pages mapped by multiple processes are not 4379 * migrated as it is expected they are cache replicated. Avoid 4380 * hinting faults in read-only file-backed mappings or the vDSO 4381 * as migrating the pages will be of marginal benefit. 4382 */ 4383 if (!vma->vm_mm || 4384 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) { 4385 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SHARED_RO); 4386 continue; 4387 } 4388 4389 /* 4390 * Skip inaccessible VMAs to avoid any confusion between 4391 * PROT_NONE and NUMA hinting PTEs 4392 */ 4393 if (!vma_is_accessible(vma)) { 4394 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_INACCESSIBLE); 4395 continue; 4396 } 4397 4398 /* Initialise new per-VMA NUMAB state. */ 4399 if (!vma->numab_state) { 4400 struct vma_numab_state *ptr; 4401 4402 ptr = kzalloc_obj(*ptr); 4403 if (!ptr) 4404 continue; 4405 4406 if (cmpxchg(&vma->numab_state, NULL, ptr)) { 4407 kfree(ptr); 4408 continue; 4409 } 4410 4411 vma->numab_state->start_scan_seq = mm->numa_scan_seq; 4412 4413 vma->numab_state->next_scan = now + 4414 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 4415 4416 /* Reset happens after 4 times scan delay of scan start */ 4417 vma->numab_state->pids_active_reset = vma->numab_state->next_scan + 4418 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 4419 4420 /* 4421 * Ensure prev_scan_seq does not match numa_scan_seq, 4422 * to prevent VMAs being skipped prematurely on the 4423 * first scan: 4424 */ 4425 vma->numab_state->prev_scan_seq = mm->numa_scan_seq - 1; 4426 } 4427 4428 /* 4429 * Scanning the VMAs of short lived tasks add more overhead. So 4430 * delay the scan for new VMAs. 4431 */ 4432 if (mm->numa_scan_seq && time_before(jiffies, 4433 vma->numab_state->next_scan)) { 4434 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SCAN_DELAY); 4435 continue; 4436 } 4437 4438 /* RESET access PIDs regularly for old VMAs. */ 4439 if (mm->numa_scan_seq && 4440 time_after(jiffies, vma->numab_state->pids_active_reset)) { 4441 vma->numab_state->pids_active_reset = vma->numab_state->pids_active_reset + 4442 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 4443 vma->numab_state->pids_active[0] = READ_ONCE(vma->numab_state->pids_active[1]); 4444 vma->numab_state->pids_active[1] = 0; 4445 } 4446 4447 /* Do not rescan VMAs twice within the same sequence. */ 4448 if (vma->numab_state->prev_scan_seq == mm->numa_scan_seq) { 4449 mm->numa_scan_offset = vma->vm_end; 4450 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SEQ_COMPLETED); 4451 continue; 4452 } 4453 4454 /* 4455 * Do not scan the VMA if task has not accessed it, unless no other 4456 * VMA candidate exists. 4457 */ 4458 if (!vma_pids_forced && !vma_is_accessed(mm, vma)) { 4459 vma_pids_skipped = true; 4460 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_PID_INACTIVE); 4461 continue; 4462 } 4463 4464 do { 4465 start = max(start, vma->vm_start); 4466 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 4467 end = min(end, vma->vm_end); 4468 nr_pte_updates = change_prot_numa(vma, start, end); 4469 4470 /* 4471 * Try to scan sysctl_numa_balancing_size worth of 4472 * hpages that have at least one present PTE that 4473 * is not already PTE-numa. If the VMA contains 4474 * areas that are unused or already full of prot_numa 4475 * PTEs, scan up to virtpages, to skip through those 4476 * areas faster. 4477 */ 4478 if (nr_pte_updates) 4479 pages -= (end - start) >> PAGE_SHIFT; 4480 virtpages -= (end - start) >> PAGE_SHIFT; 4481 4482 start = end; 4483 if (pages <= 0 || virtpages <= 0) 4484 goto out; 4485 4486 cond_resched(); 4487 } while (end != vma->vm_end); 4488 4489 /* VMA scan is complete, do not scan until next sequence. */ 4490 vma->numab_state->prev_scan_seq = mm->numa_scan_seq; 4491 4492 /* 4493 * Only force scan within one VMA at a time, to limit the 4494 * cost of scanning a potentially uninteresting VMA. 4495 */ 4496 if (vma_pids_forced) 4497 break; 4498 } 4499 4500 /* 4501 * If no VMAs are remaining and VMAs were skipped due to the PID 4502 * not accessing the VMA previously, then force a scan to ensure 4503 * forward progress: 4504 */ 4505 if (!vma && !vma_pids_forced && vma_pids_skipped) { 4506 vma_pids_forced = true; 4507 goto retry_pids; 4508 } 4509 4510 out: 4511 /* 4512 * It is possible to reach the end of the VMA list but the last few 4513 * VMAs are not guaranteed to the vma_migratable. If they are not, we 4514 * would find the !migratable VMA on the next scan but not reset the 4515 * scanner to the start so check it now. 4516 */ 4517 if (vma) 4518 mm->numa_scan_offset = start; 4519 else 4520 reset_ptenuma_scan(p); 4521 mmap_read_unlock(mm); 4522 4523 /* 4524 * Make sure tasks use at least 32x as much time to run other code 4525 * than they used here, to limit NUMA PTE scanning overhead to 3% max. 4526 * Usually update_task_scan_period slows down scanning enough; on an 4527 * overloaded system we need to limit overhead on a per task basis. 4528 */ 4529 if (unlikely(p->se.sum_exec_runtime != runtime)) { 4530 u64 diff = p->se.sum_exec_runtime - runtime; 4531 p->node_stamp += 32 * diff; 4532 } 4533 } 4534 4535 void init_numa_balancing(u64 clone_flags, struct task_struct *p) 4536 { 4537 int mm_users = 0; 4538 struct mm_struct *mm = p->mm; 4539 4540 if (mm) { 4541 mm_users = atomic_read(&mm->mm_users); 4542 if (mm_users == 1) { 4543 mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 4544 mm->numa_scan_seq = 0; 4545 } 4546 } 4547 p->node_stamp = 0; 4548 p->numa_scan_seq = mm ? mm->numa_scan_seq : 0; 4549 p->numa_scan_period = sysctl_numa_balancing_scan_delay; 4550 p->numa_migrate_retry = 0; 4551 /* Protect against double add, see task_tick_numa and task_numa_work */ 4552 p->numa_work.next = &p->numa_work; 4553 p->numa_faults = NULL; 4554 p->numa_pages_migrated = 0; 4555 p->total_numa_faults = 0; 4556 RCU_INIT_POINTER(p->numa_group, NULL); 4557 p->last_task_numa_placement = 0; 4558 p->last_sum_exec_runtime = 0; 4559 4560 init_task_work(&p->numa_work, task_numa_work); 4561 4562 /* New address space, reset the preferred nid */ 4563 if (!(clone_flags & CLONE_VM)) { 4564 p->numa_preferred_nid = NUMA_NO_NODE; 4565 return; 4566 } 4567 4568 /* 4569 * New thread, keep existing numa_preferred_nid which should be copied 4570 * already by arch_dup_task_struct but stagger when scans start. 4571 */ 4572 if (mm) { 4573 unsigned int delay; 4574 4575 delay = min_t(unsigned int, task_scan_max(current), 4576 current->numa_scan_period * mm_users * NSEC_PER_MSEC); 4577 delay += 2 * TICK_NSEC; 4578 p->node_stamp = delay; 4579 } 4580 } 4581 4582 /* 4583 * Drive the periodic memory faults.. 4584 */ 4585 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 4586 { 4587 struct callback_head *work = &curr->numa_work; 4588 u64 period, now; 4589 4590 /* 4591 * We don't care about NUMA placement if we don't have memory. 4592 */ 4593 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work) 4594 return; 4595 4596 /* 4597 * Using runtime rather than walltime has the dual advantage that 4598 * we (mostly) drive the selection from busy threads and that the 4599 * task needs to have done some actual work before we bother with 4600 * NUMA placement. 4601 */ 4602 now = curr->se.sum_exec_runtime; 4603 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; 4604 4605 if (now > curr->node_stamp + period) { 4606 if (!curr->node_stamp) 4607 curr->numa_scan_period = task_scan_start(curr); 4608 curr->node_stamp += period; 4609 4610 if (!time_before(jiffies, curr->mm->numa_next_scan)) 4611 task_work_add(curr, work, TWA_RESUME); 4612 } 4613 } 4614 4615 static void update_scan_period(struct task_struct *p, int new_cpu) 4616 { 4617 int src_nid = cpu_to_node(task_cpu(p)); 4618 int dst_nid = cpu_to_node(new_cpu); 4619 4620 if (!static_branch_likely(&sched_numa_balancing)) 4621 return; 4622 4623 if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING)) 4624 return; 4625 4626 if (src_nid == dst_nid) 4627 return; 4628 4629 /* 4630 * Allow resets if faults have been trapped before one scan 4631 * has completed. This is most likely due to a new task that 4632 * is pulled cross-node due to wakeups or load balancing. 4633 */ 4634 if (p->numa_scan_seq) { 4635 /* 4636 * Avoid scan adjustments if moving to the preferred 4637 * node or if the task was not previously running on 4638 * the preferred node. 4639 */ 4640 if (dst_nid == p->numa_preferred_nid || 4641 (p->numa_preferred_nid != NUMA_NO_NODE && 4642 src_nid != p->numa_preferred_nid)) 4643 return; 4644 } 4645 4646 p->numa_scan_period = task_scan_start(p); 4647 } 4648 4649 #else /* !CONFIG_NUMA_BALANCING: */ 4650 4651 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 4652 { 4653 } 4654 4655 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) 4656 { 4657 } 4658 4659 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) 4660 { 4661 } 4662 4663 static inline void update_scan_period(struct task_struct *p, int new_cpu) 4664 { 4665 } 4666 4667 #endif /* !CONFIG_NUMA_BALANCING */ 4668 4669 static void 4670 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 4671 { 4672 WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); 4673 update_load_add(&cfs_rq->load, se->load.weight); 4674 if (entity_is_task(se)) { 4675 struct task_struct *p = task_of(se); 4676 struct rq *rq = rq_of(cfs_rq); 4677 4678 account_numa_enqueue(rq, p); 4679 account_llc_enqueue(rq, p); 4680 list_add(&se->group_node, &rq->cfs_tasks); 4681 } 4682 cfs_rq->nr_queued++; 4683 } 4684 4685 static void 4686 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 4687 { 4688 WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); 4689 update_load_sub(&cfs_rq->load, se->load.weight); 4690 if (entity_is_task(se)) { 4691 struct task_struct *p = task_of(se); 4692 struct rq *rq = rq_of(cfs_rq); 4693 4694 account_numa_dequeue(rq, p); 4695 account_llc_dequeue(rq, p); 4696 list_del_init(&se->group_node); 4697 } 4698 cfs_rq->nr_queued--; 4699 } 4700 4701 /* 4702 * Signed add and clamp on underflow. 4703 * 4704 * Explicitly do a load-store to ensure the intermediate value never hits 4705 * memory. This allows lockless observations without ever seeing the negative 4706 * values. 4707 */ 4708 #define add_positive(_ptr, _val) do { \ 4709 typeof(_ptr) ptr = (_ptr); \ 4710 __signed_scalar_typeof(*ptr) val = (_val); \ 4711 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 4712 \ 4713 res = var + val; \ 4714 \ 4715 if (val < 0 && res > var) \ 4716 res = 0; \ 4717 \ 4718 WRITE_ONCE(*ptr, res); \ 4719 } while (0) 4720 4721 /* 4722 * Remove and clamp on negative, from a local variable. 4723 * 4724 * A variant of sub_positive(), which does not use explicit load-store 4725 * and is thus optimized for local variable updates. 4726 */ 4727 #define lsub_positive(_ptr, _val) do { \ 4728 typeof(_ptr) ptr = (_ptr); \ 4729 *ptr -= min_t(typeof(*ptr), *ptr, _val); \ 4730 } while (0) 4731 4732 4733 /* 4734 * Because of rounding, se->util_sum might ends up being +1 more than 4735 * cfs->util_sum. Although this is not a problem by itself, detaching 4736 * a lot of tasks with the rounding problem between 2 updates of 4737 * util_avg (~1ms) can make cfs->util_sum becoming null whereas 4738 * cfs_util_avg is not. 4739 * 4740 * Check that util_sum is still above its lower bound for the new 4741 * util_avg. Given that period_contrib might have moved since the last 4742 * sync, we are only sure that util_sum must be above or equal to 4743 * util_avg * minimum possible divider 4744 */ 4745 #define __update_sa(sa, name, delta_avg, delta_sum) do { \ 4746 add_positive(&(sa)->name##_avg, delta_avg); \ 4747 add_positive(&(sa)->name##_sum, delta_sum); \ 4748 (sa)->name##_sum = max_t(typeof((sa)->name##_sum), \ 4749 (sa)->name##_sum, \ 4750 (sa)->name##_avg * PELT_MIN_DIVIDER); \ 4751 } while (0) 4752 4753 static inline void 4754 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4755 { 4756 __update_sa(&cfs_rq->avg, load, se->avg.load_avg, 4757 se_weight(se) * se->avg.load_sum); 4758 } 4759 4760 static inline void 4761 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4762 { 4763 __update_sa(&cfs_rq->avg, load, -se->avg.load_avg, 4764 se_weight(se) * -se->avg.load_sum); 4765 } 4766 4767 static void 4768 rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot) 4769 { 4770 long old_weight = se->h_load.weight; 4771 4772 /* 4773 * VRUNTIME 4774 * -------- 4775 * 4776 * COROLLARY #1: The virtual runtime of the entity needs to be 4777 * adjusted if re-weight at !0-lag point. 4778 * 4779 * Proof: For contradiction assume this is not true, so we can 4780 * re-weight without changing vruntime at !0-lag point. 4781 * 4782 * Weight VRuntime Avg-VRuntime 4783 * before w v V 4784 * after w' v' V' 4785 * 4786 * Since lag needs to be preserved through re-weight: 4787 * 4788 * lag = (V - v)*w = (V'- v')*w', where v = v' 4789 * ==> V' = (V - v)*w/w' + v (1) 4790 * 4791 * Let W be the total weight of the entities before reweight, 4792 * since V' is the new weighted average of entities: 4793 * 4794 * V' = (WV + w'v - wv) / (W + w' - w) (2) 4795 * 4796 * by using (1) & (2) we obtain: 4797 * 4798 * (WV + w'v - wv) / (W + w' - w) = (V - v)*w/w' + v 4799 * ==> (WV-Wv+Wv+w'v-wv)/(W+w'-w) = (V - v)*w/w' + v 4800 * ==> (WV - Wv)/(W + w' - w) + v = (V - v)*w/w' + v 4801 * ==> (V - v)*W/(W + w' - w) = (V - v)*w/w' (3) 4802 * 4803 * Since we are doing at !0-lag point which means V != v, we 4804 * can simplify (3): 4805 * 4806 * ==> W / (W + w' - w) = w / w' 4807 * ==> Ww' = Ww + ww' - ww 4808 * ==> W * (w' - w) = w * (w' - w) 4809 * ==> W = w (re-weight indicates w' != w) 4810 * 4811 * So the cfs_rq contains only one entity, hence vruntime of 4812 * the entity @v should always equal to the cfs_rq's weighted 4813 * average vruntime @V, which means we will always re-weight 4814 * at 0-lag point, thus breach assumption. Proof completed. 4815 * 4816 * 4817 * COROLLARY #2: Re-weight does NOT affect weighted average 4818 * vruntime of all the entities. 4819 * 4820 * Proof: According to corollary #1, Eq. (1) should be: 4821 * 4822 * (V - v)*w = (V' - v')*w' 4823 * ==> v' = V' - (V - v)*w/w' (4) 4824 * 4825 * According to the weighted average formula, we have: 4826 * 4827 * V' = (WV - wv + w'v') / (W - w + w') 4828 * = (WV - wv + w'(V' - (V - v)w/w')) / (W - w + w') 4829 * = (WV - wv + w'V' - Vw + wv) / (W - w + w') 4830 * = (WV + w'V' - Vw) / (W - w + w') 4831 * 4832 * ==> V'*(W - w + w') = WV + w'V' - Vw 4833 * ==> V' * (W - w) = (W - w) * V (5) 4834 * 4835 * If the entity is the only one in the cfs_rq, then reweight 4836 * always occurs at 0-lag point, so V won't change. Or else 4837 * there are other entities, hence W != w, then Eq. (5) turns 4838 * into V' = V. So V won't change in either case, proof done. 4839 * 4840 * 4841 * So according to corollary #1 & #2, the effect of re-weight 4842 * on vruntime should be: 4843 * 4844 * v' = V' - (V - v) * w / w' (4) 4845 * = V - (V - v) * w / w' 4846 * = V - vl * w / w' 4847 * = V - vl' 4848 */ 4849 se->vlag = div64_long(se->vlag * old_weight, weight); 4850 4851 /* 4852 * DEADLINE 4853 * -------- 4854 * 4855 * When the weight changes, the virtual time slope changes and 4856 * we should adjust the relative virtual deadline accordingly. 4857 * 4858 * d' = v' + (d - v)*w/w' 4859 * = V' - (V - v)*w/w' + (d - v)*w/w' 4860 * = V - (V - v)*w/w' + (d - v)*w/w' 4861 * = V + (d - V)*w/w' 4862 */ 4863 if (se->rel_deadline) 4864 se->deadline = div64_long(se->deadline * old_weight, weight); 4865 4866 if (rel_vprot) 4867 se->vprot = div64_long(se->vprot * old_weight, weight); 4868 } 4869 4870 static void reweight_eevdf(struct cfs_rq *cfs_rq, struct sched_entity *se, 4871 unsigned long weight, bool on_rq) 4872 { 4873 bool curr = cfs_rq->curr == se; 4874 bool rel_vprot = false; 4875 u64 avruntime = 0; 4876 4877 if (se->h_load.weight == weight) 4878 return; 4879 4880 if (on_rq) { 4881 avruntime = avg_vruntime(cfs_rq); 4882 se->vlag = entity_lag(cfs_rq, se, avruntime); 4883 se->deadline -= avruntime; 4884 se->rel_deadline = 1; 4885 if (curr && protect_slice(se)) { 4886 se->vprot -= avruntime; 4887 rel_vprot = true; 4888 } 4889 4890 cfs_rq->h_nr_queued--; 4891 if (!curr) 4892 __dequeue_entity(cfs_rq, se); 4893 } 4894 4895 rescale_entity(se, weight, rel_vprot); 4896 4897 update_load_set(&se->h_load, weight); 4898 4899 if (on_rq) { 4900 if (rel_vprot) 4901 se->vprot += avruntime; 4902 se->deadline += avruntime; 4903 se->rel_deadline = 0; 4904 se->vruntime = avruntime - se->vlag; 4905 4906 if (!curr) 4907 __enqueue_entity(cfs_rq, se); 4908 cfs_rq->h_nr_queued++; 4909 } 4910 } 4911 4912 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 4913 unsigned long weight) 4914 { 4915 if (se->load.weight == weight) 4916 return; 4917 4918 if (se->on_rq) { 4919 WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); 4920 update_load_sub(&cfs_rq->load, se->load.weight); 4921 } 4922 dequeue_load_avg(cfs_rq, se); 4923 4924 update_load_set(&se->load, weight); 4925 4926 do { 4927 u32 divider = get_pelt_divider(&se->avg); 4928 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 4929 } while (0); 4930 4931 enqueue_load_avg(cfs_rq, se); 4932 4933 if (se->on_rq) 4934 update_load_add(&cfs_rq->load, se->load.weight); 4935 } 4936 4937 /* 4938 * weight = NICE_0_LOAD; 4939 * for_each_entity_se(se) 4940 * weight = __calc_prop_weight(cfs_rq_of(se), se, weight); 4941 */ 4942 static __always_inline 4943 unsigned long __calc_prop_weight(struct cfs_rq *cfs_rq, struct sched_entity *se, 4944 unsigned long weight) 4945 { 4946 weight *= se->load.weight; 4947 if (parent_entity(se)) 4948 weight /= cfs_rq->load.weight; 4949 else 4950 weight /= NICE_0_LOAD; 4951 4952 return max(weight, MIN_SHARES); 4953 } 4954 4955 static void reweight_task_fair(struct rq *rq, struct task_struct *p, 4956 const struct load_weight *lw) 4957 { 4958 struct sched_entity *se = &p->se; 4959 unsigned long weight = NICE_0_LOAD; 4960 4961 if (se->on_rq) 4962 update_curr_fair(rq); 4963 4964 reweight_entity(cfs_rq_of(se), se, lw->weight); 4965 se->load.inv_weight = lw->inv_weight; 4966 4967 if (!se->on_rq) 4968 return; 4969 4970 for_each_sched_entity(se) 4971 weight = __calc_prop_weight(cfs_rq_of(se), se, weight); 4972 4973 reweight_eevdf(&rq->cfs, &p->se, weight, p->se.on_rq); 4974 } 4975 4976 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); 4977 4978 #ifdef CONFIG_FAIR_GROUP_SCHED 4979 /* 4980 * All this does is approximate the hierarchical proportion which includes that 4981 * global sum we all love to hate. 4982 * 4983 * That is, the weight of a group entity, is the proportional share of the 4984 * group weight based on the group runqueue weights. That is: 4985 * 4986 * tg->weight * grq->load.weight 4987 * ge->load.weight = ----------------------------- (1) 4988 * \Sum grq->load.weight 4989 * 4990 * Now, because computing that sum is prohibitively expensive to compute (been 4991 * there, done that) we approximate it with this average stuff. The average 4992 * moves slower and therefore the approximation is cheaper and more stable. 4993 * 4994 * So instead of the above, we substitute: 4995 * 4996 * grq->load.weight -> grq->avg.load_avg (2) 4997 * 4998 * which yields the following: 4999 * 5000 * tg->weight * grq->avg.load_avg 5001 * ge->load.weight = ------------------------------ (3) 5002 * tg->load_avg 5003 * 5004 * Where: tg->load_avg ~= \Sum grq->avg.load_avg 5005 * 5006 * That is shares_avg, and it is right (given the approximation (2)). 5007 * 5008 * The problem with it is that because the average is slow -- it was designed 5009 * to be exactly that of course -- this leads to transients in boundary 5010 * conditions. In specific, the case where the group was idle and we start the 5011 * one task. It takes time for our CPU's grq->avg.load_avg to build up, 5012 * yielding bad latency etc.. 5013 * 5014 * Now, in that special case (1) reduces to: 5015 * 5016 * tg->weight * grq->load.weight 5017 * ge->load.weight = ----------------------------- = tg->weight (4) 5018 * grp->load.weight 5019 * 5020 * That is, the sum collapses because all other CPUs are idle; the UP scenario. 5021 * 5022 * So what we do is modify our approximation (3) to approach (4) in the (near) 5023 * UP case, like: 5024 * 5025 * ge->load.weight = 5026 * 5027 * tg->weight * grq->load.weight 5028 * --------------------------------------------------- (5) 5029 * tg->load_avg - grq->avg.load_avg + grq->load.weight 5030 * 5031 * But because grq->load.weight can drop to 0, resulting in a divide by zero, 5032 * we need to use grq->avg.load_avg as its lower bound, which then gives: 5033 * 5034 * 5035 * tg->weight * grq->load.weight 5036 * ge->load.weight = ----------------------------- (6) 5037 * tg_load_avg' 5038 * 5039 * Where: 5040 * 5041 * tg_load_avg' = tg->load_avg - grq->avg.load_avg + 5042 * max(grq->load.weight, grq->avg.load_avg) 5043 * 5044 * And that is shares_weight and is icky. In the (near) UP case it approaches 5045 * (4) while in the normal case it approaches (3). It consistently 5046 * overestimates the ge->load.weight and therefore: 5047 * 5048 * \Sum ge->load.weight >= tg->weight 5049 * 5050 * hence icky! 5051 */ 5052 static long __calc_smp_shares(struct cfs_rq *cfs_rq, long tg_shares, long shares_max) 5053 { 5054 struct task_group *tg = cfs_rq->tg; 5055 long tg_weight, load, shares; 5056 5057 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 5058 5059 tg_weight = atomic_long_read(&tg->load_avg); 5060 5061 /* Ensure tg_weight >= load */ 5062 tg_weight -= cfs_rq->tg_load_avg_contrib; 5063 tg_weight += load; 5064 5065 shares = (tg_shares * load); 5066 if (tg_weight) 5067 shares /= tg_weight; 5068 5069 /* 5070 * MIN_SHARES has to be unscaled here to support per-CPU partitioning 5071 * of a group with small tg->shares value. It is a floor value which is 5072 * assigned as a minimum load.weight to the sched_entity representing 5073 * the group on a CPU. 5074 * 5075 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 5076 * on an 8-core system with 8 tasks each runnable on one CPU shares has 5077 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In 5078 * case no task is runnable on a CPU MIN_SHARES=2 should be returned 5079 * instead of 0. 5080 */ 5081 return clamp_t(long, shares, MIN_SHARES, shares_max); 5082 } 5083 5084 static int tg_cpus(struct task_group *tg) 5085 { 5086 int nr = num_online_cpus(); 5087 5088 if (cpusets_enabled()) { 5089 struct cgroup *cgrp = tg->css.cgroup; 5090 if (cgrp) 5091 nr = cpuset_num_cpus(cgrp); 5092 } 5093 5094 /* 5095 * An empty cpuset would propagate a 0 shares_max into 5096 * __calc_smp_shares(), where clamp() yields hi when hi < lo and so 5097 * defeats the MIN_SHARES floor. Match tg_tasks(), which floors at 1. 5098 */ 5099 return max(nr, 1); 5100 } 5101 5102 static inline int tg_tasks(struct task_group *tg) 5103 { 5104 return max(1, atomic_long_read(&tg->runnable_avg) >> SCHED_CAPACITY_SHIFT); 5105 } 5106 5107 /* 5108 * Func: fraction(nr_tasks * tg->shares) 5109 * 5110 * Scale tg->shares by the number of tasks. 5111 */ 5112 static long calc_tasks_shares(struct cfs_rq *cfs_rq) 5113 { 5114 struct task_group *tg = cfs_rq->tg; 5115 int nr = tg_tasks(tg); 5116 long tg_shares = READ_ONCE(tg->shares); 5117 return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); 5118 } 5119 5120 /* 5121 * Func: min(fraction(nr_cpus * tg->shares), nice -20) 5122 * 5123 * Scale tg->shares by the maximal number of CPUs; but clip the max shares at 5124 * nice -20, otherwise a single spinner on a 512 CPU machine would result in 5125 * 512*NICE_0_LOAD, which is also crazy. 5126 */ 5127 static long calc_max_shares(struct cfs_rq *cfs_rq) 5128 { 5129 struct task_group *tg = cfs_rq->tg; 5130 int nr = tg_cpus(tg); 5131 long tg_shares = READ_ONCE(tg->shares); 5132 long max_shares = scale_load(sched_prio_to_weight[0]); 5133 return __calc_smp_shares(cfs_rq, tg_shares * nr, max_shares); 5134 } 5135 5136 /* 5137 * Func: fraction(nr * tg->shares); nr = min(nr_tasks, nr_cpus) 5138 * 5139 * Scales between "smp" and "max" in a natural way. No longer needs clipping 5140 * since there are no unnatural inflations like with "max". 5141 */ 5142 static long calc_concur_shares(struct cfs_rq *cfs_rq) 5143 { 5144 struct task_group *tg = cfs_rq->tg; 5145 int nr = min(tg_tasks(tg), tg_cpus(tg)); 5146 long tg_shares = READ_ONCE(tg->shares); 5147 return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); 5148 } 5149 5150 /* 5151 * Func: fraction(tg->shares) 5152 * 5153 * This infamously results in tiny shares when you have many CPUs. 5154 */ 5155 static long calc_smp_shares(struct cfs_rq *cfs_rq) 5156 { 5157 struct task_group *tg = cfs_rq->tg; 5158 long tg_shares = READ_ONCE(tg->shares); 5159 return __calc_smp_shares(cfs_rq, tg_shares, tg_shares); 5160 } 5161 5162 /* 5163 * Ignore this pesky SMP stuff, use (4). 5164 */ 5165 static long calc_up_shares(struct cfs_rq *cfs_rq) 5166 { 5167 struct task_group *tg = cfs_rq->tg; 5168 return READ_ONCE(tg->shares); 5169 } 5170 5171 DEFINE_STATIC_CALL(calc_group_shares, calc_concur_shares); 5172 5173 void __sched_cgroup_mode_update(int mode) 5174 { 5175 long (*func)(struct cfs_rq *); 5176 switch (mode) { 5177 case 0: 5178 func = &calc_up_shares; 5179 break; 5180 case 1: 5181 func = &calc_smp_shares; 5182 break; 5183 case 2: 5184 default: 5185 func = &calc_concur_shares; 5186 break; 5187 case 3: 5188 func = &calc_max_shares; 5189 break; 5190 case 4: 5191 func = &calc_tasks_shares; 5192 break; 5193 } 5194 static_call_update(calc_group_shares, func); 5195 } 5196 5197 /* 5198 * Recomputes the group entity based on the current state of its group 5199 * runqueue. 5200 */ 5201 static void update_cfs_group(struct sched_entity *se) 5202 { 5203 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 5204 long shares; 5205 5206 /* 5207 * When a group becomes empty, preserve its weight. This matters for 5208 * DELAY_DEQUEUE. 5209 */ 5210 if (!gcfs_rq || !gcfs_rq->load.weight) 5211 return; 5212 5213 shares = static_call(calc_group_shares)(gcfs_rq); 5214 reweight_entity(cfs_rq_of(se), se, shares); 5215 } 5216 5217 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 5218 static inline void update_cfs_group(struct sched_entity *se) 5219 { 5220 } 5221 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 5222 5223 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) 5224 { 5225 struct rq *rq = rq_of(cfs_rq); 5226 5227 if (&rq->cfs == cfs_rq) { 5228 /* 5229 * There are a few boundary cases this might miss but it should 5230 * get called often enough that that should (hopefully) not be 5231 * a real problem. 5232 * 5233 * It will not get called when we go idle, because the idle 5234 * thread is a different class (!fair), nor will the utilization 5235 * number include things like RT tasks. 5236 * 5237 * As is, the util number is not freq-invariant (we'd have to 5238 * implement arch_scale_freq_capacity() for that). 5239 * 5240 * See cpu_util_cfs(). 5241 */ 5242 cpufreq_update_util(rq, flags); 5243 } 5244 } 5245 5246 static inline bool load_avg_is_decayed(struct sched_avg *sa) 5247 { 5248 if (sa->load_sum) 5249 return false; 5250 5251 if (sa->util_sum) 5252 return false; 5253 5254 if (sa->runnable_sum) 5255 return false; 5256 5257 /* 5258 * _avg must be null when _sum are null because _avg = _sum / divider 5259 * Make sure that rounding and/or propagation of PELT values never 5260 * break this. 5261 */ 5262 WARN_ON_ONCE(sa->load_avg || 5263 sa->util_avg || 5264 sa->runnable_avg); 5265 5266 return true; 5267 } 5268 5269 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 5270 { 5271 return u64_u32_load_copy(cfs_rq->avg.last_update_time, 5272 cfs_rq->last_update_time_copy); 5273 } 5274 #ifdef CONFIG_FAIR_GROUP_SCHED 5275 /* 5276 * Because list_add_leaf_cfs_rq always places a child cfs_rq on the list 5277 * immediately before a parent cfs_rq, and cfs_rqs are removed from the list 5278 * bottom-up, we only have to test whether the cfs_rq before us on the list 5279 * is our child. 5280 * If cfs_rq is not on the list, test whether a child needs its to be added to 5281 * connect a branch to the tree * (see list_add_leaf_cfs_rq() for details). 5282 */ 5283 static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq) 5284 { 5285 struct cfs_rq *prev_cfs_rq; 5286 struct list_head *prev; 5287 struct rq *rq = rq_of(cfs_rq); 5288 5289 if (cfs_rq->on_list) { 5290 prev = cfs_rq->leaf_cfs_rq_list.prev; 5291 } else { 5292 prev = rq->tmp_alone_branch; 5293 } 5294 5295 if (prev == &rq->leaf_cfs_rq_list) 5296 return false; 5297 5298 prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list); 5299 5300 return (prev_cfs_rq->tg->parent == cfs_rq->tg); 5301 } 5302 5303 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 5304 { 5305 if (cfs_rq->load.weight) 5306 return false; 5307 5308 if (!load_avg_is_decayed(&cfs_rq->avg)) 5309 return false; 5310 5311 if (child_cfs_rq_on_list(cfs_rq)) 5312 return false; 5313 5314 if (cfs_rq->tg_load_avg_contrib) 5315 return false; 5316 5317 return true; 5318 } 5319 5320 /** 5321 * update_tg_load_avg - update the tg's load avg 5322 * @cfs_rq: the cfs_rq whose avg changed 5323 * 5324 * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. 5325 * However, because tg->load_avg is a global value there are performance 5326 * considerations. 5327 * 5328 * In order to avoid having to look at the other cfs_rq's, we use a 5329 * differential update where we store the last value we propagated. This in 5330 * turn allows skipping updates if the differential is 'small'. 5331 * 5332 * Updating tg's load_avg is necessary before update_cfs_group(). 5333 */ 5334 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) 5335 { 5336 long dl, dr; 5337 u64 now; 5338 5339 /* 5340 * No need to update load_avg for root_task_group as it is not used. 5341 */ 5342 if (cfs_rq->tg == &root_task_group) 5343 return; 5344 5345 /* rq has been offline and doesn't contribute to the share anymore: */ 5346 if (!cpu_active(cpu_of(rq_of(cfs_rq)))) 5347 return; 5348 5349 /* 5350 * For migration heavy workloads, access to tg->load_avg can be 5351 * unbound. Limit the update rate to at most once per ms. 5352 */ 5353 now = rq_clock(rq_of(cfs_rq)); 5354 if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) 5355 return; 5356 5357 dl = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 5358 dr = cfs_rq->avg.runnable_avg - cfs_rq->tg_runnable_avg_contrib; 5359 if (abs(dl) > cfs_rq->tg_load_avg_contrib / 64 || 5360 abs(dr) > cfs_rq->tg_runnable_avg_contrib / 64) { 5361 atomic_long_add(dl, &cfs_rq->tg->load_avg); 5362 atomic_long_add(dr, &cfs_rq->tg->runnable_avg); 5363 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 5364 cfs_rq->tg_runnable_avg_contrib = cfs_rq->avg.runnable_avg; 5365 cfs_rq->last_update_tg_load_avg = now; 5366 } 5367 } 5368 5369 static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) 5370 { 5371 long dl, dr; 5372 u64 now; 5373 5374 /* 5375 * No need to update load_avg for root_task_group, as it is not used. 5376 */ 5377 if (cfs_rq->tg == &root_task_group) 5378 return; 5379 5380 now = rq_clock(rq_of(cfs_rq)); 5381 dl = 0 - cfs_rq->tg_load_avg_contrib; 5382 dr = 0 - cfs_rq->tg_runnable_avg_contrib; 5383 atomic_long_add(dl, &cfs_rq->tg->load_avg); 5384 atomic_long_add(dr, &cfs_rq->tg->runnable_avg); 5385 cfs_rq->tg_load_avg_contrib = 0; 5386 cfs_rq->tg_runnable_avg_contrib = 0; 5387 cfs_rq->last_update_tg_load_avg = now; 5388 } 5389 5390 /* CPU offline callback: */ 5391 static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq) 5392 { 5393 struct task_group *tg; 5394 5395 lockdep_assert_rq_held(rq); 5396 5397 /* 5398 * The rq clock has already been updated in 5399 * set_rq_offline(), so we should skip updating 5400 * the rq clock again in unthrottle_cfs_rq(). 5401 */ 5402 rq_clock_start_loop_update(rq); 5403 5404 guard(rcu)(); 5405 5406 list_for_each_entry_rcu(tg, &task_groups, list) { 5407 struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); 5408 5409 clear_tg_load_avg(cfs_rq); 5410 } 5411 5412 rq_clock_stop_loop_update(rq); 5413 } 5414 5415 /* 5416 * Called within set_task_rq() right before setting a task's CPU. The 5417 * caller only guarantees p->pi_lock is held; no other assumptions, 5418 * including the state of rq->lock, should be made. 5419 */ 5420 void set_task_rq_fair(struct sched_entity *se, 5421 struct cfs_rq *prev, struct cfs_rq *next) 5422 { 5423 u64 p_last_update_time; 5424 u64 n_last_update_time; 5425 5426 if (!sched_feat(ATTACH_AGE_LOAD)) 5427 return; 5428 5429 /* 5430 * We are supposed to update the task to "current" time, then its up to 5431 * date and ready to go to new CPU/cfs_rq. But we have difficulty in 5432 * getting what current time is, so simply throw away the out-of-date 5433 * time. This will result in the wakee task is less decayed, but giving 5434 * the wakee more load sounds not bad. 5435 */ 5436 if (!(se->avg.last_update_time && prev)) 5437 return; 5438 5439 p_last_update_time = cfs_rq_last_update_time(prev); 5440 n_last_update_time = cfs_rq_last_update_time(next); 5441 5442 __update_load_avg_blocked_se(p_last_update_time, se); 5443 se->avg.last_update_time = n_last_update_time; 5444 } 5445 5446 /* 5447 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to 5448 * propagate its contribution. The key to this propagation is the invariant 5449 * that for each group: 5450 * 5451 * ge->avg == grq->avg (1) 5452 * 5453 * _IFF_ we look at the pure running and runnable sums. Because they 5454 * represent the very same entity, just at different points in the hierarchy. 5455 * 5456 * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial 5457 * and simply copies the running/runnable sum over (but still wrong, because 5458 * the group entity and group rq do not have their PELT windows aligned). 5459 * 5460 * However, update_tg_cfs_load() is more complex. So we have: 5461 * 5462 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2) 5463 * 5464 * And since, like util, the runnable part should be directly transferable, 5465 * the following would _appear_ to be the straight forward approach: 5466 * 5467 * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3) 5468 * 5469 * And per (1) we have: 5470 * 5471 * ge->avg.runnable_avg == grq->avg.runnable_avg 5472 * 5473 * Which gives: 5474 * 5475 * ge->load.weight * grq->avg.load_avg 5476 * ge->avg.load_avg = ----------------------------------- (4) 5477 * grq->load.weight 5478 * 5479 * Except that is wrong! 5480 * 5481 * Because while for entities historical weight is not important and we 5482 * really only care about our future and therefore can consider a pure 5483 * runnable sum, runqueues can NOT do this. 5484 * 5485 * We specifically want runqueues to have a load_avg that includes 5486 * historical weights. Those represent the blocked load, the load we expect 5487 * to (shortly) return to us. This only works by keeping the weights as 5488 * integral part of the sum. We therefore cannot decompose as per (3). 5489 * 5490 * Another reason this doesn't work is that runnable isn't a 0-sum entity. 5491 * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the 5492 * rq itself is runnable anywhere between 2/3 and 1 depending on how the 5493 * runnable section of these tasks overlap (or not). If they were to perfectly 5494 * align the rq as a whole would be runnable 2/3 of the time. If however we 5495 * always have at least 1 runnable task, the rq as a whole is always runnable. 5496 * 5497 * So we'll have to approximate.. :/ 5498 * 5499 * Given the constraint: 5500 * 5501 * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX 5502 * 5503 * We can construct a rule that adds runnable to a rq by assuming minimal 5504 * overlap. 5505 * 5506 * On removal, we'll assume each task is equally runnable; which yields: 5507 * 5508 * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight 5509 * 5510 * XXX: only do this for the part of runnable > running ? 5511 * 5512 */ 5513 static inline void 5514 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 5515 { 5516 long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg; 5517 u32 new_sum, divider; 5518 5519 /* Nothing to update */ 5520 if (!delta_avg) 5521 return; 5522 5523 /* 5524 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 5525 * See ___update_load_avg() for details. 5526 */ 5527 divider = get_pelt_divider(&cfs_rq->avg); 5528 5529 /* Set new sched_entity's utilization */ 5530 se->avg.util_avg = gcfs_rq->avg.util_avg; 5531 new_sum = se->avg.util_avg * divider; 5532 delta_sum = (long)new_sum - (long)se->avg.util_sum; 5533 se->avg.util_sum = new_sum; 5534 5535 /* Update parent cfs_rq utilization */ 5536 __update_sa(&cfs_rq->avg, util, delta_avg, delta_sum); 5537 } 5538 5539 static inline void 5540 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 5541 { 5542 long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; 5543 u64 new_sum; 5544 u32 divider; 5545 5546 /* Nothing to update */ 5547 if (!delta_avg) 5548 return; 5549 5550 /* 5551 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 5552 * See ___update_load_avg() for details. 5553 */ 5554 divider = get_pelt_divider(&cfs_rq->avg); 5555 5556 /* Set new sched_entity's runnable */ 5557 se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; 5558 new_sum = (u64)se->avg.runnable_avg * divider; 5559 delta_sum = (long)new_sum - (long)se->avg.runnable_sum; 5560 se->avg.runnable_sum = new_sum; 5561 5562 /* Update parent cfs_rq runnable */ 5563 __update_sa(&cfs_rq->avg, runnable, delta_avg, delta_sum); 5564 } 5565 5566 static inline void 5567 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 5568 { 5569 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 5570 unsigned long load_avg; 5571 u64 load_sum = 0; 5572 s64 delta_sum; 5573 u32 divider; 5574 5575 if (!runnable_sum) 5576 return; 5577 5578 gcfs_rq->prop_runnable_sum = 0; 5579 5580 /* 5581 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 5582 * See ___update_load_avg() for details. 5583 */ 5584 divider = get_pelt_divider(&cfs_rq->avg); 5585 5586 if (runnable_sum >= 0) { 5587 /* 5588 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until 5589 * the CPU is saturated running == runnable. 5590 */ 5591 runnable_sum += se->avg.load_sum; 5592 runnable_sum = min_t(long, runnable_sum, divider); 5593 } else { 5594 /* 5595 * Estimate the new unweighted runnable_sum of the gcfs_rq by 5596 * assuming all tasks are equally runnable. 5597 */ 5598 if (scale_load_down(gcfs_rq->load.weight)) { 5599 load_sum = div_u64(gcfs_rq->avg.load_sum, 5600 scale_load_down(gcfs_rq->load.weight)); 5601 } 5602 5603 /* But make sure to not inflate se's runnable */ 5604 runnable_sum = min(se->avg.load_sum, load_sum); 5605 } 5606 5607 /* 5608 * runnable_sum can't be lower than running_sum 5609 * Rescale running sum to be in the same range as runnable sum 5610 * running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT] 5611 * runnable_sum is in [0 : LOAD_AVG_MAX] 5612 */ 5613 running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 5614 runnable_sum = max(runnable_sum, running_sum); 5615 5616 load_sum = se_weight(se) * runnable_sum; 5617 load_avg = div_u64(load_sum, divider); 5618 5619 delta_avg = load_avg - se->avg.load_avg; 5620 if (!delta_avg) 5621 return; 5622 5623 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 5624 5625 se->avg.load_sum = runnable_sum; 5626 se->avg.load_avg = load_avg; 5627 __update_sa(&cfs_rq->avg, load, delta_avg, delta_sum); 5628 } 5629 5630 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) 5631 { 5632 cfs_rq->propagate = 1; 5633 cfs_rq->prop_runnable_sum += runnable_sum; 5634 } 5635 5636 /* Update task and its cfs_rq load average */ 5637 static inline int propagate_entity_load_avg(struct sched_entity *se) 5638 { 5639 struct cfs_rq *cfs_rq, *gcfs_rq; 5640 5641 if (entity_is_task(se)) 5642 return 0; 5643 5644 gcfs_rq = group_cfs_rq(se); 5645 if (!gcfs_rq->propagate) 5646 return 0; 5647 5648 gcfs_rq->propagate = 0; 5649 5650 cfs_rq = cfs_rq_of(se); 5651 5652 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 5653 5654 update_tg_cfs_util(cfs_rq, se, gcfs_rq); 5655 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); 5656 update_tg_cfs_load(cfs_rq, se, gcfs_rq); 5657 5658 trace_pelt_cfs_tp(cfs_rq); 5659 trace_pelt_se_tp(se); 5660 5661 return 1; 5662 } 5663 5664 /* 5665 * Check if we need to update the load and the utilization of a blocked 5666 * group_entity: 5667 */ 5668 static inline bool skip_blocked_update(struct sched_entity *se) 5669 { 5670 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 5671 5672 /* 5673 * If sched_entity still have not zero load or utilization, we have to 5674 * decay it: 5675 */ 5676 if (se->avg.load_avg || se->avg.util_avg) 5677 return false; 5678 5679 /* 5680 * If there is a pending propagation, we have to update the load and 5681 * the utilization of the sched_entity: 5682 */ 5683 if (gcfs_rq->propagate) 5684 return false; 5685 5686 /* 5687 * Otherwise, the load and the utilization of the sched_entity is 5688 * already zero and there is no pending propagation, so it will be a 5689 * waste of time to try to decay it: 5690 */ 5691 return true; 5692 } 5693 5694 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 5695 5696 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {} 5697 5698 static inline void clear_tg_offline_cfs_rqs(struct rq *rq) {} 5699 5700 static inline int propagate_entity_load_avg(struct sched_entity *se) 5701 { 5702 return 0; 5703 } 5704 5705 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} 5706 5707 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 5708 5709 #ifdef CONFIG_NO_HZ_COMMON 5710 static inline void migrate_se_pelt_lag(struct sched_entity *se) 5711 { 5712 u64 throttled = 0, now, lut; 5713 struct cfs_rq *cfs_rq; 5714 struct rq *rq; 5715 bool is_idle; 5716 5717 if (load_avg_is_decayed(&se->avg)) 5718 return; 5719 5720 cfs_rq = cfs_rq_of(se); 5721 rq = rq_of(cfs_rq); 5722 5723 rcu_read_lock(); 5724 is_idle = is_idle_task(rcu_dereference_all(rq->curr)); 5725 rcu_read_unlock(); 5726 5727 /* 5728 * The lag estimation comes with a cost we don't want to pay all the 5729 * time. Hence, limiting to the case where the source CPU is idle and 5730 * we know we are at the greatest risk to have an outdated clock. 5731 */ 5732 if (!is_idle) 5733 return; 5734 5735 /* 5736 * Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where: 5737 * 5738 * last_update_time (the cfs_rq's last_update_time) 5739 * = cfs_rq_clock_pelt()@cfs_rq_idle 5740 * = rq_clock_pelt()@cfs_rq_idle 5741 * - cfs->throttled_clock_pelt_time@cfs_rq_idle 5742 * 5743 * cfs_idle_lag (delta between rq's update and cfs_rq's update) 5744 * = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle 5745 * 5746 * rq_idle_lag (delta between now and rq's update) 5747 * = sched_clock_cpu() - rq_clock()@rq_idle 5748 * 5749 * We can then write: 5750 * 5751 * now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time + 5752 * sched_clock_cpu() - rq_clock()@rq_idle 5753 * Where: 5754 * rq_clock_pelt()@rq_idle is rq->clock_pelt_idle 5755 * rq_clock()@rq_idle is rq->clock_idle 5756 * cfs->throttled_clock_pelt_time@cfs_rq_idle 5757 * is cfs_rq->throttled_pelt_idle 5758 */ 5759 5760 #ifdef CONFIG_CFS_BANDWIDTH 5761 throttled = u64_u32_load(cfs_rq->throttled_pelt_idle); 5762 /* The clock has been stopped for throttling */ 5763 if (throttled == U64_MAX) 5764 return; 5765 #endif 5766 now = u64_u32_load(rq->clock_pelt_idle); 5767 /* 5768 * Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case 5769 * is observed the old clock_pelt_idle value and the new clock_idle, 5770 * which lead to an underestimation. The opposite would lead to an 5771 * overestimation. 5772 */ 5773 smp_rmb(); 5774 lut = cfs_rq_last_update_time(cfs_rq); 5775 5776 now -= throttled; 5777 if (now < lut) 5778 /* 5779 * cfs_rq->avg.last_update_time is more recent than our 5780 * estimation, let's use it. 5781 */ 5782 now = lut; 5783 else 5784 now += sched_clock_cpu(cpu_of(rq)) - u64_u32_load(rq->clock_idle); 5785 5786 __update_load_avg_blocked_se(now, se); 5787 } 5788 #else /* !CONFIG_NO_HZ_COMMON: */ 5789 static void migrate_se_pelt_lag(struct sched_entity *se) {} 5790 #endif /* !CONFIG_NO_HZ_COMMON */ 5791 5792 /** 5793 * update_cfs_rq_load_avg - update the cfs_rq's load/util averages 5794 * @now: current time, as per cfs_rq_clock_pelt() 5795 * @cfs_rq: cfs_rq to update 5796 * 5797 * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) 5798 * avg. The immediate corollary is that all (fair) tasks must be attached. 5799 * 5800 * cfs_rq->avg is used for task_h_load() and update_cfs_group() for example. 5801 * 5802 * Return: true if the load decayed or we removed load. 5803 * 5804 * Since both these conditions indicate a changed cfs_rq->avg.load we should 5805 * call update_tg_load_avg() when this function returns true. 5806 */ 5807 static inline int 5808 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 5809 { 5810 unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0; 5811 struct sched_avg *sa = &cfs_rq->avg; 5812 int decayed = 0; 5813 5814 if (cfs_rq->removed.nr) { 5815 unsigned long r; 5816 u32 divider = get_pelt_divider(&cfs_rq->avg); 5817 5818 raw_spin_lock(&cfs_rq->removed.lock); 5819 swap(cfs_rq->removed.util_avg, removed_util); 5820 swap(cfs_rq->removed.load_avg, removed_load); 5821 swap(cfs_rq->removed.runnable_avg, removed_runnable); 5822 cfs_rq->removed.nr = 0; 5823 raw_spin_unlock(&cfs_rq->removed.lock); 5824 5825 r = removed_load; 5826 __update_sa(sa, load, -r, -r*divider); 5827 5828 r = removed_util; 5829 __update_sa(sa, util, -r, -r*divider); 5830 5831 r = removed_runnable; 5832 __update_sa(sa, runnable, -r, -r*divider); 5833 5834 /* 5835 * removed_runnable is the unweighted version of removed_load so we 5836 * can use it to estimate removed_load_sum. 5837 */ 5838 add_tg_cfs_propagate(cfs_rq, 5839 -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT); 5840 5841 decayed = 1; 5842 } 5843 5844 decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 5845 u64_u32_store_copy(sa->last_update_time, 5846 cfs_rq->last_update_time_copy, 5847 sa->last_update_time); 5848 return decayed; 5849 } 5850 5851 /** 5852 * attach_entity_load_avg - attach this entity to its cfs_rq load avg 5853 * @cfs_rq: cfs_rq to attach to 5854 * @se: sched_entity to attach 5855 * 5856 * Must call update_cfs_rq_load_avg() before this, since we rely on 5857 * cfs_rq->avg.last_update_time being current. 5858 */ 5859 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 5860 { 5861 /* 5862 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 5863 * See ___update_load_avg() for details. 5864 */ 5865 u32 divider = get_pelt_divider(&cfs_rq->avg); 5866 5867 /* 5868 * When we attach the @se to the @cfs_rq, we must align the decay 5869 * window because without that, really weird and wonderful things can 5870 * happen. 5871 * 5872 * XXX illustrate 5873 */ 5874 se->avg.last_update_time = cfs_rq->avg.last_update_time; 5875 se->avg.period_contrib = cfs_rq->avg.period_contrib; 5876 5877 /* 5878 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new 5879 * period_contrib. This isn't strictly correct, but since we're 5880 * entirely outside of the PELT hierarchy, nobody cares if we truncate 5881 * _sum a little. 5882 */ 5883 se->avg.util_sum = se->avg.util_avg * divider; 5884 5885 se->avg.runnable_sum = se->avg.runnable_avg * divider; 5886 5887 se->avg.load_sum = se->avg.load_avg * divider; 5888 if (se_weight(se) < se->avg.load_sum) 5889 se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se)); 5890 else 5891 se->avg.load_sum = 1; 5892 5893 enqueue_load_avg(cfs_rq, se); 5894 cfs_rq->avg.util_avg += se->avg.util_avg; 5895 cfs_rq->avg.util_sum += se->avg.util_sum; 5896 cfs_rq->avg.runnable_avg += se->avg.runnable_avg; 5897 cfs_rq->avg.runnable_sum += se->avg.runnable_sum; 5898 5899 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); 5900 5901 cfs_rq_util_change(cfs_rq, 0); 5902 5903 trace_pelt_cfs_tp(cfs_rq); 5904 } 5905 5906 /** 5907 * detach_entity_load_avg - detach this entity from its cfs_rq load avg 5908 * @cfs_rq: cfs_rq to detach from 5909 * @se: sched_entity to detach 5910 * 5911 * Must call update_cfs_rq_load_avg() before this, since we rely on 5912 * cfs_rq->avg.last_update_time being current. 5913 */ 5914 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 5915 { 5916 dequeue_load_avg(cfs_rq, se); 5917 __update_sa(&cfs_rq->avg, util, -se->avg.util_avg, -se->avg.util_sum); 5918 __update_sa(&cfs_rq->avg, runnable, -se->avg.runnable_avg, -se->avg.runnable_sum); 5919 5920 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); 5921 5922 cfs_rq_util_change(cfs_rq, 0); 5923 5924 trace_pelt_cfs_tp(cfs_rq); 5925 } 5926 5927 #define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) 5928 5929 static inline void util_est_update(struct sched_entity *se) 5930 { 5931 unsigned int ewma, dequeued, last_ewma_diff; 5932 5933 if (!sched_feat(UTIL_EST)) 5934 return; 5935 5936 /* Get current estimate of utilization */ 5937 ewma = READ_ONCE(se->avg.util_est); 5938 5939 /* 5940 * If the PELT values haven't changed since enqueue time, 5941 * skip the util_est update. 5942 */ 5943 if (ewma & UTIL_AVG_UNCHANGED) 5944 return; 5945 5946 /* Get utilization at dequeue */ 5947 dequeued = READ_ONCE(se->avg.util_avg); 5948 5949 /* 5950 * Reset EWMA on utilization increases, the moving average is used only 5951 * to smooth utilization decreases. 5952 */ 5953 if (ewma <= dequeued) { 5954 ewma = dequeued; 5955 goto done; 5956 } 5957 5958 /* 5959 * Skip update of task's estimated utilization when its members are 5960 * already ~1% close to its last activation value. 5961 */ 5962 last_ewma_diff = ewma - dequeued; 5963 if (last_ewma_diff < UTIL_EST_MARGIN) 5964 goto done; 5965 5966 /* 5967 * To avoid underestimate of task utilization, skip updates of EWMA if 5968 * we cannot grant that thread got all CPU time it wanted. 5969 */ 5970 if ((dequeued + UTIL_EST_MARGIN) < READ_ONCE(se->avg.runnable_avg)) 5971 goto done; 5972 5973 /* 5974 * Update Task's estimated utilization 5975 * 5976 * When *p completes an activation we can consolidate another sample 5977 * of the task size. This is done by using this value to update the 5978 * Exponential Weighted Moving Average (EWMA): 5979 * 5980 * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) 5981 * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) 5982 * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) 5983 * = w * ( -last_ewma_diff ) + ewma(t-1) 5984 * = w * (-last_ewma_diff + ewma(t-1) / w) 5985 * 5986 * Where 'w' is the weight of new samples, which is configured to be 5987 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) 5988 */ 5989 ewma <<= UTIL_EST_WEIGHT_SHIFT; 5990 ewma -= last_ewma_diff; 5991 ewma >>= UTIL_EST_WEIGHT_SHIFT; 5992 done: 5993 ewma |= UTIL_AVG_UNCHANGED; 5994 WRITE_ONCE(se->avg.util_est, ewma); 5995 5996 trace_sched_util_est_se_tp(se); 5997 } 5998 5999 /* 6000 * Optional action to be done while updating the load average 6001 */ 6002 #define UPDATE_TG 0x01 6003 #define SKIP_AGE_LOAD 0x02 6004 #define DO_ATTACH 0x04 6005 #define DO_DETACH 0x08 6006 #define UPDATE_UTIL_EST 0x10 6007 6008 /* Update task and its cfs_rq load average */ 6009 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 6010 { 6011 u64 now = cfs_rq_clock_pelt(cfs_rq); 6012 int decayed; 6013 6014 /* 6015 * Track task load average for carrying it to new CPU after migrated, and 6016 * track group sched_entity load average for task_h_load calculation in migration 6017 */ 6018 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 6019 __update_load_avg_se(now, cfs_rq, se); 6020 6021 decayed = update_cfs_rq_load_avg(now, cfs_rq); 6022 decayed |= propagate_entity_load_avg(se); 6023 6024 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 6025 6026 /* 6027 * DO_ATTACH means we're here from enqueue_entity(). 6028 * !last_update_time means we've passed through 6029 * migrate_task_rq_fair() indicating we migrated. 6030 * 6031 * IOW we're enqueueing a task on a new CPU. 6032 */ 6033 attach_entity_load_avg(cfs_rq, se); 6034 update_tg_load_avg(cfs_rq); 6035 6036 } else if (flags & DO_DETACH) { 6037 /* 6038 * DO_DETACH means we're here from dequeue_entity() 6039 * and we are migrating task out of the CPU. 6040 */ 6041 detach_entity_load_avg(cfs_rq, se); 6042 update_tg_load_avg(cfs_rq); 6043 } else if (decayed) { 6044 cfs_rq_util_change(cfs_rq, 0); 6045 6046 if (flags & UPDATE_TG) 6047 update_tg_load_avg(cfs_rq); 6048 } 6049 6050 if (flags & UPDATE_UTIL_EST) 6051 util_est_update(se); 6052 } 6053 6054 /* 6055 * Synchronize entity load avg of dequeued entity without locking 6056 * the previous rq. 6057 */ 6058 static void sync_entity_load_avg(struct sched_entity *se) 6059 { 6060 struct cfs_rq *cfs_rq = cfs_rq_of(se); 6061 u64 last_update_time; 6062 6063 last_update_time = cfs_rq_last_update_time(cfs_rq); 6064 __update_load_avg_blocked_se(last_update_time, se); 6065 } 6066 6067 /* 6068 * Task first catches up with cfs_rq, and then subtract 6069 * itself from the cfs_rq (task must be off the queue now). 6070 */ 6071 static void remove_entity_load_avg(struct sched_entity *se) 6072 { 6073 struct cfs_rq *cfs_rq = cfs_rq_of(se); 6074 unsigned long flags; 6075 6076 /* 6077 * tasks cannot exit without having gone through wake_up_new_task() -> 6078 * enqueue_task_fair() which will have added things to the cfs_rq, 6079 * so we can remove unconditionally. 6080 */ 6081 6082 sync_entity_load_avg(se); 6083 6084 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); 6085 ++cfs_rq->removed.nr; 6086 cfs_rq->removed.util_avg += se->avg.util_avg; 6087 cfs_rq->removed.load_avg += se->avg.load_avg; 6088 cfs_rq->removed.runnable_avg += se->avg.runnable_avg; 6089 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); 6090 } 6091 6092 static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq) 6093 { 6094 return cfs_rq->avg.runnable_avg; 6095 } 6096 6097 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 6098 { 6099 return cfs_rq->avg.load_avg; 6100 } 6101 6102 static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) 6103 __must_hold(__rq_lockp(this_rq)); 6104 6105 static inline unsigned long task_util(struct task_struct *p) 6106 { 6107 return READ_ONCE(p->se.avg.util_avg); 6108 } 6109 6110 static inline unsigned long _task_util_est(struct task_struct *p) 6111 { 6112 return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED; 6113 } 6114 6115 static inline unsigned long task_util_est(struct task_struct *p) 6116 { 6117 return max(task_util(p), _task_util_est(p)); 6118 } 6119 6120 static inline void util_est_enqueue(struct cfs_rq *cfs_rq, 6121 struct task_struct *p) 6122 { 6123 unsigned int enqueued; 6124 6125 if (!sched_feat(UTIL_EST)) 6126 return; 6127 6128 /* Update root cfs_rq's estimated utilization */ 6129 enqueued = cfs_rq->avg.util_est; 6130 enqueued += _task_util_est(p); 6131 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); 6132 6133 trace_sched_util_est_cfs_tp(cfs_rq); 6134 } 6135 6136 static inline void util_est_dequeue(struct cfs_rq *cfs_rq, 6137 struct task_struct *p) 6138 { 6139 unsigned int enqueued; 6140 6141 if (!sched_feat(UTIL_EST)) 6142 return; 6143 6144 /* Update root cfs_rq's estimated utilization */ 6145 enqueued = cfs_rq->avg.util_est; 6146 enqueued -= min_t(unsigned int, enqueued, _task_util_est(p)); 6147 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); 6148 6149 trace_sched_util_est_cfs_tp(cfs_rq); 6150 } 6151 6152 static inline unsigned long get_actual_cpu_capacity(int cpu) 6153 { 6154 unsigned long capacity = arch_scale_cpu_capacity(cpu); 6155 6156 capacity -= max(hw_load_avg(cpu_rq(cpu)), cpufreq_get_pressure(cpu)); 6157 6158 return capacity; 6159 } 6160 6161 static inline int util_fits_cpu(unsigned long util, 6162 unsigned long uclamp_min, 6163 unsigned long uclamp_max, 6164 int cpu) 6165 { 6166 unsigned long capacity = capacity_of(cpu); 6167 unsigned long capacity_orig; 6168 bool fits, uclamp_max_fits; 6169 6170 /* 6171 * Check if the real util fits without any uclamp boost/cap applied. 6172 */ 6173 fits = fits_capacity(util, capacity); 6174 6175 if (!uclamp_is_used()) 6176 return fits; 6177 6178 /* 6179 * We must use arch_scale_cpu_capacity() for comparing against uclamp_min and 6180 * uclamp_max. We only care about capacity pressure (by using 6181 * capacity_of()) for comparing against the real util. 6182 * 6183 * If a task is boosted to 1024 for example, we don't want a tiny 6184 * pressure to skew the check whether it fits a CPU or not. 6185 * 6186 * Similarly if a task is capped to arch_scale_cpu_capacity(little_cpu), it 6187 * should fit a little cpu even if there's some pressure. 6188 * 6189 * Only exception is for HW or cpufreq pressure since it has a direct impact 6190 * on available OPP of the system. 6191 * 6192 * We honour it for uclamp_min only as a drop in performance level 6193 * could result in not getting the requested minimum performance level. 6194 * 6195 * For uclamp_max, we can tolerate a drop in performance level as the 6196 * goal is to cap the task. So it's okay if it's getting less. 6197 */ 6198 capacity_orig = arch_scale_cpu_capacity(cpu); 6199 6200 /* 6201 * We want to force a task to fit a cpu as implied by uclamp_max. 6202 * But we do have some corner cases to cater for.. 6203 * 6204 * 6205 * C=z 6206 * | ___ 6207 * | C=y | | 6208 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 6209 * | C=x | | | | 6210 * | ___ | | | | 6211 * | | | | | | | (util somewhere in this region) 6212 * | | | | | | | 6213 * | | | | | | | 6214 * +---------------------------------------- 6215 * CPU0 CPU1 CPU2 6216 * 6217 * In the above example if a task is capped to a specific performance 6218 * point, y, then when: 6219 * 6220 * * util = 80% of x then it does not fit on CPU0 and should migrate 6221 * to CPU1 6222 * * util = 80% of y then it is forced to fit on CPU1 to honour 6223 * uclamp_max request. 6224 * 6225 * which is what we're enforcing here. A task always fits if 6226 * uclamp_max <= capacity_orig. But when uclamp_max > capacity_orig, 6227 * the normal upmigration rules should withhold still. 6228 * 6229 * Only exception is when we are on max capacity, then we need to be 6230 * careful not to block overutilized state. This is so because: 6231 * 6232 * 1. There's no concept of capping at max_capacity! We can't go 6233 * beyond this performance level anyway. 6234 * 2. The system is being saturated when we're operating near 6235 * max capacity, it doesn't make sense to block overutilized. 6236 */ 6237 uclamp_max_fits = (capacity_orig == SCHED_CAPACITY_SCALE) && (uclamp_max == SCHED_CAPACITY_SCALE); 6238 uclamp_max_fits = !uclamp_max_fits && (uclamp_max <= capacity_orig); 6239 fits = fits || uclamp_max_fits; 6240 6241 /* 6242 * 6243 * C=z 6244 * | ___ (region a, capped, util >= uclamp_max) 6245 * | C=y | | 6246 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 6247 * | C=x | | | | 6248 * | ___ | | | | (region b, uclamp_min <= util <= uclamp_max) 6249 * |_ _ _|_ _|_ _ _ _| _ | _ _ _| _ | _ _ _ _ _ uclamp_min 6250 * | | | | | | | 6251 * | | | | | | | (region c, boosted, util < uclamp_min) 6252 * +---------------------------------------- 6253 * CPU0 CPU1 CPU2 6254 * 6255 * a) If util > uclamp_max, then we're capped, we don't care about 6256 * actual fitness value here. We only care if uclamp_max fits 6257 * capacity without taking margin/pressure into account. 6258 * See comment above. 6259 * 6260 * b) If uclamp_min <= util <= uclamp_max, then the normal 6261 * fits_capacity() rules apply. Except we need to ensure that we 6262 * enforce we remain within uclamp_max, see comment above. 6263 * 6264 * c) If util < uclamp_min, then we are boosted. Same as (b) but we 6265 * need to take into account the boosted value fits the CPU without 6266 * taking margin/pressure into account. 6267 * 6268 * Cases (a) and (b) are handled in the 'fits' variable already. We 6269 * just need to consider an extra check for case (c) after ensuring we 6270 * handle the case uclamp_min > uclamp_max. 6271 */ 6272 uclamp_min = min(uclamp_min, uclamp_max); 6273 if (fits && (util < uclamp_min) && 6274 (uclamp_min > get_actual_cpu_capacity(cpu))) 6275 return -1; 6276 6277 return fits; 6278 } 6279 6280 static inline int task_fits_cpu(struct task_struct *p, int cpu) 6281 { 6282 unsigned long uclamp_min = uclamp_eff_value(p, UCLAMP_MIN); 6283 unsigned long uclamp_max = uclamp_eff_value(p, UCLAMP_MAX); 6284 unsigned long util = task_util_est(p); 6285 /* 6286 * Return true only if the cpu fully fits the task requirements, which 6287 * include the utilization but also the performance hints. 6288 */ 6289 return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0); 6290 } 6291 6292 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) 6293 { 6294 int cpu = cpu_of(rq); 6295 6296 if (!sched_asym_cpucap_active()) 6297 return; 6298 6299 /* 6300 * Affinity allows us to go somewhere higher? Or are we on biggest 6301 * available CPU already? Or do we fit into this CPU ? 6302 */ 6303 if (!p || (p->nr_cpus_allowed == 1) || 6304 (arch_scale_cpu_capacity(cpu) == p->max_allowed_capacity) || 6305 task_fits_cpu(p, cpu)) { 6306 6307 rq->misfit_task_load = 0; 6308 return; 6309 } 6310 6311 /* 6312 * Make sure that misfit_task_load will not be null even if 6313 * task_h_load() returns 0. 6314 */ 6315 rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); 6316 } 6317 6318 void __setparam_fair(struct task_struct *p, const struct sched_attr *attr) 6319 { 6320 struct sched_entity *se = &p->se; 6321 6322 p->static_prio = NICE_TO_PRIO(attr->sched_nice); 6323 if (attr->sched_runtime) { 6324 se->custom_slice = 1; 6325 se->slice = clamp_t(u64, attr->sched_runtime, 6326 NSEC_PER_MSEC/10, /* HZ=1000 * 10 */ 6327 NSEC_PER_MSEC*100); /* HZ=100 / 10 */ 6328 } else { 6329 se->custom_slice = 0; 6330 se->slice = sysctl_sched_base_slice; 6331 } 6332 } 6333 6334 static void 6335 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 6336 { 6337 u64 vslice, vruntime = avg_vruntime(cfs_rq); 6338 unsigned int nr_queued = cfs_rq->h_nr_queued; 6339 bool update_zero = false; 6340 s64 lag = 0; 6341 6342 if (!se->custom_slice) 6343 se->slice = sysctl_sched_base_slice; 6344 vslice = calc_delta_fair(se->slice, se); 6345 6346 if (flags & ENQUEUE_QUEUED) 6347 nr_queued -= 1; 6348 6349 /* 6350 * Due to how V is constructed as the weighted average of entities, 6351 * adding tasks with positive lag, or removing tasks with negative lag 6352 * will move 'time' backwards, this can screw around with the lag of 6353 * other tasks. 6354 * 6355 * EEVDF: placement strategy #1 / #2 6356 */ 6357 if (sched_feat(PLACE_LAG) && nr_queued && se->vlag) { 6358 struct sched_entity *curr = cfs_rq->curr; 6359 long load, weight; 6360 6361 lag = se->vlag; 6362 6363 /* 6364 * If we want to place a task and preserve lag, we have to 6365 * consider the effect of the new entity on the weighted 6366 * average and compensate for this, otherwise lag can quickly 6367 * evaporate. 6368 * 6369 * Lag is defined as: 6370 * 6371 * lag_i = S - s_i = w_i * (V - v_i) 6372 * 6373 * To avoid the 'w_i' term all over the place, we only track 6374 * the virtual lag: 6375 * 6376 * vl_i = V - v_i <=> v_i = V - vl_i 6377 * 6378 * And we take V to be the weighted average of all v: 6379 * 6380 * V = (\Sum w_j*v_j) / W 6381 * 6382 * Where W is: \Sum w_j 6383 * 6384 * Then, the weighted average after adding an entity with lag 6385 * vl_i is given by: 6386 * 6387 * V' = (\Sum w_j*v_j + w_i*v_i) / (W + w_i) 6388 * = (W*V + w_i*(V - vl_i)) / (W + w_i) 6389 * = (W*V + w_i*V - w_i*vl_i) / (W + w_i) 6390 * = (V*(W + w_i) - w_i*vl_i) / (W + w_i) 6391 * = V - w_i*vl_i / (W + w_i) 6392 * 6393 * And the actual lag after adding an entity with vl_i is: 6394 * 6395 * vl'_i = V' - v_i 6396 * = V - w_i*vl_i / (W + w_i) - (V - vl_i) 6397 * = vl_i - w_i*vl_i / (W + w_i) 6398 * 6399 * Which is strictly less than vl_i. So in order to preserve lag 6400 * we should inflate the lag before placement such that the 6401 * effective lag after placement comes out right. 6402 * 6403 * As such, invert the above relation for vl'_i to get the vl_i 6404 * we need to use such that the lag after placement is the lag 6405 * we computed before dequeue. 6406 * 6407 * vl'_i = vl_i - w_i*vl_i / (W + w_i) 6408 * = ((W + w_i)*vl_i - w_i*vl_i) / (W + w_i) 6409 * 6410 * (W + w_i)*vl'_i = (W + w_i)*vl_i - w_i*vl_i 6411 * = W*vl_i 6412 * 6413 * vl_i = (W + w_i)*vl'_i / W 6414 */ 6415 load = cfs_rq->sum_weight; 6416 if (curr && curr->on_rq) 6417 load += avg_vruntime_weight(cfs_rq, curr->h_load.weight); 6418 if (WARN_ON_ONCE(!load)) 6419 load = 1; 6420 6421 weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); 6422 6423 /* 6424 * Rewrite, because the product (W + w_i)*vl'_i might 6425 * overflow s64: 6426 * 6427 * vl_i = (W + w_i)*vl'_i / W 6428 * = (W*vl'_i + w_i*vl'_i) / W 6429 * = vl'_i + w_i*vl'_i/W 6430 */ 6431 lag += div64_long(lag * weight, load); 6432 6433 /* 6434 * A heavy entity (relative to the tree) will pull the 6435 * avg_vruntime close to its vruntime position on enqueue. But 6436 * the zero_vruntime point is only updated at the next 6437 * update_deadline()/place_entity()/update_entity_lag(). 6438 * 6439 * Specifically (see the comment near avg_vruntime_weight()): 6440 * 6441 * sum_w_vruntime = \Sum (v_i - v0) * w_i 6442 * 6443 * Note that if v0 is near a light entity, both terms will be 6444 * small for the light entity, while in that case both terms 6445 * are large for the heavy entity, leading to risk of 6446 * overflow. 6447 * 6448 * OTOH if v0 is near the heavy entity, then the difference is 6449 * larger for the light entity, but the factor is small, while 6450 * for the heavy entity the difference is small but the factor 6451 * is large. Avoiding the multiplication overflow. 6452 */ 6453 if (weight > load) 6454 update_zero = true; 6455 } 6456 6457 se->vruntime = vruntime - lag; 6458 6459 if (update_zero) 6460 update_zero_vruntime(cfs_rq, -lag); 6461 6462 if (sched_feat(PLACE_REL_DEADLINE) && se->rel_deadline) { 6463 se->deadline += se->vruntime; 6464 se->rel_deadline = 0; 6465 return; 6466 } 6467 6468 /* 6469 * When joining the competition; the existing tasks will be, 6470 * on average, halfway through their slice, as such start tasks 6471 * off with half a slice to ease into the competition. 6472 */ 6473 if (sched_feat(PLACE_DEADLINE_INITIAL) && (flags & ENQUEUE_INITIAL)) 6474 vslice /= 2; 6475 6476 /* 6477 * EEVDF: vd_i = ve_i + r_i/w_i 6478 */ 6479 se->deadline = se->vruntime + vslice; 6480 } 6481 6482 static void check_enqueue_throttle(struct cfs_rq *cfs_rq); 6483 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); 6484 6485 static void 6486 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 6487 { 6488 /* 6489 * When enqueuing a sched_entity, we must: 6490 * - Update loads to have both entity and cfs_rq synced with now. 6491 * - For group_entity, update its runnable_weight to reflect the new 6492 * h_nr_runnable of its group cfs_rq. 6493 * - For group_entity, update its weight to reflect the new share of 6494 * its group cfs_rq 6495 * - Add its new weight to cfs_rq->load.weight 6496 */ 6497 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); 6498 se_update_runnable(se); 6499 /* 6500 * XXX update_load_avg() above will have attached us to the pelt sum; 6501 * but update_cfs_group() here will re-adjust the weight and have to 6502 * undo/redo all that. Seems wasteful. 6503 */ 6504 update_cfs_group(se); 6505 6506 account_entity_enqueue(cfs_rq, se); 6507 6508 /* Entity has migrated, no longer consider this task hot */ 6509 if (flags & ENQUEUE_MIGRATED) 6510 se->exec_start = 0; 6511 6512 check_schedstat_required(); 6513 update_stats_enqueue_fair(cfs_rq, se, flags); 6514 se->on_rq = 1; 6515 6516 if (cfs_rq->nr_queued == 1) { 6517 check_enqueue_throttle(cfs_rq); 6518 list_add_leaf_cfs_rq(cfs_rq); 6519 #ifdef CONFIG_CFS_BANDWIDTH 6520 if (cfs_rq->pelt_clock_throttled) { 6521 struct rq *rq = rq_of(cfs_rq); 6522 6523 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 6524 cfs_rq->throttled_clock_pelt; 6525 cfs_rq->pelt_clock_throttled = 0; 6526 } 6527 #endif 6528 } 6529 } 6530 6531 static void set_next_buddy(struct cfs_rq *cfs_rq, struct sched_entity *se) 6532 { 6533 if (WARN_ON_ONCE(!se->on_rq || se->sched_delayed)) 6534 return; 6535 if (se_is_idle(se)) 6536 return; 6537 cfs_rq->next = se; 6538 } 6539 6540 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 6541 { 6542 if (cfs_rq->next == se) 6543 cfs_rq->next = NULL; 6544 } 6545 6546 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 6547 6548 static void set_delayed(struct sched_entity *se) 6549 { 6550 /* 6551 * Delayed se of cfs_rq have no tasks queued on them. 6552 * Do not adjust h_nr_runnable since __dequeue_task() 6553 * will account it for blocked tasks. 6554 * 6555 * This check can be removed because when flat pick 6556 * patches get merged as only task can get delayed, 6557 * same for clear_delayed(). 6558 */ 6559 if (!entity_is_task(se)) { 6560 se->sched_delayed = 1; 6561 return; 6562 } 6563 6564 /* 6565 * Drop a task leaving the runnable set. 6566 * Needs to be called before sched_delayed is set. 6567 * clear_delayed() mirrors this after clearing the flag. 6568 */ 6569 pref_llc_running_dec(rq_of(cfs_rq_of(se)), task_of(se)); 6570 se->sched_delayed = 1; 6571 6572 for_each_sched_entity(se) { 6573 struct cfs_rq *cfs_rq = cfs_rq_of(se); 6574 6575 cfs_rq->h_nr_runnable--; 6576 } 6577 } 6578 6579 static void clear_delayed(struct sched_entity *se) 6580 { 6581 se->sched_delayed = 0; 6582 6583 /* 6584 * Delayed se of cfs_rq have no tasks queued on them. 6585 * Do not adjust h_nr_runnable since a dequeue has 6586 * already accounted for it or an enqueue of a task 6587 * below it will account for it in enqueue_task_fair(). 6588 */ 6589 if (!entity_is_task(se)) 6590 return; 6591 6592 /* 6593 * Re-add on wake, after sched_delayed is cleared. On a final delayed 6594 * dequeue account_llc_dequeue() already cleared pref_llc_queued, so 6595 * this does nothing. 6596 */ 6597 pref_llc_running_inc(rq_of(cfs_rq_of(se)), task_of(se)); 6598 6599 for_each_sched_entity(se) { 6600 struct cfs_rq *cfs_rq = cfs_rq_of(se); 6601 6602 cfs_rq->h_nr_runnable++; 6603 } 6604 } 6605 6606 static void 6607 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 6608 { 6609 int action = UPDATE_TG; 6610 6611 if (entity_is_task(se)) { 6612 if (task_on_rq_migrating(task_of(se))) 6613 action |= DO_DETACH; 6614 6615 if ((flags & DEQUEUE_SLEEP) && !(flags & DEQUEUE_DELAYED)) 6616 action |= UPDATE_UTIL_EST; 6617 } 6618 6619 /* 6620 * When dequeuing a sched_entity, we must: 6621 * - Update loads to have both entity and cfs_rq synced with now. 6622 * - For group_entity, update its runnable_weight to reflect the new 6623 * h_nr_runnable of its group cfs_rq. 6624 * - Subtract its previous weight from cfs_rq->load.weight. 6625 * - For group entity, update its weight to reflect the new share 6626 * of its group cfs_rq. 6627 */ 6628 update_load_avg(cfs_rq, se, action); 6629 se_update_runnable(se); 6630 6631 update_stats_dequeue_fair(cfs_rq, se, flags); 6632 6633 se->on_rq = 0; 6634 account_entity_dequeue(cfs_rq, se); 6635 6636 /* return excess runtime on last dequeue */ 6637 return_cfs_rq_runtime(cfs_rq); 6638 6639 update_cfs_group(se); 6640 6641 if (cfs_rq->nr_queued == 0) { 6642 update_idle_cfs_rq_clock_pelt(cfs_rq); 6643 #ifdef CONFIG_CFS_BANDWIDTH 6644 if (throttled_hierarchy(cfs_rq)) { 6645 struct rq *rq = rq_of(cfs_rq); 6646 6647 list_del_leaf_cfs_rq(cfs_rq); 6648 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 6649 cfs_rq->pelt_clock_throttled = 1; 6650 } 6651 #endif 6652 } 6653 } 6654 6655 static void 6656 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 6657 { 6658 /* 'current' is not kept within the tree. */ 6659 if (se->on_rq) { 6660 /* 6661 * Any task has to be enqueued before it get to execute on 6662 * a CPU. So account for the time it spent waiting on the 6663 * runqueue. 6664 */ 6665 update_stats_wait_end_fair(cfs_rq, se); 6666 update_load_avg(cfs_rq, se, UPDATE_TG); 6667 } 6668 6669 update_stats_curr_start(cfs_rq, se); 6670 WARN_ON_ONCE(cfs_rq->h_curr); 6671 cfs_rq->h_curr = se; 6672 6673 /* 6674 * Track our maximum slice length, if the CPU's load is at 6675 * least twice that of our own weight (i.e. don't track it 6676 * when there are only lesser-weight tasks around): 6677 */ 6678 if (schedstat_enabled() && 6679 rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) { 6680 struct sched_statistics *stats; 6681 6682 stats = __schedstats_from_se(se); 6683 __schedstat_set(stats->slice_max, 6684 max((u64)stats->slice_max, 6685 se->sum_exec_runtime - se->prev_sum_exec_runtime)); 6686 } 6687 6688 se->prev_sum_exec_runtime = se->sum_exec_runtime; 6689 } 6690 6691 static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags); 6692 6693 static struct sched_entity * 6694 pick_next_entity(struct rq *rq, bool protect) 6695 { 6696 struct cfs_rq *cfs_rq = &rq->cfs; 6697 struct sched_entity *se; 6698 6699 se = pick_eevdf(cfs_rq, protect); 6700 if (se->sched_delayed) { 6701 __dequeue_task(rq, task_of(se), DEQUEUE_SLEEP | DEQUEUE_DELAYED); 6702 /* 6703 * Must not reference @se again, see __block_task(). 6704 */ 6705 return NULL; 6706 } 6707 return se; 6708 } 6709 6710 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) 6711 { 6712 /* 6713 * If still on the runqueue then deactivate_task() 6714 * was not called and update_curr() has to be done: 6715 */ 6716 if (prev->on_rq) 6717 update_curr(cfs_rq); 6718 6719 if (prev->on_rq) { 6720 update_stats_wait_start_fair(cfs_rq, prev); 6721 /* in !on_rq case, update occurred at dequeue */ 6722 update_load_avg(cfs_rq, prev, 0); 6723 } 6724 WARN_ON_ONCE(cfs_rq->h_curr != prev); 6725 cfs_rq->h_curr = NULL; 6726 } 6727 6728 static void 6729 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 6730 { 6731 /* 6732 * Update run-time statistics of the 'current'. 6733 */ 6734 update_curr(cfs_rq); 6735 6736 /* 6737 * Ensure that runnable average is periodically updated. 6738 */ 6739 update_load_avg(cfs_rq, curr, UPDATE_TG); 6740 update_cfs_group(curr); 6741 6742 #ifdef CONFIG_SCHED_HRTICK 6743 /* 6744 * queued ticks are scheduled to match the slice, so don't bother 6745 * validating it and just reschedule. 6746 */ 6747 if (queued) { 6748 resched_curr(rq_of(cfs_rq)); 6749 return; 6750 } 6751 #endif 6752 } 6753 6754 6755 /************************************************** 6756 * CFS bandwidth control machinery 6757 */ 6758 6759 #ifdef CONFIG_CFS_BANDWIDTH 6760 6761 #ifdef CONFIG_JUMP_LABEL 6762 static struct static_key __cfs_bandwidth_used; 6763 6764 static inline bool cfs_bandwidth_used(void) 6765 { 6766 return static_key_false(&__cfs_bandwidth_used); 6767 } 6768 6769 void cfs_bandwidth_usage_inc(void) 6770 { 6771 static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used); 6772 } 6773 6774 void cfs_bandwidth_usage_dec(void) 6775 { 6776 static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); 6777 } 6778 #else /* !CONFIG_JUMP_LABEL: */ 6779 static bool cfs_bandwidth_used(void) 6780 { 6781 return true; 6782 } 6783 6784 void cfs_bandwidth_usage_inc(void) {} 6785 void cfs_bandwidth_usage_dec(void) {} 6786 #endif /* !CONFIG_JUMP_LABEL */ 6787 6788 static inline u64 sched_cfs_bandwidth_slice(void) 6789 { 6790 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; 6791 } 6792 6793 /* 6794 * Replenish runtime according to assigned quota. We use sched_clock_cpu 6795 * directly instead of rq->clock to avoid adding additional synchronization 6796 * around rq->lock. 6797 * 6798 * requires cfs_b->lock 6799 */ 6800 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) 6801 { 6802 s64 runtime; 6803 6804 if (unlikely(cfs_b->quota == RUNTIME_INF)) 6805 return; 6806 6807 cfs_b->runtime += cfs_b->quota; 6808 runtime = cfs_b->runtime_snap - cfs_b->runtime; 6809 if (runtime > 0) { 6810 cfs_b->burst_time += runtime; 6811 cfs_b->nr_burst++; 6812 } 6813 6814 cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst); 6815 cfs_b->runtime_snap = cfs_b->runtime; 6816 } 6817 6818 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 6819 { 6820 return &tg->cfs_bandwidth; 6821 } 6822 6823 /* returns 0 on failure to allocate runtime */ 6824 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, 6825 struct cfs_rq *cfs_rq, u64 target_runtime) 6826 { 6827 u64 min_amount, amount = 0; 6828 6829 lockdep_assert_held(&cfs_b->lock); 6830 6831 /* note: this is a positive sum as runtime_remaining <= 0 */ 6832 min_amount = target_runtime - cfs_rq->runtime_remaining; 6833 6834 if (cfs_b->quota == RUNTIME_INF) 6835 amount = min_amount; 6836 else { 6837 start_cfs_bandwidth(cfs_b); 6838 6839 if (cfs_b->runtime > 0) { 6840 amount = min(cfs_b->runtime, min_amount); 6841 cfs_b->runtime -= amount; 6842 cfs_b->idle = 0; 6843 } 6844 } 6845 6846 cfs_rq->runtime_remaining += amount; 6847 6848 return cfs_rq->runtime_remaining > 0; 6849 } 6850 6851 static bool throttle_cfs_rq(struct cfs_rq *cfs_rq); 6852 6853 static bool __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 6854 { 6855 /* dock delta_exec before expiring quota (as it could span periods) */ 6856 cfs_rq->runtime_remaining -= delta_exec; 6857 6858 if (likely(cfs_rq->runtime_remaining > 0)) 6859 return false; 6860 6861 if (cfs_rq->throttled) 6862 return true; 6863 /* 6864 * throttle_cfs_rq() will try to extend the runtime first 6865 * before throttling the hierarchy. 6866 */ 6867 return throttle_cfs_rq(cfs_rq); 6868 } 6869 6870 static __always_inline 6871 bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 6872 { 6873 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) 6874 return false; 6875 6876 return __account_cfs_rq_runtime(cfs_rq, delta_exec); 6877 } 6878 6879 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 6880 { 6881 return cfs_bandwidth_used() && cfs_rq->throttled; 6882 } 6883 6884 static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq) 6885 { 6886 return cfs_bandwidth_used() && cfs_rq->pelt_clock_throttled; 6887 } 6888 6889 /* check whether cfs_rq, or any parent, is throttled */ 6890 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 6891 { 6892 return cfs_bandwidth_used() && cfs_rq->throttle_count; 6893 } 6894 6895 static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu) 6896 { 6897 return throttled_hierarchy(tg_cfs_rq(task_group(p), dst_cpu)); 6898 } 6899 6900 static inline bool task_is_throttled(struct task_struct *p) 6901 { 6902 return cfs_bandwidth_used() && p->throttled; 6903 } 6904 6905 static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags); 6906 static void throttle_cfs_rq_work(struct callback_head *work) 6907 { 6908 struct task_struct *p = container_of(work, struct task_struct, sched_throttle_work); 6909 struct sched_entity *se; 6910 struct cfs_rq *cfs_rq; 6911 struct rq *rq; 6912 6913 WARN_ON_ONCE(p != current); 6914 p->sched_throttle_work.next = &p->sched_throttle_work; 6915 6916 /* 6917 * If task is exiting, then there won't be a return to userspace, so we 6918 * don't have to bother with any of this. 6919 */ 6920 if ((p->flags & PF_EXITING)) 6921 return; 6922 6923 scoped_guard(task_rq_lock, p) { 6924 se = &p->se; 6925 cfs_rq = cfs_rq_of(se); 6926 6927 /* Raced, forget */ 6928 if (p->sched_class != &fair_sched_class) 6929 return; 6930 6931 /* 6932 * If not in limbo, then either replenish has happened or this 6933 * task got migrated out of the throttled cfs_rq, move along. 6934 */ 6935 if (!cfs_rq->throttle_count) 6936 return; 6937 rq = scope.rq; 6938 update_rq_clock(rq); 6939 WARN_ON_ONCE(p->throttled || !list_empty(&p->throttle_node)); 6940 dequeue_task_fair(rq, p, DEQUEUE_SLEEP | DEQUEUE_THROTTLE); 6941 list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list); 6942 /* 6943 * Must not set throttled before dequeue or dequeue will 6944 * mistakenly regard this task as an already throttled one. 6945 */ 6946 p->throttled = true; 6947 resched_curr(rq); 6948 } 6949 } 6950 6951 void init_cfs_throttle_work(struct task_struct *p) 6952 { 6953 init_task_work(&p->sched_throttle_work, throttle_cfs_rq_work); 6954 /* Protect against double add, see throttle_cfs_rq() and throttle_cfs_rq_work() */ 6955 p->sched_throttle_work.next = &p->sched_throttle_work; 6956 INIT_LIST_HEAD(&p->throttle_node); 6957 } 6958 6959 /* 6960 * Task is throttled and someone wants to dequeue it again: 6961 * it could be sched/core when core needs to do things like 6962 * task affinity change, task group change, task sched class 6963 * change etc. and in these cases, DEQUEUE_SLEEP is not set; 6964 * or the task is blocked after throttled due to freezer etc. 6965 * and in these cases, DEQUEUE_SLEEP is set. 6966 */ 6967 static void detach_task_cfs_rq(struct task_struct *p); 6968 static void dequeue_throttled_task(struct task_struct *p, int flags) 6969 { 6970 WARN_ON_ONCE(p->se.on_rq); 6971 list_del_init(&p->throttle_node); 6972 6973 /* Task blocked after throttling or is leaving the fair class: */ 6974 if (flags & (DEQUEUE_SLEEP | DEQUEUE_CLASS)) { 6975 p->throttled = false; 6976 return; 6977 } 6978 6979 /* 6980 * task is migrating off its old cfs_rq, detach 6981 * the task's load from its old cfs_rq. 6982 */ 6983 if (task_on_rq_migrating(p)) 6984 detach_task_cfs_rq(p); 6985 } 6986 6987 static bool enqueue_throttled_task(struct task_struct *p) 6988 { 6989 struct cfs_rq *cfs_rq = cfs_rq_of(&p->se); 6990 6991 /* @p should have gone through dequeue_throttled_task() first */ 6992 WARN_ON_ONCE(!list_empty(&p->throttle_node)); 6993 6994 /* 6995 * If the throttled task @p is enqueued to a throttled cfs_rq, 6996 * take the fast path by directly putting the task on the 6997 * target cfs_rq's limbo list. 6998 * 6999 * Do not do that when @p is current because the following race can 7000 * cause @p's group_node to be incorectly re-insterted in its rq's 7001 * cfs_tasks list, despite being throttled: 7002 * 7003 * cpuX cpuY 7004 * p ret2user 7005 * throttle_cfs_rq_work() sched_move_task(p) 7006 * LOCK task_rq_lock 7007 * dequeue_task_fair(p) 7008 * UNLOCK task_rq_lock 7009 * LOCK task_rq_lock 7010 * task_current_donor(p) == true 7011 * task_on_rq_queued(p) == true 7012 * dequeue_task(p) 7013 * put_prev_task(p) 7014 * sched_change_group() 7015 * enqueue_task(p) -> p's new cfs_rq 7016 * is throttled, go 7017 * fast path and skip 7018 * actual enqueue 7019 * set_next_task(p) 7020 * list_move(&se->group_node, &rq->cfs_tasks); // bug 7021 * schedule() 7022 * 7023 * In the above race case, @p current cfs_rq is in the same rq as 7024 * its previous cfs_rq because sched_move_task() only moves a task 7025 * to a different group from the same rq, so we can use its current 7026 * cfs_rq to derive rq and test if the task is current. 7027 */ 7028 if (throttled_hierarchy(cfs_rq) && 7029 !task_current_donor(rq_of(cfs_rq), p)) { 7030 list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list); 7031 return true; 7032 } 7033 7034 /* we can't take the fast path, do an actual enqueue*/ 7035 p->throttled = false; 7036 return false; 7037 } 7038 7039 static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags); 7040 static int tg_unthrottle_up(struct task_group *tg, void *data) 7041 { 7042 struct rq *rq = data; 7043 struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); 7044 struct task_struct *p, *tmp; 7045 LIST_HEAD(throttled_tasks); 7046 7047 /* 7048 * If cfs_rq->curr is set, the cfs_rq might not have caught up 7049 * since the last clock update. Do it now before we begin 7050 * queueing task onto it to save the need for unnecessarily 7051 * unthrottle the hierarchy for this cfs_rq to be throttled 7052 * right back again. 7053 */ 7054 update_curr(cfs_rq); 7055 7056 if (--cfs_rq->throttle_count) 7057 return 0; 7058 7059 if (cfs_rq->pelt_clock_throttled) { 7060 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 7061 cfs_rq->throttled_clock_pelt; 7062 cfs_rq->pelt_clock_throttled = 0; 7063 } 7064 7065 if (cfs_rq->throttled_clock_self) { 7066 u64 delta = rq_clock(rq) - cfs_rq->throttled_clock_self; 7067 7068 cfs_rq->throttled_clock_self = 0; 7069 7070 if (WARN_ON_ONCE((s64)delta < 0)) 7071 delta = 0; 7072 7073 cfs_rq->throttled_clock_self_time += delta; 7074 } 7075 7076 /* 7077 * Move the tasks to a local list since an update_curr() during 7078 * enqueue_task_fair() can throttle a higher cfs_rq, and it can 7079 * see the "throttled_limbo_list" being non-empty in 7080 * tg_throttle_down() if throttle_count turned 0 above. 7081 */ 7082 list_splice_init(&cfs_rq->throttled_limbo_list, &throttled_tasks); 7083 7084 /* Re-enqueue the tasks that have been throttled at this level. */ 7085 list_for_each_entry_safe(p, tmp, &throttled_tasks, throttle_node) { 7086 /* 7087 * Back to being throttled! Break out and put the remaining 7088 * tasks back onto the limbo_list to prevent running them 7089 * unnecessarily. 7090 */ 7091 if (cfs_rq->throttle_count) 7092 break; 7093 7094 list_del_init(&p->throttle_node); 7095 p->throttled = false; 7096 enqueue_task_fair(rq, p, ENQUEUE_WAKEUP); 7097 } 7098 7099 list_splice(&throttled_tasks, &cfs_rq->throttled_limbo_list); 7100 7101 /* Add cfs_rq with load or one or more already running entities to the list */ 7102 if (!cfs_rq_is_decayed(cfs_rq)) 7103 list_add_leaf_cfs_rq(cfs_rq); 7104 7105 return 0; 7106 } 7107 7108 static inline bool task_has_throttle_work(struct task_struct *p) 7109 { 7110 return p->sched_throttle_work.next != &p->sched_throttle_work; 7111 } 7112 7113 static inline void task_throttle_setup_work(struct task_struct *p) 7114 { 7115 if (task_has_throttle_work(p)) 7116 return; 7117 7118 /* 7119 * Kthreads and exiting tasks don't return to userspace, so adding the 7120 * work is pointless 7121 */ 7122 if ((p->flags & (PF_EXITING | PF_KTHREAD))) 7123 return; 7124 7125 task_work_add(p, &p->sched_throttle_work, TWA_RESUME); 7126 } 7127 7128 static void record_throttle_clock(struct cfs_rq *cfs_rq) 7129 { 7130 struct rq *rq = rq_of(cfs_rq); 7131 7132 if (cfs_rq_throttled(cfs_rq) && !cfs_rq->throttled_clock) 7133 cfs_rq->throttled_clock = rq_clock(rq); 7134 7135 if (!cfs_rq->throttled_clock_self) 7136 cfs_rq->throttled_clock_self = rq_clock(rq); 7137 } 7138 7139 static int tg_throttle_down(struct task_group *tg, void *data) 7140 { 7141 struct rq *rq = data; 7142 struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); 7143 7144 if (cfs_rq->throttle_count++) 7145 return 0; 7146 7147 /* 7148 * For cfs_rqs that still have entities enqueued, PELT clock 7149 * stop happens at dequeue time when all entities are dequeued. 7150 */ 7151 if (!cfs_rq->nr_queued) { 7152 list_del_leaf_cfs_rq(cfs_rq); 7153 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 7154 cfs_rq->pelt_clock_throttled = 1; 7155 } 7156 7157 WARN_ON_ONCE(cfs_rq->throttled_clock_self); 7158 WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_limbo_list)); 7159 return 0; 7160 } 7161 7162 static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) 7163 { 7164 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 7165 struct sched_entity *curr = cfs_rq->h_curr; 7166 struct rq *rq = rq_of(cfs_rq); 7167 7168 scoped_guard(raw_spinlock, &cfs_b->lock) { 7169 u64 target_runtime = 1; 7170 7171 /* 7172 * If cfs_rq->h_curr is still runnable, we are here from an 7173 * update_curr(). Request sysctl_sched_cfs_bandwidth_slice 7174 * worth of bandwidth to continue running. 7175 * 7176 * If the curr is not runnable, just request enough bandwidth 7177 * to be runnable next time the pick selects this cfs_rq. 7178 */ 7179 if (curr && curr->on_rq) 7180 target_runtime = sched_cfs_bandwidth_slice(); 7181 7182 /* 7183 * Check if We have raced with bandwidth becoming available. If 7184 * we actually throttled the timer might not unthrottle us for 7185 * an entire period. We additionally needed to make sure that 7186 * any subsequent check_cfs_rq_runtime calls agree not to 7187 * throttle us, as we may commit to do cfs put_prev+pick_next, 7188 * so we ask for 1ns of runtime rather than just check cfs_b. 7189 * 7190 * This will start the period timer if necessary. 7191 */ 7192 if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, target_runtime)) 7193 return false; 7194 7195 /* 7196 * No bandwidth available; Add ourselves on the list to be 7197 * unthrottled later. 7198 */ 7199 list_add_tail_rcu(&cfs_rq->throttled_list, 7200 &cfs_b->throttled_cfs_rq); 7201 } 7202 7203 /* freeze hierarchy runnable averages while throttled */ 7204 scoped_guard(rcu) 7205 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); 7206 7207 /* 7208 * Note: distribution will already see us throttled via the 7209 * throttled-list. rq->lock protects completion. 7210 */ 7211 cfs_rq->throttled = 1; 7212 WARN_ON_ONCE(cfs_rq->throttled_clock); 7213 7214 /* 7215 * If current hierarchy was throttled, add throttle work to the 7216 * current donor. In case of proxy-execution, the execution 7217 * context cannot exit to the userspace while holding a mutex 7218 * and the rule of throttle deferral to only throttle the 7219 * throttled context at exit to userspace is still preserved. 7220 */ 7221 if (curr && curr->on_rq) 7222 task_throttle_setup_work(rq->donor); 7223 7224 return true; 7225 } 7226 7227 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) 7228 { 7229 struct rq *rq = rq_of(cfs_rq); 7230 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 7231 struct sched_entity *se = cfs_rq_se(cfs_rq); 7232 7233 /* 7234 * It's possible we are called with runtime_remaining < 0 due to things 7235 * like async unthrottled us with a positive runtime_remaining but other 7236 * still running entities consumed those runtime before we reached here. 7237 * 7238 * We can't unthrottle this cfs_rq without any runtime remaining because 7239 * any enqueue in tg_unthrottle_up() will immediately trigger a throttle, 7240 * which is not supposed to happen on unthrottle path. 7241 * 7242 * Catch up on the remaining runtime since last clock update before 7243 * checking runtime remaining. 7244 */ 7245 update_curr(cfs_rq); 7246 if (cfs_rq->runtime_enabled && cfs_rq->runtime_remaining <= 0) 7247 return; 7248 7249 cfs_rq->throttled = 0; 7250 7251 scoped_guard(raw_spinlock, &cfs_b->lock) { 7252 list_del_rcu(&cfs_rq->throttled_list); 7253 7254 if (!cfs_rq->throttled_clock) 7255 break; 7256 7257 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; 7258 cfs_rq->throttled_clock = 0; 7259 } 7260 7261 /* update hierarchical throttle state */ 7262 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); 7263 7264 if (!cfs_rq->load.weight) { 7265 if (!cfs_rq->on_list) 7266 return; 7267 /* 7268 * Nothing to run but something to decay (on_list)? 7269 * Complete the branch. 7270 */ 7271 for_each_sched_entity(se) { 7272 if (list_add_leaf_cfs_rq(cfs_rq_of(se))) 7273 break; 7274 } 7275 } 7276 7277 assert_list_leaf_cfs_rq(rq); 7278 7279 /* Determine whether we need to wake up potentially idle CPU: */ 7280 if (rq->curr == rq->idle && rq->cfs.h_nr_queued) 7281 resched_curr(rq); 7282 } 7283 7284 static void __cfsb_csd_unthrottle(void *arg) 7285 { 7286 struct cfs_rq *cursor, *tmp; 7287 struct rq *rq = arg; 7288 7289 guard(rq_lock)(rq); 7290 7291 /* 7292 * Iterating over the list can trigger several call to 7293 * update_rq_clock() in unthrottle_cfs_rq(). 7294 * Do it once and skip the potential next ones. 7295 */ 7296 update_rq_clock(rq); 7297 rq_clock_start_loop_update(rq); 7298 7299 /* 7300 * Since we hold rq lock we're safe from concurrent manipulation of 7301 * the CSD list. However, this RCU critical section annotates the 7302 * fact that we pair with sched_free_group_rcu(), so that we cannot 7303 * race with group being freed in the window between removing it 7304 * from the list and advancing to the next entry in the list. 7305 */ 7306 guard(rcu)(); 7307 7308 list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list, 7309 throttled_csd_list) { 7310 list_del_init(&cursor->throttled_csd_list); 7311 7312 if (cfs_rq_throttled(cursor)) 7313 unthrottle_cfs_rq(cursor); 7314 } 7315 7316 rq_clock_stop_loop_update(rq); 7317 } 7318 7319 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 7320 { 7321 struct rq *rq = rq_of(cfs_rq); 7322 bool first; 7323 7324 if (rq == this_rq()) { 7325 update_rq_clock(rq); 7326 unthrottle_cfs_rq(cfs_rq); 7327 return; 7328 } 7329 7330 /* Already enqueued */ 7331 if (WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_csd_list))) 7332 return; 7333 7334 first = list_empty(&rq->cfsb_csd_list); 7335 list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list); 7336 if (first) 7337 smp_call_function_single_async(cpu_of(rq), &rq->cfsb_csd); 7338 } 7339 7340 static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 7341 { 7342 lockdep_assert_rq_held(rq_of(cfs_rq)); 7343 7344 if (WARN_ON_ONCE(!cfs_rq_throttled(cfs_rq) || 7345 cfs_rq->runtime_remaining <= 0)) 7346 return; 7347 7348 __unthrottle_cfs_rq_async(cfs_rq); 7349 } 7350 7351 static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) 7352 { 7353 bool throttled = false, unthrottle_local = false; 7354 int this_cpu = smp_processor_id(); 7355 u64 runtime, remaining = 1; 7356 struct cfs_rq *cfs_rq; 7357 struct rq *rq; 7358 7359 guard(rcu)(); 7360 7361 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, 7362 throttled_list) { 7363 rq = rq_of(cfs_rq); 7364 7365 if (!remaining) { 7366 throttled = true; 7367 break; 7368 } 7369 7370 guard(rq_lock_irqsave)(rq); 7371 7372 if (!cfs_rq_throttled(cfs_rq)) 7373 continue; 7374 7375 /* Already queued for async unthrottle */ 7376 if (!list_empty(&cfs_rq->throttled_csd_list)) 7377 continue; 7378 7379 if (cfs_rq->h_curr) { 7380 update_rq_clock(rq); 7381 update_curr(cfs_rq); 7382 } 7383 7384 /* By the above checks, this should never be true */ 7385 WARN_ON_ONCE(cfs_rq->runtime_remaining > 0); 7386 7387 scoped_guard(raw_spinlock, &cfs_b->lock) { 7388 runtime = -cfs_rq->runtime_remaining + 1; 7389 if (runtime > cfs_b->runtime) 7390 runtime = cfs_b->runtime; 7391 cfs_b->runtime -= runtime; 7392 remaining = cfs_b->runtime; 7393 } 7394 7395 cfs_rq->runtime_remaining += runtime; 7396 7397 /* 7398 * Ran out of bandwidth during distribution! 7399 * Indicate throttled entities and break early. 7400 */ 7401 if (cfs_rq->runtime_remaining <= 0) { 7402 throttled = true; 7403 break; 7404 } 7405 7406 /* we check whether we're throttled above */ 7407 if (cpu_of(rq) != this_cpu) { 7408 unthrottle_cfs_rq_async(cfs_rq); 7409 continue; 7410 } 7411 7412 /* 7413 * Allow a parallel async unthrottle to unthrottle 7414 * this cfs_rq too via __cfsb_csd_unthrottle(). 7415 * If we are first, do it ourselves at the end and 7416 * save on an IPI from remote CPUs. 7417 */ 7418 unthrottle_local = list_empty(&rq->cfsb_csd_list); 7419 list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list); 7420 } 7421 7422 if (unthrottle_local) { 7423 /* 7424 * Protect against an IPI that is also trying to flush 7425 * the unthrottled cfs_rq(s) from this CPU's csd_list. 7426 */ 7427 scoped_guard(irqsave) 7428 __cfsb_csd_unthrottle(cpu_rq(this_cpu)); 7429 } 7430 7431 return throttled; 7432 } 7433 7434 /* 7435 * Responsible for refilling a task_group's bandwidth and unthrottling its 7436 * cfs_rqs as appropriate. If there has been no activity within the last 7437 * period the timer is deactivated until scheduling resumes; cfs_b->idle is 7438 * used to track this state. 7439 */ 7440 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) 7441 __must_hold(&cfs_b->lock) 7442 { 7443 int throttled; 7444 7445 /* no need to continue the timer with no bandwidth constraint */ 7446 if (cfs_b->quota == RUNTIME_INF) 7447 goto out_deactivate; 7448 7449 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 7450 cfs_b->nr_periods += overrun; 7451 7452 /* Refill extra burst quota even if cfs_b->idle */ 7453 __refill_cfs_bandwidth_runtime(cfs_b); 7454 7455 /* 7456 * idle depends on !throttled (for the case of a large deficit), and if 7457 * we're going inactive then everything else can be deferred 7458 */ 7459 if (cfs_b->idle && !throttled) 7460 goto out_deactivate; 7461 7462 if (!throttled) { 7463 /* mark as potentially idle for the upcoming period */ 7464 cfs_b->idle = 1; 7465 return 0; 7466 } 7467 7468 /* account preceding periods in which throttling occurred */ 7469 cfs_b->nr_throttled += overrun; 7470 7471 /* 7472 * This check is repeated as we release cfs_b->lock while we unthrottle. 7473 */ 7474 while (throttled && cfs_b->runtime > 0) { 7475 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 7476 /* we can't nest cfs_b->lock while distributing bandwidth */ 7477 throttled = distribute_cfs_runtime(cfs_b); 7478 raw_spin_lock_irqsave(&cfs_b->lock, flags); 7479 } 7480 7481 /* 7482 * While we are ensured activity in the period following an 7483 * unthrottle, this also covers the case in which the new bandwidth is 7484 * insufficient to cover the existing bandwidth deficit. (Forcing the 7485 * timer to remain active while there are any throttled entities.) 7486 */ 7487 cfs_b->idle = 0; 7488 7489 return 0; 7490 7491 out_deactivate: 7492 return 1; 7493 } 7494 7495 /* a cfs_rq won't donate quota below this amount */ 7496 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; 7497 /* minimum remaining period time to redistribute slack quota */ 7498 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; 7499 /* how long we wait to gather additional slack before distributing */ 7500 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; 7501 7502 /* 7503 * Are we near the end of the current quota period? 7504 * 7505 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the 7506 * hrtimer base being cleared by hrtimer_start. In the case of 7507 * migrate_hrtimers, base is never cleared, so we are fine. 7508 */ 7509 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) 7510 { 7511 struct hrtimer *refresh_timer = &cfs_b->period_timer; 7512 s64 remaining; 7513 7514 /* if the call-back is running a quota refresh is already occurring */ 7515 if (hrtimer_callback_running(refresh_timer)) 7516 return 1; 7517 7518 /* is a quota refresh about to occur? */ 7519 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); 7520 if (remaining < (s64)min_expire) 7521 return 1; 7522 7523 return 0; 7524 } 7525 7526 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) 7527 { 7528 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; 7529 7530 /* if there's a quota refresh soon don't bother with slack */ 7531 if (runtime_refresh_within(cfs_b, min_left)) 7532 return; 7533 7534 /* don't push forwards an existing deferred unthrottle */ 7535 if (cfs_b->slack_started) 7536 return; 7537 cfs_b->slack_started = true; 7538 7539 hrtimer_start(&cfs_b->slack_timer, 7540 ns_to_ktime(cfs_bandwidth_slack_period), 7541 HRTIMER_MODE_REL); 7542 } 7543 7544 /* we know any runtime found here is valid as update_curr() precedes return */ 7545 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 7546 { 7547 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 7548 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; 7549 7550 if (slack_runtime <= 0) 7551 return; 7552 7553 guard(raw_spinlock)(&cfs_b->lock); 7554 7555 if (cfs_b->quota != RUNTIME_INF) { 7556 cfs_b->runtime += slack_runtime; 7557 7558 /* we are under rq->lock, defer unthrottling using a timer */ 7559 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && 7560 !list_empty(&cfs_b->throttled_cfs_rq)) 7561 start_cfs_slack_bandwidth(cfs_b); 7562 } 7563 7564 /* even if it's not valid for return we don't want to try again */ 7565 cfs_rq->runtime_remaining -= slack_runtime; 7566 } 7567 7568 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 7569 { 7570 if (!cfs_bandwidth_used()) 7571 return; 7572 7573 if (!cfs_rq->runtime_enabled || cfs_rq->nr_queued) 7574 return; 7575 7576 __return_cfs_rq_runtime(cfs_rq); 7577 } 7578 7579 /* 7580 * This is done with a timer (instead of inline with bandwidth return) since 7581 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. 7582 */ 7583 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 7584 { 7585 /* confirm we're still not at a refresh boundary */ 7586 scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) { 7587 u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); 7588 7589 cfs_b->slack_started = false; 7590 7591 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) 7592 return; 7593 7594 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) 7595 runtime = cfs_b->runtime; 7596 7597 if (!runtime) 7598 return; 7599 } 7600 7601 distribute_cfs_runtime(cfs_b); 7602 } 7603 7604 /* 7605 * When a group wakes up we want to make sure that its quota is not already 7606 * expired/exceeded, otherwise it may be allowed to steal additional ticks of 7607 * runtime as update_curr() throttling can not trigger until it's on-rq. 7608 */ 7609 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) 7610 { 7611 if (!cfs_bandwidth_used()) 7612 return; 7613 7614 /* an active group must be handled by the update_curr() path */ 7615 if (!cfs_rq->runtime_enabled || cfs_rq->h_curr) 7616 return; 7617 7618 /* ensure the group is not already throttled */ 7619 if (cfs_rq_throttled(cfs_rq)) 7620 return; 7621 7622 /* update runtime allocation */ 7623 account_cfs_rq_runtime(cfs_rq, 0); 7624 } 7625 7626 static void sync_throttle(struct task_group *tg, int cpu) 7627 { 7628 struct cfs_rq *pcfs_rq, *cfs_rq; 7629 7630 if (!cfs_bandwidth_used()) 7631 return; 7632 7633 if (!tg->parent) 7634 return; 7635 7636 cfs_rq = tg_cfs_rq(tg, cpu); 7637 pcfs_rq = tg_cfs_rq(tg->parent, cpu); 7638 7639 cfs_rq->throttle_count = pcfs_rq->throttle_count; 7640 cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu)); 7641 7642 /* 7643 * It is not enough to sync the "pelt_clock_throttled" indicator 7644 * with the parent cfs_rq when the hierarchy is not queued. 7645 * Always join a throttled hierarchy with PELT clock throttled 7646 * and leaf it to the first enqueue, or distribution to 7647 * unthrottle the PELT clock. 7648 */ 7649 if (cfs_rq->throttle_count) 7650 cfs_rq->pelt_clock_throttled = 1; 7651 } 7652 7653 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) 7654 { 7655 struct cfs_bandwidth *cfs_b = 7656 container_of(timer, struct cfs_bandwidth, slack_timer); 7657 7658 do_sched_cfs_slack_timer(cfs_b); 7659 7660 return HRTIMER_NORESTART; 7661 } 7662 7663 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) 7664 { 7665 struct cfs_bandwidth *cfs_b = 7666 container_of(timer, struct cfs_bandwidth, period_timer); 7667 int overrun; 7668 int idle = 0; 7669 int count = 0; 7670 7671 CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock); 7672 7673 for (;;) { 7674 overrun = hrtimer_forward_now(timer, cfs_b->period); 7675 if (!overrun) 7676 break; 7677 7678 idle = do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags); 7679 7680 if (++count > 3) { 7681 u64 new, old = ktime_to_ns(cfs_b->period); 7682 7683 /* 7684 * Grow period by a factor of 2 to avoid losing precision. 7685 * Precision loss in the quota/period ratio can cause __cfs_schedulable 7686 * to fail. 7687 */ 7688 new = old * 2; 7689 if (new < max_bw_quota_period_us * NSEC_PER_USEC) { 7690 cfs_b->period = ns_to_ktime(new); 7691 cfs_b->quota *= 2; 7692 cfs_b->burst *= 2; 7693 7694 pr_warn_ratelimited( 7695 "cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n", 7696 smp_processor_id(), 7697 div_u64(new, NSEC_PER_USEC), 7698 div_u64(cfs_b->quota, NSEC_PER_USEC)); 7699 } else { 7700 pr_warn_ratelimited( 7701 "cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n", 7702 smp_processor_id(), 7703 div_u64(old, NSEC_PER_USEC), 7704 div_u64(cfs_b->quota, NSEC_PER_USEC)); 7705 } 7706 7707 /* reset count so we don't come right back in here */ 7708 count = 0; 7709 } 7710 } 7711 7712 if (idle) { 7713 cfs_b->period_active = 0; 7714 return HRTIMER_NORESTART; 7715 } 7716 7717 return HRTIMER_RESTART; 7718 } 7719 7720 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) 7721 { 7722 raw_spin_lock_init(&cfs_b->lock); 7723 cfs_b->runtime = 0; 7724 cfs_b->quota = RUNTIME_INF; 7725 cfs_b->period = us_to_ktime(default_bw_period_us()); 7726 cfs_b->burst = 0; 7727 cfs_b->hierarchical_quota = parent ? parent->hierarchical_quota : RUNTIME_INF; 7728 7729 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); 7730 hrtimer_setup(&cfs_b->period_timer, sched_cfs_period_timer, CLOCK_MONOTONIC, 7731 HRTIMER_MODE_ABS_PINNED); 7732 7733 /* Add a random offset so that timers interleave */ 7734 hrtimer_set_expires(&cfs_b->period_timer, 7735 get_random_u32_below(cfs_b->period)); 7736 hrtimer_setup(&cfs_b->slack_timer, sched_cfs_slack_timer, CLOCK_MONOTONIC, 7737 HRTIMER_MODE_REL); 7738 cfs_b->slack_started = false; 7739 } 7740 7741 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 7742 { 7743 cfs_rq->runtime_enabled = 0; 7744 INIT_LIST_HEAD(&cfs_rq->throttled_list); 7745 INIT_LIST_HEAD(&cfs_rq->throttled_csd_list); 7746 INIT_LIST_HEAD(&cfs_rq->throttled_limbo_list); 7747 } 7748 7749 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 7750 { 7751 lockdep_assert_held(&cfs_b->lock); 7752 7753 if (cfs_b->period_active) 7754 return; 7755 7756 cfs_b->period_active = 1; 7757 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); 7758 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); 7759 } 7760 7761 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 7762 { 7763 int __maybe_unused i; 7764 7765 /* init_cfs_bandwidth() was not called */ 7766 if (!cfs_b->throttled_cfs_rq.next) 7767 return; 7768 7769 hrtimer_cancel(&cfs_b->period_timer); 7770 hrtimer_cancel(&cfs_b->slack_timer); 7771 7772 /* 7773 * It is possible that we still have some cfs_rq's pending on a CSD 7774 * list, though this race is very rare. In order for this to occur, we 7775 * must have raced with the last task leaving the group while there 7776 * exist throttled cfs_rq(s), and the period_timer must have queued the 7777 * CSD item but the remote cpu has not yet processed it. To handle this, 7778 * we can simply flush all pending CSD work inline here. We're 7779 * guaranteed at this point that no additional cfs_rq of this group can 7780 * join a CSD list. 7781 */ 7782 for_each_possible_cpu(i) { 7783 struct rq *rq = cpu_rq(i); 7784 7785 if (list_empty(&rq->cfsb_csd_list)) 7786 continue; 7787 7788 scoped_guard(irqsave) 7789 __cfsb_csd_unthrottle(rq); 7790 } 7791 } 7792 7793 /* 7794 * Both these CPU hotplug callbacks race against unregister_fair_sched_group() 7795 * 7796 * The race is harmless, since modifying bandwidth settings of unhooked group 7797 * bits doesn't do much. 7798 */ 7799 7800 /* cpu online callback */ 7801 static void __maybe_unused update_runtime_enabled(struct rq *rq) 7802 { 7803 struct task_group *tg; 7804 7805 lockdep_assert_rq_held(rq); 7806 7807 guard(rcu)(); 7808 7809 list_for_each_entry_rcu(tg, &task_groups, list) { 7810 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 7811 struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); 7812 7813 scoped_guard(raw_spinlock, &cfs_b->lock) 7814 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; 7815 } 7816 } 7817 7818 /* cpu offline callback */ 7819 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) 7820 { 7821 struct task_group *tg; 7822 7823 lockdep_assert_rq_held(rq); 7824 7825 // Do not unthrottle for an active CPU 7826 if (cpumask_test_cpu(cpu_of(rq), cpu_active_mask)) 7827 return; 7828 7829 /* 7830 * The rq clock has already been updated in the 7831 * set_rq_offline(), so we should skip updating 7832 * the rq clock again in unthrottle_cfs_rq(). 7833 */ 7834 rq_clock_start_loop_update(rq); 7835 7836 guard(rcu)(); 7837 7838 list_for_each_entry_rcu(tg, &task_groups, list) { 7839 struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); 7840 7841 if (!cfs_rq->runtime_enabled) 7842 continue; 7843 7844 /* 7845 * Offline rq is schedulable till CPU is completely disabled 7846 * in take_cpu_down(), so we prevent new cfs throttling here. 7847 */ 7848 cfs_rq->runtime_enabled = 0; 7849 7850 if (!cfs_rq_throttled(cfs_rq)) 7851 continue; 7852 7853 /* 7854 * clock_task is not advancing so we just need to make sure 7855 * there's some valid quota amount 7856 */ 7857 cfs_rq->runtime_remaining = 1; 7858 unthrottle_cfs_rq(cfs_rq); 7859 } 7860 7861 rq_clock_stop_loop_update(rq); 7862 } 7863 7864 bool cfs_task_bw_constrained(struct task_struct *p) 7865 { 7866 struct cfs_rq *cfs_rq = task_cfs_rq(p); 7867 7868 if (!cfs_bandwidth_used()) 7869 return false; 7870 7871 if (cfs_rq->runtime_enabled || 7872 tg_cfs_bandwidth(cfs_rq->tg)->hierarchical_quota != RUNTIME_INF) 7873 return true; 7874 7875 return false; 7876 } 7877 7878 #ifdef CONFIG_NO_HZ_FULL 7879 /* called from pick_next_task_fair() */ 7880 static void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) 7881 { 7882 int cpu = cpu_of(rq); 7883 7884 if (!cfs_bandwidth_used()) 7885 return; 7886 7887 if (!tick_nohz_full_cpu(cpu)) 7888 return; 7889 7890 if (rq->nr_running != 1) 7891 return; 7892 7893 /* 7894 * We know there is only one task runnable and we've just picked it. The 7895 * normal enqueue path will have cleared TICK_DEP_BIT_SCHED if we will 7896 * be otherwise able to stop the tick. Just need to check if we are using 7897 * bandwidth control. 7898 */ 7899 if (cfs_task_bw_constrained(p)) 7900 tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED); 7901 } 7902 #endif /* CONFIG_NO_HZ_FULL */ 7903 7904 #else /* !CONFIG_CFS_BANDWIDTH: */ 7905 7906 static bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { return false; } 7907 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 7908 static inline void sync_throttle(struct task_group *tg, int cpu) {} 7909 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 7910 static void task_throttle_setup_work(struct task_struct *p) {} 7911 static bool task_is_throttled(struct task_struct *p) { return false; } 7912 static void dequeue_throttled_task(struct task_struct *p, int flags) {} 7913 static bool enqueue_throttled_task(struct task_struct *p) { return false; } 7914 static void record_throttle_clock(struct cfs_rq *cfs_rq) {} 7915 7916 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 7917 { 7918 return 0; 7919 } 7920 7921 static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq) 7922 { 7923 return false; 7924 } 7925 7926 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 7927 { 7928 return 0; 7929 } 7930 7931 static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu) 7932 { 7933 return 0; 7934 } 7935 7936 #ifdef CONFIG_FAIR_GROUP_SCHED 7937 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) {} 7938 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 7939 #endif 7940 7941 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 7942 { 7943 return NULL; 7944 } 7945 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 7946 static inline void update_runtime_enabled(struct rq *rq) {} 7947 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} 7948 #ifdef CONFIG_CGROUP_SCHED 7949 bool cfs_task_bw_constrained(struct task_struct *p) 7950 { 7951 return false; 7952 } 7953 #endif 7954 #endif /* !CONFIG_CFS_BANDWIDTH */ 7955 7956 #if !defined(CONFIG_CFS_BANDWIDTH) || !defined(CONFIG_NO_HZ_FULL) 7957 static inline void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) {} 7958 #endif 7959 7960 /************************************************** 7961 * CFS operations on tasks: 7962 */ 7963 7964 #ifdef CONFIG_SCHED_HRTICK 7965 static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 7966 { 7967 struct sched_entity *se = &p->se; 7968 unsigned long scale = 1024; 7969 unsigned long util = 0; 7970 u64 vdelta; 7971 u64 delta; 7972 7973 WARN_ON_ONCE(task_rq(p) != rq); 7974 7975 if (rq->cfs.h_nr_queued <= 1) 7976 return; 7977 7978 /* 7979 * Compute time until virtual deadline 7980 */ 7981 vdelta = se->deadline - se->vruntime; 7982 if ((s64)vdelta < 0) { 7983 if (task_current_donor(rq, p)) 7984 resched_curr(rq); 7985 return; 7986 } 7987 delta = (se->h_load.weight * vdelta) / NICE_0_LOAD; 7988 7989 /* 7990 * Correct for instantaneous load of other classes. 7991 */ 7992 util += cpu_util_irq(rq); 7993 if (util && util < 1024) { 7994 scale *= 1024; 7995 scale /= (1024 - util); 7996 } 7997 7998 hrtick_start(rq, (scale * delta) / 1024); 7999 } 8000 8001 /* 8002 * Called on enqueue to start the hrtick when h_nr_queued becomes more than 1. 8003 */ 8004 static void hrtick_update(struct rq *rq) 8005 { 8006 struct task_struct *donor = rq->donor; 8007 8008 if (!hrtick_enabled_fair(rq) || donor->sched_class != &fair_sched_class) 8009 return; 8010 8011 if (hrtick_active(rq)) 8012 return; 8013 8014 hrtick_start_fair(rq, donor); 8015 } 8016 #else /* !CONFIG_SCHED_HRTICK: */ 8017 static inline void 8018 hrtick_start_fair(struct rq *rq, struct task_struct *p) 8019 { 8020 } 8021 8022 static inline void hrtick_update(struct rq *rq) 8023 { 8024 } 8025 #endif /* !CONFIG_SCHED_HRTICK */ 8026 8027 static inline bool cpu_overutilized(int cpu) 8028 { 8029 unsigned long rq_util_max; 8030 8031 if (!sched_energy_enabled()) 8032 return false; 8033 8034 rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX); 8035 8036 /* Return true only if the utilization doesn't fit CPU's capacity */ 8037 return !util_fits_cpu(cpu_util_cfs(cpu), 0, rq_util_max, cpu); 8038 } 8039 8040 /* 8041 * overutilized value make sense only if EAS is enabled 8042 */ 8043 static inline bool is_rd_overutilized(struct root_domain *rd) 8044 { 8045 return !sched_energy_enabled() || READ_ONCE(rd->overutilized); 8046 } 8047 8048 static inline void set_rd_overutilized(struct root_domain *rd, bool flag) 8049 { 8050 if (!sched_energy_enabled()) 8051 return; 8052 8053 WRITE_ONCE(rd->overutilized, flag); 8054 trace_sched_overutilized_tp(rd, flag); 8055 } 8056 8057 static inline void check_update_overutilized_status(struct rq *rq) 8058 { 8059 /* 8060 * overutilized field is used for load balancing decisions only 8061 * if energy aware scheduler is being used 8062 */ 8063 8064 if (!is_rd_overutilized(rq->rd) && cpu_overutilized(rq->cpu)) 8065 set_rd_overutilized(rq->rd, 1); 8066 } 8067 8068 /* Runqueue only has SCHED_IDLE tasks enqueued */ 8069 static int sched_idle_rq(struct rq *rq) 8070 { 8071 return unlikely(rq->nr_running == rq->cfs.h_nr_idle && 8072 rq->nr_running); 8073 } 8074 8075 static int choose_sched_idle_rq(struct rq *rq, struct task_struct *p) 8076 { 8077 return sched_idle_rq(rq) && !task_has_idle_policy(p); 8078 } 8079 8080 static int choose_idle_cpu(int cpu, struct task_struct *p) 8081 { 8082 return available_idle_cpu(cpu) || 8083 choose_sched_idle_rq(cpu_rq(cpu), p); 8084 } 8085 8086 static void 8087 requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 8088 { 8089 /* 8090 * se->sched_delayed should imply: se->on_rq == 1. 8091 * Because a delayed entity is one that is still on 8092 * the runqueue competing until elegibility. 8093 */ 8094 WARN_ON_ONCE(!se->sched_delayed); 8095 WARN_ON_ONCE(!se->on_rq); 8096 8097 if (update_entity_lag(cfs_rq, se)) { 8098 cfs_rq->h_nr_queued--; 8099 if (se != cfs_rq->curr) 8100 __dequeue_entity(cfs_rq, se); 8101 place_entity(cfs_rq, se, 0); 8102 if (se != cfs_rq->curr) 8103 __enqueue_entity(cfs_rq, se); 8104 cfs_rq->h_nr_queued++; 8105 } 8106 8107 update_load_avg(cfs_rq, se, 0); 8108 clear_delayed(se); 8109 } 8110 8111 static unsigned long enqueue_hierarchy(struct task_struct *p, int flags) 8112 { 8113 unsigned long weight = NICE_0_LOAD; 8114 int task_new = !(flags & ENQUEUE_WAKEUP); 8115 struct sched_entity *se = &p->se; 8116 int h_nr_idle = task_has_idle_policy(p); 8117 int h_nr_runnable = 1; 8118 8119 if (task_new && se->sched_delayed) 8120 h_nr_runnable = 0; 8121 8122 for_each_sched_entity(se) { 8123 struct cfs_rq *cfs_rq = cfs_rq_of(se); 8124 8125 update_curr(cfs_rq); 8126 8127 if (!se->on_rq) { 8128 enqueue_entity(cfs_rq, se, flags); 8129 } else { 8130 update_load_avg(cfs_rq, se, UPDATE_TG); 8131 se_update_runnable(se); 8132 update_cfs_group(se); 8133 } 8134 8135 cfs_rq->h_nr_runnable += h_nr_runnable; 8136 cfs_rq->h_nr_queued++; 8137 cfs_rq->h_nr_idle += h_nr_idle; 8138 8139 if (cfs_rq_is_idle(cfs_rq)) 8140 h_nr_idle = 1; 8141 8142 weight = __calc_prop_weight(cfs_rq, se, weight); 8143 8144 flags = ENQUEUE_WAKEUP; 8145 } 8146 8147 return weight; 8148 } 8149 8150 /* Update curr's vruntime before placing entity or updating lag */ 8151 static inline void update_curr_eevdf(struct cfs_rq *cfs_rq) 8152 { 8153 if (!cfs_rq->curr) 8154 return; 8155 8156 update_curr(cfs_rq_of(cfs_rq->curr)); 8157 } 8158 8159 /* 8160 * The enqueue_task method is called before nr_running is 8161 * increased. Here we update the fair scheduling stats and 8162 * then put the task into the rbtree: 8163 */ 8164 static void 8165 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 8166 { 8167 int rq_h_nr_queued = rq->cfs.h_nr_queued; 8168 int task_new = !(flags & ENQUEUE_WAKEUP); 8169 struct sched_entity *se = &p->se; 8170 struct cfs_rq *cfs_rq = &rq->cfs; 8171 unsigned long weight; 8172 bool curr; 8173 8174 if (task_is_throttled(p) && enqueue_throttled_task(p)) 8175 return; 8176 8177 /* 8178 * The code below (indirectly) updates schedutil which looks at 8179 * the cfs_rq utilization to select a frequency. 8180 * Let's add the task's estimated utilization to the cfs_rq's 8181 * estimated utilization, before we update schedutil. 8182 */ 8183 if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED)) 8184 util_est_enqueue(cfs_rq, p); 8185 8186 update_curr_eevdf(cfs_rq); 8187 8188 if (flags & ENQUEUE_DELAYED) { 8189 requeue_delayed_entity(cfs_rq, se); 8190 return; 8191 } 8192 8193 /* 8194 * If in_iowait is set, the code below may not trigger any cpufreq 8195 * utilization updates, so do it here explicitly with the IOWAIT flag 8196 * passed. 8197 */ 8198 if (p->in_iowait) 8199 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); 8200 8201 curr = (cfs_rq->curr == se); 8202 /* 8203 * This can actually happen with CFS_BANDWIDTH, a task can be throttled 8204 * and dequeued, unthrottled from another CPU and then enqueued again 8205 * before schedule(). 8206 */ 8207 if (curr) 8208 place_entity(cfs_rq, se, flags); 8209 8210 if (se->on_rq && se->sched_delayed) 8211 requeue_delayed_entity(cfs_rq, se); 8212 8213 weight = enqueue_hierarchy(p, flags); 8214 8215 if (!curr) { 8216 reweight_eevdf(cfs_rq, se, weight, false); 8217 place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED); 8218 __enqueue_entity(cfs_rq, se); 8219 } 8220 8221 if (!rq_h_nr_queued && rq->cfs.h_nr_queued) 8222 dl_server_start(&rq->fair_server); 8223 8224 /* At this point se is NULL and we are at root level*/ 8225 add_nr_running(rq, 1); 8226 8227 /* 8228 * Since new tasks are assigned an initial util_avg equal to 8229 * half of the spare capacity of their CPU, tiny tasks have the 8230 * ability to cross the overutilized threshold, which will 8231 * result in the load balancer ruining all the task placement 8232 * done by EAS. As a way to mitigate that effect, do not account 8233 * for the first enqueue operation of new tasks during the 8234 * overutilized flag detection. 8235 * 8236 * A better way of solving this problem would be to wait for 8237 * the PELT signals of tasks to converge before taking them 8238 * into account, but that is not straightforward to implement, 8239 * and the following generally works well enough in practice. 8240 */ 8241 if (!task_new) 8242 check_update_overutilized_status(rq); 8243 8244 assert_list_leaf_cfs_rq(rq); 8245 8246 hrtick_update(rq); 8247 } 8248 8249 static void dequeue_hierarchy(struct task_struct *p, int flags) 8250 { 8251 struct sched_entity *se = &p->se; 8252 bool task_sleep = flags & DEQUEUE_SLEEP; 8253 bool task_delayed = flags & DEQUEUE_DELAYED; 8254 bool task_throttled = flags & DEQUEUE_THROTTLE; 8255 int h_nr_runnable = 0; 8256 int h_nr_idle = task_has_idle_policy(p); 8257 bool dequeue = true; 8258 8259 if (task_sleep || task_delayed || !se->sched_delayed) 8260 h_nr_runnable = 1; 8261 8262 for_each_sched_entity(se) { 8263 struct cfs_rq *cfs_rq = cfs_rq_of(se); 8264 8265 update_curr(cfs_rq); 8266 8267 if (dequeue) { 8268 dequeue_entity(cfs_rq, se, flags); 8269 /* Don't dequeue parent if it has other entities besides us */ 8270 if (cfs_rq->load.weight) 8271 dequeue = false; 8272 } else { 8273 update_load_avg(cfs_rq, se, UPDATE_TG); 8274 se_update_runnable(se); 8275 update_cfs_group(se); 8276 } 8277 8278 cfs_rq->h_nr_runnable -= h_nr_runnable; 8279 cfs_rq->h_nr_queued--; 8280 cfs_rq->h_nr_idle -= h_nr_idle; 8281 8282 if (cfs_rq_is_idle(cfs_rq)) 8283 h_nr_idle = 1; 8284 8285 if (throttled_hierarchy(cfs_rq) && task_throttled) 8286 record_throttle_clock(cfs_rq); 8287 8288 flags |= DEQUEUE_SLEEP; 8289 flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL); 8290 } 8291 } 8292 8293 /* 8294 * The part of dequeue_task_fair() that is needed to dequeue delayed tasks. 8295 * 8296 * Returns: 8297 * true - dequeued 8298 * false - delayed 8299 */ 8300 static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags) 8301 { 8302 struct sched_entity *se = &p->se; 8303 struct cfs_rq *cfs_rq = &rq->cfs; 8304 bool was_sched_idle = sched_idle_rq(rq); 8305 bool task_sleep = flags & DEQUEUE_SLEEP; 8306 bool task_delayed = flags & DEQUEUE_DELAYED; 8307 8308 clear_buddies(cfs_rq, se); 8309 8310 update_curr_eevdf(cfs_rq); 8311 update_entity_lag(cfs_rq, se); 8312 8313 if (flags & DEQUEUE_DELAYED) { 8314 WARN_ON_ONCE(!se->sched_delayed); 8315 } else { 8316 bool delay = task_sleep; 8317 /* 8318 * DELAY_DEQUEUE relies on spurious wakeups, special task 8319 * states must not suffer spurious wakeups, excempt them. 8320 */ 8321 if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) 8322 delay = false; 8323 8324 WARN_ON_ONCE(delay && se->sched_delayed); 8325 8326 if (sched_feat(DELAY_DEQUEUE) && delay && 8327 !entity_eligible(cfs_rq, se)) { 8328 update_load_avg(cfs_rq_of(se), se, UPDATE_UTIL_EST); 8329 set_delayed(se); 8330 return false; 8331 } 8332 } 8333 8334 dequeue_hierarchy(p, flags); 8335 8336 if (sched_feat(PLACE_REL_DEADLINE) && !task_sleep) { 8337 se->deadline -= se->vruntime; 8338 se->rel_deadline = 1; 8339 } 8340 if (se != cfs_rq->curr) 8341 __dequeue_entity(cfs_rq, se); 8342 8343 sub_nr_running(rq, 1); 8344 8345 /* balance early to pull high priority tasks */ 8346 if (unlikely(!was_sched_idle && sched_idle_rq(rq))) 8347 rq->next_balance = jiffies; 8348 8349 if (task_delayed) { 8350 clear_delayed(se); 8351 8352 WARN_ON_ONCE(!task_sleep); 8353 WARN_ON_ONCE(p->on_rq != 1); 8354 8355 /* 8356 * Fix-up what block_task() skipped. 8357 * 8358 * Must be last, @p might not be valid after this. 8359 */ 8360 __block_task(rq, p); 8361 } 8362 8363 return true; 8364 } 8365 8366 /* 8367 * The dequeue_task method is called before nr_running is 8368 * decreased. We remove the task from the rbtree and 8369 * update the fair scheduling stats: 8370 */ 8371 static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 8372 { 8373 if (task_is_throttled(p)) { 8374 dequeue_throttled_task(p, flags); 8375 return true; 8376 } 8377 8378 if (!p->se.sched_delayed) 8379 util_est_dequeue(&rq->cfs, p); 8380 8381 if (!__dequeue_task(rq, p, flags)) 8382 return false; 8383 8384 /* 8385 * Must not reference @p after __dequeue_task(DEQUEUE_DELAYED). 8386 */ 8387 return true; 8388 } 8389 8390 static inline unsigned int cfs_h_nr_delayed(struct rq *rq) 8391 { 8392 return (rq->cfs.h_nr_queued - rq->cfs.h_nr_runnable); 8393 } 8394 8395 /* Working cpumask for: sched_balance_rq(), sched_balance_newidle(). */ 8396 static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); 8397 static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask); 8398 static DEFINE_PER_CPU(cpumask_var_t, should_we_balance_tmpmask); 8399 8400 #ifdef CONFIG_NO_HZ_COMMON 8401 8402 static struct { 8403 cpumask_var_t idle_cpus_mask; 8404 int has_blocked_load; /* Idle CPUS has blocked load */ 8405 int needs_update; /* Newly idle CPUs need their next_balance collated */ 8406 unsigned long next_balance; /* in jiffy units */ 8407 unsigned long next_blocked; /* Next update of blocked load in jiffies */ 8408 } nohz ____cacheline_aligned; 8409 8410 #endif /* CONFIG_NO_HZ_COMMON */ 8411 8412 static unsigned long cpu_load(struct rq *rq) 8413 { 8414 return cfs_rq_load_avg(&rq->cfs); 8415 } 8416 8417 /* 8418 * cpu_load_without - compute CPU load without any contributions from *p 8419 * @cpu: the CPU which load is requested 8420 * @p: the task which load should be discounted 8421 * 8422 * The load of a CPU is defined by the load of tasks currently enqueued on that 8423 * CPU as well as tasks which are currently sleeping after an execution on that 8424 * CPU. 8425 * 8426 * This method returns the load of the specified CPU by discounting the load of 8427 * the specified task, whenever the task is currently contributing to the CPU 8428 * load. 8429 */ 8430 static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p) 8431 { 8432 struct cfs_rq *cfs_rq; 8433 unsigned int load; 8434 8435 /* Task has no contribution or is new */ 8436 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 8437 return cpu_load(rq); 8438 8439 cfs_rq = &rq->cfs; 8440 load = READ_ONCE(cfs_rq->avg.load_avg); 8441 8442 /* Discount task's util from CPU's util */ 8443 lsub_positive(&load, task_h_load(p)); 8444 8445 return load; 8446 } 8447 8448 static unsigned long cpu_runnable(struct rq *rq) 8449 { 8450 return cfs_rq_runnable_avg(&rq->cfs); 8451 } 8452 8453 static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p) 8454 { 8455 struct cfs_rq *cfs_rq; 8456 unsigned int runnable; 8457 8458 /* Task has no contribution or is new */ 8459 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 8460 return cpu_runnable(rq); 8461 8462 cfs_rq = &rq->cfs; 8463 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 8464 8465 /* Discount task's runnable from CPU's runnable */ 8466 lsub_positive(&runnable, p->se.avg.runnable_avg); 8467 8468 return runnable; 8469 } 8470 8471 static unsigned long capacity_of(int cpu) 8472 { 8473 return cpu_rq(cpu)->cpu_capacity; 8474 } 8475 8476 static void record_wakee(struct task_struct *p) 8477 { 8478 /* 8479 * Only decay a single time; tasks that have less then 1 wakeup per 8480 * jiffy will not have built up many flips. 8481 */ 8482 if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 8483 current->wakee_flips >>= 1; 8484 current->wakee_flip_decay_ts = jiffies; 8485 } 8486 8487 if (current->last_wakee != p) { 8488 current->last_wakee = p; 8489 current->wakee_flips++; 8490 } 8491 } 8492 8493 /* 8494 * Detect M:N waker/wakee relationships via a switching-frequency heuristic. 8495 * 8496 * A waker of many should wake a different task than the one last awakened 8497 * at a frequency roughly N times higher than one of its wakees. 8498 * 8499 * In order to determine whether we should let the load spread vs consolidating 8500 * to shared cache, we look for a minimum 'flip' frequency of llc_size in one 8501 * partner, and a factor of lls_size higher frequency in the other. 8502 * 8503 * With both conditions met, we can be relatively sure that the relationship is 8504 * non-monogamous, with partner count exceeding socket size. 8505 * 8506 * Waker/wakee being client/server, worker/dispatcher, interrupt source or 8507 * whatever is irrelevant, spread criteria is apparent partner count exceeds 8508 * socket size. 8509 */ 8510 static int wake_wide(struct task_struct *p) 8511 { 8512 unsigned int master = current->wakee_flips; 8513 unsigned int slave = p->wakee_flips; 8514 int factor = __this_cpu_read(sd_llc_size); 8515 8516 if (master < slave) 8517 swap(master, slave); 8518 if (slave < factor || master < slave * factor) 8519 return 0; 8520 return 1; 8521 } 8522 8523 /* 8524 * The purpose of wake_affine() is to quickly determine on which CPU we can run 8525 * soonest. For the purpose of speed we only consider the waking and previous 8526 * CPU. 8527 * 8528 * wake_affine_idle() - only considers 'now', it check if the waking CPU is 8529 * cache-affine and is (or will be) idle. 8530 * 8531 * wake_affine_weight() - considers the weight to reflect the average 8532 * scheduling latency of the CPUs. This seems to work 8533 * for the overloaded case. 8534 */ 8535 static int 8536 wake_affine_idle(int this_cpu, int prev_cpu, int sync) 8537 { 8538 /* 8539 * If this_cpu is idle, it implies the wakeup is from interrupt 8540 * context. Only allow the move if cache is shared. Otherwise an 8541 * interrupt intensive workload could force all tasks onto one 8542 * node depending on the IO topology or IRQ affinity settings. 8543 * 8544 * If the prev_cpu is idle and cache affine then avoid a migration. 8545 * There is no guarantee that the cache hot data from an interrupt 8546 * is more important than cache hot data on the prev_cpu and from 8547 * a cpufreq perspective, it's better to have higher utilisation 8548 * on one CPU. 8549 */ 8550 if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 8551 return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 8552 8553 if (sync) { 8554 struct rq *rq = cpu_rq(this_cpu); 8555 8556 if ((rq->nr_running - cfs_h_nr_delayed(rq)) == 1) 8557 return this_cpu; 8558 } 8559 8560 if (available_idle_cpu(prev_cpu)) 8561 return prev_cpu; 8562 8563 return nr_cpumask_bits; 8564 } 8565 8566 static int 8567 wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 8568 int this_cpu, int prev_cpu, int sync) 8569 { 8570 s64 this_eff_load, prev_eff_load; 8571 unsigned long task_load; 8572 8573 this_eff_load = cpu_load(cpu_rq(this_cpu)); 8574 8575 if (sync) { 8576 unsigned long current_load = task_h_load(current); 8577 8578 if (current_load > this_eff_load) 8579 return this_cpu; 8580 8581 this_eff_load -= current_load; 8582 } 8583 8584 task_load = task_h_load(p); 8585 8586 this_eff_load += task_load; 8587 if (sched_feat(WA_BIAS)) 8588 this_eff_load *= 100; 8589 this_eff_load *= capacity_of(prev_cpu); 8590 8591 prev_eff_load = cpu_load(cpu_rq(prev_cpu)); 8592 prev_eff_load -= task_load; 8593 if (sched_feat(WA_BIAS)) 8594 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 8595 prev_eff_load *= capacity_of(this_cpu); 8596 8597 /* 8598 * If sync, adjust the weight of prev_eff_load such that if 8599 * prev_eff == this_eff that select_idle_sibling() will consider 8600 * stacking the wakee on top of the waker if no other CPU is 8601 * idle. 8602 */ 8603 if (sync) 8604 prev_eff_load += 1; 8605 8606 return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 8607 } 8608 8609 static int wake_affine(struct sched_domain *sd, struct task_struct *p, 8610 int this_cpu, int prev_cpu, int sync) 8611 { 8612 int target = nr_cpumask_bits; 8613 8614 if (sched_feat(WA_IDLE)) 8615 target = wake_affine_idle(this_cpu, prev_cpu, sync); 8616 8617 if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 8618 target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync); 8619 8620 schedstat_inc(p->stats.nr_wakeups_affine_attempts); 8621 if (target != this_cpu) 8622 return prev_cpu; 8623 8624 schedstat_inc(sd->ttwu_move_affine); 8625 schedstat_inc(p->stats.nr_wakeups_affine); 8626 return target; 8627 } 8628 8629 static struct sched_group * 8630 sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu); 8631 8632 /* 8633 * sched_balance_find_dst_group_cpu - find the idlest CPU among the CPUs in the group. 8634 */ 8635 static int 8636 sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 8637 { 8638 unsigned long load, min_load = ULONG_MAX; 8639 u64 min_exit_latency = U64_MAX; 8640 unsigned int nr_candidates = 0; 8641 int least_loaded_cpu = this_cpu; 8642 int shallowest_idle_cpu = -1; 8643 int i; 8644 8645 /* Check if we have any choice: */ 8646 if (group->group_weight == 1) 8647 return cpumask_first(sched_group_span(group)); 8648 8649 /* Traverse only the allowed CPUs */ 8650 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 8651 struct rq *rq = cpu_rq(i); 8652 8653 if (!sched_core_cookie_match(rq, p)) 8654 continue; 8655 8656 if (choose_sched_idle_rq(rq, p)) 8657 return i; 8658 8659 if (available_idle_cpu(i)) { 8660 struct cpuidle_state *idle = idle_get_state(rq); 8661 u64 exit_latency = idle ? idle->exit_latency : U64_MAX; 8662 8663 if (shallowest_idle_cpu == -1 || exit_latency < min_exit_latency) { 8664 min_exit_latency = exit_latency; 8665 shallowest_idle_cpu = i; 8666 nr_candidates = 1; 8667 } else if (exit_latency == min_exit_latency) { 8668 nr_candidates++; 8669 if (!reciprocal_scale(sched_rng(), nr_candidates)) 8670 shallowest_idle_cpu = i; 8671 } 8672 } else if (shallowest_idle_cpu == -1) { 8673 load = cpu_load(cpu_rq(i)); 8674 if (load < min_load) { 8675 min_load = load; 8676 least_loaded_cpu = i; 8677 } 8678 } 8679 } 8680 8681 return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 8682 } 8683 8684 static inline int sched_balance_find_dst_cpu(struct sched_domain *sd, struct task_struct *p, 8685 int cpu, int prev_cpu, int sd_flag) 8686 { 8687 int new_cpu = cpu; 8688 8689 if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr)) 8690 return prev_cpu; 8691 8692 /* 8693 * We need task's util for cpu_util_without, sync it up to 8694 * prev_cpu's last_update_time. 8695 */ 8696 if (!(sd_flag & SD_BALANCE_FORK)) 8697 sync_entity_load_avg(&p->se); 8698 8699 while (sd) { 8700 struct sched_group *group; 8701 struct sched_domain *tmp; 8702 int weight; 8703 8704 if (!(sd->flags & sd_flag)) { 8705 sd = sd->child; 8706 continue; 8707 } 8708 8709 group = sched_balance_find_dst_group(sd, p, cpu); 8710 if (!group) { 8711 sd = sd->child; 8712 continue; 8713 } 8714 8715 new_cpu = sched_balance_find_dst_group_cpu(group, p, cpu); 8716 if (new_cpu == cpu) { 8717 /* Now try balancing at a lower domain level of 'cpu': */ 8718 sd = sd->child; 8719 continue; 8720 } 8721 8722 /* Now try balancing at a lower domain level of 'new_cpu': */ 8723 cpu = new_cpu; 8724 weight = sd->span_weight; 8725 sd = NULL; 8726 for_each_domain(cpu, tmp) { 8727 if (weight <= tmp->span_weight) 8728 break; 8729 if (tmp->flags & sd_flag) 8730 sd = tmp; 8731 } 8732 } 8733 8734 return new_cpu; 8735 } 8736 8737 static inline int __select_idle_cpu(int cpu, struct task_struct *p) 8738 { 8739 if (choose_idle_cpu(cpu, p) && sched_cpu_cookie_match(cpu_rq(cpu), p)) 8740 return cpu; 8741 8742 return -1; 8743 } 8744 8745 DEFINE_STATIC_KEY_FALSE(sched_smt_present); 8746 EXPORT_SYMBOL_GPL(sched_smt_present); 8747 8748 static inline void set_idle_cores(int cpu, int val) 8749 { 8750 struct sched_domain_shared *sds; 8751 8752 sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu)); 8753 if (sds) 8754 WRITE_ONCE(sds->has_idle_cores, val); 8755 } 8756 8757 static inline bool test_idle_cores(int cpu) 8758 { 8759 struct sched_domain_shared *sds; 8760 8761 sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu)); 8762 if (sds) 8763 return READ_ONCE(sds->has_idle_cores); 8764 8765 return false; 8766 } 8767 8768 /* 8769 * Redirect a CPU to a higher-priority available sibling in its SMT domain, 8770 * subject to task affinity. 8771 */ 8772 static inline int select_idle_smt_cpu(struct task_struct *p, int cpu) 8773 { 8774 struct sched_domain *sd; 8775 int best = cpu; 8776 int sibling; 8777 8778 if (!sched_smt_active()) 8779 return cpu; 8780 8781 sd = rcu_dereference_all(cpu_rq(cpu)->sd); 8782 if (!sd || !(sd->flags & SD_SHARE_CPUCAPACITY) || 8783 !(sd->flags & SD_ASYM_PACKING)) 8784 return cpu; 8785 8786 for_each_cpu_and(sibling, sched_domain_span(sd), p->cpus_ptr) { 8787 if (sibling == best || !choose_idle_cpu(sibling, p)) 8788 continue; 8789 8790 if (sched_asym_prefer(sibling, best)) 8791 best = sibling; 8792 } 8793 8794 return best; 8795 } 8796 8797 /* 8798 * Scans the local SMT mask to see if the entire core is idle, and records this 8799 * information in sd_balance_shared->has_idle_cores. 8800 * 8801 * Since SMT siblings share all cache levels, inspecting this limited remote 8802 * state should be fairly cheap. 8803 */ 8804 void __update_idle_core(struct rq *rq) 8805 { 8806 int core = cpu_of(rq); 8807 int cpu; 8808 8809 rcu_read_lock(); 8810 if (test_idle_cores(core)) 8811 goto unlock; 8812 8813 for_each_cpu(cpu, cpu_smt_mask(core)) { 8814 if (cpu == core) 8815 continue; 8816 8817 if (!available_idle_cpu(cpu)) 8818 goto unlock; 8819 } 8820 8821 set_idle_cores(core, 1); 8822 unlock: 8823 rcu_read_unlock(); 8824 } 8825 8826 /* 8827 * Scan the entire LLC domain for idle cores; this dynamically switches off if 8828 * there are no idle cores left in the system; tracked through 8829 * sd_balance_shared->has_idle_cores and enabled through update_idle_core() 8830 * above. 8831 */ 8832 static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 8833 { 8834 bool idle = true; 8835 int cpu; 8836 8837 for_each_cpu(cpu, cpu_smt_mask(core)) { 8838 if (!available_idle_cpu(cpu)) { 8839 idle = false; 8840 if (*idle_cpu == -1) { 8841 if (choose_sched_idle_rq(cpu_rq(cpu), p) && 8842 cpumask_test_cpu(cpu, cpus)) { 8843 *idle_cpu = cpu; 8844 break; 8845 } 8846 continue; 8847 } 8848 break; 8849 } 8850 if (*idle_cpu == -1 && cpumask_test_cpu(cpu, cpus)) 8851 *idle_cpu = cpu; 8852 } 8853 8854 if (idle) 8855 return core; 8856 8857 cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); 8858 return -1; 8859 } 8860 8861 /* 8862 * Scan the local SMT mask for idle CPUs. 8863 */ 8864 static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 8865 { 8866 int cpu; 8867 8868 for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) { 8869 if (cpu == target) 8870 continue; 8871 /* 8872 * Check if the CPU is in the LLC scheduling domain of @target. 8873 * Due to isolcpus, there is no guarantee that all the siblings are in the domain. 8874 */ 8875 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 8876 continue; 8877 if (choose_idle_cpu(cpu, p)) 8878 return cpu; 8879 } 8880 8881 return -1; 8882 } 8883 8884 /* 8885 * Scan the LLC domain for idle CPUs; this is dynamically regulated by 8886 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the 8887 * average idle time for this rq (as found in rq->avg_idle). 8888 */ 8889 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target) 8890 { 8891 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 8892 int i, cpu, idle_cpu = -1, nr = INT_MAX; 8893 8894 if (sched_feat(SIS_UTIL) && sd->shared) { 8895 /* 8896 * Increment because !--nr is the condition to stop scan. 8897 * 8898 * Since "sd" is "sd_llc" for target CPU dereferenced in the 8899 * caller, it is safe to directly dereference "sd->shared". 8900 * Topology bits always ensure it assigned for "sd_llc" abd it 8901 * cannot disappear as long as we have a RCU protected 8902 * reference to one the associated "sd" here. 8903 */ 8904 nr = READ_ONCE(sd->shared->nr_idle_scan) + 1; 8905 /* overloaded LLC is unlikely to have idle cpu/core */ 8906 if (nr == 1) 8907 return -1; 8908 } 8909 8910 if (!cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr)) 8911 return -1; 8912 8913 if (static_branch_unlikely(&sched_cluster_active)) { 8914 struct sched_group *sg = sd->groups; 8915 8916 if (sg->flags & SD_CLUSTER) { 8917 for_each_cpu_wrap(cpu, sched_group_span(sg), target + 1) { 8918 if (!cpumask_test_cpu(cpu, cpus)) 8919 continue; 8920 8921 if (has_idle_core) { 8922 i = select_idle_core(p, cpu, cpus, &idle_cpu); 8923 if ((unsigned int)i < nr_cpumask_bits) 8924 return i; 8925 } else { 8926 if (--nr <= 0) 8927 return -1; 8928 idle_cpu = __select_idle_cpu(cpu, p); 8929 if ((unsigned int)idle_cpu < nr_cpumask_bits) 8930 return idle_cpu; 8931 } 8932 } 8933 cpumask_andnot(cpus, cpus, sched_group_span(sg)); 8934 } 8935 } 8936 8937 for_each_cpu_wrap(cpu, cpus, target + 1) { 8938 if (has_idle_core) { 8939 i = select_idle_core(p, cpu, cpus, &idle_cpu); 8940 if ((unsigned int)i < nr_cpumask_bits) 8941 return i; 8942 8943 } else { 8944 if (--nr <= 0) 8945 return -1; 8946 idle_cpu = __select_idle_cpu(cpu, p); 8947 if ((unsigned int)idle_cpu < nr_cpumask_bits) 8948 break; 8949 } 8950 } 8951 8952 if (has_idle_core) 8953 set_idle_cores(target, false); 8954 8955 return idle_cpu; 8956 } 8957 8958 /* 8959 * Idle-capacity scan converts util_fits_cpu() outcomes into preference ranks, 8960 * where lower values indicate a better fit - see select_idle_capacity(). 8961 * 8962 * A CPU that both fits the task and sits on a fully-idle SMT core is returned 8963 * immediately and is never assigned one of these ranks. On !SMT every CPU is 8964 * its own "core", so the early return covers all fits-and-idle cases and the 8965 * core-tier ranks below become unreachable. 8966 * 8967 * Rank Val Tier Meaning 8968 * ------------------------------ --- ------ --------------------------- 8969 * ASYM_IDLE_UCLAMP_MISFIT -4 core Idle core; capacity fits 8970 * util but uclamp_min misses. 8971 * ASYM_IDLE_COMPLETE_MISFIT -3 core Idle core; capacity does 8972 * not fit. Still beats every 8973 * thread-tier rank: a busy 8974 * sibling cuts effective 8975 * capacity more than a 8976 * misfit hurts a quiet core. 8977 * ASYM_IDLE_THREAD_FITS -2 thread Busy SMT sibling; capacity 8978 * fits util + uclamp. 8979 * ASYM_IDLE_THREAD_UCLAMP_MISFIT -1 thread Busy SMT sibling; capacity 8980 * fits but uclamp_min misses 8981 * (native util_fits_cpu() 8982 * return value). 8983 * ASYM_IDLE_THREAD_MISFIT 0 thread Busy SMT sibling; capacity 8984 * does not fit. 8985 * 8986 * ASYM_IDLE_CORE_BIAS (-3) is an offset, not a state. On an idle core, 8987 * fits += ASYM_IDLE_CORE_BIAS rebases thread-tier ranks into the core tier: 8988 * 8989 * ASYM_IDLE_THREAD_UCLAMP_MISFIT (-1) + BIAS -> ASYM_IDLE_UCLAMP_MISFIT (-4) 8990 * ASYM_IDLE_THREAD_MISFIT (0) + BIAS -> ASYM_IDLE_COMPLETE_MISFIT (-3) 8991 * 8992 * ASYM_IDLE_THREAD_FITS (-2) is never rebased because a fully-fitting idle-core 8993 * candidate early-returns from select_idle_capacity(). 8994 */ 8995 enum asym_fits_state { 8996 ASYM_IDLE_UCLAMP_MISFIT = -4, 8997 ASYM_IDLE_COMPLETE_MISFIT, 8998 ASYM_IDLE_THREAD_FITS, 8999 ASYM_IDLE_THREAD_UCLAMP_MISFIT, 9000 ASYM_IDLE_THREAD_MISFIT, 9001 9002 /* util_fits_cpu() bias for idle core */ 9003 ASYM_IDLE_CORE_BIAS = -3, 9004 }; 9005 9006 /* 9007 * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which 9008 * the task fits. If no CPU is big enough, but there are idle ones, try to 9009 * maximize capacity. 9010 */ 9011 static int 9012 select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) 9013 { 9014 /* 9015 * On !SMT systems, has_idle_core is always false and preferred_core 9016 * is always true (CPU == core), so the SMT preference logic below 9017 * collapses to the plain capacity scan. 9018 */ 9019 bool has_idle_core = sched_smt_active() && test_idle_cores(target); 9020 unsigned long task_util, util_min, util_max, best_cap = 0; 9021 int fits, best_fits = ASYM_IDLE_THREAD_MISFIT; 9022 int cpu, best_cpu = -1; 9023 struct cpumask *cpus; 9024 int nr = INT_MAX; 9025 9026 cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 9027 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 9028 9029 task_util = task_util_est(p); 9030 util_min = uclamp_eff_value(p, UCLAMP_MIN); 9031 util_max = uclamp_eff_value(p, UCLAMP_MAX); 9032 9033 if (sched_feat(SIS_UTIL) && sd->shared) { 9034 /* 9035 * Same nr_idle_scan hint as select_idle_cpu(), nr only limits 9036 * the scan when not preferring an idle core. 9037 */ 9038 nr = READ_ONCE(sd->shared->nr_idle_scan) + 1; 9039 /* overloaded domain is unlikely to have idle cpu/core */ 9040 if (nr == 1) 9041 return -1; 9042 } 9043 9044 for_each_cpu_wrap(cpu, cpus, target) { 9045 bool preferred_core = !has_idle_core || is_core_idle(cpu); 9046 unsigned long cpu_cap = capacity_of(cpu); 9047 9048 /* 9049 * Stop when the nr_idle_scan is exhausted (mirrors 9050 * select_idle_cpu() logic). 9051 */ 9052 if (!has_idle_core && --nr <= 0) 9053 return best_cpu; 9054 9055 if (!choose_idle_cpu(cpu, p)) 9056 continue; 9057 9058 fits = util_fits_cpu(task_util, util_min, util_max, cpu); 9059 9060 /* 9061 * Perfect fit: capacity satisfies util + uclamp and the CPU 9062 * sits on a fully-idle SMT core, this is a !SMT system, or 9063 * there is no idle core to find. 9064 * Short-circuit the rank-based selection and return 9065 * immediately. 9066 */ 9067 if (fits > 0 && preferred_core) 9068 return cpu; 9069 /* 9070 * Only the min performance hint (i.e. uclamp_min) doesn't fit. 9071 * Look for the CPU with best capacity. 9072 */ 9073 else if (fits < 0) 9074 cpu_cap = get_actual_cpu_capacity(cpu); 9075 /* 9076 * fits > 0 implies we are not on a preferred core, but the util 9077 * fits CPU capacity. Set fits to ASYM_IDLE_THREAD_FITS 9078 * so the effective range becomes 9079 * [ASYM_IDLE_THREAD_FITS, ASYM_IDLE_THREAD_MISFIT], where: 9080 * ASYM_IDLE_THREAD_MISFIT - does not fit 9081 * ASYM_IDLE_THREAD_UCLAMP_MISFIT - fits with the exception of UCLAMP_MIN 9082 * ASYM_IDLE_THREAD_FITS - fits with the exception of preferred_core 9083 */ 9084 else if (fits > 0) 9085 fits = ASYM_IDLE_THREAD_FITS; 9086 9087 /* 9088 * If we are on a preferred core, translate the range of fits 9089 * of [ASYM_IDLE_THREAD_UCLAMP_MISFIT, ASYM_IDLE_THREAD_MISFIT] to 9090 * [ASYM_IDLE_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT]. 9091 * This ensures that an idle core is always given priority over 9092 * (partially) busy core. 9093 * 9094 * A fully fitting idle core would have returned early and hence 9095 * fits > 0 for preferred_core need not be dealt with. 9096 */ 9097 if (preferred_core) 9098 fits += ASYM_IDLE_CORE_BIAS; 9099 9100 /* 9101 * First, select CPU which fits better (lower is more preferred). 9102 * Then, select the one with best capacity at same level. 9103 */ 9104 if ((fits < best_fits) || 9105 ((fits == best_fits) && (cpu_cap > best_cap))) { 9106 best_cap = cpu_cap; 9107 best_cpu = cpu; 9108 best_fits = fits; 9109 } 9110 } 9111 9112 /* 9113 * A value in the [ASYM_IDLE_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT] 9114 * range means the chosen CPU is in a fully idle SMT core. Values above 9115 * ASYM_IDLE_COMPLETE_MISFIT mean we never ranked such a CPU best. 9116 * 9117 * The asym-capacity wakeup path returns from select_idle_sibling() 9118 * after this function and never runs select_idle_cpu(), so the usual 9119 * select_idle_cpu() tail that clears idle cores must live here when the 9120 * idle-core preference did not win. 9121 */ 9122 if (has_idle_core && best_fits > ASYM_IDLE_COMPLETE_MISFIT) 9123 set_idle_cores(target, false); 9124 9125 return best_cpu; 9126 } 9127 9128 static inline bool asym_fits_cpu(unsigned long util, 9129 unsigned long util_min, 9130 unsigned long util_max, 9131 int cpu) 9132 { 9133 if (sched_asym_cpucap_active()) { 9134 /* 9135 * Return true only if the cpu fully fits the task requirements 9136 * which include the utilization and the performance hints. 9137 * 9138 * When SMT is active, also require that the core has no busy 9139 * siblings. 9140 * 9141 * Note: gating on is_core_idle() also makes the early-bailout 9142 * candidates in select_idle_sibling() (target, prev, 9143 * recent_used_cpu) idle-core-aware on ASYM+SMT, which the 9144 * NO_ASYM path does not do. 9145 */ 9146 return (!sched_smt_active() || is_core_idle(cpu)) && 9147 (util_fits_cpu(util, util_min, util_max, cpu) > 0); 9148 } 9149 9150 return true; 9151 } 9152 9153 /* 9154 * Try and locate an idle core/thread in the LLC cache domain. 9155 */ 9156 static int select_idle_sibling(struct task_struct *p, int prev, int target) 9157 { 9158 bool has_idle_core = false; 9159 struct sched_domain *sd; 9160 unsigned long task_util, util_min, util_max; 9161 int i, recent_used_cpu, prev_aff = -1; 9162 9163 /* 9164 * On asymmetric system, update task utilization because we will check 9165 * that the task fits with CPU's capacity. 9166 */ 9167 if (sched_asym_cpucap_active()) { 9168 sync_entity_load_avg(&p->se); 9169 task_util = task_util_est(p); 9170 util_min = uclamp_eff_value(p, UCLAMP_MIN); 9171 util_max = uclamp_eff_value(p, UCLAMP_MAX); 9172 } 9173 9174 /* 9175 * per-cpu select_rq_mask usage 9176 */ 9177 lockdep_assert_irqs_disabled(); 9178 9179 if (choose_idle_cpu(target, p) && 9180 asym_fits_cpu(task_util, util_min, util_max, target)) 9181 goto select_smt_priority; 9182 9183 /* 9184 * If the previous CPU is cache affine and idle, don't be stupid: 9185 */ 9186 if (prev != target && cpus_share_cache(prev, target) && 9187 choose_idle_cpu(prev, p) && 9188 asym_fits_cpu(task_util, util_min, util_max, prev)) { 9189 9190 if (!static_branch_unlikely(&sched_cluster_active) || 9191 cpus_share_resources(prev, target)) { 9192 target = prev; 9193 goto select_smt_priority; 9194 } 9195 9196 prev_aff = prev; 9197 } 9198 9199 /* 9200 * Allow a per-cpu kthread to stack with the wakee if the 9201 * kworker thread and the tasks previous CPUs are the same. 9202 * The assumption is that the wakee queued work for the 9203 * per-cpu kthread that is now complete and the wakeup is 9204 * essentially a sync wakeup. An obvious example of this 9205 * pattern is IO completions. 9206 */ 9207 if (is_per_cpu_kthread(current) && 9208 in_task() && 9209 prev == smp_processor_id() && 9210 this_rq()->nr_running <= 1 && 9211 asym_fits_cpu(task_util, util_min, util_max, prev)) { 9212 target = prev; 9213 goto select_smt_priority; 9214 } 9215 9216 /* Check a recently used CPU as a potential idle candidate: */ 9217 recent_used_cpu = p->recent_used_cpu; 9218 p->recent_used_cpu = prev; 9219 if (recent_used_cpu != prev && 9220 recent_used_cpu != target && 9221 cpus_share_cache(recent_used_cpu, target) && 9222 choose_idle_cpu(recent_used_cpu, p) && 9223 cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) && 9224 asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { 9225 9226 if (!static_branch_unlikely(&sched_cluster_active) || 9227 cpus_share_resources(recent_used_cpu, target)) { 9228 target = recent_used_cpu; 9229 goto select_smt_priority; 9230 } 9231 9232 } else { 9233 recent_used_cpu = -1; 9234 } 9235 9236 /* 9237 * For asymmetric CPU capacity systems, our domain of interest is 9238 * sd_asym_cpucapacity rather than sd_llc. 9239 */ 9240 if (sched_asym_cpucap_active()) { 9241 sd = rcu_dereference_all(per_cpu(sd_asym_cpucapacity, target)); 9242 /* 9243 * On an asymmetric CPU capacity system where an exclusive 9244 * cpuset defines a symmetric island (i.e. one unique 9245 * capacity_orig value through the cpuset), the key will be set 9246 * but the CPUs within that cpuset will not have a domain with 9247 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric 9248 * capacity path. 9249 */ 9250 if (sd) { 9251 i = select_idle_capacity(p, sd, target); 9252 if ((unsigned int)i < nr_cpumask_bits) { 9253 target = i; 9254 goto select_smt_priority; 9255 } 9256 return target; 9257 } 9258 } 9259 9260 sd = rcu_dereference_all(per_cpu(sd_llc, target)); 9261 if (!sd) 9262 return target; 9263 9264 if (sched_smt_active()) { 9265 has_idle_core = test_idle_cores(target); 9266 9267 if (!has_idle_core && cpus_share_cache(prev, target)) { 9268 i = select_idle_smt(p, sd, prev); 9269 if ((unsigned int)i < nr_cpumask_bits) { 9270 target = i; 9271 goto select_smt_priority; 9272 } 9273 } 9274 } 9275 9276 i = select_idle_cpu(p, sd, has_idle_core, target); 9277 if ((unsigned int)i < nr_cpumask_bits) { 9278 target = i; 9279 goto select_smt_priority; 9280 } 9281 9282 /* 9283 * For cluster machines which have lower sharing cache like L2 or 9284 * LLC Tag, we tend to find an idle CPU in the target's cluster 9285 * first. But prev_cpu or recent_used_cpu may also be a good candidate, 9286 * use them if possible when no idle CPU found in select_idle_cpu(). 9287 */ 9288 if ((unsigned int)prev_aff < nr_cpumask_bits) { 9289 target = prev_aff; 9290 goto select_smt_priority; 9291 } 9292 if ((unsigned int)recent_used_cpu < nr_cpumask_bits) { 9293 target = recent_used_cpu; 9294 goto select_smt_priority; 9295 } 9296 9297 return target; 9298 9299 select_smt_priority: 9300 return select_idle_smt_cpu(p, target); 9301 } 9302 9303 /** 9304 * cpu_util() - Estimates the amount of CPU capacity used by CFS tasks. 9305 * @cpu: the CPU to get the utilization for 9306 * @p: task for which the CPU utilization should be predicted or NULL 9307 * @dst_cpu: CPU @p migrates to, -1 if @p moves from @cpu or @p == NULL 9308 * @boost: 1 to enable boosting, otherwise 0 9309 * 9310 * The unit of the return value must be the same as the one of CPU capacity 9311 * so that CPU utilization can be compared with CPU capacity. 9312 * 9313 * CPU utilization is the sum of running time of runnable tasks plus the 9314 * recent utilization of currently non-runnable tasks on that CPU. 9315 * It represents the amount of CPU capacity currently used by CFS tasks in 9316 * the range [0..max CPU capacity] with max CPU capacity being the CPU 9317 * capacity at f_max. 9318 * 9319 * The estimated CPU utilization is defined as the maximum between CPU 9320 * utilization and sum of the estimated utilization of the currently 9321 * runnable tasks on that CPU. It preserves a utilization "snapshot" of 9322 * previously-executed tasks, which helps better deduce how busy a CPU will 9323 * be when a long-sleeping task wakes up. The contribution to CPU utilization 9324 * of such a task would be significantly decayed at this point of time. 9325 * 9326 * Boosted CPU utilization is defined as max(CPU runnable, CPU utilization). 9327 * CPU contention for CFS tasks can be detected by CPU runnable > CPU 9328 * utilization. Boosting is implemented in cpu_util() so that internal 9329 * users (e.g. EAS) can use it next to external users (e.g. schedutil), 9330 * latter via cpu_util_cfs_boost(). 9331 * 9332 * CPU utilization can be higher than the current CPU capacity 9333 * (f_curr/f_max * max CPU capacity) or even the max CPU capacity because 9334 * of rounding errors as well as task migrations or wakeups of new tasks. 9335 * CPU utilization has to be capped to fit into the [0..max CPU capacity] 9336 * range. Otherwise a group of CPUs (CPU0 util = 121% + CPU1 util = 80%) 9337 * could be seen as over-utilized even though CPU1 has 20% of spare CPU 9338 * capacity. CPU utilization is allowed to overshoot current CPU capacity 9339 * though since this is useful for predicting the CPU capacity required 9340 * after task migrations (scheduler-driven DVFS). 9341 * 9342 * Return: (Boosted) (estimated) utilization for the specified CPU. 9343 */ 9344 static unsigned long 9345 cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost) 9346 { 9347 bool add_task = p && task_cpu(p) != cpu && dst_cpu == cpu; 9348 bool sub_task = p && task_cpu(p) == cpu && dst_cpu != cpu; 9349 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; 9350 unsigned long util = READ_ONCE(cfs_rq->avg.util_avg); 9351 unsigned long runnable; 9352 9353 /* 9354 * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its 9355 * contribution. If @p migrates from another CPU to @cpu add its 9356 * contribution. In all the other cases @cpu is not impacted by the 9357 * migration so its util_avg is already correct. 9358 */ 9359 if (add_task) 9360 util += task_util(p); 9361 else if (sub_task) 9362 lsub_positive(&util, task_util(p)); 9363 9364 if (boost) { 9365 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 9366 if (add_task) 9367 runnable += READ_ONCE(p->se.avg.runnable_avg); 9368 else if (sub_task) 9369 lsub_positive(&runnable, 9370 READ_ONCE(p->se.avg.runnable_avg)); 9371 util = max(util, runnable); 9372 } 9373 9374 if (sched_feat(UTIL_EST)) { 9375 unsigned long util_est; 9376 9377 util_est = READ_ONCE(cfs_rq->avg.util_est); 9378 9379 /* 9380 * During wake-up @p isn't enqueued yet and doesn't contribute 9381 * to any cpu_rq(cpu)->cfs.avg.util_est. 9382 * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p 9383 * has been enqueued. 9384 * 9385 * During exec (@dst_cpu = -1) @p is enqueued and does 9386 * contribute to cpu_rq(cpu)->cfs.util_est. 9387 * Remove it to "simulate" cpu_util without @p's contribution. 9388 * 9389 * Despite the task_on_rq_queued(@p) check there is still a 9390 * small window for a possible race when an exec 9391 * select_task_rq_fair() races with LB's detach_task(). 9392 * 9393 * detach_task() 9394 * deactivate_task() 9395 * p->on_rq = TASK_ON_RQ_MIGRATING; 9396 * -------------------------------- A 9397 * dequeue_task() \ 9398 * dequeue_task_fair() + Race Time 9399 * util_est_dequeue() / 9400 * -------------------------------- B 9401 * 9402 * The additional check "current == p" is required to further 9403 * reduce the race window. 9404 */ 9405 if (dst_cpu == cpu) 9406 util_est += _task_util_est(p); 9407 else if (p && unlikely(task_on_rq_queued(p) || current == p)) 9408 lsub_positive(&util_est, _task_util_est(p)); 9409 9410 util = max(util, util_est); 9411 } 9412 9413 return min(util, arch_scale_cpu_capacity(cpu)); 9414 } 9415 9416 unsigned long cpu_util_cfs(int cpu) 9417 { 9418 return cpu_util(cpu, NULL, -1, 0); 9419 } 9420 9421 unsigned long cpu_util_cfs_boost(int cpu) 9422 { 9423 return cpu_util(cpu, NULL, -1, 1); 9424 } 9425 9426 /* 9427 * cpu_util_without: compute cpu utilization without any contributions from *p 9428 * @cpu: the CPU which utilization is requested 9429 * @p: the task which utilization should be discounted 9430 * 9431 * The utilization of a CPU is defined by the utilization of tasks currently 9432 * enqueued on that CPU as well as tasks which are currently sleeping after an 9433 * execution on that CPU. 9434 * 9435 * This method returns the utilization of the specified CPU by discounting the 9436 * utilization of the specified task, whenever the task is currently 9437 * contributing to the CPU utilization. 9438 */ 9439 static unsigned long cpu_util_without(int cpu, struct task_struct *p) 9440 { 9441 /* Task has no contribution or is new */ 9442 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 9443 p = NULL; 9444 9445 return cpu_util(cpu, p, -1, 0); 9446 } 9447 9448 /* 9449 * This function computes an effective utilization for the given CPU, to be 9450 * used for frequency selection given the linear relation: f = u * f_max. 9451 * 9452 * The scheduler tracks the following metrics: 9453 * 9454 * cpu_util_{cfs,rt,dl,irq}() 9455 * cpu_bw_dl() 9456 * 9457 * Where the cfs,rt and dl util numbers are tracked with the same metric and 9458 * synchronized windows and are thus directly comparable. 9459 * 9460 * The cfs,rt,dl utilization are the running times measured with rq->clock_task 9461 * which excludes things like IRQ and steal-time. These latter are then accrued 9462 * in the IRQ utilization. 9463 * 9464 * The DL bandwidth number OTOH is not a measured metric but a value computed 9465 * based on the task model parameters and gives the minimal utilization 9466 * required to meet deadlines. 9467 */ 9468 unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, 9469 unsigned long *min, 9470 unsigned long *max) 9471 { 9472 unsigned long util, irq, scale; 9473 struct rq *rq = cpu_rq(cpu); 9474 9475 scale = arch_scale_cpu_capacity(cpu); 9476 9477 /* 9478 * Early check to see if IRQ/steal time saturates the CPU, can be 9479 * because of inaccuracies in how we track these -- see 9480 * update_irq_load_avg(). 9481 */ 9482 irq = cpu_util_irq(rq); 9483 if (unlikely(irq >= scale)) { 9484 if (min) 9485 *min = scale; 9486 if (max) 9487 *max = scale; 9488 return scale; 9489 } 9490 9491 if (min) { 9492 /* 9493 * The minimum utilization returns the highest level between: 9494 * - the computed DL bandwidth needed with the IRQ pressure which 9495 * steals time to the deadline task. 9496 * - The minimum performance requirement for CFS and/or RT. 9497 */ 9498 *min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN)); 9499 9500 /* 9501 * When an RT task is runnable and uclamp is not used, we must 9502 * ensure that the task will run at maximum compute capacity. 9503 */ 9504 if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt)) 9505 *min = max(*min, scale); 9506 } 9507 9508 /* 9509 * Because the time spend on RT/DL tasks is visible as 'lost' time to 9510 * CFS tasks and we use the same metric to track the effective 9511 * utilization (PELT windows are synchronized) we can directly add them 9512 * to obtain the CPU's actual utilization. 9513 */ 9514 util = util_cfs + cpu_util_rt(rq); 9515 util += cpu_util_dl(rq); 9516 9517 /* 9518 * The maximum hint is a soft bandwidth requirement, which can be lower 9519 * than the actual utilization because of uclamp_max requirements. 9520 */ 9521 if (max) 9522 *max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX)); 9523 9524 if (util >= scale) 9525 return scale; 9526 9527 /* 9528 * There is still idle time; further improve the number by using the 9529 * IRQ metric. Because IRQ/steal time is hidden from the task clock we 9530 * need to scale the task numbers: 9531 * 9532 * max - irq 9533 * U' = irq + --------- * U 9534 * max 9535 */ 9536 util = scale_irq_capacity(util, irq, scale); 9537 util += irq; 9538 9539 return min(scale, util); 9540 } 9541 9542 unsigned long sched_cpu_util(int cpu) 9543 { 9544 return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL); 9545 } 9546 9547 /* 9548 * energy_env - Utilization landscape for energy estimation. 9549 * @task_busy_time: Utilization contribution by the task for which we test the 9550 * placement. Given by eenv_task_busy_time(). 9551 * @pd_busy_time: Utilization of the whole perf domain without the task 9552 * contribution. Given by eenv_pd_busy_time(). 9553 * @cpu_cap: Maximum CPU capacity for the perf domain. 9554 * @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap). 9555 */ 9556 struct energy_env { 9557 unsigned long task_busy_time; 9558 unsigned long pd_busy_time; 9559 unsigned long cpu_cap; 9560 unsigned long pd_cap; 9561 }; 9562 9563 /* 9564 * Compute the task busy time for compute_energy(). This time cannot be 9565 * injected directly into effective_cpu_util() because of the IRQ scaling. 9566 * The latter only makes sense with the most recent CPUs where the task has 9567 * run. 9568 */ 9569 static inline void eenv_task_busy_time(struct energy_env *eenv, 9570 struct task_struct *p, int prev_cpu) 9571 { 9572 unsigned long busy_time, max_cap = arch_scale_cpu_capacity(prev_cpu); 9573 unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu)); 9574 9575 if (unlikely(irq >= max_cap)) 9576 busy_time = max_cap; 9577 else 9578 busy_time = scale_irq_capacity(task_util_est(p), irq, max_cap); 9579 9580 eenv->task_busy_time = busy_time; 9581 } 9582 9583 /* 9584 * Compute the perf_domain (PD) busy time for compute_energy(). Based on the 9585 * utilization for each @pd_cpus, it however doesn't take into account 9586 * clamping since the ratio (utilization / cpu_capacity) is already enough to 9587 * scale the EM reported power consumption at the (eventually clamped) 9588 * cpu_capacity. 9589 * 9590 * The contribution of the task @p for which we want to estimate the 9591 * energy cost is removed (by cpu_util()) and must be calculated 9592 * separately (see eenv_task_busy_time). This ensures: 9593 * 9594 * - A stable PD utilization, no matter which CPU of that PD we want to place 9595 * the task on. 9596 * 9597 * - A fair comparison between CPUs as the task contribution (task_util()) 9598 * will always be the same no matter which CPU utilization we rely on 9599 * (util_avg or util_est). 9600 * 9601 * Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't 9602 * exceed @eenv->pd_cap. 9603 */ 9604 static inline void eenv_pd_busy_time(struct energy_env *eenv, 9605 struct cpumask *pd_cpus, 9606 struct task_struct *p) 9607 { 9608 unsigned long busy_time = 0; 9609 int cpu; 9610 9611 for_each_cpu(cpu, pd_cpus) { 9612 unsigned long util = cpu_util(cpu, p, -1, 0); 9613 9614 busy_time += effective_cpu_util(cpu, util, NULL, NULL); 9615 } 9616 9617 eenv->pd_busy_time = min(eenv->pd_cap, busy_time); 9618 } 9619 9620 /* 9621 * Compute the maximum utilization for compute_energy() when the task @p 9622 * is placed on the cpu @dst_cpu. 9623 * 9624 * Returns the maximum utilization among @eenv->cpus. This utilization can't 9625 * exceed @eenv->cpu_cap. 9626 */ 9627 static inline unsigned long 9628 eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus, 9629 struct task_struct *p, int dst_cpu) 9630 { 9631 unsigned long max_util = 0; 9632 int cpu; 9633 9634 for_each_cpu(cpu, pd_cpus) { 9635 struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL; 9636 unsigned long util = cpu_util(cpu, p, dst_cpu, 1); 9637 unsigned long eff_util, min, max; 9638 9639 /* 9640 * Performance domain frequency: utilization clamping 9641 * must be considered since it affects the selection 9642 * of the performance domain frequency. 9643 * NOTE: in case RT tasks are running, by default the min 9644 * utilization can be max OPP. 9645 */ 9646 eff_util = effective_cpu_util(cpu, util, &min, &max); 9647 9648 /* Task's uclamp can modify min and max value */ 9649 if (tsk && uclamp_is_used()) { 9650 min = max(min, uclamp_eff_value(p, UCLAMP_MIN)); 9651 9652 /* 9653 * If there is no active max uclamp constraint, 9654 * directly use task's one, otherwise keep max. 9655 */ 9656 if (uclamp_rq_is_idle(cpu_rq(cpu))) 9657 max = uclamp_eff_value(p, UCLAMP_MAX); 9658 else 9659 max = max(max, uclamp_eff_value(p, UCLAMP_MAX)); 9660 } 9661 9662 eff_util = sugov_effective_cpu_perf(cpu, eff_util, min, max); 9663 max_util = max(max_util, eff_util); 9664 } 9665 9666 return min(max_util, eenv->cpu_cap); 9667 } 9668 9669 /* 9670 * compute_energy(): Use the Energy Model to estimate the energy that @pd would 9671 * consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task 9672 * contribution is ignored. 9673 */ 9674 static inline unsigned long 9675 compute_energy(struct energy_env *eenv, struct perf_domain *pd, 9676 struct cpumask *pd_cpus, struct task_struct *p, int dst_cpu) 9677 { 9678 unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu); 9679 unsigned long busy_time = eenv->pd_busy_time; 9680 unsigned long energy; 9681 9682 if (dst_cpu >= 0) 9683 busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time); 9684 9685 energy = em_cpu_energy(pd->em_pd, max_util, busy_time, eenv->cpu_cap); 9686 9687 trace_sched_compute_energy_tp(p, dst_cpu, energy, max_util, busy_time); 9688 9689 return energy; 9690 } 9691 9692 /* 9693 * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the 9694 * waking task. find_energy_efficient_cpu() looks for the CPU with maximum 9695 * spare capacity in each performance domain and uses it as a potential 9696 * candidate to execute the task. Then, it uses the Energy Model to figure 9697 * out which of the CPU candidates is the most energy-efficient. 9698 * 9699 * The rationale for this heuristic is as follows. In a performance domain, 9700 * all the most energy efficient CPU candidates (according to the Energy 9701 * Model) are those for which we'll request a low frequency. When there are 9702 * several CPUs for which the frequency request will be the same, we don't 9703 * have enough data to break the tie between them, because the Energy Model 9704 * only includes active power costs. With this model, if we assume that 9705 * frequency requests follow utilization (e.g. using schedutil), the CPU with 9706 * the maximum spare capacity in a performance domain is guaranteed to be among 9707 * the best candidates of the performance domain. 9708 * 9709 * In practice, it could be preferable from an energy standpoint to pack 9710 * small tasks on a CPU in order to let other CPUs go in deeper idle states, 9711 * but that could also hurt our chances to go cluster idle, and we have no 9712 * ways to tell with the current Energy Model if this is actually a good 9713 * idea or not. So, find_energy_efficient_cpu() basically favors 9714 * cluster-packing, and spreading inside a cluster. That should at least be 9715 * a good thing for latency, and this is consistent with the idea that most 9716 * of the energy savings of EAS come from the asymmetry of the system, and 9717 * not so much from breaking the tie between identical CPUs. That's also the 9718 * reason why EAS is enabled in the topology code only for systems where 9719 * SD_ASYM_CPUCAPACITY is set. 9720 * 9721 * NOTE: Forkees are not accepted in the energy-aware wake-up path because 9722 * they don't have any useful utilization data yet and it's not possible to 9723 * forecast their impact on energy consumption. Consequently, they will be 9724 * placed by sched_balance_find_dst_cpu() on the least loaded CPU, which might turn out 9725 * to be energy-inefficient in some use-cases. The alternative would be to 9726 * bias new tasks towards specific types of CPUs first, or to try to infer 9727 * their util_avg from the parent task, but those heuristics could hurt 9728 * other use-cases too. So, until someone finds a better way to solve this, 9729 * let's keep things simple by re-using the existing slow path. 9730 */ 9731 static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 9732 { 9733 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 9734 unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; 9735 unsigned long p_util_min = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MIN) : 0; 9736 unsigned long p_util_max = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MAX) : 1024; 9737 struct root_domain *rd = this_rq()->rd; 9738 int cpu, best_energy_cpu, target = -1; 9739 int prev_fits = -1, best_fits = -1; 9740 unsigned long best_actual_cap = 0; 9741 unsigned long prev_actual_cap = 0; 9742 struct sched_domain *sd; 9743 struct perf_domain *pd; 9744 struct energy_env eenv; 9745 9746 pd = rcu_dereference_all(rd->pd); 9747 if (!pd) 9748 return target; 9749 9750 /* 9751 * Energy-aware wake-up happens on the lowest sched_domain starting 9752 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu. 9753 */ 9754 sd = rcu_dereference_all(*this_cpu_ptr(&sd_asym_cpucapacity)); 9755 while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 9756 sd = sd->parent; 9757 if (!sd) 9758 return target; 9759 9760 target = prev_cpu; 9761 9762 sync_entity_load_avg(&p->se); 9763 if (!task_util_est(p) && p_util_min == 0) 9764 return target; 9765 9766 eenv_task_busy_time(&eenv, p, prev_cpu); 9767 9768 for (; pd; pd = pd->next) { 9769 unsigned long util_min = p_util_min, util_max = p_util_max; 9770 unsigned long cpu_cap, cpu_actual_cap, util; 9771 long prev_spare_cap = -1, max_spare_cap = -1; 9772 unsigned long rq_util_min, rq_util_max; 9773 unsigned long cur_delta, base_energy; 9774 int max_spare_cap_cpu = -1; 9775 int fits, max_fits = -1; 9776 9777 if (!cpumask_and(cpus, perf_domain_span(pd), cpu_online_mask)) 9778 continue; 9779 9780 /* Account external pressure for the energy estimation */ 9781 cpu = cpumask_first(cpus); 9782 cpu_actual_cap = get_actual_cpu_capacity(cpu); 9783 9784 eenv.cpu_cap = cpu_actual_cap; 9785 eenv.pd_cap = 0; 9786 9787 for_each_cpu(cpu, cpus) { 9788 struct rq *rq = cpu_rq(cpu); 9789 9790 eenv.pd_cap += cpu_actual_cap; 9791 9792 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 9793 continue; 9794 9795 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 9796 continue; 9797 9798 util = cpu_util(cpu, p, cpu, 0); 9799 cpu_cap = capacity_of(cpu); 9800 9801 /* 9802 * Skip CPUs that cannot satisfy the capacity request. 9803 * IOW, placing the task there would make the CPU 9804 * overutilized. Take uclamp into account to see how 9805 * much capacity we can get out of the CPU; this is 9806 * aligned with sched_cpu_util(). 9807 */ 9808 if (uclamp_is_used() && !uclamp_rq_is_idle(rq)) { 9809 /* 9810 * Open code uclamp_rq_util_with() except for 9811 * the clamp() part. I.e.: apply max aggregation 9812 * only. util_fits_cpu() logic requires to 9813 * operate on non clamped util but must use the 9814 * max-aggregated uclamp_{min, max}. 9815 */ 9816 rq_util_min = uclamp_rq_get(rq, UCLAMP_MIN); 9817 rq_util_max = uclamp_rq_get(rq, UCLAMP_MAX); 9818 9819 util_min = max(rq_util_min, p_util_min); 9820 util_max = max(rq_util_max, p_util_max); 9821 } 9822 9823 fits = util_fits_cpu(util, util_min, util_max, cpu); 9824 if (!fits) 9825 continue; 9826 9827 lsub_positive(&cpu_cap, util); 9828 9829 if (cpu == prev_cpu) { 9830 /* Always use prev_cpu as a candidate. */ 9831 prev_spare_cap = cpu_cap; 9832 prev_fits = fits; 9833 } else if ((fits > max_fits) || 9834 ((fits == max_fits) && ((long)cpu_cap > max_spare_cap))) { 9835 /* 9836 * Find the CPU with the maximum spare capacity 9837 * among the remaining CPUs in the performance 9838 * domain. 9839 */ 9840 max_spare_cap = cpu_cap; 9841 max_spare_cap_cpu = cpu; 9842 max_fits = fits; 9843 } 9844 } 9845 9846 if (max_spare_cap_cpu < 0 && prev_spare_cap < 0) 9847 continue; 9848 9849 eenv_pd_busy_time(&eenv, cpus, p); 9850 /* Compute the 'base' energy of the pd, without @p */ 9851 base_energy = compute_energy(&eenv, pd, cpus, p, -1); 9852 9853 /* Evaluate the energy impact of using prev_cpu. */ 9854 if (prev_spare_cap > -1) { 9855 prev_delta = compute_energy(&eenv, pd, cpus, p, 9856 prev_cpu); 9857 /* CPU utilization has changed */ 9858 if (prev_delta < base_energy) 9859 return target; 9860 prev_delta -= base_energy; 9861 prev_actual_cap = cpu_actual_cap; 9862 best_delta = min(best_delta, prev_delta); 9863 } 9864 9865 /* Evaluate the energy impact of using max_spare_cap_cpu. */ 9866 if (max_spare_cap_cpu >= 0 && max_spare_cap > prev_spare_cap) { 9867 /* Current best energy cpu fits better */ 9868 if (max_fits < best_fits) 9869 continue; 9870 9871 /* 9872 * Both don't fit performance hint (i.e. uclamp_min) 9873 * but best energy cpu has better capacity. 9874 */ 9875 if ((max_fits < 0) && 9876 (cpu_actual_cap <= best_actual_cap)) 9877 continue; 9878 9879 cur_delta = compute_energy(&eenv, pd, cpus, p, 9880 max_spare_cap_cpu); 9881 /* CPU utilization has changed */ 9882 if (cur_delta < base_energy) 9883 return target; 9884 cur_delta -= base_energy; 9885 9886 /* 9887 * Both fit for the task but best energy cpu has lower 9888 * energy impact. 9889 */ 9890 if ((max_fits > 0) && (best_fits > 0) && 9891 (cur_delta >= best_delta)) 9892 continue; 9893 9894 best_delta = cur_delta; 9895 best_energy_cpu = max_spare_cap_cpu; 9896 best_fits = max_fits; 9897 best_actual_cap = cpu_actual_cap; 9898 } 9899 } 9900 9901 if ((best_fits > prev_fits) || 9902 ((best_fits > 0) && (best_delta < prev_delta)) || 9903 ((best_fits < 0) && (best_actual_cap > prev_actual_cap))) 9904 target = best_energy_cpu; 9905 9906 return target; 9907 } 9908 9909 /* 9910 * select_task_rq_fair: Select target runqueue for the waking task in domains 9911 * that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE, 9912 * SD_BALANCE_FORK, or SD_BALANCE_EXEC. 9913 * 9914 * Balances load by selecting the idlest CPU in the idlest group, or under 9915 * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set. 9916 * 9917 * Returns the target CPU number. 9918 */ 9919 static int 9920 select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) 9921 { 9922 int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 9923 struct sched_domain *tmp, *sd = NULL; 9924 int cpu = smp_processor_id(); 9925 int new_cpu = prev_cpu; 9926 int want_affine = 0; 9927 /* SD_flags and WF_flags share the first nibble */ 9928 int sd_flag = wake_flags & 0xF; 9929 9930 /* 9931 * required for stable ->cpus_allowed 9932 */ 9933 lockdep_assert_held(&p->pi_lock); 9934 if (wake_flags & WF_TTWU) { 9935 record_wakee(p); 9936 9937 if ((wake_flags & WF_CURRENT_CPU) && 9938 cpumask_test_cpu(cpu, p->cpus_ptr)) 9939 return cpu; 9940 9941 if (!is_rd_overutilized(this_rq()->rd)) { 9942 new_cpu = find_energy_efficient_cpu(p, prev_cpu); 9943 if (new_cpu >= 0) 9944 return new_cpu; 9945 new_cpu = prev_cpu; 9946 } 9947 9948 want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr); 9949 } 9950 9951 for_each_domain(cpu, tmp) { 9952 /* 9953 * If both 'cpu' and 'prev_cpu' are part of this domain, 9954 * cpu is a valid SD_WAKE_AFFINE target. 9955 */ 9956 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && 9957 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { 9958 if (cpu != prev_cpu) 9959 new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync); 9960 9961 sd = NULL; /* Prefer wake_affine over balance flags */ 9962 break; 9963 } 9964 9965 /* 9966 * Usually only true for WF_EXEC and WF_FORK, as sched_domains 9967 * usually do not have SD_BALANCE_WAKE set. That means wakeup 9968 * will usually go to the fast path. 9969 */ 9970 if (tmp->flags & sd_flag) 9971 sd = tmp; 9972 else if (!want_affine) 9973 break; 9974 } 9975 9976 /* Slow path */ 9977 if (unlikely(sd)) { 9978 new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag); 9979 return select_idle_smt_cpu(p, new_cpu); 9980 } 9981 9982 /* Fast path */ 9983 if (wake_flags & WF_TTWU) 9984 return select_idle_sibling(p, prev_cpu, new_cpu); 9985 9986 return new_cpu; 9987 } 9988 9989 /* 9990 * Called immediately before a task is migrated to a new CPU; task_cpu(p) and 9991 * cfs_rq_of(p) references at time of call are still valid and identify the 9992 * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held. 9993 */ 9994 static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) 9995 { 9996 struct sched_entity *se = &p->se; 9997 9998 if (!task_on_rq_migrating(p)) { 9999 remove_entity_load_avg(se); 10000 10001 /* 10002 * Here, the task's PELT values have been updated according to 10003 * the current rq's clock. But if that clock hasn't been 10004 * updated in a while, a substantial idle time will be missed, 10005 * leading to an inflation after wake-up on the new rq. 10006 * 10007 * Estimate the missing time from the cfs_rq last_update_time 10008 * and update sched_avg to improve the PELT continuity after 10009 * migration. 10010 */ 10011 migrate_se_pelt_lag(se); 10012 } 10013 10014 /* Tell new CPU we are migrated */ 10015 se->avg.last_update_time = 0; 10016 10017 update_scan_period(p, new_cpu); 10018 } 10019 10020 static void task_dead_fair(struct task_struct *p) 10021 { 10022 struct sched_entity *se = &p->se; 10023 remove_entity_load_avg(se); 10024 } 10025 10026 /* 10027 * Set the max capacity the task is allowed to run at for misfit detection. 10028 */ 10029 static void set_task_max_allowed_capacity(struct task_struct *p) 10030 { 10031 struct asym_cap_data *entry; 10032 10033 if (!sched_asym_cpucap_active()) 10034 return; 10035 10036 rcu_read_lock(); 10037 list_for_each_entry_rcu(entry, &asym_cap_list, link) { 10038 cpumask_t *cpumask; 10039 10040 cpumask = cpu_capacity_span(entry); 10041 if (!cpumask_intersects(p->cpus_ptr, cpumask)) 10042 continue; 10043 10044 p->max_allowed_capacity = entry->capacity; 10045 break; 10046 } 10047 rcu_read_unlock(); 10048 } 10049 10050 static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context *ctx) 10051 { 10052 set_cpus_allowed_common(p, ctx); 10053 set_task_max_allowed_capacity(p); 10054 } 10055 10056 enum preempt_wakeup_action { 10057 PREEMPT_WAKEUP_NONE, /* No preemption. */ 10058 PREEMPT_WAKEUP_SHORT, /* Ignore slice protection. */ 10059 PREEMPT_WAKEUP_PICK, /* Let pick_eevdf() decide. */ 10060 PREEMPT_WAKEUP_RESCHED, /* Force reschedule. */ 10061 }; 10062 10063 static inline bool set_preempt_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse) 10064 { 10065 /* 10066 * Keep existing buddy if the deadline is sooner than pse. 10067 * The older buddy may be cache cold and completely unrelated 10068 * to the current wakeup but that is unpredictable where as 10069 * obeying the deadline is more in line with EEVDF objectives. 10070 */ 10071 if (cfs_rq->next && entity_before(cfs_rq->next, pse)) 10072 return false; 10073 10074 set_next_buddy(cfs_rq, pse); 10075 return true; 10076 } 10077 10078 static inline bool set_short_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse) 10079 { 10080 if (cfs_rq->next) { 10081 if (cfs_rq->next->slice < pse->slice) 10082 return false; 10083 10084 if (cfs_rq->next->slice == pse->slice && 10085 entity_before(cfs_rq->next, pse)) 10086 return false; 10087 } 10088 10089 set_next_buddy(cfs_rq, pse); 10090 return true; 10091 } 10092 10093 /* 10094 * WF_SYNC|WF_TTWU indicates the waker expects to sleep but it is not 10095 * strictly enforced because the hint is either misunderstood or 10096 * multiple tasks must be woken up. 10097 */ 10098 static inline enum preempt_wakeup_action 10099 preempt_sync(struct rq *rq, int wake_flags, 10100 struct sched_entity *pse, struct sched_entity *se) 10101 { 10102 u64 threshold, delta; 10103 10104 /* 10105 * WF_SYNC without WF_TTWU is not expected so warn if it happens even 10106 * though it is likely harmless. 10107 */ 10108 WARN_ON_ONCE(!(wake_flags & WF_TTWU)); 10109 10110 threshold = sysctl_sched_migration_cost; 10111 delta = rq_clock_task(rq) - se->exec_start; 10112 if ((s64)delta < 0) 10113 delta = 0; 10114 10115 /* 10116 * WF_RQ_SELECTED implies the tasks are stacking on a CPU when they 10117 * could run on other CPUs. Reduce the threshold before preemption is 10118 * allowed to an arbitrary lower value as it is more likely (but not 10119 * guaranteed) the waker requires the wakee to finish. 10120 */ 10121 if (wake_flags & WF_RQ_SELECTED) 10122 threshold >>= 2; 10123 10124 /* 10125 * As WF_SYNC is not strictly obeyed, allow some runtime for batch 10126 * wakeups to be issued. 10127 */ 10128 if (entity_before(pse, se) && delta >= threshold) 10129 return PREEMPT_WAKEUP_RESCHED; 10130 10131 return PREEMPT_WAKEUP_NONE; 10132 } 10133 10134 /* 10135 * Preempt the current task with a newly woken task if needed: 10136 */ 10137 static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_flags) 10138 { 10139 enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK; 10140 struct task_struct *donor = rq->donor; 10141 struct sched_entity *nse, *se = &donor->se, *pse = &p->se; 10142 struct cfs_rq *cfs_rq = &rq->cfs; 10143 int cse_is_idle, pse_is_idle; 10144 10145 /* 10146 * XXX Getting preempted by higher class, try and find idle CPU? 10147 */ 10148 if (p->sched_class != &fair_sched_class || 10149 donor->sched_class != &fair_sched_class) 10150 return; 10151 10152 if (unlikely(se == pse)) 10153 return; 10154 10155 /* 10156 * This is possible from callers such as attach_tasks(), in which we 10157 * unconditionally wakeup_preempt() after an enqueue (which may have 10158 * lead to a throttle). This both saves work and prevents false 10159 * next-buddy nomination below. 10160 */ 10161 if (task_is_throttled(p)) 10162 return; 10163 10164 /* 10165 * We can come here with TIF_NEED_RESCHED already set from new task 10166 * wake up path. 10167 * 10168 * Note: this also catches the edge-case of curr being in a throttled 10169 * group (e.g. via set_curr_task), since update_curr() (in the 10170 * enqueue of curr) will have resulted in resched being set. This 10171 * prevents us from potentially nominating it as a false LAST_BUDDY 10172 * below. 10173 */ 10174 if (!sched_feat(PREEMPT_SHORT) && test_tsk_need_resched(rq->curr)) 10175 return; 10176 10177 if (!sched_feat(WAKEUP_PREEMPTION)) 10178 return; 10179 10180 WARN_ON_ONCE(!pse); 10181 10182 cse_is_idle = se_is_idle(se); 10183 pse_is_idle = se_is_idle(pse); 10184 10185 nse = se; 10186 /* 10187 * Preempt an idle entity in favor of a non-idle entity (and don't preempt 10188 * in the inverse case). 10189 */ 10190 if (cse_is_idle && !pse_is_idle) 10191 goto preempt; 10192 10193 update_curr_fair(rq); 10194 10195 if (cse_is_idle != pse_is_idle) 10196 goto update; 10197 10198 /* 10199 * BATCH and IDLE tasks do not preempt others. 10200 */ 10201 if (unlikely(!normal_policy(p->policy))) 10202 goto update; 10203 10204 /* 10205 * Do not preempt for tasks that are sched_delayed as it would violate 10206 * EEVDF to forcibly queue an ineligible task. 10207 */ 10208 if (pse->sched_delayed) 10209 goto update; 10210 10211 /* 10212 * If @p has a shorter slice than current and @p is eligible, override 10213 * current's slice protection in order to allow preemption. 10214 */ 10215 if (sched_feat(PREEMPT_SHORT) && (pse->slice < se->slice)) { 10216 preempt_action = PREEMPT_WAKEUP_SHORT; 10217 goto pick; 10218 } 10219 10220 /* 10221 * Ignore wakee preemption on WF_FORK as it is less likely that 10222 * there is shared data as exec often follow fork. 10223 */ 10224 if (wake_flags & WF_FORK) 10225 goto update; 10226 10227 /* Prefer picking wakee soon if appropriate. */ 10228 if (sched_feat(NEXT_BUDDY) && set_preempt_buddy(cfs_rq, pse)) { 10229 /* 10230 * Decide whether to obey WF_SYNC hint for a new buddy. Old 10231 * buddies are ignored as they may not be relevant to the 10232 * waker and less likely to be cache hot. 10233 */ 10234 if (wake_flags & WF_SYNC) 10235 preempt_action = preempt_sync(rq, wake_flags, pse, se); 10236 } 10237 10238 switch (preempt_action) { 10239 case PREEMPT_WAKEUP_NONE: 10240 return; 10241 case PREEMPT_WAKEUP_RESCHED: 10242 goto preempt; 10243 case PREEMPT_WAKEUP_SHORT: 10244 fallthrough; 10245 case PREEMPT_WAKEUP_PICK: 10246 break; 10247 } 10248 10249 pick: 10250 if (cfs_rq->h_nr_queued) { 10251 nse = pick_next_entity(rq, preempt_action != PREEMPT_WAKEUP_SHORT); 10252 if (unlikely(!nse)) 10253 goto pick; 10254 10255 /* If @p has become the most eligible task, force preemption */ 10256 if (nse == pse) 10257 goto preempt; 10258 } 10259 10260 /* 10261 * If @p is eligible but not the next task to run then cancel protection 10262 * to prevent large scheduling latency 10263 */ 10264 if (preempt_action == PREEMPT_WAKEUP_SHORT && entity_eligible(cfs_rq, pse)) 10265 goto preempt; 10266 update: 10267 if (sched_feat(RUN_TO_PARITY)) 10268 update_protect_slice(cfs_rq, se); 10269 10270 return; 10271 10272 preempt: 10273 cancel_protect_slice(se); 10274 10275 if (preempt_action == PREEMPT_WAKEUP_SHORT) 10276 set_short_buddy(cfs_rq, pse); 10277 10278 resched_curr_lazy(rq); 10279 } 10280 10281 struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) 10282 __must_hold(__rq_lockp(rq)) 10283 { 10284 struct cfs_rq *cfs_rq = &rq->cfs; 10285 struct sched_entity *se; 10286 struct task_struct *p; 10287 int new_tasks; 10288 10289 again: 10290 if (!cfs_rq->h_nr_queued) 10291 goto idle; 10292 10293 /* Might not have done put_prev_entity() */ 10294 if (cfs_rq->curr && cfs_rq->curr->on_rq) 10295 update_curr_eevdf(cfs_rq); 10296 10297 se = pick_next_entity(rq, true); 10298 if (!se) 10299 goto again; 10300 10301 p = task_of(se); 10302 return p; 10303 10304 idle: 10305 if (sched_core_enabled(rq)) 10306 return NULL; 10307 10308 new_tasks = sched_balance_newidle(rq, rf); 10309 if (new_tasks < 0) 10310 return RETRY_TASK; 10311 if (new_tasks > 0) 10312 goto again; 10313 return NULL; 10314 } 10315 10316 static struct task_struct * 10317 fair_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf) 10318 __must_hold(__rq_lockp(dl_se->rq)) 10319 { 10320 return pick_task_fair(dl_se->rq, rf); 10321 } 10322 10323 void fair_server_init(struct rq *rq) 10324 { 10325 struct sched_dl_entity *dl_se = &rq->fair_server; 10326 10327 init_dl_entity(dl_se); 10328 10329 dl_server_init(dl_se, rq, fair_server_pick_task); 10330 } 10331 10332 /* 10333 * Account for a descheduled task: 10334 */ 10335 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next) 10336 { 10337 struct sched_entity *se = &prev->se; 10338 struct cfs_rq *cfs_rq = &rq->cfs; 10339 struct sched_entity *nse = NULL; 10340 10341 #ifdef CONFIG_FAIR_GROUP_SCHED 10342 if (next && next->sched_class == &fair_sched_class) 10343 nse = &next->se; 10344 #endif 10345 10346 while (se) { 10347 cfs_rq = cfs_rq_of(se); 10348 if (!nse || cfs_rq->h_curr) 10349 put_prev_entity(cfs_rq, se); 10350 #ifdef CONFIG_FAIR_GROUP_SCHED 10351 if (nse) { 10352 if (is_same_group(se, nse)) 10353 break; 10354 10355 int d = nse->depth - se->depth; 10356 if (d >= 0) { 10357 /* nse has equal or greater depth, ascend */ 10358 nse = parent_entity(nse); 10359 /* if nse is the deeper, do not ascend se */ 10360 if (d > 0) 10361 continue; 10362 } 10363 } 10364 #endif 10365 se = parent_entity(se); 10366 } 10367 10368 /* Put 'current' back into the tree. */ 10369 cfs_rq = &rq->cfs; 10370 se = &prev->se; 10371 WARN_ON_ONCE(cfs_rq->curr != se); 10372 cfs_rq->curr = NULL; 10373 if (se->on_rq) 10374 __enqueue_entity(cfs_rq, se); 10375 } 10376 10377 /* 10378 * sched_yield() is very simple 10379 */ 10380 static void yield_task_fair(struct rq *rq) 10381 { 10382 struct task_struct *curr = rq->donor; 10383 struct sched_entity *se = &curr->se; 10384 struct cfs_rq *cfs_rq = &rq->cfs; 10385 10386 /* 10387 * Are we the only task in the tree? 10388 */ 10389 if (unlikely(rq->nr_running == 1)) 10390 return; 10391 10392 clear_buddies(cfs_rq, se); 10393 10394 update_rq_clock(rq); 10395 /* 10396 * Update run-time statistics of the 'current'. 10397 */ 10398 update_curr_eevdf(cfs_rq); 10399 /* 10400 * Tell update_rq_clock() that we've just updated, 10401 * so we don't do microscopic update in schedule() 10402 * and double the fastpath cost. 10403 */ 10404 rq_clock_skip_update(rq); 10405 10406 /* 10407 * Forfeit the remaining vruntime, only if the entity is eligible. This 10408 * condition is necessary because in core scheduling we prefer to run 10409 * ineligible tasks rather than force idling. If this happens we may 10410 * end up in a loop where the core scheduler picks the yielding task, 10411 * which yields immediately again; without the condition the vruntime 10412 * ends up quickly running away. 10413 */ 10414 if (entity_eligible(cfs_rq, se)) { 10415 se->vruntime = se->deadline; 10416 update_deadline(cfs_rq, se); 10417 } 10418 } 10419 10420 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) 10421 { 10422 struct sched_entity *se = &p->se; 10423 10424 /* !se->on_rq also covers throttled task */ 10425 if (!se->on_rq || se->sched_delayed) 10426 return false; 10427 10428 /* Tell the scheduler that we'd really like se to run next. */ 10429 set_next_buddy(&task_rq(p)->cfs, se); 10430 10431 yield_task_fair(rq); 10432 10433 return true; 10434 } 10435 10436 /************************************************** 10437 * Fair scheduling class load-balancing methods. 10438 * 10439 * BASICS 10440 * 10441 * The purpose of load-balancing is to achieve the same basic fairness the 10442 * per-CPU scheduler provides, namely provide a proportional amount of compute 10443 * time to each task. This is expressed in the following equation: 10444 * 10445 * W_i,n/P_i == W_j,n/P_j for all i,j (1) 10446 * 10447 * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight 10448 * W_i,0 is defined as: 10449 * 10450 * W_i,0 = \Sum_j w_i,j (2) 10451 * 10452 * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight 10453 * is derived from the nice value as per sched_prio_to_weight[]. 10454 * 10455 * The weight average is an exponential decay average of the instantaneous 10456 * weight: 10457 * 10458 * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) 10459 * 10460 * C_i is the compute capacity of CPU i, typically it is the 10461 * fraction of 'recent' time available for SCHED_OTHER task execution. But it 10462 * can also include other factors [XXX]. 10463 * 10464 * To achieve this balance we define a measure of imbalance which follows 10465 * directly from (1): 10466 * 10467 * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) 10468 * 10469 * We them move tasks around to minimize the imbalance. In the continuous 10470 * function space it is obvious this converges, in the discrete case we get 10471 * a few fun cases generally called infeasible weight scenarios. 10472 * 10473 * [XXX expand on: 10474 * - infeasible weights; 10475 * - local vs global optima in the discrete case. ] 10476 * 10477 * 10478 * SCHED DOMAINS 10479 * 10480 * In order to solve the imbalance equation (4), and avoid the obvious O(n^2) 10481 * for all i,j solution, we create a tree of CPUs that follows the hardware 10482 * topology where each level pairs two lower groups (or better). This results 10483 * in O(log n) layers. Furthermore we reduce the number of CPUs going up the 10484 * tree to only the first of the previous level and we decrease the frequency 10485 * of load-balance at each level inversely proportional to the number of CPUs in 10486 * the groups. 10487 * 10488 * This yields: 10489 * 10490 * log_2 n 1 n 10491 * \Sum { --- * --- * 2^i } = O(n) (5) 10492 * i = 0 2^i 2^i 10493 * `- size of each group 10494 * | | `- number of CPUs doing load-balance 10495 * | `- freq 10496 * `- sum over all levels 10497 * 10498 * Coupled with a limit on how many tasks we can migrate every balance pass, 10499 * this makes (5) the runtime complexity of the balancer. 10500 * 10501 * An important property here is that each CPU is still (indirectly) connected 10502 * to every other CPU in at most O(log n) steps: 10503 * 10504 * The adjacency matrix of the resulting graph is given by: 10505 * 10506 * log_2 n 10507 * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) 10508 * k = 0 10509 * 10510 * And you'll find that: 10511 * 10512 * A^(log_2 n)_i,j != 0 for all i,j (7) 10513 * 10514 * Showing there's indeed a path between every CPU in at most O(log n) steps. 10515 * The task movement gives a factor of O(m), giving a convergence complexity 10516 * of: 10517 * 10518 * O(nm log n), n := nr_cpus, m := nr_tasks (8) 10519 * 10520 * 10521 * WORK CONSERVING 10522 * 10523 * In order to avoid CPUs going idle while there's still work to do, new idle 10524 * balancing is more aggressive and has the newly idle CPU iterate up the domain 10525 * tree itself instead of relying on other CPUs to bring it work. 10526 * 10527 * This adds some complexity to both (5) and (8) but it reduces the total idle 10528 * time. 10529 * 10530 * [XXX more?] 10531 * 10532 * 10533 * CGROUPS 10534 * 10535 * Cgroups make a horror show out of (2), instead of a simple sum we get: 10536 * 10537 * s_k,i 10538 * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) 10539 * S_k 10540 * 10541 * Where 10542 * 10543 * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) 10544 * 10545 * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i. 10546 * 10547 * The big problem is S_k, its a global sum needed to compute a local (W_i) 10548 * property. 10549 * 10550 * [XXX write more on how we solve this.. _after_ merging pjt's patches that 10551 * rewrite all of this once again.] 10552 */ 10553 10554 static unsigned long __read_mostly max_load_balance_interval = HZ/10; 10555 10556 enum fbq_type { regular, remote, all }; 10557 10558 /* 10559 * 'group_type' describes the group of CPUs at the moment of load balancing. 10560 * 10561 * The enum is ordered by pulling priority, with the group with lowest priority 10562 * first so the group_type can simply be compared when selecting the busiest 10563 * group. See update_sd_pick_busiest(). 10564 */ 10565 enum group_type { 10566 /* The group has spare capacity that can be used to run more tasks. */ 10567 group_has_spare = 0, 10568 /* 10569 * The group is fully used and the tasks don't compete for more CPU 10570 * cycles. Nevertheless, some tasks might wait before running. 10571 */ 10572 group_fully_busy, 10573 /* 10574 * One task doesn't fit with CPU's capacity and must be migrated to a 10575 * more powerful CPU. 10576 */ 10577 group_misfit_task, 10578 /* 10579 * Balance SMT group that's fully busy. Can benefit from migration 10580 * a task on SMT with busy sibling to another CPU on idle core. 10581 */ 10582 group_smt_balance, 10583 /* 10584 * SD_ASYM_PACKING only: One local CPU with higher capacity is available, 10585 * and the task should be migrated to it instead of running on the 10586 * current CPU. 10587 */ 10588 group_asym_packing, 10589 /* 10590 * The tasks' affinity constraints previously prevented the scheduler 10591 * from balancing the load across the system. 10592 */ 10593 group_imbalanced, 10594 /* 10595 * There are tasks running on non-preferred LLC, possible to move 10596 * them to their preferred LLC without creating too much imbalance. 10597 * The priority of group_llc_balance is lower than that of 10598 * group_overloaded and higher than that of all other group types. 10599 * This is because group_llc_balance may exacerbate load imbalance. 10600 * If the LLC balancing attempt fails, the nr_balance_failed 10601 * mechanism will trigger other group types to rebalance the load. 10602 */ 10603 group_llc_balance, 10604 /* 10605 * The CPU is overloaded and can't provide expected CPU cycles to all 10606 * tasks. 10607 */ 10608 group_overloaded 10609 }; 10610 10611 enum migration_type { 10612 migrate_load = 0, 10613 migrate_util, 10614 migrate_task, 10615 migrate_misfit, 10616 migrate_llc_task 10617 }; 10618 10619 #define LBF_ALL_PINNED 0x01 10620 #define LBF_NEED_BREAK 0x02 10621 #define LBF_DST_PINNED 0x04 10622 #define LBF_SOME_PINNED 0x08 10623 #define LBF_ACTIVE_LB 0x10 10624 #define LBF_LLC_PINNED 0x20 10625 #define LBF_ACTIVE_LB_LLC 0x40 10626 10627 struct lb_env { 10628 struct sched_domain *sd; 10629 10630 struct rq *src_rq; 10631 int src_cpu; 10632 10633 int dst_cpu; 10634 struct rq *dst_rq; 10635 bool dst_core_idle; 10636 10637 struct cpumask *dst_grpmask; 10638 int new_dst_cpu; 10639 enum cpu_idle_type idle; 10640 long imbalance; 10641 /* The set of CPUs under consideration for load-balancing */ 10642 struct cpumask *cpus; 10643 10644 unsigned int flags; 10645 10646 unsigned int loop; 10647 unsigned int loop_break; 10648 unsigned int loop_max; 10649 10650 enum fbq_type fbq_type; 10651 enum migration_type migration_type; 10652 struct list_head tasks; 10653 }; 10654 10655 /* 10656 * Is this task likely cache-hot: 10657 */ 10658 static int task_hot(struct task_struct *p, struct lb_env *env) 10659 { 10660 s64 delta; 10661 10662 lockdep_assert_rq_held(env->src_rq); 10663 10664 if (p->sched_class != &fair_sched_class) 10665 return 0; 10666 10667 if (unlikely(task_has_idle_policy(p))) 10668 return 0; 10669 10670 /* SMT siblings share cache */ 10671 if (env->sd->flags & SD_SHARE_CPUCAPACITY) 10672 return 0; 10673 10674 /* 10675 * Buddy candidates are cache hot: 10676 */ 10677 if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 10678 (&p->se == cfs_rq_of(&p->se)->next)) 10679 return 1; 10680 10681 if (sysctl_sched_migration_cost == -1) 10682 return 1; 10683 10684 /* 10685 * Don't migrate task if the task's cookie does not match 10686 * with the destination CPU's core cookie. 10687 */ 10688 if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p)) 10689 return 1; 10690 10691 if (sysctl_sched_migration_cost == 0) 10692 return 0; 10693 10694 delta = rq_clock_task(env->src_rq) - p->se.exec_start; 10695 10696 return delta < (s64)sysctl_sched_migration_cost; 10697 } 10698 10699 #ifdef CONFIG_NUMA_BALANCING 10700 /* 10701 * Returns a positive value, if task migration degrades locality. 10702 * Returns 0, if task migration is not affected by locality. 10703 * Returns a negative value, if task migration improves locality i.e migration preferred. 10704 */ 10705 static long migrate_degrades_locality(struct task_struct *p, struct lb_env *env) 10706 { 10707 struct numa_group *numa_group = rcu_dereference_all(p->numa_group); 10708 unsigned long src_weight, dst_weight; 10709 int src_nid, dst_nid, dist; 10710 10711 if (!static_branch_likely(&sched_numa_balancing)) 10712 return 0; 10713 10714 if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) 10715 return 0; 10716 10717 src_nid = cpu_to_node(env->src_cpu); 10718 dst_nid = cpu_to_node(env->dst_cpu); 10719 10720 if (src_nid == dst_nid) 10721 return 0; 10722 10723 /* Migrating away from the preferred node is always bad. */ 10724 if (src_nid == p->numa_preferred_nid) { 10725 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) 10726 return 1; 10727 else 10728 return 0; 10729 } 10730 10731 /* Encourage migration to the preferred node. */ 10732 if (dst_nid == p->numa_preferred_nid) 10733 return -1; 10734 10735 /* Leaving a core idle is often worse than degrading locality. */ 10736 if (env->idle == CPU_IDLE) 10737 return 0; 10738 10739 dist = node_distance(src_nid, dst_nid); 10740 if (numa_group) { 10741 src_weight = group_weight(p, src_nid, dist); 10742 dst_weight = group_weight(p, dst_nid, dist); 10743 } else { 10744 src_weight = task_weight(p, src_nid, dist); 10745 dst_weight = task_weight(p, dst_nid, dist); 10746 } 10747 10748 return src_weight - dst_weight; 10749 } 10750 10751 #else /* !CONFIG_NUMA_BALANCING: */ 10752 static inline long migrate_degrades_locality(struct task_struct *p, 10753 struct lb_env *env) 10754 { 10755 return 0; 10756 } 10757 #endif /* !CONFIG_NUMA_BALANCING */ 10758 10759 /* 10760 * Check whether the task is ineligible on the destination cpu 10761 * 10762 * When the PLACE_LAG scheduling feature is enabled and 10763 * dst_cfs_rq->nr_queued is greater than 1, if the task 10764 * is ineligible, it will also be ineligible when 10765 * it is migrated to the destination cpu. 10766 */ 10767 static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu) 10768 { 10769 struct cfs_rq *dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; 10770 10771 if (sched_feat(PLACE_LAG) && dst_cfs_rq->h_nr_queued && 10772 !entity_eligible(&task_rq(p)->cfs, &p->se)) 10773 return 1; 10774 10775 return 0; 10776 } 10777 10778 #ifdef CONFIG_SCHED_CACHE 10779 /* 10780 * The margin used when comparing LLC utilization with CPU capacity. 10781 * It determines the LLC load level where active LLC aggregation is 10782 * done. 10783 * Derived from fits_capacity(). 10784 * 10785 * (default: ~50%, tunable via debugfs) 10786 */ 10787 static bool fits_llc_capacity(unsigned long util, unsigned long max) 10788 { 10789 u32 aggr_pct = llc_overaggr_pct; 10790 10791 /* 10792 * For single core systems, raise the aggregation 10793 * threshold to accommodate more tasks. 10794 */ 10795 if (cpu_smt_num_threads == 1) 10796 aggr_pct = (aggr_pct * 3 / 2); 10797 10798 return util * 100 < max * aggr_pct; 10799 } 10800 10801 /* 10802 * The margin used when comparing utilization. 10803 * is 'util1' noticeably greater than 'util2' 10804 * Derived from capacity_greater(). 10805 * Bias is in perentage. 10806 */ 10807 /* Allows dst util to be bigger than src util by up to bias percent */ 10808 #define util_greater(util1, util2) \ 10809 ((util1) * 100 > (util2) * (100 + llc_imb_pct)) 10810 10811 static __maybe_unused bool get_llc_stats(int cpu, unsigned long *util, 10812 unsigned long *cap) 10813 { 10814 struct sched_domain_shared *sd_share; 10815 10816 sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, cpu)); 10817 if (!sd_share) 10818 return false; 10819 10820 *util = READ_ONCE(sd_share->util_avg); 10821 *cap = READ_ONCE(sd_share->capacity); 10822 10823 return true; 10824 } 10825 10826 /* 10827 * Decision matrix according to the LLC utilization. To 10828 * decide whether we can do task aggregation across LLC. 10829 * 10830 * By default, 50% is the threshold for treating the LLC 10831 * as busy. The reason for choosing 50% is to avoid saturation 10832 * of SMT-2, and it is also a safe cutoff for other SMT-n 10833 * platforms. SMT-1 has higher threshold because it is 10834 * supposed to accommodate more tasks, see fits_llc_capacity(). 10835 * 10836 * 20% is the utilization imbalance percentage to decide 10837 * if the preferred LLC is busier than the non-preferred LLC. 10838 * 20 is a little higher than the LLC domain's imbalance_pct 10839 * 17. The hysteresis is used to avoid task bouncing between the 10840 * preferred LLC and the non-preferred LLC, and it will 10841 * be turned into tunable debugfs. 10842 * 10843 * 1. moving towards the preferred LLC, dst is the preferred 10844 * LLC, src is not. 10845 * 10846 * src \ dst 30% 40% 50% 60% 10847 * 30% Y Y Y N 10848 * 40% Y Y Y Y 10849 * 50% Y Y G G 10850 * 60% Y Y G G 10851 * 10852 * 2. moving out of the preferred LLC, src is the preferred 10853 * LLC, dst is not: 10854 * 10855 * src \ dst 30% 40% 50% 60% 10856 * 30% N N N N 10857 * 40% N N N N 10858 * 50% N N G G 10859 * 60% Y N G G 10860 * 10861 * src : src_util 10862 * dst : dst_util 10863 * Y : Yes, migrate 10864 * N : No, do not migrate 10865 * G : let the Generic load balance to even the load. 10866 * 10867 * The intention is that if both LLCs are quite busy, cache aware 10868 * load balance should not be performed, and generic load balance 10869 * should take effect. However, if one is busy and the other is not, 10870 * the preferred LLC capacity(50%) and imbalance criteria(20%) should 10871 * be considered to determine whether LLC aggregation should be 10872 * performed to bias the load towards the preferred LLC. 10873 */ 10874 10875 /* migration decision, 3 states are orthogonal. */ 10876 enum llc_mig { 10877 mig_forbid = 0, /* N: Don't migrate task, respect LLC preference */ 10878 mig_llc, /* Y: Do LLC preference based migration */ 10879 mig_unrestricted /* G: Don't restrict generic load balance migration */ 10880 }; 10881 10882 /* 10883 * Check if task can be moved from the source LLC to the 10884 * destination LLC without breaking cache aware preferrence. 10885 * src_cpu and dst_cpu are arbitrary CPUs within the source 10886 * and destination LLCs, respectively. 10887 */ 10888 static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu, 10889 unsigned long tsk_util, 10890 bool to_pref) 10891 { 10892 unsigned long src_util, dst_util, src_cap, dst_cap; 10893 10894 if (!get_llc_stats(src_cpu, &src_util, &src_cap) || 10895 !get_llc_stats(dst_cpu, &dst_util, &dst_cap)) 10896 return mig_unrestricted; 10897 10898 src_util = src_util < tsk_util ? 0 : src_util - tsk_util; 10899 dst_util = dst_util + tsk_util; 10900 10901 if (!fits_llc_capacity(dst_util, dst_cap) && 10902 !fits_llc_capacity(src_util, src_cap)) 10903 return mig_unrestricted; 10904 10905 if (to_pref) { 10906 /* 10907 * Don't migrate if we will get preferred LLC too 10908 * heavily loaded and if the dest is much busier 10909 * than the src, in which case migration will 10910 * increase the imbalance too much. 10911 */ 10912 if (!fits_llc_capacity(dst_util, dst_cap) && 10913 util_greater(dst_util, src_util)) 10914 return mig_forbid; 10915 } else { 10916 /* 10917 * Don't migrate if we will leave preferred LLC 10918 * too idle, or if this migration leads to the 10919 * non-preferred LLC falls within sysctl_aggr_imb percent 10920 * of preferred LLC, leading to migration again 10921 * back to preferred LLC. 10922 */ 10923 if (fits_llc_capacity(src_util, src_cap) || 10924 !util_greater(src_util, dst_util)) 10925 return mig_forbid; 10926 } 10927 return mig_llc; 10928 } 10929 10930 static inline bool task_misfits_asym_cpu(struct lb_env *env, struct task_struct *p) 10931 { 10932 /* 10933 * On asymmetric CPU capacity domains, do not let cache-aware 10934 * balancing pull the task onto a destination CPU that cannot 10935 * accommodate it. Doing so would turn the task into a misfit on 10936 * the destination, trading a cache-locality gain for a capacity 10937 * loss. If the task already does not fit its source CPU, the move 10938 * cannot make things worse, so let the LLC preference decide. 10939 */ 10940 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && p && 10941 !task_fits_cpu(p, env->dst_cpu) && 10942 task_fits_cpu(p, env->src_cpu)) 10943 return true; 10944 10945 return false; 10946 } 10947 10948 static inline bool sched_asym(struct sched_domain *sd, int dst_cpu, int src_cpu); 10949 10950 /* 10951 * Check if task p can migrate from source LLC to 10952 * destination LLC in terms of cache aware load balance. 10953 */ 10954 static enum llc_mig can_migrate_llc_task(struct lb_env *env, 10955 struct task_struct *p) 10956 { 10957 struct sched_cache_group *grp; 10958 bool to_pref; 10959 int cpu, src_cpu, dst_cpu; 10960 10961 if (task_misfits_asym_cpu(env, p)) 10962 return mig_forbid; 10963 10964 src_cpu = env->src_cpu; 10965 dst_cpu = env->dst_cpu; 10966 grp = rcu_dereference_all(p->sched_cache_grp); 10967 if (!grp) 10968 return mig_unrestricted; 10969 10970 cpu = READ_ONCE(grp->cpu); 10971 if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu)) 10972 return mig_unrestricted; 10973 10974 /* Prioritize asym packing to idle core over cache awareness */ 10975 if (env->idle && sched_asym(env->sd, dst_cpu, src_cpu)) 10976 return mig_unrestricted; 10977 10978 /* skip cache aware load balance for too many threads */ 10979 if (invalid_llc_nr(grp, p, dst_cpu) || 10980 exceed_llc_capacity(grp, dst_cpu)) { 10981 if (READ_ONCE(grp->cpu) != -1) 10982 WRITE_ONCE(grp->cpu, -1); 10983 return mig_unrestricted; 10984 } 10985 10986 if (cpus_share_cache(dst_cpu, cpu)) 10987 to_pref = true; 10988 else if (cpus_share_cache(src_cpu, cpu)) 10989 to_pref = false; 10990 else 10991 return mig_unrestricted; 10992 10993 return can_migrate_llc(src_cpu, dst_cpu, 10994 task_util(p), to_pref); 10995 } 10996 10997 /* 10998 * Check if active load balance breaks LLC locality in 10999 * terms of cache aware load balance. The load level and 11000 * imbalance do not warrant breaking LLC preference per 11001 * the can_migrate_llc() policy. Here, the benefit of 11002 * LLC locality outweighs the power efficiency gained from 11003 * migrating the only runnable task away. 11004 */ 11005 static inline bool 11006 alb_break_llc(struct lb_env *env) 11007 { 11008 if (!sched_cache_enabled()) 11009 return false; 11010 11011 if (cpus_share_cache(env->src_cpu, env->dst_cpu)) 11012 return false; 11013 /* 11014 * All tasks prefer to stay on their current CPU. 11015 * Do not pull a task from its preferred CPU if: 11016 * 1. It is the only task running and does not exceed 11017 * imbalance allowance; OR 11018 * 2. Migrating it away from its preferred LLC would violate 11019 * the cache-aware scheduling policy. 11020 */ 11021 if (env->src_rq->nr_pref_llc_running && 11022 env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable) { 11023 unsigned long util = 0; 11024 struct task_struct *cur; 11025 11026 /* 11027 * Migrating misfit tasks from current CPU 11028 * to CPU with a better fit. 11029 * Prioritize that over LLC preference. 11030 */ 11031 if (env->migration_type == migrate_misfit) 11032 return false; 11033 11034 if (env->src_rq->nr_running <= 1) 11035 return true; 11036 11037 cur = rcu_dereference_all(env->src_rq->curr); 11038 /* 11039 * Active balance only detaches CFS tasks, so a non-fair 11040 * curr is never migrated. Ignore it and avoid using its 11041 * state util in the misfit and LLC checks. 11042 */ 11043 if (cur && cur->sched_class != &fair_sched_class) 11044 cur = NULL; 11045 if (cur) 11046 util = task_util(cur); 11047 11048 if (task_misfits_asym_cpu(env, cur) || 11049 can_migrate_llc(env->src_cpu, env->dst_cpu, 11050 util, false) == mig_forbid) 11051 return true; 11052 } 11053 11054 return false; 11055 } 11056 11057 /* 11058 * Returns true if p's preferred LLC does not match the destination CPU 11059 * under migrate_llc_task semantics. Passive LB passes migrate_llc_task 11060 * in env->migration_type, while active LB carries LBF_ACTIVE_LB_LLC in 11061 * env->flags to avoid overwriting env->migration_type. 11062 */ 11063 static inline bool 11064 migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env) 11065 { 11066 return sched_cache_enabled() && 11067 (env->migration_type == migrate_llc_task || 11068 env->flags & LBF_ACTIVE_LB_LLC) && 11069 READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu); 11070 } 11071 11072 /* 11073 * Check if migrating task p from env->src_cpu to 11074 * env->dst_cpu breaks LLC localiy. 11075 */ 11076 static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) 11077 { 11078 if (!sched_cache_enabled()) 11079 return false; 11080 11081 if (task_has_sched_core(p)) 11082 return false; 11083 /* 11084 * Skip over tasks that would degrade LLC locality; 11085 * only when nr_balanced_failed is sufficiently high do we 11086 * ignore this constraint. 11087 * 11088 * Threshold of cache_nice_tries is set to 1 higher 11089 * than nr_balance_failed to avoid excessive task 11090 * migration at the same time. 11091 */ 11092 if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1) 11093 return false; 11094 11095 /* 11096 * We know the env->src_cpu has some tasks prefer to 11097 * run on env->dst_cpu, skip the tasks do not prefer 11098 * env->dst_cpu, and find the one that prefers. 11099 */ 11100 if (migrate_llc_task_wrong_dst(p, env)) 11101 return true; 11102 11103 if (can_migrate_llc_task(env, p) != mig_forbid) 11104 return false; 11105 11106 return true; 11107 } 11108 11109 #else 11110 static inline bool get_llc_stats(int cpu, unsigned long *util, 11111 unsigned long *cap) 11112 { 11113 return false; 11114 } 11115 11116 static inline bool 11117 alb_break_llc(struct lb_env *env) 11118 { 11119 return false; 11120 } 11121 11122 static inline bool 11123 migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env) 11124 { 11125 return false; 11126 } 11127 11128 static inline bool 11129 migrate_degrades_llc(struct task_struct *p, struct lb_env *env) 11130 { 11131 return false; 11132 } 11133 #endif 11134 /* 11135 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? 11136 */ 11137 static 11138 int can_migrate_task(struct task_struct *p, struct lb_env *env) 11139 { 11140 long degrades, hot; 11141 11142 lockdep_assert_rq_held(env->src_rq); 11143 if (p->sched_task_hot) 11144 p->sched_task_hot = 0; 11145 11146 /* 11147 * We do not migrate tasks that are: 11148 * 1) delayed dequeued unless we migrate load, or 11149 * 2) target cfs_rq is in throttled hierarchy, or 11150 * 3) cannot be migrated to this CPU due to cpus_ptr, or 11151 * 4) running (obviously), or 11152 * 5) are cache-hot on their current CPU, or 11153 * 6) are blocked on mutexes (if SCHED_PROXY_EXEC is enabled) 11154 */ 11155 if ((p->se.sched_delayed) && (env->migration_type != migrate_load)) 11156 return 0; 11157 11158 if (lb_throttled_hierarchy(p, env->dst_cpu)) 11159 return 0; 11160 11161 /* 11162 * We want to prioritize the migration of eligible tasks. 11163 * For ineligible tasks we soft-limit them and only allow 11164 * them to migrate when nr_balance_failed is non-zero to 11165 * avoid load-balancing trying very hard to balance the load. 11166 */ 11167 if (!env->sd->nr_balance_failed && 11168 task_is_ineligible_on_dst_cpu(p, env->dst_cpu)) 11169 return 0; 11170 11171 /* Disregard percpu kthreads; they are where they need to be. */ 11172 if (kthread_is_per_cpu(p)) 11173 return 0; 11174 11175 if (task_is_blocked(p)) 11176 return 0; 11177 11178 if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { 11179 int cpu; 11180 11181 schedstat_inc(p->stats.nr_failed_migrations_affine); 11182 11183 env->flags |= LBF_SOME_PINNED; 11184 11185 /* 11186 * Remember if this task can be migrated to any other CPU in 11187 * our sched_group. We may want to revisit it if we couldn't 11188 * meet load balance goals by pulling other tasks on src_cpu. 11189 * 11190 * Avoid computing new_dst_cpu 11191 * - for NEWLY_IDLE 11192 * - if we have already computed one in current iteration 11193 * - if it's an active balance 11194 */ 11195 if (env->idle == CPU_NEWLY_IDLE || 11196 env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB)) 11197 return 0; 11198 11199 /* Prevent to re-select dst_cpu via env's CPUs: */ 11200 cpu = cpumask_first_and_and(env->dst_grpmask, env->cpus, p->cpus_ptr); 11201 11202 if (cpu < nr_cpu_ids) { 11203 env->flags |= LBF_DST_PINNED; 11204 env->new_dst_cpu = cpu; 11205 } 11206 11207 return 0; 11208 } 11209 11210 /* Record that we found at least one task that could run on dst_cpu */ 11211 env->flags &= ~LBF_ALL_PINNED; 11212 11213 if (task_on_cpu(env->src_rq, p) || 11214 task_current_donor(env->src_rq, p)) { 11215 schedstat_inc(p->stats.nr_failed_migrations_running); 11216 return 0; 11217 } 11218 11219 /* 11220 * Aggressive migration if: 11221 * 1) active balance 11222 * 2) destination numa is preferred 11223 * 3) task is cache cold, or 11224 * 4) too many balance attempts have failed. 11225 */ 11226 if (env->flags & LBF_ACTIVE_LB) 11227 return !migrate_llc_task_wrong_dst(p, env); 11228 11229 degrades = migrate_degrades_locality(p, env); 11230 if (!degrades) { 11231 /* 11232 * If the NUMA locality is not broken, 11233 * further check if migration would hurt 11234 * LLC locality. 11235 */ 11236 if (migrate_degrades_llc(p, env)) { 11237 /* 11238 * If regular load balancing fails to pull a task 11239 * due to LLC locality, this is expected behavior 11240 * and we set LBF_LLC_PINNED so we don't increase 11241 * nr_balance_failed unecessarily. 11242 */ 11243 if (env->migration_type != migrate_llc_task) 11244 env->flags |= LBF_LLC_PINNED; 11245 11246 return 0; 11247 } 11248 11249 hot = task_hot(p, env); 11250 } else { 11251 hot = degrades > 0; 11252 } 11253 11254 if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) { 11255 if (hot) 11256 p->sched_task_hot = 1; 11257 return 1; 11258 } 11259 11260 schedstat_inc(p->stats.nr_failed_migrations_hot); 11261 return 0; 11262 } 11263 11264 /* 11265 * detach_task() -- detach the task for the migration specified in env 11266 */ 11267 static void detach_task(struct task_struct *p, struct lb_env *env) 11268 { 11269 lockdep_assert_rq_held(env->src_rq); 11270 11271 if (p->sched_task_hot) { 11272 p->sched_task_hot = 0; 11273 schedstat_inc(env->sd->lb_hot_gained[env->idle]); 11274 schedstat_inc(p->stats.nr_forced_migrations); 11275 } 11276 11277 WARN_ON(task_current(env->src_rq, p)); 11278 WARN_ON(task_current_donor(env->src_rq, p)); 11279 11280 deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 11281 set_task_cpu(p, env->dst_cpu); 11282 } 11283 11284 /* 11285 * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as 11286 * part of active balancing operations within "domain". 11287 * 11288 * Returns a task if successful and NULL otherwise. 11289 */ 11290 static struct task_struct *detach_one_task(struct lb_env *env) 11291 { 11292 struct task_struct *p; 11293 11294 lockdep_assert_rq_held(env->src_rq); 11295 11296 list_for_each_entry_reverse(p, 11297 &env->src_rq->cfs_tasks, se.group_node) { 11298 if (!can_migrate_task(p, env)) 11299 continue; 11300 11301 detach_task(p, env); 11302 11303 /* 11304 * Right now, this is only the second place where 11305 * lb_gained[env->idle] is updated (other is detach_tasks) 11306 * so we can safely collect stats here rather than 11307 * inside detach_tasks(). 11308 */ 11309 schedstat_inc(env->sd->lb_gained[env->idle]); 11310 return p; 11311 } 11312 return NULL; 11313 } 11314 11315 /* 11316 * detach_tasks() -- tries to detach up to imbalance load/util/tasks from 11317 * busiest_rq, as part of a balancing operation within domain "sd". 11318 * 11319 * Returns number of detached tasks if successful and 0 otherwise. 11320 */ 11321 static int detach_tasks(struct lb_env *env) 11322 { 11323 struct list_head *tasks = &env->src_rq->cfs_tasks; 11324 unsigned long util, load; 11325 struct task_struct *p; 11326 int detached = 0; 11327 11328 lockdep_assert_rq_held(env->src_rq); 11329 11330 /* 11331 * Source run queue has been emptied by another CPU, clear 11332 * LBF_ALL_PINNED flag as we will not test any task. 11333 */ 11334 if (env->src_rq->nr_running <= 1) { 11335 env->flags &= ~LBF_ALL_PINNED; 11336 return 0; 11337 } 11338 11339 if (env->imbalance <= 0) 11340 return 0; 11341 11342 while (!list_empty(tasks)) { 11343 /* 11344 * We don't want to steal all, otherwise we may be treated likewise, 11345 * which could at worst lead to a livelock crash. 11346 */ 11347 if (env->idle && env->src_rq->nr_running <= 1) 11348 break; 11349 11350 env->loop++; 11351 /* We've more or less seen every task there is, call it quits */ 11352 if (env->loop > env->loop_max) 11353 break; 11354 11355 /* take a breather every nr_migrate tasks */ 11356 if (env->loop > env->loop_break) { 11357 env->loop_break += SCHED_NR_MIGRATE_BREAK; 11358 env->flags |= LBF_NEED_BREAK; 11359 break; 11360 } 11361 11362 p = list_last_entry(tasks, struct task_struct, se.group_node); 11363 11364 if (!can_migrate_task(p, env)) 11365 goto next; 11366 11367 switch (env->migration_type) { 11368 case migrate_load: 11369 /* 11370 * Depending of the number of CPUs and tasks and the 11371 * cgroup hierarchy, task_h_load() can return a null 11372 * value. Make sure that env->imbalance decreases 11373 * otherwise detach_tasks() will stop only after 11374 * detaching up to loop_max tasks. 11375 */ 11376 load = max_t(unsigned long, task_h_load(p), 1); 11377 11378 if (sched_feat(LB_MIN) && 11379 load < 16 && !env->sd->nr_balance_failed) 11380 goto next; 11381 11382 /* 11383 * Make sure that we don't migrate too much load. 11384 * Nevertheless, let relax the constraint if 11385 * scheduler fails to find a good waiting task to 11386 * migrate. 11387 */ 11388 if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance) 11389 goto next; 11390 11391 env->imbalance -= load; 11392 break; 11393 11394 case migrate_util: 11395 util = task_util_est(p); 11396 11397 if (shr_bound(util, env->sd->nr_balance_failed) > env->imbalance) 11398 goto next; 11399 11400 env->imbalance -= util; 11401 break; 11402 11403 case migrate_task: 11404 env->imbalance--; 11405 break; 11406 11407 case migrate_misfit: 11408 /* This is not a misfit task */ 11409 if (task_fits_cpu(p, env->src_cpu)) 11410 goto next; 11411 11412 env->imbalance = 0; 11413 break; 11414 11415 case migrate_llc_task: 11416 env->imbalance--; 11417 break; 11418 } 11419 11420 detach_task(p, env); 11421 list_add(&p->se.group_node, &env->tasks); 11422 11423 detached++; 11424 11425 #ifdef CONFIG_PREEMPTION 11426 /* 11427 * NEWIDLE balancing is a source of latency, so preemptible 11428 * kernels will stop after the first task is detached to minimize 11429 * the critical section. 11430 */ 11431 if (env->idle == CPU_NEWLY_IDLE) 11432 break; 11433 #endif 11434 11435 /* 11436 * We only want to steal up to the prescribed amount of 11437 * load/util/tasks. 11438 */ 11439 if (env->imbalance <= 0) 11440 break; 11441 11442 continue; 11443 next: 11444 if (p->sched_task_hot) 11445 schedstat_inc(p->stats.nr_failed_migrations_hot); 11446 11447 list_move(&p->se.group_node, tasks); 11448 } 11449 11450 /* 11451 * Right now, this is one of only two places we collect this stat 11452 * so we can safely collect detach_one_task() stats here rather 11453 * than inside detach_one_task(). 11454 */ 11455 schedstat_add(env->sd->lb_gained[env->idle], detached); 11456 11457 return detached; 11458 } 11459 11460 /* 11461 * attach_tasks() -- attaches all tasks detached by detach_tasks() to their 11462 * new rq. 11463 */ 11464 static void attach_tasks(struct lb_env *env) 11465 { 11466 __attach_tasks(env->dst_rq, &env->tasks); 11467 } 11468 11469 #ifdef CONFIG_NO_HZ_COMMON 11470 static inline bool cfs_rq_has_blocked_load(struct cfs_rq *cfs_rq) 11471 { 11472 if (cfs_rq->avg.load_avg) 11473 return true; 11474 11475 if (cfs_rq->avg.util_avg) 11476 return true; 11477 11478 return false; 11479 } 11480 11481 static inline bool others_have_blocked(struct rq *rq) 11482 { 11483 if (cpu_util_rt(rq)) 11484 return true; 11485 11486 if (cpu_util_dl(rq)) 11487 return true; 11488 11489 if (hw_load_avg(rq)) 11490 return true; 11491 11492 if (cpu_util_irq(rq)) 11493 return true; 11494 11495 return false; 11496 } 11497 11498 static inline void update_blocked_load_tick(struct rq *rq) 11499 { 11500 WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies); 11501 } 11502 11503 static inline void update_has_blocked_load_status(struct rq *rq, bool has_blocked_load) 11504 { 11505 if (!has_blocked_load) 11506 rq->has_blocked_load = 0; 11507 } 11508 #else /* !CONFIG_NO_HZ_COMMON: */ 11509 static inline bool cfs_rq_has_blocked_load(struct cfs_rq *cfs_rq) { return false; } 11510 static inline bool others_have_blocked(struct rq *rq) { return false; } 11511 static inline void update_blocked_load_tick(struct rq *rq) {} 11512 static inline void update_has_blocked_load_status(struct rq *rq, bool has_blocked_load) {} 11513 #endif /* !CONFIG_NO_HZ_COMMON */ 11514 11515 static bool __update_blocked_others(struct rq *rq, bool *done) 11516 { 11517 bool updated; 11518 11519 /* 11520 * update_load_avg() can call cpufreq_update_util(). Make sure that RT, 11521 * DL and IRQ signals have been updated before updating CFS. 11522 */ 11523 updated = update_other_load_avgs(rq); 11524 11525 if (others_have_blocked(rq)) 11526 *done = false; 11527 11528 return updated; 11529 } 11530 11531 #ifdef CONFIG_FAIR_GROUP_SCHED 11532 11533 static bool __update_blocked_fair(struct rq *rq, bool *done) 11534 { 11535 struct cfs_rq *cfs_rq, *pos; 11536 bool decayed = false; 11537 11538 /* 11539 * Iterates the task_group tree in a bottom up fashion, see 11540 * list_add_leaf_cfs_rq() for details. 11541 */ 11542 for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 11543 struct sched_entity *se; 11544 11545 if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { 11546 update_tg_load_avg(cfs_rq); 11547 11548 if (cfs_rq->nr_queued == 0) 11549 update_idle_cfs_rq_clock_pelt(cfs_rq); 11550 11551 if (cfs_rq == &rq->cfs) 11552 decayed = true; 11553 } 11554 11555 /* Propagate pending load changes to the parent, if any: */ 11556 se = cfs_rq_se(cfs_rq); 11557 if (se && !skip_blocked_update(se)) 11558 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 11559 11560 /* 11561 * There can be a lot of idle CPU cgroups. Don't let fully 11562 * decayed cfs_rqs linger on the list. 11563 */ 11564 if (cfs_rq_is_decayed(cfs_rq)) 11565 list_del_leaf_cfs_rq(cfs_rq); 11566 11567 /* Don't need periodic decay once load/util_avg are null */ 11568 if (cfs_rq_has_blocked_load(cfs_rq)) 11569 *done = false; 11570 } 11571 11572 return decayed; 11573 } 11574 11575 /* 11576 * Compute the hierarchical load factor for cfs_rq and all its ascendants. 11577 * This needs to be done in a top-down fashion because the load of a child 11578 * group is a fraction of its parents load. 11579 */ 11580 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) 11581 { 11582 struct sched_entity *se = cfs_rq_se(cfs_rq); 11583 unsigned long now = jiffies; 11584 unsigned long load; 11585 11586 if (cfs_rq->last_h_load_update == now) 11587 return; 11588 11589 WRITE_ONCE(cfs_rq->h_load_next, NULL); 11590 for_each_sched_entity(se) { 11591 cfs_rq = cfs_rq_of(se); 11592 WRITE_ONCE(cfs_rq->h_load_next, se); 11593 if (cfs_rq->last_h_load_update == now) 11594 break; 11595 } 11596 11597 if (!se) { 11598 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); 11599 cfs_rq->last_h_load_update = now; 11600 } 11601 11602 while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { 11603 load = cfs_rq->h_load; 11604 load = div64_ul(load * se->avg.load_avg, 11605 cfs_rq_load_avg(cfs_rq) + 1); 11606 cfs_rq = group_cfs_rq(se); 11607 cfs_rq->h_load = load; 11608 cfs_rq->last_h_load_update = now; 11609 } 11610 } 11611 11612 static unsigned long task_h_load(struct task_struct *p) 11613 { 11614 struct cfs_rq *cfs_rq = task_cfs_rq(p); 11615 11616 update_cfs_rq_h_load(cfs_rq); 11617 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 11618 cfs_rq_load_avg(cfs_rq) + 1); 11619 } 11620 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 11621 static bool __update_blocked_fair(struct rq *rq, bool *done) 11622 { 11623 struct cfs_rq *cfs_rq = &rq->cfs; 11624 bool decayed; 11625 11626 decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq); 11627 if (cfs_rq_has_blocked_load(cfs_rq)) 11628 *done = false; 11629 11630 return decayed; 11631 } 11632 11633 static unsigned long task_h_load(struct task_struct *p) 11634 { 11635 return p->se.avg.load_avg; 11636 } 11637 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 11638 11639 static void __sched_balance_update_blocked_averages(struct rq *rq) 11640 { 11641 bool decayed = false, done = true; 11642 11643 update_blocked_load_tick(rq); 11644 11645 decayed |= __update_blocked_others(rq, &done); 11646 decayed |= __update_blocked_fair(rq, &done); 11647 11648 update_has_blocked_load_status(rq, !done); 11649 if (decayed) 11650 cpufreq_update_util(rq, 0); 11651 } 11652 11653 static void sched_balance_update_blocked_averages(int cpu) 11654 { 11655 struct rq *rq = cpu_rq(cpu); 11656 11657 guard(rq_lock_irqsave)(rq); 11658 update_rq_clock(rq); 11659 __sched_balance_update_blocked_averages(rq); 11660 } 11661 11662 /********** Helpers for sched_balance_find_src_group ************************/ 11663 11664 /* 11665 * sg_lb_stats - stats of a sched_group required for load-balancing: 11666 */ 11667 struct sg_lb_stats { 11668 unsigned long avg_load; /* Avg load over the CPUs of the group */ 11669 unsigned long group_load; /* Total load over the CPUs of the group */ 11670 unsigned long group_capacity; /* Capacity over the CPUs of the group */ 11671 unsigned long group_util; /* Total utilization over the CPUs of the group */ 11672 unsigned long group_runnable; /* Total runnable time over the CPUs of the group */ 11673 unsigned int sum_nr_running; /* Nr of all tasks running in the group */ 11674 unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */ 11675 unsigned int idle_cpus; /* Nr of idle CPUs in the group */ 11676 unsigned int group_weight; 11677 enum group_type group_type; 11678 unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */ 11679 unsigned int group_smt_balance; /* Task on busy SMT be moved */ 11680 unsigned int group_llc_balance; /* Tasks should be moved to preferred LLC */ 11681 unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ 11682 unsigned int group_overutilized; /* At least one CPU is overutilized in the group */ 11683 #ifdef CONFIG_NUMA_BALANCING 11684 unsigned int nr_numa_running; 11685 unsigned int nr_preferred_running; 11686 #endif 11687 #ifdef CONFIG_SCHED_CACHE 11688 unsigned int nr_pref_dst_llc; 11689 #endif 11690 }; 11691 11692 /* 11693 * sd_lb_stats - stats of a sched_domain required for load-balancing: 11694 */ 11695 struct sd_lb_stats { 11696 struct sched_group *busiest; /* Busiest group in this sd */ 11697 struct sched_group *local; /* Local group in this sd */ 11698 unsigned long total_load; /* Total load of all groups in sd */ 11699 unsigned long total_capacity; /* Total capacity of all groups in sd */ 11700 unsigned long avg_load; /* Average load across all groups in sd */ 11701 unsigned int prefer_sibling; /* Tasks should go to sibling first */ 11702 11703 struct sg_lb_stats busiest_stat; /* Statistics of the busiest group */ 11704 struct sg_lb_stats local_stat; /* Statistics of the local group */ 11705 }; 11706 11707 static inline void init_sd_lb_stats(struct sd_lb_stats *sds) 11708 { 11709 /* 11710 * Skimp on the clearing to avoid duplicate work. We can avoid clearing 11711 * local_stat because update_sg_lb_stats() does a full clear/assignment. 11712 * We must however set busiest_stat::group_type and 11713 * busiest_stat::idle_cpus to the worst busiest group because 11714 * update_sd_pick_busiest() reads these before assignment. 11715 */ 11716 *sds = (struct sd_lb_stats){ 11717 .busiest = NULL, 11718 .local = NULL, 11719 .total_load = 0UL, 11720 .total_capacity = 0UL, 11721 .busiest_stat = { 11722 .idle_cpus = UINT_MAX, 11723 .group_type = group_has_spare, 11724 }, 11725 }; 11726 } 11727 11728 static unsigned long scale_rt_capacity(int cpu) 11729 { 11730 unsigned long max = get_actual_cpu_capacity(cpu); 11731 struct rq *rq = cpu_rq(cpu); 11732 unsigned long used, free; 11733 unsigned long irq; 11734 11735 irq = cpu_util_irq(rq); 11736 11737 if (unlikely(irq >= max)) 11738 return 1; 11739 11740 /* 11741 * avg_rt.util_avg and avg_dl.util_avg track binary signals 11742 * (running and not running) with weights 0 and 1024 respectively. 11743 */ 11744 used = cpu_util_rt(rq); 11745 used += cpu_util_dl(rq); 11746 11747 if (unlikely(used >= max)) 11748 return 1; 11749 11750 free = max - used; 11751 11752 return scale_irq_capacity(free, irq, max); 11753 } 11754 11755 static void update_cpu_capacity(struct sched_domain *sd, int cpu) 11756 { 11757 unsigned long capacity = scale_rt_capacity(cpu); 11758 struct sched_group *sdg = sd->groups; 11759 11760 if (!capacity) 11761 capacity = 1; 11762 11763 cpu_rq(cpu)->cpu_capacity = capacity; 11764 trace_sched_cpu_capacity_tp(cpu_rq(cpu)); 11765 11766 sdg->sgc->capacity = capacity; 11767 sdg->sgc->min_capacity = capacity; 11768 sdg->sgc->max_capacity = capacity; 11769 } 11770 11771 void update_group_capacity(struct sched_domain *sd, int cpu) 11772 { 11773 struct sched_domain *child = sd->child; 11774 struct sched_group *group, *sdg = sd->groups; 11775 unsigned long capacity, min_capacity, max_capacity; 11776 unsigned long interval; 11777 11778 interval = msecs_to_jiffies(sd->balance_interval); 11779 interval = clamp(interval, 1UL, max_load_balance_interval); 11780 sdg->sgc->next_update = jiffies + interval; 11781 11782 if (!child) { 11783 update_cpu_capacity(sd, cpu); 11784 return; 11785 } 11786 11787 capacity = 0; 11788 min_capacity = ULONG_MAX; 11789 max_capacity = 0; 11790 11791 if (child->flags & SD_NUMA) { 11792 /* 11793 * SD_NUMA domains cannot assume that child groups 11794 * span the current group. 11795 */ 11796 11797 for_each_cpu(cpu, sched_group_span(sdg)) { 11798 unsigned long cpu_cap = capacity_of(cpu); 11799 11800 capacity += cpu_cap; 11801 min_capacity = min(cpu_cap, min_capacity); 11802 max_capacity = max(cpu_cap, max_capacity); 11803 } 11804 } else { 11805 /* 11806 * !SD_NUMA domains can assume that child groups 11807 * span the current group. 11808 */ 11809 11810 group = child->groups; 11811 do { 11812 struct sched_group_capacity *sgc = group->sgc; 11813 11814 capacity += sgc->capacity; 11815 min_capacity = min(sgc->min_capacity, min_capacity); 11816 max_capacity = max(sgc->max_capacity, max_capacity); 11817 group = group->next; 11818 } while (group != child->groups); 11819 } 11820 11821 sdg->sgc->capacity = capacity; 11822 sdg->sgc->min_capacity = min_capacity; 11823 sdg->sgc->max_capacity = max_capacity; 11824 } 11825 11826 /* 11827 * Check whether the capacity of the rq has been noticeably reduced by side 11828 * activity. The imbalance_pct is used for the threshold. 11829 * Return true is the capacity is reduced 11830 */ 11831 static inline int 11832 check_cpu_capacity(struct rq *rq, struct sched_domain *sd) 11833 { 11834 return ((rq->cpu_capacity * sd->imbalance_pct) < 11835 (arch_scale_cpu_capacity(cpu_of(rq)) * 100)); 11836 } 11837 11838 /* Check if the rq has a misfit task */ 11839 static inline bool check_misfit_status(struct rq *rq) 11840 { 11841 return rq->misfit_task_load; 11842 } 11843 11844 /* 11845 * Group imbalance indicates (and tries to solve) the problem where balancing 11846 * groups is inadequate due to ->cpus_ptr constraints. 11847 * 11848 * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a 11849 * cpumask covering 1 CPU of the first group and 3 CPUs of the second group. 11850 * Something like: 11851 * 11852 * { 0 1 2 3 } { 4 5 6 7 } 11853 * * * * * 11854 * 11855 * If we were to balance group-wise we'd place two tasks in the first group and 11856 * two tasks in the second group. Clearly this is undesired as it will overload 11857 * cpu 3 and leave one of the CPUs in the second group unused. 11858 * 11859 * The current solution to this issue is detecting the skew in the first group 11860 * by noticing the lower domain failed to reach balance and had difficulty 11861 * moving tasks due to affinity constraints. 11862 * 11863 * When this is so detected; this group becomes a candidate for busiest; see 11864 * update_sd_pick_busiest(). And calculate_imbalance() and 11865 * sched_balance_find_src_group() avoid some of the usual balance conditions to allow it 11866 * to create an effective group imbalance. 11867 * 11868 * This is a somewhat tricky proposition since the next run might not find the 11869 * group imbalance and decide the groups need to be balanced again. A most 11870 * subtle and fragile situation. 11871 */ 11872 11873 static inline int sg_imbalanced(struct sched_group *group) 11874 { 11875 return group->sgc->imbalance; 11876 } 11877 11878 /* 11879 * group_has_capacity returns true if the group has spare capacity that could 11880 * be used by some tasks. 11881 * We consider that a group has spare capacity if the number of task is 11882 * smaller than the number of CPUs or if the utilization is lower than the 11883 * available capacity for CFS tasks. 11884 * For the latter, we use a threshold to stabilize the state, to take into 11885 * account the variance of the tasks' load and to return true if the available 11886 * capacity in meaningful for the load balancer. 11887 * As an example, an available capacity of 1% can appear but it doesn't make 11888 * any benefit for the load balance. 11889 */ 11890 static inline bool 11891 group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 11892 { 11893 if (sgs->sum_nr_running < sgs->group_weight) 11894 return true; 11895 11896 if ((sgs->group_capacity * imbalance_pct) < 11897 (sgs->group_runnable * 100)) 11898 return false; 11899 11900 if ((sgs->group_capacity * 100) > 11901 (sgs->group_util * imbalance_pct)) 11902 return true; 11903 11904 return false; 11905 } 11906 11907 /* 11908 * group_is_overloaded returns true if the group has more tasks than it can 11909 * handle. 11910 * group_is_overloaded is not equals to !group_has_capacity because a group 11911 * with the exact right number of tasks, has no more spare capacity but is not 11912 * overloaded so both group_has_capacity and group_is_overloaded return 11913 * false. 11914 */ 11915 static inline bool 11916 group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 11917 { 11918 /* 11919 * With EAS and uclamp, 1 CPU in the group must be overutilized to 11920 * consider the group overloaded. 11921 */ 11922 if (sched_energy_enabled() && !sgs->group_overutilized) 11923 return false; 11924 11925 if (sgs->sum_nr_running <= sgs->group_weight) 11926 return false; 11927 11928 if ((sgs->group_capacity * 100) < 11929 (sgs->group_util * imbalance_pct)) 11930 return true; 11931 11932 if ((sgs->group_capacity * imbalance_pct) < 11933 (sgs->group_runnable * 100)) 11934 return true; 11935 11936 return false; 11937 } 11938 11939 static inline enum 11940 group_type group_classify(unsigned int imbalance_pct, 11941 struct sched_group *group, 11942 struct sg_lb_stats *sgs) 11943 { 11944 if (group_is_overloaded(imbalance_pct, sgs)) 11945 return group_overloaded; 11946 11947 if (sgs->group_llc_balance) 11948 return group_llc_balance; 11949 11950 if (sg_imbalanced(group)) 11951 return group_imbalanced; 11952 11953 if (sgs->group_asym_packing) 11954 return group_asym_packing; 11955 11956 if (sgs->group_smt_balance) 11957 return group_smt_balance; 11958 11959 if (sgs->group_misfit_task_load) 11960 return group_misfit_task; 11961 11962 if (!group_has_capacity(imbalance_pct, sgs)) 11963 return group_fully_busy; 11964 11965 return group_has_spare; 11966 } 11967 11968 /** 11969 * sched_use_asym_prio - Check whether asym_packing priority must be used 11970 * @sd: The scheduling domain of the load balancing 11971 * @cpu: A CPU 11972 * 11973 * Always use CPU priority when balancing load between SMT siblings. When 11974 * balancing load between cores, it is not sufficient that @cpu is idle. Only 11975 * use CPU priority if the whole core is idle. 11976 * 11977 * Returns: True if the priority of @cpu must be followed. False otherwise. 11978 */ 11979 static bool sched_use_asym_prio(struct sched_domain *sd, int cpu) 11980 { 11981 if (!(sd->flags & SD_ASYM_PACKING)) 11982 return false; 11983 11984 if (!sched_smt_active()) 11985 return true; 11986 11987 return sd->flags & SD_SHARE_CPUCAPACITY || is_core_idle(cpu); 11988 } 11989 11990 static inline bool sched_asym(struct sched_domain *sd, int dst_cpu, int src_cpu) 11991 { 11992 /* 11993 * First check if @dst_cpu can do asym_packing load balance. Only do it 11994 * if it has higher priority than @src_cpu. 11995 */ 11996 return sched_use_asym_prio(sd, dst_cpu) && 11997 sched_asym_prefer(dst_cpu, src_cpu); 11998 } 11999 12000 /** 12001 * sched_group_asym - Check if the destination CPU can do asym_packing balance 12002 * @env: The load balancing environment 12003 * @sgs: Load-balancing statistics of the candidate busiest group 12004 * @group: The candidate busiest group 12005 * 12006 * @env::dst_cpu can do asym_packing if it has higher priority than the 12007 * preferred CPU of @group. 12008 * 12009 * Return: true if @env::dst_cpu can do with asym_packing load balance. False 12010 * otherwise. 12011 */ 12012 static inline bool 12013 sched_group_asym(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group *group) 12014 { 12015 /* 12016 * CPU priorities do not make sense for SMT cores with more than one 12017 * busy sibling. 12018 */ 12019 if ((group->flags & SD_SHARE_CPUCAPACITY) && 12020 (sgs->group_weight - sgs->idle_cpus != 1)) 12021 return false; 12022 12023 return sched_asym(env->sd, env->dst_cpu, READ_ONCE(group->asym_prefer_cpu)); 12024 } 12025 12026 /* One group has more than one SMT CPU while the other group does not */ 12027 static inline bool smt_vs_nonsmt_groups(struct sched_group *sg1, 12028 struct sched_group *sg2) 12029 { 12030 if (!sg1 || !sg2) 12031 return false; 12032 12033 return (sg1->flags & SD_SHARE_CPUCAPACITY) != 12034 (sg2->flags & SD_SHARE_CPUCAPACITY); 12035 } 12036 12037 static inline bool smt_balance(struct lb_env *env, struct sg_lb_stats *sgs, 12038 struct sched_group *group) 12039 { 12040 if (!env->idle) 12041 return false; 12042 12043 /* 12044 * For SMT source group, it is better to move a task 12045 * to a CPU that doesn't have multiple tasks sharing its CPU capacity. 12046 * Note that if a group has a single SMT, SD_SHARE_CPUCAPACITY 12047 * will not be on. 12048 */ 12049 if (group->flags & SD_SHARE_CPUCAPACITY && 12050 sgs->sum_h_nr_running > 1) 12051 return true; 12052 12053 return false; 12054 } 12055 12056 static inline long sibling_imbalance(struct lb_env *env, 12057 struct sd_lb_stats *sds, 12058 struct sg_lb_stats *busiest, 12059 struct sg_lb_stats *local) 12060 { 12061 int ncores_busiest, ncores_local; 12062 long imbalance; 12063 12064 if (!env->idle || !busiest->sum_nr_running) 12065 return 0; 12066 12067 ncores_busiest = sds->busiest->cores; 12068 ncores_local = sds->local->cores; 12069 12070 if (ncores_busiest == ncores_local) { 12071 imbalance = busiest->sum_nr_running; 12072 lsub_positive(&imbalance, local->sum_nr_running); 12073 return imbalance; 12074 } 12075 12076 /* Balance such that nr_running/ncores ratio are same on both groups */ 12077 imbalance = ncores_local * busiest->sum_nr_running; 12078 lsub_positive(&imbalance, ncores_busiest * local->sum_nr_running); 12079 /* Normalize imbalance and do rounding on normalization */ 12080 imbalance = 2 * imbalance + ncores_local + ncores_busiest; 12081 imbalance /= ncores_local + ncores_busiest; 12082 12083 /* Take advantage of resource in an empty sched group */ 12084 if (imbalance <= 1 && local->sum_nr_running == 0 && 12085 busiest->sum_nr_running > 1) 12086 imbalance = 2; 12087 12088 return imbalance; 12089 } 12090 12091 static inline bool 12092 sched_reduced_capacity(struct rq *rq, struct sched_domain *sd) 12093 { 12094 /* 12095 * When there is more than 1 task, the group_overloaded case already 12096 * takes care of cpu with reduced capacity 12097 */ 12098 if (rq->cfs.h_nr_runnable != 1) 12099 return false; 12100 12101 return check_cpu_capacity(rq, sd); 12102 } 12103 12104 #ifdef CONFIG_SCHED_CACHE 12105 /* 12106 * Record the statistics for this scheduler group for later 12107 * use. These values guide load balancing on aggregating tasks 12108 * to a LLC. 12109 */ 12110 static void record_sg_llc_stats(struct lb_env *env, 12111 struct sg_lb_stats *sgs, 12112 struct sched_group *group) 12113 { 12114 struct sched_domain_shared *sd_share; 12115 int cpu; 12116 12117 if (!sched_cache_enabled() || env->idle == CPU_NEWLY_IDLE) 12118 return; 12119 12120 /* Only care about sched domain spanning multiple LLCs */ 12121 if (env->sd->child != rcu_dereference_all(per_cpu(sd_llc, env->dst_cpu))) 12122 return; 12123 12124 /* 12125 * A redo repeats stats published on the first pass. Hotplug can 12126 * still leave env->cpus partial, so util would not match capacity. 12127 */ 12128 if ((env->flags & LBF_ALL_PINNED) || 12129 !cpumask_subset(sched_group_span(group), env->cpus)) 12130 return; 12131 12132 /* 12133 * At this point we know this group spans a LLC domain. 12134 * Record the statistic of this group in its corresponding 12135 * shared LLC domain. 12136 * Note: sd_share cannot be obtained via sd->child->shared, 12137 * because the latter refers to the domain that covers the 12138 * local group. Instead, sd_share should be located using 12139 * the first CPU of the LLC group. 12140 */ 12141 cpu = cpumask_first(sched_group_span(group)); 12142 sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, cpu)); 12143 if (!sd_share) 12144 return; 12145 12146 if (READ_ONCE(sd_share->util_avg) != sgs->group_util) 12147 WRITE_ONCE(sd_share->util_avg, sgs->group_util); 12148 12149 if (unlikely(READ_ONCE(sd_share->capacity) != sgs->group_capacity)) 12150 WRITE_ONCE(sd_share->capacity, sgs->group_capacity); 12151 } 12152 12153 /* 12154 * Do LLC balance on sched group that contains LLC, and have tasks preferring 12155 * to run on LLC in idle dst_cpu. 12156 */ 12157 static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs, 12158 struct sched_group *group) 12159 { 12160 if (!sched_cache_enabled()) 12161 return false; 12162 12163 if (env->sd->flags & SD_SHARE_LLC) 12164 return false; 12165 12166 /* 12167 * On asymmetric domains, group_misfit_task_load 12168 * should be prioritized to move tasks to CPU that fit them 12169 * over aggregating tasks to their preferred LLC. 12170 */ 12171 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 12172 sgs->group_misfit_task_load) 12173 return false; 12174 12175 /* 12176 * On asym packing domains, if the destination CPU 12177 * has higher priority than all CPUs in the source group, 12178 * prioritize asym packing. 12179 */ 12180 if (sgs->group_asym_packing) 12181 return false; 12182 12183 /* 12184 * Skip cache aware tagging if nr_balanced_failed is sufficiently high. 12185 * Threshold of cache_nice_tries is set to 1 higher than nr_balance_failed 12186 * to avoid excessive task migration at the same time. 12187 */ 12188 if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1) 12189 return false; 12190 12191 if (sgs->nr_pref_dst_llc && 12192 can_migrate_llc(cpumask_first(sched_group_span(group)), 12193 env->dst_cpu, 0, true) == mig_llc) 12194 return true; 12195 12196 return false; 12197 } 12198 12199 static bool update_llc_busiest(struct lb_env *env, 12200 struct sg_lb_stats *busiest, 12201 struct sg_lb_stats *sgs) 12202 { 12203 /* 12204 * There are more tasks that want to run on dst_cpu's LLC. 12205 */ 12206 return sgs->nr_pref_dst_llc > busiest->nr_pref_dst_llc; 12207 } 12208 #else 12209 static inline void record_sg_llc_stats(struct lb_env *env, struct sg_lb_stats *sgs, 12210 struct sched_group *group) 12211 { 12212 } 12213 12214 static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs, 12215 struct sched_group *group) 12216 { 12217 return false; 12218 } 12219 12220 static bool update_llc_busiest(struct lb_env *env, 12221 struct sg_lb_stats *busiest, 12222 struct sg_lb_stats *sgs) 12223 { 12224 return false; 12225 } 12226 #endif 12227 12228 /** 12229 * update_sg_lb_stats - Update sched_group's statistics for load balancing. 12230 * @env: The load balancing environment. 12231 * @sds: Load-balancing data with statistics of the local group. 12232 * @group: sched_group whose statistics are to be updated. 12233 * @sgs: variable to hold the statistics for this group. 12234 * @sg_overloaded: sched_group is overloaded 12235 */ 12236 static inline void update_sg_lb_stats(struct lb_env *env, 12237 struct sd_lb_stats *sds, 12238 struct sched_group *group, 12239 struct sg_lb_stats *sgs, 12240 bool *sg_overloaded) 12241 { 12242 int i, nr_running, local_group, sd_flags = env->sd->flags; 12243 bool balancing_at_rd = !env->sd->parent; 12244 12245 memset(sgs, 0, sizeof(*sgs)); 12246 12247 local_group = group == sds->local; 12248 12249 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 12250 struct rq *rq = cpu_rq(i); 12251 unsigned long load = cpu_load(rq); 12252 12253 sgs->group_load += load; 12254 sgs->group_util += cpu_util_cfs(i); 12255 sgs->group_runnable += cpu_runnable(rq); 12256 sgs->sum_h_nr_running += rq->cfs.h_nr_runnable; 12257 12258 nr_running = rq->nr_running; 12259 sgs->sum_nr_running += nr_running; 12260 12261 if (cpu_overutilized(i)) 12262 sgs->group_overutilized = 1; 12263 12264 #ifdef CONFIG_SCHED_CACHE 12265 if (sched_cache_enabled()) { 12266 struct sched_domain *sd_tmp; 12267 int dst_llc; 12268 12269 dst_llc = llc_id(env->dst_cpu); 12270 if (llc_id(i) != dst_llc) { 12271 sd_tmp = rcu_dereference_all(rq->sd); 12272 if (sd_tmp && (unsigned int)dst_llc < sd_tmp->llc_max) 12273 sgs->nr_pref_dst_llc += sd_tmp->llc_counts[dst_llc]; 12274 } 12275 } 12276 #endif 12277 12278 /* 12279 * No need to call idle_cpu() if nr_running is not 0 12280 */ 12281 if (!nr_running && idle_cpu(i)) { 12282 sgs->idle_cpus++; 12283 /* Idle cpu can't have misfit task */ 12284 continue; 12285 } 12286 12287 /* Overload indicator is only updated at root domain */ 12288 if (balancing_at_rd && nr_running > 1) 12289 *sg_overloaded = 1; 12290 12291 #ifdef CONFIG_NUMA_BALANCING 12292 /* Only fbq_classify_group() uses this to classify NUMA groups */ 12293 if (sd_flags & SD_NUMA) { 12294 sgs->nr_numa_running += rq->nr_numa_running; 12295 sgs->nr_preferred_running += rq->nr_preferred_running; 12296 } 12297 #endif 12298 if (local_group) 12299 continue; 12300 12301 if (sd_flags & SD_ASYM_CPUCAPACITY) { 12302 if (rq->misfit_task_load) { 12303 /* 12304 * Always mark the root domain overloaded so big 12305 * CPUs can pick up misfit tasks via newly idle 12306 * balance. 12307 */ 12308 if (balancing_at_rd) 12309 *sg_overloaded = 1; 12310 12311 /* 12312 * Only account misfit load if @dst_cpu can 12313 * help; otherwise, the group may be classified 12314 * as misfit_task and update_sd_pick_busiest() 12315 * will skip it. 12316 */ 12317 if (capacity_greater(capacity_of(env->dst_cpu), 12318 group->sgc->max_capacity) && 12319 (sgs->group_misfit_task_load < rq->misfit_task_load)) 12320 sgs->group_misfit_task_load = rq->misfit_task_load; 12321 } 12322 } else if (env->idle && sched_reduced_capacity(rq, env->sd)) { 12323 /* Check for a task running on a CPU with reduced capacity */ 12324 if (sgs->group_misfit_task_load < load) 12325 sgs->group_misfit_task_load = load; 12326 } 12327 } 12328 12329 sgs->group_capacity = group->sgc->capacity; 12330 12331 sgs->group_weight = group->group_weight; 12332 12333 if (!local_group) { 12334 /* Check if dst CPU is idle and preferred to this group */ 12335 if (env->idle && sgs->sum_h_nr_running && 12336 sched_group_asym(env, sgs, group)) 12337 sgs->group_asym_packing = 1; 12338 12339 /* Check for loaded SMT group to be balanced to dst CPU */ 12340 if (smt_balance(env, sgs, group)) 12341 sgs->group_smt_balance = 1; 12342 12343 /* Check for tasks in this group can be moved to their preferred LLC */ 12344 if (llc_balance(env, sgs, group)) 12345 sgs->group_llc_balance = 1; 12346 } 12347 12348 sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 12349 12350 record_sg_llc_stats(env, sgs, group); 12351 /* Computing avg_load makes sense only when group is overloaded */ 12352 if (sgs->group_type == group_overloaded) 12353 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 12354 sgs->group_capacity; 12355 } 12356 12357 /** 12358 * update_sd_pick_busiest - return 1 on busiest group 12359 * @env: The load balancing environment. 12360 * @sds: sched_domain statistics 12361 * @sg: sched_group candidate to be checked for being the busiest 12362 * @sgs: sched_group statistics 12363 * 12364 * Determine if @sg is a busier group than the previously selected 12365 * busiest group. 12366 * 12367 * Return: %true if @sg is a busier group than the previously selected 12368 * busiest group. %false otherwise. 12369 */ 12370 static bool update_sd_pick_busiest(struct lb_env *env, 12371 struct sd_lb_stats *sds, 12372 struct sched_group *sg, 12373 struct sg_lb_stats *sgs) 12374 { 12375 struct sg_lb_stats *busiest = &sds->busiest_stat; 12376 12377 /* Make sure that there is at least one task to pull */ 12378 if (!sgs->sum_h_nr_running) 12379 return false; 12380 12381 /* 12382 * Don't try to pull misfit tasks we can't help. 12383 * We can use max_capacity here as reduction in capacity on some 12384 * CPUs in the group should either be possible to resolve 12385 * internally or be covered by avg_load imbalance (eventually). 12386 * 12387 * When SMT is active, only pull a misfit to dst_cpu if it is on a 12388 * fully idle core; otherwise the effective capacity of the core is 12389 * reduced and we may not actually provide more capacity than the 12390 * source. 12391 */ 12392 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 12393 (sgs->group_type == group_misfit_task) && 12394 (!env->dst_core_idle || 12395 !capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || 12396 sds->local_stat.group_type != group_has_spare)) 12397 return false; 12398 12399 /* 12400 * Candidate sg has no more than one task per CPU and has higher 12401 * per-CPU capacity. Migrating tasks to less capable CPUs may harm 12402 * throughput. Maximize throughput, power/energy consequences are not 12403 * considered. 12404 */ 12405 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 12406 (sgs->group_type <= group_fully_busy) && 12407 (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) 12408 return false; 12409 12410 if (sgs->group_type > busiest->group_type) 12411 return true; 12412 12413 if (sgs->group_type < busiest->group_type) 12414 return false; 12415 12416 /* 12417 * The candidate and the current busiest group are the same type of 12418 * group. Let check which one is the busiest according to the type. 12419 */ 12420 12421 switch (sgs->group_type) { 12422 case group_overloaded: 12423 /* Select the overloaded group with highest avg_load. */ 12424 return sgs->avg_load > busiest->avg_load; 12425 12426 case group_llc_balance: 12427 /* Select the group with most tasks preferring dst LLC */ 12428 return update_llc_busiest(env, busiest, sgs); 12429 12430 case group_imbalanced: 12431 /* 12432 * Select the 1st imbalanced group as we don't have any way to 12433 * choose one more than another. 12434 */ 12435 return false; 12436 12437 case group_asym_packing: 12438 /* Prefer to move from lowest priority CPU's work */ 12439 return sched_asym_prefer(READ_ONCE(sds->busiest->asym_prefer_cpu), 12440 READ_ONCE(sg->asym_prefer_cpu)); 12441 12442 case group_misfit_task: 12443 /* 12444 * If we have more than one misfit sg go with the biggest 12445 * misfit. 12446 */ 12447 return sgs->group_misfit_task_load > busiest->group_misfit_task_load; 12448 12449 case group_smt_balance: 12450 /* 12451 * Check if we have spare CPUs on either SMT group to 12452 * choose has spare or fully busy handling. 12453 */ 12454 if (sgs->idle_cpus != 0 || busiest->idle_cpus != 0) 12455 goto has_spare; 12456 12457 fallthrough; 12458 12459 case group_fully_busy: 12460 /* 12461 * Select the fully busy group with highest avg_load. In 12462 * theory, there is no need to pull task from such kind of 12463 * group because tasks have all compute capacity that they need 12464 * but we can still improve the overall throughput by reducing 12465 * contention when accessing shared HW resources. 12466 * 12467 * XXX for now avg_load is not computed and always 0 so we 12468 * select the 1st one, except if @sg is composed of SMT 12469 * siblings. 12470 */ 12471 12472 if (sgs->avg_load < busiest->avg_load) 12473 return false; 12474 12475 if (sgs->avg_load == busiest->avg_load) { 12476 /* 12477 * SMT sched groups need more help than non-SMT groups. 12478 * If @sg happens to also be SMT, either choice is good. 12479 */ 12480 if (sds->busiest->flags & SD_SHARE_CPUCAPACITY) 12481 return false; 12482 } 12483 12484 break; 12485 12486 case group_has_spare: 12487 /* 12488 * Do not pick sg with SMT CPUs over sg with pure CPUs, 12489 * as we do not want to pull task off SMT core with one task 12490 * and make the core idle. 12491 */ 12492 if (smt_vs_nonsmt_groups(sds->busiest, sg)) { 12493 if (sg->flags & SD_SHARE_CPUCAPACITY && sgs->sum_h_nr_running <= 1) 12494 return false; 12495 else 12496 return true; 12497 } 12498 has_spare: 12499 12500 /* 12501 * Select not overloaded group with lowest number of idle CPUs 12502 * and highest number of running tasks. We could also compare 12503 * the spare capacity which is more stable but it can end up 12504 * that the group has less spare capacity but finally more idle 12505 * CPUs which means less opportunity to pull tasks. 12506 */ 12507 if (sgs->idle_cpus > busiest->idle_cpus) 12508 return false; 12509 else if ((sgs->idle_cpus == busiest->idle_cpus) && 12510 (sgs->sum_nr_running <= busiest->sum_nr_running)) 12511 return false; 12512 12513 break; 12514 } 12515 12516 return true; 12517 } 12518 12519 #ifdef CONFIG_NUMA_BALANCING 12520 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 12521 { 12522 if (sgs->sum_h_nr_running > sgs->nr_numa_running) 12523 return regular; 12524 if (sgs->sum_h_nr_running > sgs->nr_preferred_running) 12525 return remote; 12526 return all; 12527 } 12528 12529 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 12530 { 12531 if (rq->nr_running > rq->nr_numa_running) 12532 return regular; 12533 if (rq->nr_running > rq->nr_preferred_running) 12534 return remote; 12535 return all; 12536 } 12537 #else /* !CONFIG_NUMA_BALANCING: */ 12538 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 12539 { 12540 return all; 12541 } 12542 12543 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 12544 { 12545 return regular; 12546 } 12547 #endif /* !CONFIG_NUMA_BALANCING */ 12548 12549 12550 struct sg_lb_stats; 12551 12552 /* 12553 * task_running_on_cpu - return 1 if @p is running on @cpu. 12554 */ 12555 12556 static unsigned int task_running_on_cpu(int cpu, struct task_struct *p) 12557 { 12558 /* Task has no contribution or is new */ 12559 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 12560 return 0; 12561 12562 if (task_on_rq_queued(p)) 12563 return 1; 12564 12565 return 0; 12566 } 12567 12568 /** 12569 * idle_cpu_without - would a given CPU be idle without p ? 12570 * @cpu: the processor on which idleness is tested. 12571 * @p: task which should be ignored. 12572 * 12573 * Return: 1 if the CPU would be idle. 0 otherwise. 12574 */ 12575 static int idle_cpu_without(int cpu, struct task_struct *p) 12576 { 12577 struct rq *rq = cpu_rq(cpu); 12578 12579 if (rq->curr != rq->idle && rq->curr != p) 12580 return 0; 12581 12582 /* 12583 * rq->nr_running can't be used but an updated version without the 12584 * impact of p on cpu must be used instead. The updated nr_running 12585 * be computed and tested before calling idle_cpu_without(). 12586 */ 12587 12588 if (rq->ttwu_pending) 12589 return 0; 12590 12591 return 1; 12592 } 12593 12594 /* 12595 * update_sg_wakeup_stats - Update sched_group's statistics for wakeup. 12596 * @sd: The sched_domain level to look for idlest group. 12597 * @group: sched_group whose statistics are to be updated. 12598 * @sgs: variable to hold the statistics for this group. 12599 * @p: The task for which we look for the idlest group/CPU. 12600 */ 12601 static inline void update_sg_wakeup_stats(struct sched_domain *sd, 12602 struct sched_group *group, 12603 struct sg_lb_stats *sgs, 12604 struct task_struct *p) 12605 { 12606 int i, nr_running; 12607 12608 memset(sgs, 0, sizeof(*sgs)); 12609 12610 /* Assume that task can't fit any CPU of the group */ 12611 if (sd->flags & SD_ASYM_CPUCAPACITY) 12612 sgs->group_misfit_task_load = 1; 12613 12614 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 12615 struct rq *rq = cpu_rq(i); 12616 unsigned int local; 12617 12618 sgs->group_load += cpu_load_without(rq, p); 12619 sgs->group_util += cpu_util_without(i, p); 12620 sgs->group_runnable += cpu_runnable_without(rq, p); 12621 local = task_running_on_cpu(i, p); 12622 sgs->sum_h_nr_running += rq->cfs.h_nr_runnable - local; 12623 12624 nr_running = rq->nr_running - local; 12625 sgs->sum_nr_running += nr_running; 12626 12627 /* 12628 * No need to call idle_cpu_without() if nr_running is not 0 12629 */ 12630 if (!nr_running && idle_cpu_without(i, p)) 12631 sgs->idle_cpus++; 12632 12633 /* Check if task fits in the CPU */ 12634 if (sd->flags & SD_ASYM_CPUCAPACITY && 12635 sgs->group_misfit_task_load && 12636 task_fits_cpu(p, i)) 12637 sgs->group_misfit_task_load = 0; 12638 12639 } 12640 12641 sgs->group_capacity = group->sgc->capacity; 12642 12643 sgs->group_weight = group->group_weight; 12644 12645 sgs->group_type = group_classify(sd->imbalance_pct, group, sgs); 12646 12647 /* 12648 * Computing avg_load makes sense only when group is fully busy or 12649 * overloaded 12650 */ 12651 if (sgs->group_type == group_fully_busy || 12652 sgs->group_type == group_overloaded) 12653 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 12654 sgs->group_capacity; 12655 } 12656 12657 static bool update_pick_idlest(struct sched_group *idlest, 12658 struct sg_lb_stats *idlest_sgs, 12659 struct sched_group *group, 12660 struct sg_lb_stats *sgs) 12661 { 12662 if (sgs->group_type < idlest_sgs->group_type) 12663 return true; 12664 12665 if (sgs->group_type > idlest_sgs->group_type) 12666 return false; 12667 12668 /* 12669 * The candidate and the current idlest group are the same type of 12670 * group. Let check which one is the idlest according to the type. 12671 */ 12672 12673 switch (sgs->group_type) { 12674 case group_overloaded: 12675 case group_fully_busy: 12676 /* Select the group with lowest avg_load. */ 12677 if (idlest_sgs->avg_load <= sgs->avg_load) 12678 return false; 12679 break; 12680 12681 case group_llc_balance: 12682 case group_imbalanced: 12683 case group_asym_packing: 12684 case group_smt_balance: 12685 /* Those types are not used in the slow wakeup path */ 12686 return false; 12687 12688 case group_misfit_task: 12689 /* Select group with the highest max capacity */ 12690 if (idlest->sgc->max_capacity >= group->sgc->max_capacity) 12691 return false; 12692 break; 12693 12694 case group_has_spare: 12695 /* Select group with most idle CPUs */ 12696 if (idlest_sgs->idle_cpus > sgs->idle_cpus) 12697 return false; 12698 12699 /* Select group with lowest group_util */ 12700 if (idlest_sgs->idle_cpus == sgs->idle_cpus && 12701 idlest_sgs->group_util <= sgs->group_util) 12702 return false; 12703 12704 break; 12705 } 12706 12707 return true; 12708 } 12709 12710 /* 12711 * sched_balance_find_dst_group() finds and returns the least busy CPU group within the 12712 * domain. 12713 * 12714 * Assumes p is allowed on at least one CPU in sd. 12715 */ 12716 static struct sched_group * 12717 sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) 12718 { 12719 struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups; 12720 struct sg_lb_stats local_sgs, tmp_sgs; 12721 struct sg_lb_stats *sgs; 12722 unsigned long imbalance; 12723 struct sg_lb_stats idlest_sgs = { 12724 .avg_load = UINT_MAX, 12725 .group_type = group_overloaded, 12726 }; 12727 12728 do { 12729 int local_group; 12730 12731 /* Skip over this group if it has no CPUs allowed */ 12732 if (!cpumask_intersects(sched_group_span(group), 12733 p->cpus_ptr)) 12734 continue; 12735 12736 /* Skip over this group if no cookie matched */ 12737 if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group)) 12738 continue; 12739 12740 local_group = cpumask_test_cpu(this_cpu, 12741 sched_group_span(group)); 12742 12743 if (local_group) { 12744 sgs = &local_sgs; 12745 local = group; 12746 } else { 12747 sgs = &tmp_sgs; 12748 } 12749 12750 update_sg_wakeup_stats(sd, group, sgs, p); 12751 12752 if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) { 12753 idlest = group; 12754 idlest_sgs = *sgs; 12755 } 12756 12757 } while (group = group->next, group != sd->groups); 12758 12759 12760 /* There is no idlest group to push tasks to */ 12761 if (!idlest) 12762 return NULL; 12763 12764 /* The local group has been skipped because of CPU affinity */ 12765 if (!local) 12766 return idlest; 12767 12768 /* 12769 * If the local group is idler than the selected idlest group 12770 * don't try and push the task. 12771 */ 12772 if (local_sgs.group_type < idlest_sgs.group_type) 12773 return NULL; 12774 12775 /* 12776 * If the local group is busier than the selected idlest group 12777 * try and push the task. 12778 */ 12779 if (local_sgs.group_type > idlest_sgs.group_type) 12780 return idlest; 12781 12782 switch (local_sgs.group_type) { 12783 case group_overloaded: 12784 case group_fully_busy: 12785 12786 /* Calculate allowed imbalance based on load */ 12787 imbalance = scale_load_down(NICE_0_LOAD) * 12788 (sd->imbalance_pct-100) / 100; 12789 12790 /* 12791 * When comparing groups across NUMA domains, it's possible for 12792 * the local domain to be very lightly loaded relative to the 12793 * remote domains but "imbalance" skews the comparison making 12794 * remote CPUs look much more favourable. When considering 12795 * cross-domain, add imbalance to the load on the remote node 12796 * and consider staying local. 12797 */ 12798 12799 if ((sd->flags & SD_NUMA) && 12800 ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load)) 12801 return NULL; 12802 12803 /* 12804 * If the local group is less loaded than the selected 12805 * idlest group don't try and push any tasks. 12806 */ 12807 if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance)) 12808 return NULL; 12809 12810 if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load) 12811 return NULL; 12812 break; 12813 12814 case group_llc_balance: 12815 case group_imbalanced: 12816 case group_asym_packing: 12817 case group_smt_balance: 12818 /* Those type are not used in the slow wakeup path */ 12819 return NULL; 12820 12821 case group_misfit_task: 12822 /* Select group with the highest max capacity */ 12823 if (local->sgc->max_capacity >= idlest->sgc->max_capacity) 12824 return NULL; 12825 break; 12826 12827 case group_has_spare: 12828 #ifdef CONFIG_NUMA 12829 if (sd->flags & SD_NUMA) { 12830 int imb_numa_nr = sd->imb_numa_nr; 12831 #ifdef CONFIG_NUMA_BALANCING 12832 int idlest_cpu; 12833 /* 12834 * If there is spare capacity at NUMA, try to select 12835 * the preferred node 12836 */ 12837 if (cpu_to_node(this_cpu) == p->numa_preferred_nid) 12838 return NULL; 12839 12840 idlest_cpu = cpumask_first(sched_group_span(idlest)); 12841 if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid) 12842 return idlest; 12843 #endif /* CONFIG_NUMA_BALANCING */ 12844 /* 12845 * Otherwise, keep the task close to the wakeup source 12846 * and improve locality if the number of running tasks 12847 * would remain below threshold where an imbalance is 12848 * allowed while accounting for the possibility the 12849 * task is pinned to a subset of CPUs. If there is a 12850 * real need of migration, periodic load balance will 12851 * take care of it. 12852 */ 12853 if (p->nr_cpus_allowed != NR_CPUS) { 12854 unsigned int w = cpumask_weight_and(p->cpus_ptr, 12855 sched_group_span(local)); 12856 imb_numa_nr = min(w, sd->imb_numa_nr); 12857 } 12858 12859 imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus); 12860 if (!adjust_numa_imbalance(imbalance, 12861 local_sgs.sum_nr_running + 1, 12862 imb_numa_nr)) { 12863 return NULL; 12864 } 12865 } 12866 #endif /* CONFIG_NUMA */ 12867 12868 /* 12869 * Select group with highest number of idle CPUs. We could also 12870 * compare the utilization which is more stable but it can end 12871 * up that the group has less spare capacity but finally more 12872 * idle CPUs which means more opportunity to run task. 12873 */ 12874 if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus) 12875 return NULL; 12876 break; 12877 } 12878 12879 return idlest; 12880 } 12881 12882 static void update_idle_cpu_scan(struct lb_env *env, 12883 unsigned long sum_util) 12884 { 12885 struct sched_domain_shared *sd_share; 12886 struct sched_domain *sd = env->sd; 12887 int llc_weight, pct; 12888 u64 x, y, tmp; 12889 /* 12890 * Update the number of CPUs to scan in LLC domain, which could 12891 * be used as a hint in select_idle_cpu(). The update of sd_share 12892 * could be expensive because it is within a shared cache line. 12893 * So the write of this hint only occurs during periodic load 12894 * balancing, rather than CPU_NEWLY_IDLE, because the latter 12895 * can fire way more frequently than the former. 12896 */ 12897 if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE) 12898 return; 12899 12900 sd_share = sd->shared; 12901 if (!sd_share) 12902 return; 12903 12904 /* 12905 * The number of CPUs to search drops as sum_util increases, when 12906 * sum_util hits 85% or above, the scan stops. 12907 * The reason to choose 85% as the threshold is because this is the 12908 * imbalance_pct(117) when a LLC sched group is overloaded. 12909 * 12910 * let y = SCHED_CAPACITY_SCALE - p * x^2 [1] 12911 * and y'= y / SCHED_CAPACITY_SCALE 12912 * 12913 * x is the ratio of sum_util compared to the CPU capacity: 12914 * x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE) 12915 * y' is the ratio of CPUs to be scanned in the LLC domain, 12916 * and the number of CPUs to scan is calculated by: 12917 * 12918 * nr_scan = llc_weight * y' [2] 12919 * 12920 * When x hits the threshold of overloaded, AKA, when 12921 * x = 100 / pct, y drops to 0. According to [1], 12922 * p should be SCHED_CAPACITY_SCALE * pct^2 / 10000 12923 * 12924 * Scale x by SCHED_CAPACITY_SCALE: 12925 * x' = sum_util / llc_weight; [3] 12926 * 12927 * and finally [1] becomes: 12928 * y = SCHED_CAPACITY_SCALE - 12929 * x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4] 12930 * 12931 */ 12932 /* equation [3] */ 12933 x = sum_util; 12934 llc_weight = sd->span_weight; 12935 do_div(x, llc_weight); 12936 12937 /* equation [4] */ 12938 pct = sd->imbalance_pct; 12939 tmp = x * x * pct * pct; 12940 do_div(tmp, 10000 * SCHED_CAPACITY_SCALE); 12941 tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE); 12942 y = SCHED_CAPACITY_SCALE - tmp; 12943 12944 /* equation [2] */ 12945 y *= llc_weight; 12946 do_div(y, SCHED_CAPACITY_SCALE); 12947 if ((int)y != sd_share->nr_idle_scan) 12948 WRITE_ONCE(sd_share->nr_idle_scan, (int)y); 12949 } 12950 12951 /** 12952 * update_sd_lb_stats - Update sched_domain's statistics for load balancing. 12953 * @env: The load balancing environment. 12954 * @sds: variable to hold the statistics for this sched_domain. 12955 */ 12956 12957 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) 12958 { 12959 struct sched_group *sg = env->sd->groups; 12960 struct sg_lb_stats *local = &sds->local_stat; 12961 struct sg_lb_stats tmp_sgs; 12962 unsigned long sum_util = 0; 12963 bool sg_overloaded = 0, sg_overutilized = 0; 12964 12965 env->dst_core_idle = !sched_smt_active() || is_core_idle(env->dst_cpu); 12966 12967 do { 12968 struct sg_lb_stats *sgs = &tmp_sgs; 12969 int local_group; 12970 12971 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg)); 12972 if (local_group) { 12973 sds->local = sg; 12974 sgs = local; 12975 12976 if (env->idle != CPU_NEWLY_IDLE || 12977 time_after_eq(jiffies, sg->sgc->next_update)) 12978 update_group_capacity(env->sd, env->dst_cpu); 12979 } 12980 12981 update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded); 12982 12983 if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) { 12984 sds->busiest = sg; 12985 sds->busiest_stat = *sgs; 12986 } 12987 12988 sg_overutilized |= sgs->group_overutilized; 12989 12990 /* Now, start updating sd_lb_stats */ 12991 sds->total_load += sgs->group_load; 12992 sds->total_capacity += sgs->group_capacity; 12993 12994 sum_util += sgs->group_util; 12995 sg = sg->next; 12996 } while (sg != env->sd->groups); 12997 12998 /* 12999 * Indicate that the child domain of the busiest group prefers tasks 13000 * go to a child's sibling domains first. NB the flags of a sched group 13001 * are those of the child domain. 13002 */ 13003 if (sds->busiest) 13004 sds->prefer_sibling = !!(sds->busiest->flags & SD_PREFER_SIBLING); 13005 13006 13007 if (env->sd->flags & SD_NUMA) 13008 env->fbq_type = fbq_classify_group(&sds->busiest_stat); 13009 13010 if (!env->sd->parent) { 13011 /* update overload indicator if we are at root domain */ 13012 set_rd_overloaded(env->dst_rq->rd, sg_overloaded); 13013 13014 /* Update over-utilization (tipping point, U >= 0) indicator */ 13015 set_rd_overutilized(env->dst_rq->rd, sg_overutilized); 13016 } else if (sg_overutilized) { 13017 set_rd_overutilized(env->dst_rq->rd, sg_overutilized); 13018 } 13019 13020 update_idle_cpu_scan(env, sum_util); 13021 } 13022 13023 /** 13024 * calculate_imbalance - Calculate the amount of imbalance present within the 13025 * groups of a given sched_domain during load balance. 13026 * @env: load balance environment 13027 * @sds: statistics of the sched_domain whose imbalance is to be calculated. 13028 */ 13029 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) 13030 { 13031 struct sg_lb_stats *local, *busiest; 13032 13033 local = &sds->local_stat; 13034 busiest = &sds->busiest_stat; 13035 13036 if (busiest->group_type == group_misfit_task) { 13037 if (env->sd->flags & SD_ASYM_CPUCAPACITY) { 13038 /* Set imbalance to allow misfit tasks to be balanced. */ 13039 env->migration_type = migrate_misfit; 13040 env->imbalance = 1; 13041 } else { 13042 /* 13043 * Set load imbalance to allow moving task from cpu 13044 * with reduced capacity. 13045 */ 13046 env->migration_type = migrate_load; 13047 env->imbalance = busiest->group_misfit_task_load; 13048 } 13049 return; 13050 } 13051 13052 if (busiest->group_type == group_asym_packing) { 13053 /* 13054 * In case of asym capacity, we will try to migrate all load to 13055 * the preferred CPU. 13056 */ 13057 env->migration_type = migrate_task; 13058 env->imbalance = busiest->sum_h_nr_running; 13059 return; 13060 } 13061 13062 if (busiest->group_type == group_smt_balance) { 13063 /* Reduce number of tasks sharing CPU capacity */ 13064 env->migration_type = migrate_task; 13065 env->imbalance = 1; 13066 return; 13067 } 13068 13069 #ifdef CONFIG_SCHED_CACHE 13070 if (busiest->group_type == group_llc_balance) { 13071 /* Move a task that prefer local LLC */ 13072 env->migration_type = migrate_llc_task; 13073 env->imbalance = 1; 13074 return; 13075 } 13076 #endif 13077 13078 if (busiest->group_type == group_imbalanced) { 13079 /* 13080 * In the group_imb case we cannot rely on group-wide averages 13081 * to ensure CPU-load equilibrium, try to move any task to fix 13082 * the imbalance. The next load balance will take care of 13083 * balancing back the system. 13084 */ 13085 env->migration_type = migrate_task; 13086 env->imbalance = 1; 13087 return; 13088 } 13089 13090 /* 13091 * Try to use spare capacity of local group without overloading it or 13092 * emptying busiest. 13093 */ 13094 if (local->group_type == group_has_spare) { 13095 if ((busiest->group_type > group_fully_busy) && 13096 !(env->sd->flags & SD_SHARE_LLC)) { 13097 /* 13098 * If busiest is overloaded, try to fill spare 13099 * capacity. This might end up creating spare capacity 13100 * in busiest or busiest still being overloaded but 13101 * there is no simple way to directly compute the 13102 * amount of load to migrate in order to balance the 13103 * system. 13104 */ 13105 env->migration_type = migrate_util; 13106 env->imbalance = max(local->group_capacity, local->group_util) - 13107 local->group_util; 13108 13109 /* 13110 * In some cases, the group's utilization is max or even 13111 * higher than capacity because of migrations but the 13112 * local CPU is (newly) idle. There is at least one 13113 * waiting task in this overloaded busiest group. Let's 13114 * try to pull it. 13115 */ 13116 if (env->idle && env->imbalance == 0) { 13117 env->migration_type = migrate_task; 13118 env->imbalance = 1; 13119 } 13120 13121 return; 13122 } 13123 13124 if (busiest->group_weight == 1 || sds->prefer_sibling) { 13125 /* 13126 * When prefer sibling, evenly spread running tasks on 13127 * groups. 13128 */ 13129 env->migration_type = migrate_task; 13130 env->imbalance = sibling_imbalance(env, sds, busiest, local); 13131 } else { 13132 13133 /* 13134 * If there is no overload, we just want to even the number of 13135 * idle CPUs. 13136 */ 13137 env->migration_type = migrate_task; 13138 env->imbalance = max_t(long, 0, 13139 (local->idle_cpus - busiest->idle_cpus)); 13140 } 13141 13142 #ifdef CONFIG_NUMA 13143 /* Consider allowing a small imbalance between NUMA groups */ 13144 if (env->sd->flags & SD_NUMA) { 13145 env->imbalance = adjust_numa_imbalance(env->imbalance, 13146 local->sum_nr_running + 1, 13147 env->sd->imb_numa_nr); 13148 } 13149 #endif 13150 13151 /* Number of tasks to move to restore balance */ 13152 env->imbalance >>= 1; 13153 13154 return; 13155 } 13156 13157 /* 13158 * Local is fully busy but has to take more load to relieve the 13159 * busiest group 13160 */ 13161 if (local->group_type < group_overloaded) { 13162 /* 13163 * Local will become overloaded so the avg_load metrics are 13164 * finally needed. 13165 */ 13166 13167 local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 13168 local->group_capacity; 13169 13170 /* 13171 * If the local group is more loaded than the selected 13172 * busiest group don't try to pull any tasks. 13173 */ 13174 if (local->avg_load >= busiest->avg_load) { 13175 env->imbalance = 0; 13176 return; 13177 } 13178 13179 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 13180 sds->total_capacity; 13181 13182 /* 13183 * If the local group is more loaded than the average system 13184 * load, don't try to pull any tasks. 13185 */ 13186 if (local->avg_load >= sds->avg_load) { 13187 env->imbalance = 0; 13188 return; 13189 } 13190 13191 } 13192 13193 /* 13194 * Both group are or will become overloaded and we're trying to get all 13195 * the CPUs to the average_load, so we don't want to push ourselves 13196 * above the average load, nor do we wish to reduce the max loaded CPU 13197 * below the average load. At the same time, we also don't want to 13198 * reduce the group load below the group capacity. Thus we look for 13199 * the minimum possible imbalance. 13200 */ 13201 env->migration_type = migrate_load; 13202 env->imbalance = min( 13203 (busiest->avg_load - sds->avg_load) * busiest->group_capacity, 13204 (sds->avg_load - local->avg_load) * local->group_capacity 13205 ) / SCHED_CAPACITY_SCALE; 13206 } 13207 13208 /******* sched_balance_find_src_group() helpers end here *********************/ 13209 13210 /* 13211 * Decision matrix according to the local and busiest group type: 13212 * 13213 * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded 13214 * has_spare nr_idle balanced N/A N/A balanced balanced 13215 * fully_busy nr_idle nr_idle N/A N/A balanced balanced 13216 * misfit_task force N/A N/A N/A N/A N/A 13217 * asym_packing force force N/A N/A force force 13218 * imbalanced force force N/A N/A force force 13219 * overloaded force force N/A N/A force avg_load 13220 * 13221 * N/A : Not Applicable because already filtered while updating 13222 * statistics. 13223 * balanced : The system is balanced for these 2 groups. 13224 * force : Calculate the imbalance as load migration is probably needed. 13225 * avg_load : Only if imbalance is significant enough. 13226 * nr_idle : dst_cpu is not busy and the number of idle CPUs is quite 13227 * different in groups. 13228 */ 13229 13230 /** 13231 * sched_balance_find_src_group - Returns the busiest group within the sched_domain 13232 * if there is an imbalance. 13233 * @env: The load balancing environment. 13234 * 13235 * Also calculates the amount of runnable load which should be moved 13236 * to restore balance. 13237 * 13238 * Return: - The busiest group if imbalance exists. 13239 */ 13240 static struct sched_group *sched_balance_find_src_group(struct lb_env *env) 13241 { 13242 struct sg_lb_stats *local, *busiest; 13243 struct sd_lb_stats sds; 13244 13245 init_sd_lb_stats(&sds); 13246 13247 /* 13248 * Compute the various statistics relevant for load balancing at 13249 * this level. 13250 */ 13251 update_sd_lb_stats(env, &sds); 13252 13253 /* There is no busy sibling group to pull tasks from */ 13254 if (!sds.busiest) 13255 goto out_balanced; 13256 13257 busiest = &sds.busiest_stat; 13258 13259 /* Misfit tasks should be dealt with regardless of the avg load */ 13260 if (busiest->group_type == group_misfit_task) 13261 goto force_balance; 13262 13263 if (!is_rd_overutilized(env->dst_rq->rd) && 13264 rcu_dereference_all(env->dst_rq->rd->pd)) 13265 goto out_balanced; 13266 13267 /* ASYM feature bypasses nice load balance check */ 13268 if (busiest->group_type == group_asym_packing) 13269 goto force_balance; 13270 13271 /* 13272 * If the busiest group is imbalanced the below checks don't 13273 * work because they assume all things are equal, which typically 13274 * isn't true due to cpus_ptr constraints and the like. 13275 */ 13276 if (busiest->group_type == group_imbalanced) 13277 goto force_balance; 13278 13279 local = &sds.local_stat; 13280 /* 13281 * If the local group is busier than the selected busiest group 13282 * don't try and pull any tasks. 13283 */ 13284 if (local->group_type > busiest->group_type) 13285 goto out_balanced; 13286 13287 /* 13288 * When groups are overloaded, use the avg_load to ensure fairness 13289 * between tasks. 13290 */ 13291 if (local->group_type == group_overloaded) { 13292 /* 13293 * If the local group is more loaded than the selected 13294 * busiest group don't try to pull any tasks. 13295 */ 13296 if (local->avg_load >= busiest->avg_load) 13297 goto out_balanced; 13298 13299 /* XXX broken for overlapping NUMA groups */ 13300 sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) / 13301 sds.total_capacity; 13302 13303 /* 13304 * Don't pull any tasks if this group is already above the 13305 * domain average load. 13306 */ 13307 if (local->avg_load >= sds.avg_load) 13308 goto out_balanced; 13309 13310 /* 13311 * If the busiest group is more loaded, use imbalance_pct to be 13312 * conservative. 13313 */ 13314 if (100 * busiest->avg_load <= 13315 env->sd->imbalance_pct * local->avg_load) 13316 goto out_balanced; 13317 } 13318 13319 /* 13320 * Try to move all excess tasks to a sibling domain of the busiest 13321 * group's child domain. 13322 */ 13323 if (sds.prefer_sibling && local->group_type == group_has_spare && 13324 (busiest->group_type == group_llc_balance || 13325 sibling_imbalance(env, &sds, busiest, local) > 1)) 13326 goto force_balance; 13327 13328 if (busiest->group_type != group_overloaded) { 13329 if (!env->idle) { 13330 /* 13331 * If the busiest group is not overloaded (and as a 13332 * result the local one too) but this CPU is already 13333 * busy, let another idle CPU try to pull task. 13334 */ 13335 goto out_balanced; 13336 } 13337 13338 if (busiest->group_type == group_smt_balance && 13339 smt_vs_nonsmt_groups(sds.local, sds.busiest)) { 13340 /* Let non SMT CPU pull from SMT CPU sharing with sibling */ 13341 goto force_balance; 13342 } 13343 13344 if (busiest->group_weight > 1 && 13345 local->idle_cpus <= (busiest->idle_cpus + 1)) { 13346 /* 13347 * If the busiest group is not overloaded 13348 * and there is no imbalance between this and busiest 13349 * group wrt idle CPUs, it is balanced. The imbalance 13350 * becomes significant if the diff is greater than 1 13351 * otherwise we might end up to just move the imbalance 13352 * on another group. Of course this applies only if 13353 * there is more than 1 CPU per group. 13354 */ 13355 goto out_balanced; 13356 } 13357 13358 if (busiest->sum_h_nr_running == 1) { 13359 /* 13360 * busiest doesn't have any tasks waiting to run 13361 */ 13362 goto out_balanced; 13363 } 13364 } 13365 13366 force_balance: 13367 /* Looks like there is an imbalance. Compute it */ 13368 calculate_imbalance(env, &sds); 13369 return env->imbalance ? sds.busiest : NULL; 13370 13371 out_balanced: 13372 env->imbalance = 0; 13373 return NULL; 13374 } 13375 13376 /* 13377 * sched_balance_find_src_rq - find the busiest runqueue among the CPUs in the group. 13378 */ 13379 static struct rq *sched_balance_find_src_rq(struct lb_env *env, 13380 struct sched_group *group) 13381 { 13382 struct rq *busiest = NULL, *rq; 13383 unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; 13384 unsigned int __maybe_unused busiest_pref_llc = 0; 13385 struct sched_domain __maybe_unused *sd_tmp; 13386 unsigned int busiest_nr = 0; 13387 int __maybe_unused dst_llc; 13388 int i; 13389 13390 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 13391 unsigned long capacity, load, util; 13392 unsigned int nr_running; 13393 enum fbq_type rt; 13394 13395 rq = cpu_rq(i); 13396 rt = fbq_classify_rq(rq); 13397 13398 /* 13399 * We classify groups/runqueues into three groups: 13400 * - regular: there are !numa tasks 13401 * - remote: there are numa tasks that run on the 'wrong' node 13402 * - all: there is no distinction 13403 * 13404 * In order to avoid migrating ideally placed numa tasks, 13405 * ignore those when there's better options. 13406 * 13407 * If we ignore the actual busiest queue to migrate another 13408 * task, the next balance pass can still reduce the busiest 13409 * queue by moving tasks around inside the node. 13410 * 13411 * If we cannot move enough load due to this classification 13412 * the next pass will adjust the group classification and 13413 * allow migration of more tasks. 13414 * 13415 * Both cases only affect the total convergence complexity. 13416 */ 13417 if (rt > env->fbq_type) 13418 continue; 13419 13420 nr_running = rq->cfs.h_nr_runnable; 13421 if (!nr_running) 13422 continue; 13423 13424 capacity = capacity_of(i); 13425 13426 /* 13427 * For ASYM_CPUCAPACITY domains, don't pick a CPU that could 13428 * eventually lead to active_balancing high->low capacity. 13429 * Higher per-CPU capacity is considered better than balancing 13430 * average load. 13431 */ 13432 if (env->sd->flags & SD_ASYM_CPUCAPACITY && 13433 nr_running == 1) { 13434 bool cluster_equal_cap = static_branch_unlikely(&sched_cluster_active) && 13435 (get_actual_cpu_capacity(env->dst_cpu) == 13436 get_actual_cpu_capacity(i)); 13437 bool smt_degraded_cap = sched_smt_active() && !is_core_idle(i); 13438 13439 /* 13440 * Busy SMT siblings reduce the capacity of CPU @i. Do 13441 * not skip it in this case. 13442 * 13443 * CONFIG_SCHED_CLUSTER requires balancing load across 13444 * clusters of identical capacity, accounting for 13445 * hardware and cpufreq pressure. 13446 */ 13447 if (!smt_degraded_cap && !cluster_equal_cap && 13448 !capacity_greater(capacity_of(env->dst_cpu), capacity)) 13449 continue; 13450 } 13451 13452 /* 13453 * Make sure we only pull tasks from a CPU of lower priority 13454 * when balancing between SMT siblings. 13455 * 13456 * If balancing between cores, let lower priority CPUs help 13457 * SMT cores with more than one busy sibling. 13458 */ 13459 if (sched_asym(env->sd, i, env->dst_cpu) && nr_running == 1) 13460 continue; 13461 13462 switch (env->migration_type) { 13463 case migrate_load: 13464 /* 13465 * When comparing with load imbalance, use cpu_load() 13466 * which is not scaled with the CPU capacity. 13467 */ 13468 load = cpu_load(rq); 13469 13470 if (nr_running == 1 && load > env->imbalance && 13471 !check_cpu_capacity(rq, env->sd)) 13472 break; 13473 13474 /* 13475 * For the load comparisons with the other CPUs, 13476 * consider the cpu_load() scaled with the CPU 13477 * capacity, so that the load can be moved away 13478 * from the CPU that is potentially running at a 13479 * lower capacity. 13480 * 13481 * Thus we're looking for max(load_i / capacity_i), 13482 * crosswise multiplication to rid ourselves of the 13483 * division works out to: 13484 * load_i * capacity_j > load_j * capacity_i; 13485 * where j is our previous maximum. 13486 */ 13487 if (load * busiest_capacity > busiest_load * capacity) { 13488 busiest_load = load; 13489 busiest_capacity = capacity; 13490 busiest = rq; 13491 } 13492 break; 13493 13494 case migrate_util: 13495 util = cpu_util_cfs_boost(i); 13496 13497 /* 13498 * Don't try to pull utilization from a CPU with one 13499 * running task. Whatever its utilization, we will fail 13500 * detach the task. 13501 */ 13502 if (nr_running <= 1) 13503 continue; 13504 13505 if (busiest_util < util) { 13506 busiest_util = util; 13507 busiest = rq; 13508 } 13509 break; 13510 13511 case migrate_task: 13512 if (busiest_nr < nr_running) { 13513 busiest_nr = nr_running; 13514 busiest = rq; 13515 } 13516 break; 13517 13518 case migrate_misfit: 13519 /* 13520 * For ASYM_CPUCAPACITY domains with misfit tasks we 13521 * simply seek the "biggest" misfit task. 13522 */ 13523 if (rq->misfit_task_load > busiest_load) { 13524 busiest_load = rq->misfit_task_load; 13525 busiest = rq; 13526 } 13527 13528 break; 13529 13530 case migrate_llc_task: 13531 #ifdef CONFIG_SCHED_CACHE 13532 sd_tmp = rcu_dereference_all(rq->sd); 13533 dst_llc = llc_id(env->dst_cpu); 13534 13535 if (sd_tmp && (unsigned)dst_llc < sd_tmp->llc_max) { 13536 unsigned int this_pref_llc = 13537 sd_tmp->llc_counts[dst_llc]; 13538 13539 if (busiest_pref_llc < this_pref_llc) { 13540 busiest_pref_llc = this_pref_llc; 13541 busiest = rq; 13542 } 13543 } 13544 #endif 13545 break; 13546 13547 } 13548 } 13549 13550 return busiest; 13551 } 13552 13553 /* 13554 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but 13555 * so long as it is large enough. 13556 */ 13557 #define MAX_PINNED_INTERVAL 512 13558 13559 static inline bool 13560 asym_active_balance(struct lb_env *env) 13561 { 13562 /* 13563 * ASYM_PACKING needs to force migrate tasks from busy but lower 13564 * priority CPUs in order to pack all tasks in the highest priority 13565 * CPUs. When done between cores, do it only if the whole core if the 13566 * whole core is idle. 13567 * 13568 * If @env::src_cpu is an SMT core with busy siblings, let 13569 * the lower priority @env::dst_cpu help it. Do not follow 13570 * CPU priority. 13571 */ 13572 return env->idle && sched_use_asym_prio(env->sd, env->dst_cpu) && 13573 (sched_asym_prefer(env->dst_cpu, env->src_cpu) || 13574 !sched_use_asym_prio(env->sd, env->src_cpu)); 13575 } 13576 13577 static inline bool 13578 imbalanced_active_balance(struct lb_env *env) 13579 { 13580 struct sched_domain *sd = env->sd; 13581 13582 /* 13583 * The imbalanced case includes the case of pinned tasks preventing a fair 13584 * distribution of the load on the system but also the even distribution of the 13585 * threads on a system with spare capacity 13586 */ 13587 if ((env->migration_type == migrate_task) && 13588 (sd->nr_balance_failed > sd->cache_nice_tries+2)) 13589 return 1; 13590 13591 return 0; 13592 } 13593 13594 static int need_active_balance(struct lb_env *env) 13595 { 13596 struct sched_domain *sd = env->sd; 13597 13598 if (asym_active_balance(env)) 13599 return 1; 13600 13601 if (alb_break_llc(env)) 13602 return 0; 13603 13604 if (imbalanced_active_balance(env)) 13605 return 1; 13606 13607 /* 13608 * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. 13609 * It's worth migrating the task if the src_cpu's capacity is reduced 13610 * because of other sched_class or IRQs if more capacity stays 13611 * available on dst_cpu. 13612 */ 13613 if (env->idle && 13614 (env->src_rq->cfs.h_nr_runnable == 1)) { 13615 if ((check_cpu_capacity(env->src_rq, sd)) && 13616 (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) 13617 return 1; 13618 } 13619 13620 if (env->migration_type == migrate_misfit || 13621 env->migration_type == migrate_llc_task) 13622 return 1; 13623 13624 return 0; 13625 } 13626 13627 static int active_load_balance_cpu_stop(void *data); 13628 static int active_load_balance_llc_cpu_stop(void *data); 13629 13630 /* 13631 * migration_type is checked elsewhere to decide migration policy, so 13632 * it shouldn't be repurposed just to flag an LLC-directed active 13633 * balance across the stopper. Pick the callback here instead. 13634 */ 13635 static inline cpu_stop_fn_t alb_stop_fn(struct lb_env *env) 13636 { 13637 if (env->migration_type == migrate_llc_task) 13638 return active_load_balance_llc_cpu_stop; 13639 13640 return active_load_balance_cpu_stop; 13641 } 13642 13643 static int should_we_balance(struct lb_env *env) 13644 { 13645 struct cpumask *swb_cpus = this_cpu_cpumask_var_ptr(should_we_balance_tmpmask); 13646 struct sched_group *sg = env->sd->groups; 13647 int cpu, idle_smt = -1; 13648 13649 /* 13650 * Ensure the balancing environment is consistent; can happen 13651 * when the softirq triggers 'during' hotplug. 13652 */ 13653 if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 13654 return 0; 13655 13656 /* 13657 * In the newly idle case, we will allow all the CPUs 13658 * to do the newly idle load balance. 13659 * 13660 * However, we bail out if we already have tasks or a wakeup pending, 13661 * to optimize wakeup latency. 13662 */ 13663 if (env->idle == CPU_NEWLY_IDLE) { 13664 if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending) 13665 return 0; 13666 return 1; 13667 } 13668 13669 cpumask_copy(swb_cpus, group_balance_mask(sg)); 13670 /* Try to find first idle CPU */ 13671 for_each_cpu_and(cpu, swb_cpus, env->cpus) { 13672 if (!idle_cpu(cpu)) 13673 continue; 13674 13675 /* 13676 * Don't balance to idle SMT in busy core right away when 13677 * balancing cores, but remember the first idle SMT CPU for 13678 * later consideration. Find CPU on an idle core first. 13679 */ 13680 if (sched_smt_active() && 13681 !(env->sd->flags & SD_SHARE_CPUCAPACITY) && 13682 !is_core_idle(cpu)) { 13683 if (idle_smt == -1) 13684 idle_smt = cpu; 13685 /* 13686 * If the core is not idle, and first SMT sibling which is 13687 * idle has been found, then its not needed to check other 13688 * SMT siblings for idleness: 13689 */ 13690 cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu)); 13691 continue; 13692 } 13693 13694 /* 13695 * Are we the first idle core in a non-SMT domain or higher, 13696 * or the first idle CPU in a SMT domain? 13697 */ 13698 return cpu == env->dst_cpu; 13699 } 13700 13701 /* Are we the first idle CPU with busy siblings? */ 13702 if (idle_smt != -1) 13703 return idle_smt == env->dst_cpu; 13704 13705 /* Are we the first CPU of this group ? */ 13706 return group_balance_cpu(sg) == env->dst_cpu; 13707 } 13708 13709 static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd, 13710 enum cpu_idle_type idle) 13711 { 13712 if (!schedstat_enabled()) 13713 return; 13714 13715 switch (env->migration_type) { 13716 case migrate_load: 13717 __schedstat_add(sd->lb_imbalance_load[idle], env->imbalance); 13718 break; 13719 case migrate_util: 13720 __schedstat_add(sd->lb_imbalance_util[idle], env->imbalance); 13721 break; 13722 case migrate_task: 13723 __schedstat_add(sd->lb_imbalance_task[idle], env->imbalance); 13724 break; 13725 case migrate_misfit: 13726 __schedstat_add(sd->lb_imbalance_misfit[idle], env->imbalance); 13727 break; 13728 case migrate_llc_task: 13729 break; 13730 } 13731 } 13732 13733 /* 13734 * This flag serializes load-balancing passes over large domains 13735 * (above the NODE topology level) - only one load-balancing instance 13736 * may run at a time, to reduce overhead on very large systems with 13737 * lots of CPUs and large NUMA distances. 13738 * 13739 * - Note that load-balancing passes triggered while another one 13740 * is executing are skipped and not re-tried. 13741 * 13742 * - Also note that this does not serialize rebalance_domains() 13743 * execution, as non-SD_SERIALIZE domains will still be 13744 * load-balanced in parallel. 13745 */ 13746 static atomic_t sched_balance_running = ATOMIC_INIT(0); 13747 13748 /* 13749 * Check this_cpu to ensure it is balanced within domain. Attempt to move 13750 * tasks if there is an imbalance. 13751 */ 13752 static int sched_balance_rq(int this_cpu, struct rq *this_rq, 13753 struct sched_domain *sd, enum cpu_idle_type idle, 13754 int *continue_balancing) 13755 { 13756 int ld_moved, cur_ld_moved, active_balance = 0; 13757 struct sched_domain *sd_parent = sd->parent; 13758 struct sched_group *group; 13759 struct rq *busiest; 13760 struct rq_flags rf; 13761 struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 13762 struct lb_env env = { 13763 .sd = sd, 13764 .dst_cpu = this_cpu, 13765 .dst_rq = this_rq, 13766 .dst_grpmask = group_balance_mask(sd->groups), 13767 .idle = idle, 13768 .loop_break = SCHED_NR_MIGRATE_BREAK, 13769 .cpus = cpus, 13770 .fbq_type = all, 13771 .tasks = LIST_HEAD_INIT(env.tasks), 13772 }; 13773 bool need_unlock = false; 13774 13775 cpumask_and(cpus, sched_domain_span(sd), cpu_preferred_mask); 13776 13777 schedstat_inc(sd->lb_count[idle]); 13778 13779 redo: 13780 if (!should_we_balance(&env)) { 13781 *continue_balancing = 0; 13782 goto out_balanced; 13783 } 13784 13785 if (!need_unlock && (sd->flags & SD_SERIALIZE)) { 13786 int zero = 0; 13787 if (!atomic_try_cmpxchg_acquire(&sched_balance_running, &zero, 1)) 13788 goto out_balanced; 13789 13790 need_unlock = true; 13791 } 13792 13793 group = sched_balance_find_src_group(&env); 13794 if (!group) { 13795 schedstat_inc(sd->lb_nobusyg[idle]); 13796 goto out_balanced; 13797 } 13798 13799 busiest = sched_balance_find_src_rq(&env, group); 13800 if (!busiest) { 13801 schedstat_inc(sd->lb_nobusyq[idle]); 13802 goto out_balanced; 13803 } 13804 13805 WARN_ON_ONCE(busiest == env.dst_rq); 13806 13807 update_lb_imbalance_stat(&env, sd, idle); 13808 13809 env.src_cpu = busiest->cpu; 13810 env.src_rq = busiest; 13811 13812 ld_moved = 0; 13813 /* Clear this flag as soon as we find a pullable task */ 13814 env.flags |= LBF_ALL_PINNED; 13815 if (busiest->nr_running > 1) { 13816 /* 13817 * Attempt to move tasks. If sched_balance_find_src_group has found 13818 * an imbalance but busiest->nr_running <= 1, the group is 13819 * still unbalanced. ld_moved simply stays zero, so it is 13820 * correctly treated as an imbalance. 13821 */ 13822 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 13823 13824 more_balance: 13825 rq_lock_irqsave(busiest, &rf); 13826 update_rq_clock(busiest); 13827 13828 /* 13829 * cur_ld_moved - load moved in current iteration 13830 * ld_moved - cumulative load moved across iterations 13831 */ 13832 cur_ld_moved = detach_tasks(&env); 13833 13834 /* 13835 * We've detached some tasks from busiest_rq. Every 13836 * task is masked "TASK_ON_RQ_MIGRATING", so we can safely 13837 * unlock busiest->lock, and we are able to be sure 13838 * that nobody can manipulate the tasks in parallel. 13839 * See task_rq_lock() family for the details. 13840 */ 13841 13842 rq_unlock(busiest, &rf); 13843 13844 if (cur_ld_moved) { 13845 attach_tasks(&env); 13846 ld_moved += cur_ld_moved; 13847 } 13848 13849 local_irq_restore(rf.flags); 13850 13851 if (env.flags & LBF_NEED_BREAK) { 13852 env.flags &= ~LBF_NEED_BREAK; 13853 goto more_balance; 13854 } 13855 13856 /* 13857 * Revisit (affine) tasks on src_cpu that couldn't be moved to 13858 * us and move them to an alternate dst_cpu in our sched_group 13859 * where they can run. The upper limit on how many times we 13860 * iterate on same src_cpu is dependent on number of CPUs in our 13861 * sched_group. 13862 * 13863 * This changes load balance semantics a bit on who can move 13864 * load to a given_cpu. In addition to the given_cpu itself 13865 * (or a ilb_cpu acting on its behalf where given_cpu is 13866 * nohz-idle), we now have balance_cpu in a position to move 13867 * load to given_cpu. In rare situations, this may cause 13868 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding 13869 * _independently_ and at _same_ time to move some load to 13870 * given_cpu) causing excess load to be moved to given_cpu. 13871 * This however should not happen so much in practice and 13872 * moreover subsequent load balance cycles should correct the 13873 * excess load moved. 13874 */ 13875 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { 13876 13877 /* Prevent to re-select dst_cpu via env's CPUs */ 13878 __cpumask_clear_cpu(env.dst_cpu, env.cpus); 13879 13880 env.dst_rq = cpu_rq(env.new_dst_cpu); 13881 env.dst_cpu = env.new_dst_cpu; 13882 env.flags &= ~LBF_DST_PINNED; 13883 env.loop = 0; 13884 env.loop_break = SCHED_NR_MIGRATE_BREAK; 13885 13886 /* 13887 * Go back to "more_balance" rather than "redo" since we 13888 * need to continue with same src_cpu. 13889 */ 13890 goto more_balance; 13891 } 13892 13893 /* 13894 * We failed to reach balance because of affinity. 13895 */ 13896 if (sd_parent) { 13897 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 13898 13899 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) 13900 *group_imbalance = 1; 13901 } 13902 13903 /* All tasks on this runqueue were pinned by CPU affinity */ 13904 if (unlikely(env.flags & LBF_ALL_PINNED)) { 13905 __cpumask_clear_cpu(cpu_of(busiest), cpus); 13906 /* 13907 * Attempting to continue load balancing at the current 13908 * sched_domain level only makes sense if there are 13909 * active CPUs remaining as possible busiest CPUs to 13910 * pull load from which are not contained within the 13911 * destination group that is receiving any migrated 13912 * load. 13913 */ 13914 if (!cpumask_subset(cpus, env.dst_grpmask)) { 13915 env.loop = 0; 13916 env.loop_break = SCHED_NR_MIGRATE_BREAK; 13917 goto redo; 13918 } 13919 goto out_all_pinned; 13920 } 13921 } 13922 13923 if (ld_moved) { 13924 sd->nr_balance_failed = 0; 13925 goto out_unbalanced; 13926 } 13927 13928 schedstat_inc(sd->lb_failed[idle]); 13929 /* 13930 * Increment the failure counter only on periodic balance. 13931 * We do not want newidle balance, which can be very 13932 * frequent, pollute the failure counter causing 13933 * excessive cache_hot migrations and active balances. 13934 * 13935 * Similarly for migration_misfit which is not related to 13936 * load/util migration, don't pollute nr_balance_failed. 13937 * 13938 * The same for cache aware scheduling's allowance for 13939 * load imbalance. If regular load balance does not 13940 * migrate task due to LLC locality, it is a expected 13941 * behavior and don't pollute nr_balance_failed. 13942 * See can_migrate_task(). 13943 */ 13944 if (idle != CPU_NEWLY_IDLE && 13945 env.migration_type != migrate_misfit && 13946 !(env.flags & LBF_LLC_PINNED)) 13947 sd->nr_balance_failed++; 13948 13949 if (!need_active_balance(&env)) 13950 goto out_unbalanced; 13951 13952 scoped_guard (raw_spin_rq_lock_irqsave, busiest) { 13953 /* 13954 * Don't kick the active_load_balance_cpu_stop, 13955 * if the curr task on busiest CPU can't be 13956 * moved to this_cpu: 13957 */ 13958 if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) 13959 goto out_one_pinned; 13960 13961 /* Record that we found at least one task that could run on this_cpu */ 13962 env.flags &= ~LBF_ALL_PINNED; 13963 13964 /* 13965 * ->active_balance synchronizes accesses to 13966 * ->active_balance_work. Once set, it's cleared 13967 * only after active load balance is finished. 13968 */ 13969 if (busiest->active_balance) 13970 goto out_unbalanced; 13971 13972 /* 13973 * @busiest dropped its rq_lock in the middle of 13974 * scheduling out its ->curr task (->on_rq := 0), no 13975 * need to forcefully punt it away with active balance. 13976 */ 13977 if (!busiest->curr->on_rq) 13978 goto out_unbalanced; 13979 13980 busiest->active_balance = 1; 13981 busiest->push_cpu = this_cpu; 13982 active_balance = 1; 13983 preempt_disable(); 13984 } 13985 if (active_balance) { 13986 stop_one_cpu_nowait(cpu_of(busiest), 13987 alb_stop_fn(&env), busiest, 13988 &busiest->active_balance_work); 13989 } 13990 preempt_enable(); 13991 13992 out_unbalanced: 13993 /* We were unbalanced, so reset the balancing interval */ 13994 sd->balance_interval = sd->min_interval; 13995 goto out; 13996 13997 out_balanced: 13998 /* 13999 * We reach balance although we may have faced some affinity 14000 * constraints. Clear the imbalance flag only if other tasks got 14001 * a chance to move and fix the imbalance. 14002 */ 14003 if (sd_parent && !(env.flags & LBF_ALL_PINNED)) { 14004 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 14005 14006 if (*group_imbalance) 14007 *group_imbalance = 0; 14008 } 14009 14010 out_all_pinned: 14011 /* 14012 * We reach balance because all tasks are pinned at this level so 14013 * we can't migrate them. Let the imbalance flag set so parent level 14014 * can try to migrate them. 14015 */ 14016 schedstat_inc(sd->lb_balanced[idle]); 14017 14018 sd->nr_balance_failed = 0; 14019 14020 out_one_pinned: 14021 ld_moved = 0; 14022 14023 /* 14024 * sched_balance_newidle() disregards balance intervals, so we could 14025 * repeatedly reach this code, which would lead to balance_interval 14026 * skyrocketing in a short amount of time. Skip the balance_interval 14027 * increase logic to avoid that. 14028 * 14029 * Similarly misfit migration which is not necessarily an indication of 14030 * the system being busy and requires lb to backoff to let it settle 14031 * down. 14032 */ 14033 if (env.idle == CPU_NEWLY_IDLE || 14034 env.migration_type == migrate_misfit) 14035 goto out; 14036 14037 /* tune up the balancing interval */ 14038 if ((env.flags & LBF_ALL_PINNED && 14039 sd->balance_interval < MAX_PINNED_INTERVAL) || 14040 sd->balance_interval < sd->max_interval) 14041 sd->balance_interval *= 2; 14042 out: 14043 if (need_unlock) 14044 atomic_set_release(&sched_balance_running, 0); 14045 14046 return ld_moved; 14047 } 14048 14049 static inline unsigned long 14050 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) 14051 { 14052 unsigned long interval = sd->balance_interval; 14053 14054 if (cpu_busy) 14055 interval *= sd->busy_factor; 14056 14057 /* scale ms to jiffies */ 14058 interval = msecs_to_jiffies(interval); 14059 14060 /* 14061 * Reduce likelihood of busy balancing at higher domains racing with 14062 * balancing at lower domains by preventing their balancing periods 14063 * from being multiples of each other. 14064 */ 14065 if (cpu_busy) 14066 interval -= 1; 14067 14068 interval = clamp(interval, 1UL, max_load_balance_interval); 14069 14070 return interval; 14071 } 14072 14073 static inline void 14074 update_next_balance(struct sched_domain *sd, unsigned long *next_balance) 14075 { 14076 unsigned long interval, next; 14077 14078 /* used by idle balance, so cpu_busy = 0 */ 14079 interval = get_sd_balance_interval(sd, 0); 14080 next = sd->last_balance + interval; 14081 14082 if (time_after(*next_balance, next)) 14083 *next_balance = next; 14084 } 14085 14086 /* 14087 * active_load_balance_cpu_stop is run by the CPU stopper. It pushes 14088 * running tasks off the busiest CPU onto idle CPUs. It requires at 14089 * least 1 task to be running on each physical CPU where possible, and 14090 * avoids physical / logical imbalances. 14091 */ 14092 static int __active_load_balance_cpu_stop(void *data, unsigned int lb_flags) 14093 { 14094 struct rq *busiest_rq = data; 14095 int busiest_cpu = cpu_of(busiest_rq); 14096 int target_cpu = busiest_rq->push_cpu; 14097 struct rq *target_rq = cpu_rq(target_cpu); 14098 struct sched_domain *sd; 14099 struct task_struct *p = NULL; 14100 struct rq_flags rf; 14101 14102 rq_lock_irq(busiest_rq, &rf); 14103 /* 14104 * Between queueing the stop-work and running it is a hole in which 14105 * CPUs can become inactive. We should not move tasks from or to 14106 * inactive CPUs. 14107 */ 14108 if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu)) 14109 goto out_unlock; 14110 14111 /* Make sure the requested CPU hasn't gone down in the meantime: */ 14112 if (unlikely(busiest_cpu != smp_processor_id() || 14113 !busiest_rq->active_balance)) 14114 goto out_unlock; 14115 14116 /* Is there any task to move? */ 14117 if (busiest_rq->nr_running <= 1) 14118 goto out_unlock; 14119 14120 /* 14121 * This condition is "impossible", if it occurs 14122 * we need to fix it. Originally reported by 14123 * Bjorn Helgaas on a 128-CPU setup. 14124 */ 14125 WARN_ON_ONCE(busiest_rq == target_rq); 14126 14127 /* Search for an sd spanning us and the target CPU. */ 14128 rcu_read_lock(); 14129 for_each_domain(target_cpu, sd) { 14130 if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) 14131 break; 14132 } 14133 14134 if (likely(sd)) { 14135 struct lb_env env = { 14136 .sd = sd, 14137 .dst_cpu = target_cpu, 14138 .dst_rq = target_rq, 14139 .src_cpu = busiest_rq->cpu, 14140 .src_rq = busiest_rq, 14141 .idle = CPU_IDLE, 14142 .flags = LBF_ACTIVE_LB | lb_flags, 14143 }; 14144 14145 schedstat_inc(sd->alb_count); 14146 update_rq_clock(busiest_rq); 14147 14148 p = detach_one_task(&env); 14149 if (p) { 14150 schedstat_inc(sd->alb_pushed); 14151 /* Active balancing done, reset the failure counter. */ 14152 sd->nr_balance_failed = 0; 14153 } else { 14154 schedstat_inc(sd->alb_failed); 14155 } 14156 } 14157 rcu_read_unlock(); 14158 out_unlock: 14159 busiest_rq->active_balance = 0; 14160 rq_unlock(busiest_rq, &rf); 14161 14162 if (p) 14163 attach_one_task(target_rq, p); 14164 14165 local_irq_enable(); 14166 14167 return 0; 14168 } 14169 14170 static int active_load_balance_cpu_stop(void *data) 14171 { 14172 return __active_load_balance_cpu_stop(data, 0); 14173 } 14174 14175 static int active_load_balance_llc_cpu_stop(void *data) 14176 { 14177 return __active_load_balance_cpu_stop(data, LBF_ACTIVE_LB_LLC); 14178 } 14179 14180 /* 14181 * Scale the max sched_balance_rq interval with the number of CPUs in the system. 14182 * This trades load-balance latency on larger machines for less cross talk. 14183 */ 14184 void update_max_interval(void) 14185 { 14186 max_load_balance_interval = HZ*num_online_cpus()/10; 14187 } 14188 14189 static inline void update_newidle_stats(struct sched_domain *sd, unsigned int success) 14190 { 14191 sd->newidle_call++; 14192 sd->newidle_success += success; 14193 14194 if (sd->newidle_call >= 1024) { 14195 u64 now = sched_clock(); 14196 s64 delta = now - sd->newidle_stamp; 14197 sd->newidle_stamp = now; 14198 int ratio = 0; 14199 14200 if (delta < 0) 14201 delta = 0; 14202 14203 if (sched_feat(NI_RATE)) { 14204 /* 14205 * ratio delta freq 14206 * 14207 * 1024 - 4 s - 128 Hz 14208 * 512 - 2 s - 256 Hz 14209 * 256 - 1 s - 512 Hz 14210 * 128 - .5 s - 1024 Hz 14211 * 64 - .25 s - 2048 Hz 14212 */ 14213 ratio = delta >> 22; 14214 } 14215 14216 ratio += sd->newidle_success; 14217 14218 sd->newidle_ratio = min(1024, ratio); 14219 sd->newidle_call /= 2; 14220 sd->newidle_success /= 2; 14221 } 14222 } 14223 14224 static inline bool 14225 update_newidle_cost(struct sched_domain *sd, u64 cost, unsigned int success) 14226 { 14227 unsigned long next_decay = sd->last_decay_max_lb_cost + HZ; 14228 unsigned long now = jiffies; 14229 14230 if (cost) 14231 update_newidle_stats(sd, success); 14232 14233 if (cost > sd->max_newidle_lb_cost) { 14234 /* 14235 * Track max cost of a domain to make sure to not delay the 14236 * next wakeup on the CPU. 14237 */ 14238 sd->max_newidle_lb_cost = cost; 14239 sd->last_decay_max_lb_cost = now; 14240 14241 } else if (time_after(now, next_decay)) { 14242 /* 14243 * Decay the newidle max times by ~1% per second to ensure that 14244 * it is not outdated and the current max cost is actually 14245 * shorter. 14246 */ 14247 sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256; 14248 sd->last_decay_max_lb_cost = now; 14249 return true; 14250 } 14251 14252 return false; 14253 } 14254 14255 /* 14256 * It checks each scheduling domain to see if it is due to be balanced, 14257 * and initiates a balancing operation if so. 14258 * 14259 * Balancing parameters are set up in init_sched_domains. 14260 */ 14261 static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle) 14262 { 14263 int continue_balancing = 1; 14264 int cpu = rq->cpu; 14265 int busy = idle != CPU_IDLE && !sched_idle_rq(rq); 14266 unsigned long interval; 14267 struct sched_domain *sd; 14268 /* Earliest time when we have to do rebalance again */ 14269 unsigned long next_balance = jiffies + 60*HZ; 14270 int update_next_balance = 0; 14271 int need_decay = 0; 14272 u64 max_cost = 0; 14273 14274 rcu_read_lock(); 14275 for_each_domain(cpu, sd) { 14276 /* 14277 * Decay the newidle max times here because this is a regular 14278 * visit to all the domains. 14279 */ 14280 need_decay = update_newidle_cost(sd, 0, 0); 14281 max_cost += sd->max_newidle_lb_cost; 14282 14283 /* 14284 * Stop the load balance at this level. There is another 14285 * CPU in our sched group which is doing load balancing more 14286 * actively. 14287 */ 14288 if (!continue_balancing) { 14289 if (need_decay) 14290 continue; 14291 break; 14292 } 14293 14294 interval = get_sd_balance_interval(sd, busy); 14295 if (time_after_eq(jiffies, sd->last_balance + interval)) { 14296 if (sched_balance_rq(cpu, rq, sd, idle, &continue_balancing)) { 14297 /* 14298 * The LBF_DST_PINNED logic could have changed 14299 * env->dst_cpu, so we can't know our idle 14300 * state even if we migrated tasks. Update it. 14301 */ 14302 idle = idle_cpu(cpu); 14303 busy = !idle && !sched_idle_rq(rq); 14304 } 14305 sd->last_balance = jiffies; 14306 interval = get_sd_balance_interval(sd, busy); 14307 } 14308 if (time_after(next_balance, sd->last_balance + interval)) { 14309 next_balance = sd->last_balance + interval; 14310 update_next_balance = 1; 14311 } 14312 } 14313 if (need_decay) { 14314 /* 14315 * Ensure the rq-wide value also decays but keep it at a 14316 * reasonable floor to avoid funnies with rq->avg_idle. 14317 */ 14318 rq->max_idle_balance_cost = 14319 max((u64)sysctl_sched_migration_cost, max_cost); 14320 } 14321 rcu_read_unlock(); 14322 14323 /* 14324 * next_balance will be updated only when there is a need. 14325 * When the cpu is attached to null domain for ex, it will not be 14326 * updated. 14327 */ 14328 if (likely(update_next_balance)) 14329 rq->next_balance = next_balance; 14330 14331 } 14332 14333 static inline int on_null_domain(struct rq *rq) 14334 { 14335 return unlikely(!rcu_dereference_sched(rq->sd)); 14336 } 14337 14338 #ifdef CONFIG_NO_HZ_COMMON 14339 /* 14340 * NOHZ idle load balancing (ILB) details: 14341 * 14342 * - When one of the busy CPUs notices that there may be an idle rebalancing 14343 * needed, they will kick the idle load balancer, which then does idle 14344 * load balancing for all the idle CPUs. 14345 */ 14346 static inline int find_new_ilb(void) 14347 { 14348 struct cpumask *ilb_cpus; 14349 int ilb_cpu, fallback = -1; 14350 14351 lockdep_assert_irqs_disabled(); 14352 14353 /* 14354 * Reuse the per-CPU select_rq_mask, which is protected from concurrent 14355 * use on this CPU by having interrupts disabled. 14356 */ 14357 ilb_cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 14358 cpumask_and(ilb_cpus, nohz.idle_cpus_mask, 14359 housekeeping_cpumask(HK_TYPE_KERNEL_NOISE)); 14360 14361 for_each_cpu(ilb_cpu, ilb_cpus) { 14362 if (!idle_cpu(ilb_cpu)) { 14363 /* 14364 * Once an idle fallback exists, a busy CPU proves that 14365 * this core cannot be fully idle. Skip its siblings. 14366 */ 14367 if (sched_smt_active() && fallback >= 0) 14368 cpumask_andnot(ilb_cpus, ilb_cpus, cpu_smt_mask(ilb_cpu)); 14369 continue; 14370 } 14371 14372 /* 14373 * Running the idle load balancer on an idle sibling of a busy 14374 * SMT core can reduce the capacity available to its sibling. Prefer 14375 * a CPU whose entire core is idle, but retain the first idle CPU as 14376 * a fallback so idle balancing can still make progress when no fully 14377 * idle core exists. 14378 */ 14379 if (sched_smt_active() && !is_core_idle(ilb_cpu)) { 14380 if (fallback < 0) 14381 fallback = ilb_cpu; 14382 14383 /* 14384 * The core is not idle, so there is no need to check 14385 * any of its other SMT siblings. 14386 */ 14387 cpumask_andnot(ilb_cpus, ilb_cpus, 14388 cpu_smt_mask(ilb_cpu)); 14389 continue; 14390 } 14391 14392 return ilb_cpu; 14393 } 14394 14395 return fallback; 14396 } 14397 14398 /* 14399 * Kick a CPU to do the NOHZ balancing, if it is time for it, via a cross-CPU 14400 * SMP function call (IPI). 14401 * 14402 * Prefer a CPU on a fully idle core in the HK_TYPE_KERNEL_NOISE housekeeping 14403 * set. Fall back to the first idle CPU when no fully idle core exists. 14404 */ 14405 static void kick_ilb(unsigned int flags) 14406 { 14407 int ilb_cpu; 14408 14409 /* 14410 * Increase nohz.next_balance only when if full ilb is triggered but 14411 * not if we only update stats. 14412 */ 14413 if (flags & NOHZ_BALANCE_KICK) 14414 nohz.next_balance = jiffies+1; 14415 14416 ilb_cpu = find_new_ilb(); 14417 if (ilb_cpu < 0) 14418 return; 14419 14420 /* 14421 * Don't bother if no new NOHZ balance work items for ilb_cpu, 14422 * i.e. all bits in flags are already set in ilb_cpu. 14423 */ 14424 if ((atomic_read(nohz_flags(ilb_cpu)) & flags) == flags) 14425 return; 14426 14427 /* 14428 * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets 14429 * the first flag owns it; cleared by nohz_csd_func(). 14430 */ 14431 flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu)); 14432 if (flags & NOHZ_KICK_MASK) 14433 return; 14434 14435 /* 14436 * This way we generate an IPI on the target CPU which 14437 * is idle, and the softirq performing NOHZ idle load balancing 14438 * will be run before returning from the IPI. 14439 */ 14440 smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd); 14441 } 14442 14443 /* 14444 * Current decision point for kicking the idle load balancer in the presence 14445 * of idle CPUs in the system. 14446 */ 14447 static void nohz_balancer_kick(struct rq *rq) 14448 { 14449 unsigned long now = jiffies; 14450 struct sched_domain_shared *sds; 14451 struct sched_domain *sd; 14452 int nr_busy, i, cpu = rq->cpu; 14453 unsigned int flags = 0; 14454 14455 if (unlikely(rq->idle_balance)) 14456 return; 14457 14458 /* 14459 * We may be recently in ticked or tickless idle mode. At the first 14460 * busy tick after returning from idle, we will update the busy stats. 14461 */ 14462 nohz_balance_exit_idle(rq); 14463 14464 if (READ_ONCE(nohz.has_blocked_load) && 14465 time_after(now, READ_ONCE(nohz.next_blocked))) 14466 flags = NOHZ_STATS_KICK; 14467 14468 /* 14469 * Most of the time system is not 100% busy. i.e nohz.nr_cpus > 0 14470 * Skip the read if time is not due. 14471 * 14472 * If none are in tickless mode, there maybe a narrow window 14473 * (28 jiffies, HZ=1000) where flags maybe set and kick_ilb called. 14474 * But idle load balancing is not done as find_new_ilb fails. 14475 * That's very rare. So read nohz.nr_cpus only if time is due. 14476 */ 14477 if (time_before(now, nohz.next_balance)) 14478 goto out; 14479 14480 /* 14481 * None are in tickless mode and hence no need for NOHZ idle load 14482 * balancing 14483 */ 14484 if (unlikely(cpumask_empty(nohz.idle_cpus_mask))) 14485 return; 14486 14487 if (rq->nr_running >= 2) { 14488 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 14489 goto out; 14490 } 14491 14492 sd = rcu_dereference_all(rq->sd); 14493 if (sd) { 14494 /* 14495 * If there's a runnable CFS task and the current CPU has reduced 14496 * capacity, kick the ILB to see if there's a better CPU to run on: 14497 */ 14498 if (rq->cfs.h_nr_runnable >= 1 && check_cpu_capacity(rq, sd)) { 14499 flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 14500 goto out; 14501 } 14502 } 14503 14504 sd = rcu_dereference_all(per_cpu(sd_asym_packing, cpu)); 14505 if (sd) { 14506 /* 14507 * When ASYM_PACKING; see if there's a more preferred CPU 14508 * currently idle; in which case, kick the ILB to move tasks 14509 * around. 14510 * 14511 * When balancing between cores, all the SMT siblings of the 14512 * preferred CPU must be idle. 14513 */ 14514 for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { 14515 if (sched_asym(sd, i, cpu)) { 14516 flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 14517 goto out; 14518 } 14519 } 14520 } 14521 14522 sd = rcu_dereference_all(per_cpu(sd_asym_cpucapacity, cpu)); 14523 if (sd) { 14524 /* 14525 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU 14526 * to run the misfit task on. 14527 */ 14528 if (check_misfit_status(rq)) 14529 flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 14530 14531 /* 14532 * For asymmetric systems, we do not want to nicely balance 14533 * cache use, instead we want to embrace asymmetry and only 14534 * ensure tasks have enough CPU capacity. 14535 * 14536 * Skip the LLC logic because it's not relevant in that case. 14537 */ 14538 goto out; 14539 } 14540 14541 sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu)); 14542 if (sds) { 14543 /* 14544 * If there is an imbalance between LLC domains (IOW we could 14545 * increase the overall cache utilization), we need a less-loaded LLC 14546 * domain to pull some load from. Likewise, we may need to spread 14547 * load within the current LLC domain (e.g. packed SMT cores but 14548 * other CPUs are idle). We can't really know from here how busy 14549 * the others are - so just get a NOHZ balance going if it looks 14550 * like this LLC domain has tasks we could move. 14551 */ 14552 nr_busy = atomic_read(&sds->nr_busy_cpus); 14553 if (nr_busy > 1) 14554 flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 14555 } 14556 out: 14557 if (READ_ONCE(nohz.needs_update)) 14558 flags |= NOHZ_NEXT_KICK; 14559 14560 if (flags) 14561 kick_ilb(flags); 14562 } 14563 14564 static void set_cpu_sd_state_busy(int cpu) 14565 { 14566 struct sched_domain *sd; 14567 sd = rcu_dereference_all(per_cpu(sd_llc, cpu)); 14568 14569 /* 14570 * sd->nohz_idle only pairs with nr_busy_cpus on sd->shared; if this 14571 * domain has no shared object there is nothing to clear or account. 14572 */ 14573 if (!sd || !sd->shared || !sd->nohz_idle) 14574 return; 14575 sd->nohz_idle = 0; 14576 14577 atomic_inc(&sd->shared->nr_busy_cpus); 14578 } 14579 14580 void nohz_balance_exit_idle(struct rq *rq) 14581 { 14582 WARN_ON_ONCE(rq != this_rq()); 14583 14584 if (likely(!rq->nohz_tick_stopped)) 14585 return; 14586 14587 rq->nohz_tick_stopped = 0; 14588 cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask); 14589 14590 set_cpu_sd_state_busy(rq->cpu); 14591 } 14592 14593 static void set_cpu_sd_state_idle(int cpu) 14594 { 14595 struct sched_domain *sd; 14596 sd = rcu_dereference_all(per_cpu(sd_llc, cpu)); 14597 14598 /* See set_cpu_sd_state_busy(): nohz_idle is only used with sd->shared. */ 14599 if (!sd || !sd->shared || sd->nohz_idle) 14600 return; 14601 sd->nohz_idle = 1; 14602 14603 atomic_dec(&sd->shared->nr_busy_cpus); 14604 } 14605 14606 /* 14607 * This routine will record that the CPU is going idle with tick stopped. 14608 * This info will be used in performing idle load balancing in the future. 14609 */ 14610 void nohz_balance_enter_idle(int cpu) 14611 { 14612 struct rq *rq = cpu_rq(cpu); 14613 14614 WARN_ON_ONCE(cpu != smp_processor_id()); 14615 14616 /* If this CPU is going down, then nothing needs to be done: */ 14617 if (!cpu_active(cpu)) 14618 return; 14619 14620 /* 14621 * Can be set safely without rq->lock held 14622 * If a clear happens, it will have evaluated last additions because 14623 * rq->lock is held during the check and the clear 14624 */ 14625 rq->has_blocked_load = 1; 14626 14627 /* 14628 * The tick is still stopped but load could have been added in the 14629 * meantime. We set the nohz.has_blocked_load flag to trig a check of the 14630 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear 14631 * of nohz.has_blocked_load can only happen after checking the new load 14632 */ 14633 if (rq->nohz_tick_stopped) 14634 goto out; 14635 14636 /* If we're a completely isolated CPU, we don't play: */ 14637 if (on_null_domain(rq)) 14638 return; 14639 14640 rq->nohz_tick_stopped = 1; 14641 14642 cpumask_set_cpu(cpu, nohz.idle_cpus_mask); 14643 14644 /* 14645 * Ensures that if nohz_idle_balance() fails to observe our 14646 * @idle_cpus_mask store, it must observe the @has_blocked_load 14647 * and @needs_update stores. 14648 */ 14649 smp_mb__after_atomic(); 14650 14651 set_cpu_sd_state_idle(cpu); 14652 14653 WRITE_ONCE(nohz.needs_update, 1); 14654 out: 14655 /* 14656 * Each time a cpu enter idle, we assume that it has blocked load and 14657 * enable the periodic update of the load of idle CPUs 14658 */ 14659 WRITE_ONCE(nohz.has_blocked_load, 1); 14660 } 14661 14662 static bool update_nohz_stats(struct rq *rq) 14663 { 14664 unsigned int cpu = rq->cpu; 14665 14666 if (!rq->has_blocked_load) 14667 return false; 14668 14669 if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask)) 14670 return false; 14671 14672 if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick))) 14673 return true; 14674 14675 sched_balance_update_blocked_averages(cpu); 14676 14677 return rq->has_blocked_load; 14678 } 14679 14680 /* 14681 * Internal function that runs load balance for all idle CPUs. The load balance 14682 * can be a simple update of blocked load or a complete load balance with 14683 * tasks movement depending of flags. 14684 */ 14685 static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags) 14686 { 14687 /* Earliest time when we have to do rebalance again */ 14688 unsigned long now = jiffies; 14689 unsigned long next_balance = now + 60*HZ; 14690 bool has_blocked_load = false; 14691 int update_next_balance = 0; 14692 int this_cpu = this_rq->cpu; 14693 int balance_cpu; 14694 struct rq *rq; 14695 14696 WARN_ON_ONCE((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); 14697 14698 /* 14699 * We assume there will be no idle load after this update and clear 14700 * the has_blocked_load flag. If a cpu enters idle in the mean time, it will 14701 * set the has_blocked_load flag and trigger another update of idle load. 14702 * Because a cpu that becomes idle, is added to idle_cpus_mask before 14703 * setting the flag, we are sure to not clear the state and not 14704 * check the load of an idle cpu. 14705 * 14706 * Same applies to idle_cpus_mask vs needs_update. 14707 */ 14708 if (flags & NOHZ_STATS_KICK) 14709 WRITE_ONCE(nohz.has_blocked_load, 0); 14710 if (flags & NOHZ_NEXT_KICK) 14711 WRITE_ONCE(nohz.needs_update, 0); 14712 14713 /* 14714 * Ensures that if we miss the CPU, we must see the has_blocked_load 14715 * store from nohz_balance_enter_idle(). 14716 */ 14717 smp_mb(); 14718 14719 /* 14720 * Start with the next CPU after this_cpu so we will end with this_cpu and let a 14721 * chance for other idle cpu to pull load. 14722 */ 14723 for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) { 14724 if (!idle_cpu(balance_cpu)) 14725 continue; 14726 14727 /* 14728 * If this CPU gets work to do, stop the load balancing 14729 * work being done for other CPUs. Next load 14730 * balancing owner will pick it up. 14731 */ 14732 if (!idle_cpu(this_cpu) && need_resched()) { 14733 if (flags & NOHZ_STATS_KICK) 14734 has_blocked_load = true; 14735 if (flags & NOHZ_NEXT_KICK) 14736 WRITE_ONCE(nohz.needs_update, 1); 14737 goto abort; 14738 } 14739 14740 rq = cpu_rq(balance_cpu); 14741 14742 if (flags & NOHZ_STATS_KICK) 14743 has_blocked_load |= update_nohz_stats(rq); 14744 14745 /* 14746 * If time for next balance is due, 14747 * do the balance. 14748 */ 14749 if (time_after_eq(jiffies, rq->next_balance)) { 14750 struct rq_flags rf; 14751 14752 rq_lock_irqsave(rq, &rf); 14753 update_rq_clock(rq); 14754 rq_unlock_irqrestore(rq, &rf); 14755 14756 if (flags & NOHZ_BALANCE_KICK) 14757 sched_balance_domains(rq, CPU_IDLE); 14758 } 14759 14760 if (time_after(next_balance, rq->next_balance)) { 14761 next_balance = rq->next_balance; 14762 update_next_balance = 1; 14763 } 14764 } 14765 14766 /* 14767 * next_balance will be updated only when there is a need. 14768 * When the CPU is attached to null domain for ex, it will not be 14769 * updated. 14770 */ 14771 if (likely(update_next_balance)) 14772 nohz.next_balance = next_balance; 14773 14774 if (flags & NOHZ_STATS_KICK) 14775 WRITE_ONCE(nohz.next_blocked, 14776 now + msecs_to_jiffies(LOAD_AVG_PERIOD)); 14777 14778 abort: 14779 /* There is still blocked load, enable periodic update */ 14780 if (has_blocked_load) 14781 WRITE_ONCE(nohz.has_blocked_load, 1); 14782 } 14783 14784 /* 14785 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the 14786 * rebalancing for all the CPUs for whom scheduler ticks are stopped. 14787 */ 14788 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 14789 { 14790 unsigned int flags = this_rq->nohz_idle_balance; 14791 14792 if (!flags) 14793 return false; 14794 14795 this_rq->nohz_idle_balance = 0; 14796 14797 if (idle != CPU_IDLE) 14798 return false; 14799 14800 _nohz_idle_balance(this_rq, flags); 14801 14802 return true; 14803 } 14804 14805 /* 14806 * Check if we need to directly run the ILB for updating blocked load before 14807 * entering idle state. Here we run ILB directly without issuing IPIs. 14808 * 14809 * Note that when this function is called, the tick may not yet be stopped on 14810 * this CPU yet. nohz.idle_cpus_mask is updated only when tick is stopped and 14811 * cleared on the next busy tick. In other words, nohz.idle_cpus_mask updates 14812 * don't align with CPUs enter/exit idle to avoid bottlenecks due to high idle 14813 * entry/exit rate (usec). So it is possible that _nohz_idle_balance() is 14814 * called from this function on (this) CPU that's not yet in the mask. That's 14815 * OK because the goal of nohz_run_idle_balance() is to run ILB only for 14816 * updating the blocked load of already idle CPUs without waking up one of 14817 * those idle CPUs and outside the preempt disable / IRQ off phase of the local 14818 * cpu about to enter idle, because it can take a long time. 14819 */ 14820 void nohz_run_idle_balance(int cpu) 14821 { 14822 unsigned int flags; 14823 14824 flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu)); 14825 14826 /* 14827 * Update the blocked load only if no SCHED_SOFTIRQ is about to happen 14828 * (i.e. NOHZ_STATS_KICK set) and will do the same. 14829 */ 14830 if ((flags == NOHZ_NEWILB_KICK) && !need_resched()) 14831 _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK); 14832 } 14833 14834 static void nohz_newidle_balance(struct rq *this_rq) 14835 { 14836 int this_cpu = this_rq->cpu; 14837 14838 /* Will wake up very soon. No time for doing anything else*/ 14839 if (this_rq->avg_idle < sysctl_sched_migration_cost) 14840 return; 14841 14842 /* Don't need to update blocked load of idle CPUs*/ 14843 if (!READ_ONCE(nohz.has_blocked_load) || 14844 time_before(jiffies, READ_ONCE(nohz.next_blocked))) 14845 return; 14846 14847 /* 14848 * Set the need to trigger ILB in order to update blocked load 14849 * before entering idle state. 14850 */ 14851 atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu)); 14852 } 14853 14854 #else /* !CONFIG_NO_HZ_COMMON: */ 14855 static inline void nohz_balancer_kick(struct rq *rq) { } 14856 14857 static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 14858 { 14859 return false; 14860 } 14861 14862 static inline void nohz_newidle_balance(struct rq *this_rq) { } 14863 #endif /* !CONFIG_NO_HZ_COMMON */ 14864 14865 /* 14866 * sched_balance_newidle is called by schedule() if this_cpu is about to become 14867 * idle. Attempts to pull tasks from other CPUs. 14868 * 14869 * Returns: 14870 * < 0 - we released the lock and there are !fair tasks present 14871 * 0 - failed, no new tasks 14872 * > 0 - success, new (fair) tasks present 14873 */ 14874 static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) 14875 __must_hold(__rq_lockp(this_rq)) 14876 { 14877 unsigned long next_balance = jiffies + HZ; 14878 int this_cpu = this_rq->cpu; 14879 int continue_balancing = 1; 14880 u64 t0, t1, curr_cost = 0; 14881 struct sched_domain *sd; 14882 int pulled_task = 0; 14883 14884 update_misfit_status(NULL, this_rq); 14885 14886 /* 14887 * There is a task waiting to run. No need to search for one. 14888 * Return 0; the task will be enqueued when switching to idle. 14889 */ 14890 if (this_rq->ttwu_pending) 14891 return 0; 14892 14893 /* 14894 * We must set idle_stamp _before_ calling sched_balance_rq() 14895 * for CPU_NEWLY_IDLE, such that we measure the this duration 14896 * as idle time. 14897 */ 14898 this_rq->idle_stamp = rq_clock(this_rq); 14899 14900 /* Do not pull tasks towards !preferred CPUs */ 14901 if (!cpu_preferred(this_cpu)) 14902 return 0; 14903 14904 /* 14905 * This is OK, because current is on_cpu, which avoids it being picked 14906 * for load-balance and preemption/IRQs are still disabled avoiding 14907 * further scheduler activity on it and we're being very careful to 14908 * re-start the picking loop. 14909 */ 14910 rq_unpin_lock(this_rq, rf); 14911 14912 sd = rcu_dereference_sched_domain(this_rq->sd); 14913 if (!sd) 14914 goto out; 14915 14916 if (!get_rd_overloaded(this_rq->rd) || 14917 this_rq->avg_idle < sd->max_newidle_lb_cost) { 14918 14919 update_next_balance(sd, &next_balance); 14920 goto out; 14921 } 14922 14923 /* 14924 * Include sched_balance_update_blocked_averages() in the cost 14925 * calculation because it can be quite costly -- this ensures we skip 14926 * it when avg_idle gets to be very low. 14927 */ 14928 t0 = sched_clock_cpu(this_cpu); 14929 __sched_balance_update_blocked_averages(this_rq); 14930 14931 rq_modified_begin(this_rq, &fair_sched_class); 14932 raw_spin_rq_unlock(this_rq); 14933 14934 for_each_domain(this_cpu, sd) { 14935 u64 domain_cost; 14936 14937 update_next_balance(sd, &next_balance); 14938 14939 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) 14940 break; 14941 14942 if (sd->flags & SD_BALANCE_NEWIDLE) { 14943 unsigned int weight = 1; 14944 14945 if (sched_feat(NI_RANDOM) && sd->newidle_ratio < 1024) { 14946 /* 14947 * Throw a 1k sided dice; and only run 14948 * newidle_balance according to the success 14949 * rate. 14950 */ 14951 u32 d1k = sched_rng() % 1024; 14952 weight = 1 + sd->newidle_ratio; 14953 if (d1k > weight) { 14954 update_newidle_stats(sd, 0); 14955 continue; 14956 } 14957 weight = (1024 + weight/2) / weight; 14958 } 14959 14960 pulled_task = sched_balance_rq(this_cpu, this_rq, 14961 sd, CPU_NEWLY_IDLE, 14962 &continue_balancing); 14963 14964 t1 = sched_clock_cpu(this_cpu); 14965 domain_cost = t1 - t0; 14966 curr_cost += domain_cost; 14967 t0 = t1; 14968 14969 /* 14970 * Track max cost of a domain to make sure to not delay the 14971 * next wakeup on the CPU. 14972 */ 14973 update_newidle_cost(sd, domain_cost, weight * !!pulled_task); 14974 } 14975 14976 /* 14977 * Stop searching for tasks to pull if there are 14978 * now runnable tasks on this rq. 14979 */ 14980 if (pulled_task || !continue_balancing) 14981 break; 14982 } 14983 14984 raw_spin_rq_lock(this_rq); 14985 14986 if (curr_cost > this_rq->max_idle_balance_cost) 14987 this_rq->max_idle_balance_cost = curr_cost; 14988 14989 /* 14990 * While browsing the domains, we released the rq lock, a task could 14991 * have been enqueued in the meantime. Since we're not going idle, 14992 * pretend we pulled a task. 14993 */ 14994 if (this_rq->cfs.h_nr_queued && !pulled_task) 14995 pulled_task = 1; 14996 14997 /* If a higher prio class was modified, restart the pick */ 14998 if (rq_modified_above(this_rq, &fair_sched_class)) 14999 pulled_task = -1; 15000 15001 out: 15002 /* Move the next balance forward */ 15003 if (time_after(this_rq->next_balance, next_balance)) 15004 this_rq->next_balance = next_balance; 15005 15006 if (pulled_task) 15007 this_rq->idle_stamp = 0; 15008 else 15009 nohz_newidle_balance(this_rq); 15010 15011 rq_repin_lock(this_rq, rf); 15012 15013 return pulled_task; 15014 } 15015 15016 /* 15017 * This softirq handler is triggered via SCHED_SOFTIRQ from two places: 15018 * 15019 * - directly from the local sched_tick() for periodic load balancing 15020 * 15021 * - indirectly from a remote sched_tick() for NOHZ idle balancing 15022 * through the SMP cross-call nohz_csd_func() 15023 */ 15024 static __latent_entropy void sched_balance_softirq(void) 15025 { 15026 struct rq *this_rq = this_rq(); 15027 enum cpu_idle_type idle = this_rq->idle_balance; 15028 /* 15029 * If this CPU has a pending NOHZ_BALANCE_KICK, then do the 15030 * balancing on behalf of the other idle CPUs whose ticks are 15031 * stopped. Do nohz_idle_balance *before* sched_balance_domains to 15032 * give the idle CPUs a chance to load balance. Else we may 15033 * load balance only within the local sched_domain hierarchy 15034 * and abort nohz_idle_balance altogether if we pull some load. 15035 */ 15036 if (nohz_idle_balance(this_rq, idle)) 15037 return; 15038 15039 /* normal load balance */ 15040 sched_balance_update_blocked_averages(this_rq->cpu); 15041 sched_balance_domains(this_rq, idle); 15042 } 15043 15044 /* 15045 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. 15046 */ 15047 void sched_balance_trigger(struct rq *rq) 15048 { 15049 /* 15050 * Don't need to rebalance while attached to NULL domain or 15051 * runqueue CPU is not active 15052 */ 15053 if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq)))) 15054 return; 15055 15056 if (time_after_eq(jiffies, rq->next_balance)) 15057 raise_softirq(SCHED_SOFTIRQ); 15058 15059 nohz_balancer_kick(rq); 15060 } 15061 15062 static void rq_online_fair(struct rq *rq) 15063 { 15064 update_sysctl(); 15065 15066 update_runtime_enabled(rq); 15067 } 15068 15069 static void rq_offline_fair(struct rq *rq) 15070 { 15071 update_sysctl(); 15072 15073 /* Ensure any throttled groups are reachable by pick_next_task */ 15074 unthrottle_offline_cfs_rqs(rq); 15075 15076 /* Ensure that we remove rq contribution to group share: */ 15077 clear_tg_offline_cfs_rqs(rq); 15078 } 15079 15080 #ifdef CONFIG_SCHED_CORE 15081 static inline bool 15082 __entity_slice_used(struct sched_entity *se, int min_nr_tasks) 15083 { 15084 u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime; 15085 u64 slice = se->slice; 15086 15087 return (rtime * min_nr_tasks > slice); 15088 } 15089 15090 #define MIN_NR_TASKS_DURING_FORCEIDLE 2 15091 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) 15092 { 15093 if (!sched_core_enabled(rq)) 15094 return; 15095 15096 /* 15097 * If runqueue has only one task which used up its slice and 15098 * if the sibling is forced idle, then trigger schedule to 15099 * give forced idle task a chance. 15100 * 15101 * __entity_slice_used() considers only this active rq and it gets the 15102 * whole slice. But during force idle, we have siblings acting 15103 * like a single runqueue and hence we need to consider runnable 15104 * tasks on this CPU and the forced idle CPU. Ideally, we should 15105 * go through the forced idle rq, but that would be a perf hit. 15106 * We can assume that the forced idle CPU has at least 15107 * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check 15108 * if we need to give up the CPU. 15109 */ 15110 if (rq->core->core_forceidle_count && rq->cfs.h_nr_queued == 1 && 15111 __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) 15112 resched_curr(rq); 15113 } 15114 15115 /* 15116 * Consider any infeasible weight scenario. Take for instance two tasks, 15117 * each bound to their respective sibling, one with weight 1 and one with 15118 * weight 2. Then the lower weight task will run ahead of the higher weight 15119 * task without bound. 15120 * 15121 * This utterly destroys the concept of a shared time base. 15122 * 15123 * Remember; all this is about a proportionally fair scheduling, where each 15124 * tasks receives: 15125 * 15126 * w_i 15127 * dt_i = ---------- dt (1) 15128 * \Sum_j w_j 15129 * 15130 * which we do by tracking a virtual time, s_i: 15131 * 15132 * 1 15133 * s_i = --- d[t]_i (2) 15134 * w_i 15135 * 15136 * Where d[t] is a delta of discrete time, while dt is an infinitesimal. 15137 * The immediate corollary is that the ideal schedule S, where (2) to use 15138 * an infinitesimal delta, is: 15139 * 15140 * 1 15141 * S = ---------- dt (3) 15142 * \Sum_i w_i 15143 * 15144 * From which we can define the lag, or deviation from the ideal, as: 15145 * 15146 * lag(i) = S - s_i (4) 15147 * 15148 * And since the one and only purpose is to approximate S, we get that: 15149 * 15150 * \Sum_i w_i lag(i) := 0 (5) 15151 * 15152 * If this were not so, we no longer converge to S, and we can no longer 15153 * claim our scheduler has any of the properties we derive from S. This is 15154 * exactly what you did above, you broke it! 15155 * 15156 * 15157 * Let's continue for a while though; to see if there is anything useful to 15158 * be learned. We can combine (1)-(3) or (4)-(5) and express S in s_i: 15159 * 15160 * \Sum_i w_i s_i 15161 * S = -------------- (6) 15162 * \Sum_i w_i 15163 * 15164 * Which gives us a way to compute S, given our s_i. Now, if you've read 15165 * our code, you know that we do not in fact do this, the reason for this 15166 * is two-fold. Firstly, computing S in that way requires a 64bit division 15167 * for every time we'd use it (see 12), and secondly, this only describes 15168 * the steady-state, it doesn't handle dynamics. 15169 * 15170 * Anyway, in (6): s_i -> x + (s_i - x), to get: 15171 * 15172 * \Sum_i w_i (s_i - x) 15173 * S - x = -------------------- (7) 15174 * \Sum_i w_i 15175 * 15176 * Which shows that S and s_i transform alike (which makes perfect sense 15177 * given that S is basically the (weighted) average of s_i). 15178 * 15179 * So the thing to remember is that the above is strictly UP. It is 15180 * possible to generalize to multiple runqueues -- however it gets really 15181 * yuck when you have to add affinity support, as illustrated by our very 15182 * first counter-example. 15183 * 15184 * Luckily I think we can avoid needing a full multi-queue variant for 15185 * core-scheduling (or load-balancing). The crucial observation is that we 15186 * only actually need this comparison in the presence of forced-idle; only 15187 * then do we need to tell if the stalled rq has higher priority over the 15188 * other. 15189 * 15190 * [XXX assumes SMT2; better consider the more general case, I suspect 15191 * it'll work out because our comparison is always between 2 rqs and the 15192 * answer is only interesting if one of them is forced-idle] 15193 * 15194 * And (under assumption of SMT2) when there is forced-idle, there is only 15195 * a single queue, so everything works like normal. 15196 * 15197 * Let, for our runqueue 'k': 15198 * 15199 * T_k = \Sum_i w_i s_i 15200 * W_k = \Sum_i w_i ; for all i of k (8) 15201 * 15202 * Then we can write (6) like: 15203 * 15204 * T_k 15205 * S_k = --- (9) 15206 * W_k 15207 * 15208 * From which immediately follows that: 15209 * 15210 * T_k + T_l 15211 * S_k+l = --------- (10) 15212 * W_k + W_l 15213 * 15214 * On which we can define a combined lag: 15215 * 15216 * lag_k+l(i) := S_k+l - s_i (11) 15217 * 15218 * And that gives us the tools to compare tasks across a combined runqueue. 15219 * 15220 * 15221 * Combined this gives the following: 15222 * 15223 * a) when a runqueue enters force-idle, sync it against it's sibling rq(s) 15224 * using (7); this only requires storing single 'time'-stamps. 15225 * 15226 * b) when comparing tasks between 2 runqueues of which one is forced-idle, 15227 * compare the combined lag, per (11). 15228 * 15229 * Now, of course cgroups (I so hate them) make this more interesting in 15230 * that a) seems to suggest we need to iterate all cgroup on a CPU at such 15231 * boundaries, but I think we can avoid that. The force-idle is for the 15232 * whole CPU, all it's rqs. So we can mark it in the root and lazily 15233 * propagate downward on demand. 15234 */ 15235 15236 /* 15237 * So this sync is basically a relative reset of S to 0. 15238 * 15239 * So with 2 queues, when one goes idle, we drop them both to 0 and one 15240 * then increases due to not being idle, and the idle one builds up lag to 15241 * get re-elected. So far so simple, right? 15242 * 15243 * When there's 3, we can have the situation where 2 run and one is idle, 15244 * we sync to 0 and let the idle one build up lag to get re-election. Now 15245 * suppose another one also drops idle. At this point dropping all to 0 15246 * again would destroy the built-up lag from the queue that was already 15247 * idle, not good. 15248 * 15249 * So instead of syncing everything, we can: 15250 * 15251 * less := !((s64)(s_a - s_b) <= 0) 15252 * 15253 * (v_a - S_a) - (v_b - S_b) == v_a - v_b - S_a + S_b 15254 * == v_a - (v_b - S_a + S_b) 15255 * 15256 * IOW, we can recast the (lag) comparison to a one-sided difference. 15257 * So if then, instead of syncing the whole queue, sync the idle queue 15258 * against the active queue with S_a + S_b at the point where we sync. 15259 * 15260 * (XXX consider the implication of living in a cyclic group: N / 2^n N) 15261 * 15262 * This gives us means of syncing single queues against the active queue, 15263 * and for already idle queues to preserve their build-up lag. 15264 * 15265 * Of course, then we get the situation where there's 2 active and one 15266 * going idle, who do we pick to sync against? Theory would have us sync 15267 * against the combined S, but as we've already demonstrated, there is no 15268 * such thing in infeasible weight scenarios. 15269 * 15270 * One thing I've considered; and this is where that core_active rudiment 15271 * came from, is having active queues sync up between themselves after 15272 * every tick. This limits the observed divergence due to the work 15273 * conservancy. 15274 * 15275 * On top of that, we can improve upon things by employing (10) here. 15276 */ 15277 15278 /* 15279 * se_fi_update - Update the cfs_rq->zero_vruntime_fi in a CFS hierarchy if needed. 15280 */ 15281 static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq, 15282 bool forceidle) 15283 { 15284 for_each_sched_entity(se) { 15285 struct cfs_rq *cfs_rq = cfs_rq_of(se); 15286 15287 if (forceidle) { 15288 if (cfs_rq->forceidle_seq == fi_seq) 15289 break; 15290 cfs_rq->forceidle_seq = fi_seq; 15291 } 15292 15293 cfs_rq->zero_vruntime_fi = cfs_rq->zero_vruntime; 15294 } 15295 } 15296 15297 void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi) 15298 { 15299 struct sched_entity *se = &p->se; 15300 15301 if (p->sched_class != &fair_sched_class) 15302 return; 15303 15304 se_fi_update(se, rq->core->core_forceidle_seq, in_fi); 15305 } 15306 15307 bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, 15308 bool in_fi) 15309 { 15310 struct rq *rq = task_rq(a); 15311 const struct sched_entity *sea = &a->se; 15312 const struct sched_entity *seb = &b->se; 15313 struct cfs_rq *cfs_rqa; 15314 struct cfs_rq *cfs_rqb; 15315 s64 delta; 15316 15317 WARN_ON_ONCE(task_rq(b)->core != rq->core); 15318 15319 cfs_rqa = &task_rq(a)->cfs; 15320 cfs_rqb = &task_rq(b)->cfs; 15321 15322 /* 15323 * Find delta after normalizing se's vruntime with its cfs_rq's 15324 * zero_vruntime_fi, which would have been updated in prior calls 15325 * to se_fi_update(). 15326 */ 15327 delta = vruntime_op(sea->vruntime, "-", seb->vruntime) + 15328 vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi); 15329 15330 return delta > 0; 15331 } 15332 15333 static int task_is_throttled_fair(struct task_struct *p, int cpu) 15334 { 15335 struct cfs_rq *cfs_rq; 15336 15337 #ifdef CONFIG_FAIR_GROUP_SCHED 15338 cfs_rq = tg_cfs_rq(task_group(p), cpu); 15339 #else 15340 cfs_rq = &cpu_rq(cpu)->cfs; 15341 #endif 15342 return throttled_hierarchy(cfs_rq); 15343 } 15344 #else /* !CONFIG_SCHED_CORE: */ 15345 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} 15346 #endif /* !CONFIG_SCHED_CORE */ 15347 15348 /* 15349 * scheduler tick hitting a task of our scheduling class. 15350 * 15351 * NOTE: This function can be called remotely by the tick offload that 15352 * goes along full dynticks. Therefore no local assumption can be made 15353 * and everything must be accessed through the @rq and @curr passed in 15354 * parameters. 15355 */ 15356 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) 15357 { 15358 struct sched_entity *se = &curr->se; 15359 15360 if (se->on_rq) { 15361 unsigned long weight = NICE_0_LOAD; 15362 struct cfs_rq *cfs_rq; 15363 15364 for_each_sched_entity(se) { 15365 cfs_rq = cfs_rq_of(se); 15366 entity_tick(cfs_rq, se, queued); 15367 15368 weight = __calc_prop_weight(cfs_rq, se, weight); 15369 } 15370 15371 se = &curr->se; 15372 reweight_eevdf(cfs_rq, se, weight, se->on_rq); 15373 } 15374 15375 if (queued) 15376 return; 15377 15378 if (static_branch_unlikely(&sched_numa_balancing)) 15379 task_tick_numa(rq, curr); 15380 15381 task_tick_cache(rq, curr); 15382 15383 update_misfit_status(curr, rq); 15384 check_update_overutilized_status(task_rq(curr)); 15385 15386 task_tick_core(rq, curr); 15387 } 15388 15389 /* 15390 * called on fork with the child task as argument from the parent's context 15391 * - child not yet on the tasklist 15392 * - preemption disabled 15393 */ 15394 static void task_fork_fair(struct task_struct *p) 15395 { 15396 set_task_max_allowed_capacity(p); 15397 } 15398 15399 /* 15400 * Priority of the task has changed. Check to see if we preempt 15401 * the current task. 15402 */ 15403 static void 15404 prio_changed_fair(struct rq *rq, struct task_struct *p, u64 oldprio) 15405 { 15406 if (!task_on_rq_queued(p)) 15407 return; 15408 15409 if (p->prio == oldprio) 15410 return; 15411 15412 if (rq->cfs.h_nr_queued == 1) 15413 return; 15414 15415 /* 15416 * Reschedule if we are currently running on this runqueue and 15417 * our priority decreased, or if we are not currently running on 15418 * this runqueue and our priority is higher than the current's 15419 */ 15420 if (task_current_donor(rq, p)) { 15421 if (p->prio > oldprio) 15422 resched_curr(rq); 15423 } else { 15424 wakeup_preempt(rq, p, 0); 15425 } 15426 } 15427 15428 #ifdef CONFIG_FAIR_GROUP_SCHED 15429 /* 15430 * Propagate the changes of the sched_entity across the tg tree to make it 15431 * visible to the root 15432 */ 15433 static void propagate_entity_cfs_rq(struct sched_entity *se) 15434 { 15435 struct cfs_rq *cfs_rq = cfs_rq_of(se); 15436 15437 /* 15438 * If a task gets attached to this cfs_rq and before being queued, 15439 * it gets migrated to another CPU due to reasons like affinity 15440 * change, make sure this cfs_rq stays on leaf cfs_rq list to have 15441 * that removed load decayed or it can cause faireness problem. 15442 */ 15443 if (!cfs_rq_pelt_clock_throttled(cfs_rq)) 15444 list_add_leaf_cfs_rq(cfs_rq); 15445 15446 /* Start to propagate at parent */ 15447 se = se->parent; 15448 15449 for_each_sched_entity(se) { 15450 cfs_rq = cfs_rq_of(se); 15451 15452 update_load_avg(cfs_rq, se, UPDATE_TG); 15453 15454 if (!cfs_rq_pelt_clock_throttled(cfs_rq)) 15455 list_add_leaf_cfs_rq(cfs_rq); 15456 } 15457 15458 assert_list_leaf_cfs_rq(rq_of(cfs_rq)); 15459 } 15460 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 15461 static void propagate_entity_cfs_rq(struct sched_entity *se) { } 15462 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 15463 15464 static void detach_entity_cfs_rq(struct sched_entity *se) 15465 { 15466 struct cfs_rq *cfs_rq = cfs_rq_of(se); 15467 15468 /* 15469 * In case the task sched_avg hasn't been attached: 15470 * - A forked task which hasn't been woken up by wake_up_new_task(). 15471 * - A task which has been woken up by try_to_wake_up() but is 15472 * waiting for actually being woken up by sched_ttwu_pending(). 15473 */ 15474 if (!se->avg.last_update_time) 15475 return; 15476 15477 /* Catch up with the cfs_rq and remove our load when we leave */ 15478 update_load_avg(cfs_rq, se, 0); 15479 detach_entity_load_avg(cfs_rq, se); 15480 update_tg_load_avg(cfs_rq); 15481 propagate_entity_cfs_rq(se); 15482 } 15483 15484 static void attach_entity_cfs_rq(struct sched_entity *se) 15485 { 15486 struct cfs_rq *cfs_rq = cfs_rq_of(se); 15487 15488 /* Synchronize entity with its cfs_rq */ 15489 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); 15490 attach_entity_load_avg(cfs_rq, se); 15491 update_tg_load_avg(cfs_rq); 15492 propagate_entity_cfs_rq(se); 15493 } 15494 15495 static void detach_task_cfs_rq(struct task_struct *p) 15496 { 15497 struct sched_entity *se = &p->se; 15498 15499 detach_entity_cfs_rq(se); 15500 } 15501 15502 static void attach_task_cfs_rq(struct task_struct *p) 15503 { 15504 struct sched_entity *se = &p->se; 15505 15506 attach_entity_cfs_rq(se); 15507 } 15508 15509 static void switching_from_fair(struct rq *rq, struct task_struct *p) 15510 { 15511 if (p->se.sched_delayed) 15512 dequeue_task(rq, p, DEQUEUE_SLEEP | DEQUEUE_DELAYED | DEQUEUE_NOCLOCK); 15513 } 15514 15515 static void switched_from_fair(struct rq *rq, struct task_struct *p) 15516 { 15517 detach_task_cfs_rq(p); 15518 } 15519 15520 static void switched_to_fair(struct rq *rq, struct task_struct *p) 15521 { 15522 WARN_ON_ONCE(p->se.sched_delayed); 15523 15524 attach_task_cfs_rq(p); 15525 15526 set_task_max_allowed_capacity(p); 15527 15528 if (task_on_rq_queued(p)) { 15529 /* 15530 * We were most likely switched from sched_rt, so 15531 * kick off the schedule if running, otherwise just see 15532 * if we can still preempt the current task. 15533 */ 15534 if (task_current_donor(rq, p)) 15535 resched_curr(rq); 15536 else 15537 wakeup_preempt(rq, p, 0); 15538 } 15539 } 15540 15541 static void set_next_task_fair(struct rq *rq, struct task_struct *p, enum snt_e type) 15542 { 15543 struct sched_entity *se = &p->se; 15544 struct cfs_rq *cfs_rq = &rq->cfs; 15545 unsigned long weight = NICE_0_LOAD; 15546 bool first = type == SNT_PICK; 15547 bool throttled = false; 15548 bool on_rq = se->on_rq; 15549 15550 if (type == SNT_REPICK) 15551 goto repick; 15552 15553 clear_buddies(cfs_rq, se); 15554 15555 if (on_rq) 15556 __dequeue_entity(cfs_rq, se); 15557 15558 for_each_sched_entity(se) { 15559 cfs_rq = cfs_rq_of(se); 15560 15561 if (!IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) || 15562 !first || !cfs_rq->h_curr) 15563 set_next_entity(cfs_rq, se); 15564 15565 /* ensure bandwidth has been allocated on our new cfs_rq */ 15566 throttled |= account_cfs_rq_runtime(cfs_rq, 0); 15567 15568 if (on_rq) 15569 weight = __calc_prop_weight(cfs_rq, se, weight); 15570 } 15571 15572 if (throttled) 15573 task_throttle_setup_work(p); 15574 15575 se = &p->se; 15576 cfs_rq->curr = se; 15577 15578 if (on_rq) { 15579 reweight_eevdf(cfs_rq, se, weight, se->on_rq); 15580 if (first) 15581 set_protect_slice(cfs_rq, se); 15582 } 15583 15584 if (task_on_rq_queued(p)) { 15585 /* 15586 * Move the next running task to the front of the list, so our 15587 * cfs_tasks list becomes MRU one. 15588 */ 15589 list_move(&se->group_node, &rq->cfs_tasks); 15590 } 15591 if (!first) 15592 return; 15593 15594 WARN_ON_ONCE(se->sched_delayed); 15595 15596 update_misfit_status(p, rq); 15597 sched_fair_update_stop_tick(rq, p); 15598 15599 repick: 15600 /* 15601 * A same-task repick skips put_prev_task_fair(), but 15602 * pick_task_fair() refreshed the entity hrtick_start_fair() reads 15603 * before selecting it again. rq->cfs.curr identifies that entity, 15604 * including with group scheduling. 15605 */ 15606 if (hrtick_enabled_fair(rq)) 15607 hrtick_start_fair(rq, p); 15608 } 15609 15610 void init_cfs_rq(struct cfs_rq *cfs_rq) 15611 { 15612 cfs_rq->tasks_timeline = RB_ROOT_CACHED; 15613 cfs_rq->zero_vruntime = (u64)(-(1LL << 20)); 15614 raw_spin_lock_init(&cfs_rq->removed.lock); 15615 } 15616 15617 #ifdef CONFIG_FAIR_GROUP_SCHED 15618 static void task_change_group_fair(struct task_struct *p) 15619 { 15620 /* 15621 * We couldn't detach or attach a forked task which 15622 * hasn't been woken up by wake_up_new_task(). 15623 */ 15624 if (READ_ONCE(p->__state) == TASK_NEW) 15625 return; 15626 15627 detach_task_cfs_rq(p); 15628 15629 /* Tell se's cfs_rq has been changed -- migrated */ 15630 p->se.avg.last_update_time = 0; 15631 set_task_rq(p, task_cpu(p)); 15632 attach_task_cfs_rq(p); 15633 } 15634 15635 void free_fair_sched_group(struct task_group *tg) 15636 { 15637 free_percpu(tg->cfs_rq); 15638 } 15639 15640 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 15641 { 15642 struct cfs_tg_state __percpu *state; 15643 struct sched_entity *se; 15644 struct cfs_rq *cfs_rq; 15645 int i; 15646 15647 state = alloc_percpu_gfp(struct cfs_tg_state, GFP_KERNEL); 15648 if (!state) 15649 goto err; 15650 15651 tg->cfs_rq = &state->cfs_rq; 15652 tg->shares = NICE_0_LOAD; 15653 15654 init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent)); 15655 15656 for_each_possible_cpu(i) { 15657 cfs_rq = tg_cfs_rq(tg, i); 15658 if (!cfs_rq) 15659 goto err; 15660 15661 se = tg_se(tg, i); 15662 init_cfs_rq(cfs_rq); 15663 init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i)); 15664 init_entity_runnable_average(se); 15665 } 15666 15667 return 1; 15668 15669 err: 15670 return 0; 15671 } 15672 15673 void online_fair_sched_group(struct task_group *tg) 15674 { 15675 struct sched_entity *se; 15676 struct rq_flags rf; 15677 struct rq *rq; 15678 int i; 15679 15680 for_each_possible_cpu(i) { 15681 rq = cpu_rq(i); 15682 se = tg_se(tg, i); 15683 rq_lock_irq(rq, &rf); 15684 update_rq_clock(rq); 15685 attach_entity_cfs_rq(se); 15686 sync_throttle(tg, i); 15687 rq_unlock_irq(rq, &rf); 15688 } 15689 } 15690 15691 void unregister_fair_sched_group(struct task_group *tg) 15692 { 15693 int cpu; 15694 15695 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); 15696 15697 for_each_possible_cpu(cpu) { 15698 struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu); 15699 struct sched_entity *se = tg_se(tg, cpu); 15700 struct rq *rq = cpu_rq(cpu); 15701 15702 if (se) 15703 remove_entity_load_avg(se); 15704 15705 /* 15706 * Only empty task groups can be destroyed; so we can speculatively 15707 * check on_list without danger of it being re-added. 15708 */ 15709 if (cfs_rq->on_list) { 15710 guard(rq_lock_irqsave)(rq); 15711 list_del_leaf_cfs_rq(cfs_rq); 15712 } 15713 } 15714 } 15715 15716 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, 15717 struct sched_entity *se, int cpu, 15718 struct sched_entity *parent) 15719 { 15720 struct rq *rq = cpu_rq(cpu); 15721 15722 cfs_rq->tg = tg; 15723 cfs_rq->rq = rq; 15724 init_cfs_rq_runtime(cfs_rq); 15725 15726 /* se could be NULL for root_task_group */ 15727 if (!se) 15728 return; 15729 15730 if (!parent) { 15731 se->cfs_rq = &rq->cfs; 15732 se->depth = 0; 15733 } else { 15734 se->cfs_rq = parent->my_q; 15735 se->depth = parent->depth + 1; 15736 } 15737 15738 se->my_q = cfs_rq; 15739 /* guarantee group entities always have weight */ 15740 update_load_set(&se->load, NICE_0_LOAD); 15741 se->parent = parent; 15742 } 15743 15744 static DEFINE_MUTEX(shares_mutex); 15745 15746 static int __sched_group_set_shares(struct task_group *tg, unsigned long shares) 15747 { 15748 int i; 15749 15750 lockdep_assert_held(&shares_mutex); 15751 15752 /* 15753 * We can't change the weight of the root cgroup. 15754 */ 15755 if (is_root_task_group(tg)) 15756 return -EINVAL; 15757 15758 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); 15759 15760 if (tg->shares == shares) 15761 return 0; 15762 15763 tg->shares = shares; 15764 for_each_possible_cpu(i) { 15765 struct rq *rq = cpu_rq(i); 15766 struct sched_entity *se = tg_se(tg, i); 15767 struct rq_flags rf; 15768 15769 /* Propagate contribution to hierarchy */ 15770 rq_lock_irqsave(rq, &rf); 15771 update_rq_clock(rq); 15772 for_each_sched_entity(se) { 15773 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 15774 update_cfs_group(se); 15775 } 15776 rq_unlock_irqrestore(rq, &rf); 15777 } 15778 15779 return 0; 15780 } 15781 15782 int sched_group_set_shares(struct task_group *tg, unsigned long shares) 15783 { 15784 int ret; 15785 15786 mutex_lock(&shares_mutex); 15787 if (tg_is_idle(tg)) 15788 ret = -EINVAL; 15789 else 15790 ret = __sched_group_set_shares(tg, shares); 15791 mutex_unlock(&shares_mutex); 15792 15793 return ret; 15794 } 15795 15796 int sched_group_set_idle(struct task_group *tg, long idle) 15797 { 15798 int i; 15799 15800 if (tg == &root_task_group) 15801 return -EINVAL; 15802 15803 if (idle < 0 || idle > 1) 15804 return -EINVAL; 15805 15806 mutex_lock(&shares_mutex); 15807 15808 if (tg->idle == idle) { 15809 mutex_unlock(&shares_mutex); 15810 return 0; 15811 } 15812 15813 tg->idle = idle; 15814 15815 for_each_possible_cpu(i) { 15816 struct rq *rq = cpu_rq(i); 15817 struct sched_entity *se = tg_se(tg, i); 15818 struct cfs_rq *grp_cfs_rq = tg_cfs_rq(tg, i); 15819 bool was_idle = cfs_rq_is_idle(grp_cfs_rq); 15820 long idle_task_delta; 15821 struct rq_flags rf; 15822 15823 rq_lock_irqsave(rq, &rf); 15824 15825 grp_cfs_rq->idle = idle; 15826 if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq))) 15827 goto next_cpu; 15828 15829 idle_task_delta = grp_cfs_rq->h_nr_queued - 15830 grp_cfs_rq->h_nr_idle; 15831 if (!cfs_rq_is_idle(grp_cfs_rq)) 15832 idle_task_delta *= -1; 15833 15834 for_each_sched_entity(se) { 15835 struct cfs_rq *cfs_rq = cfs_rq_of(se); 15836 15837 if (!se->on_rq) 15838 break; 15839 15840 cfs_rq->h_nr_idle += idle_task_delta; 15841 15842 /* Already accounted at parent level and above. */ 15843 if (cfs_rq_is_idle(cfs_rq)) 15844 break; 15845 } 15846 15847 next_cpu: 15848 rq_unlock_irqrestore(rq, &rf); 15849 } 15850 15851 /* Idle groups have minimum weight. */ 15852 if (tg_is_idle(tg)) 15853 __sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO)); 15854 else 15855 __sched_group_set_shares(tg, NICE_0_LOAD); 15856 15857 mutex_unlock(&shares_mutex); 15858 return 0; 15859 } 15860 15861 #endif /* CONFIG_FAIR_GROUP_SCHED */ 15862 15863 15864 static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) 15865 { 15866 struct sched_entity *se = &task->se; 15867 unsigned int rr_interval = 0; 15868 15869 /* 15870 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise 15871 * idle runqueue: 15872 */ 15873 if (rq->cfs.load.weight) 15874 rr_interval = NS_TO_JIFFIES(se->slice); 15875 15876 return rr_interval; 15877 } 15878 15879 /* 15880 * All the scheduling class methods: 15881 */ 15882 DEFINE_SCHED_CLASS(fair) = { 15883 .enqueue_task = enqueue_task_fair, 15884 .dequeue_task = dequeue_task_fair, 15885 .yield_task = yield_task_fair, 15886 .yield_to_task = yield_to_task_fair, 15887 15888 .wakeup_preempt = wakeup_preempt_fair, 15889 15890 .pick_task = pick_task_fair, 15891 .put_prev_task = put_prev_task_fair, 15892 .set_next_task = set_next_task_fair, 15893 15894 .select_task_rq = select_task_rq_fair, 15895 .migrate_task_rq = migrate_task_rq_fair, 15896 15897 .rq_online = rq_online_fair, 15898 .rq_offline = rq_offline_fair, 15899 15900 .task_dead = task_dead_fair, 15901 .set_cpus_allowed = set_cpus_allowed_fair, 15902 15903 .task_tick = task_tick_fair, 15904 .task_fork = task_fork_fair, 15905 15906 .reweight_task = reweight_task_fair, 15907 .prio_changed = prio_changed_fair, 15908 .switching_from = switching_from_fair, 15909 .switched_from = switched_from_fair, 15910 .switched_to = switched_to_fair, 15911 15912 .get_rr_interval = get_rr_interval_fair, 15913 15914 .update_curr = update_curr_fair, 15915 15916 #ifdef CONFIG_FAIR_GROUP_SCHED 15917 .task_change_group = task_change_group_fair, 15918 #endif 15919 15920 #ifdef CONFIG_SCHED_CORE 15921 .task_is_throttled = task_is_throttled_fair, 15922 #endif 15923 15924 #ifdef CONFIG_UCLAMP_TASK 15925 .uclamp_enabled = 1, 15926 #endif 15927 }; 15928 15929 void print_cfs_stats(struct seq_file *m, int cpu) 15930 { 15931 struct cfs_rq *cfs_rq, *pos; 15932 15933 rcu_read_lock(); 15934 for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos) 15935 print_cfs_rq(m, cpu, cfs_rq); 15936 rcu_read_unlock(); 15937 } 15938 15939 #ifdef CONFIG_NUMA_BALANCING 15940 void show_numa_stats(struct task_struct *p, struct seq_file *m) 15941 { 15942 int node; 15943 unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; 15944 struct numa_group *ng; 15945 15946 rcu_read_lock(); 15947 ng = rcu_dereference_all(p->numa_group); 15948 for_each_online_node(node) { 15949 if (p->numa_faults) { 15950 tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; 15951 tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; 15952 } 15953 if (ng) { 15954 gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 15955 gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 15956 } 15957 print_numa_stats(m, node, tsf, tpf, gsf, gpf); 15958 } 15959 rcu_read_unlock(); 15960 } 15961 #endif /* CONFIG_NUMA_BALANCING */ 15962 15963 __init void init_sched_fair_class(void) 15964 { 15965 int i; 15966 15967 for_each_possible_cpu(i) { 15968 zalloc_cpumask_var_node(&per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(i)); 15969 zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i)); 15970 zalloc_cpumask_var_node(&per_cpu(should_we_balance_tmpmask, i), 15971 GFP_KERNEL, cpu_to_node(i)); 15972 15973 #ifdef CONFIG_CFS_BANDWIDTH 15974 INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i)); 15975 INIT_LIST_HEAD(&cpu_rq(i)->cfsb_csd_list); 15976 #endif 15977 } 15978 15979 open_softirq(SCHED_SOFTIRQ, sched_balance_softirq); 15980 15981 #ifdef CONFIG_NO_HZ_COMMON 15982 nohz.next_balance = jiffies; 15983 nohz.next_blocked = jiffies; 15984 zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); 15985 #endif 15986 } 15987