1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) 4 * 5 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * 7 * Interactivity improvements by Mike Galbraith 8 * (C) 2007 Mike Galbraith <efault@gmx.de> 9 * 10 * Various enhancements by Dmitry Adamushko. 11 * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com> 12 * 13 * Group scheduling enhancements by Srivatsa Vaddagiri 14 * Copyright IBM Corporation, 2007 15 * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> 16 * 17 * Scaled math optimizations by Thomas Gleixner 18 * Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de> 19 * 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 */ 23 #include "sched.h" 24 25 #include <trace/events/sched.h> 26 27 /* 28 * Targeted preemption latency for CPU-bound tasks: 29 * 30 * NOTE: this latency value is not the same as the concept of 31 * 'timeslice length' - timeslices in CFS are of variable length 32 * and have no persistent notion like in traditional, time-slice 33 * based scheduling concepts. 34 * 35 * (to see the precise effective timeslice length of your workload, 36 * run vmstat and monitor the context-switches (cs) field) 37 * 38 * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds) 39 */ 40 unsigned int sysctl_sched_latency = 6000000ULL; 41 static unsigned int normalized_sysctl_sched_latency = 6000000ULL; 42 43 /* 44 * The initial- and re-scaling of tunables is configurable 45 * 46 * Options are: 47 * 48 * SCHED_TUNABLESCALING_NONE - unscaled, always *1 49 * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus) 50 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus 51 * 52 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) 53 */ 54 enum sched_tunable_scaling sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; 55 56 /* 57 * Minimal preemption granularity for CPU-bound tasks: 58 * 59 * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) 60 */ 61 unsigned int sysctl_sched_min_granularity = 750000ULL; 62 static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; 63 64 /* 65 * This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity 66 */ 67 static unsigned int sched_nr_latency = 8; 68 69 /* 70 * After fork, child runs first. If set to 0 (default) then 71 * parent will (try to) run first. 72 */ 73 unsigned int sysctl_sched_child_runs_first __read_mostly; 74 75 /* 76 * SCHED_OTHER wake-up granularity. 77 * 78 * This option delays the preemption effects of decoupled workloads 79 * and reduces their over-scheduling. Synchronous workloads will still 80 * have immediate wakeup/sleep latencies. 81 * 82 * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) 83 */ 84 unsigned int sysctl_sched_wakeup_granularity = 1000000UL; 85 static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; 86 87 const_debug unsigned int sysctl_sched_migration_cost = 500000UL; 88 89 int sched_thermal_decay_shift; 90 static int __init setup_sched_thermal_decay_shift(char *str) 91 { 92 int _shift = 0; 93 94 if (kstrtoint(str, 0, &_shift)) 95 pr_warn("Unable to set scheduler thermal pressure decay shift parameter\n"); 96 97 sched_thermal_decay_shift = clamp(_shift, 0, 10); 98 return 1; 99 } 100 __setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift); 101 102 #ifdef CONFIG_SMP 103 /* 104 * For asym packing, by default the lower numbered CPU has higher priority. 105 */ 106 int __weak arch_asym_cpu_priority(int cpu) 107 { 108 return -cpu; 109 } 110 111 /* 112 * The margin used when comparing utilization with CPU capacity. 113 * 114 * (default: ~20%) 115 */ 116 #define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024) 117 118 #endif 119 120 #ifdef CONFIG_CFS_BANDWIDTH 121 /* 122 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool 123 * each time a cfs_rq requests quota. 124 * 125 * Note: in the case that the slice exceeds the runtime remaining (either due 126 * to consumption or the quota being specified to be smaller than the slice) 127 * we will always only issue the remaining available time. 128 * 129 * (default: 5 msec, units: microseconds) 130 */ 131 unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; 132 #endif 133 134 static inline void update_load_add(struct load_weight *lw, unsigned long inc) 135 { 136 lw->weight += inc; 137 lw->inv_weight = 0; 138 } 139 140 static inline void update_load_sub(struct load_weight *lw, unsigned long dec) 141 { 142 lw->weight -= dec; 143 lw->inv_weight = 0; 144 } 145 146 static inline void update_load_set(struct load_weight *lw, unsigned long w) 147 { 148 lw->weight = w; 149 lw->inv_weight = 0; 150 } 151 152 /* 153 * Increase the granularity value when there are more CPUs, 154 * because with more CPUs the 'effective latency' as visible 155 * to users decreases. But the relationship is not linear, 156 * so pick a second-best guess by going with the log2 of the 157 * number of CPUs. 158 * 159 * This idea comes from the SD scheduler of Con Kolivas: 160 */ 161 static unsigned int get_update_sysctl_factor(void) 162 { 163 unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); 164 unsigned int factor; 165 166 switch (sysctl_sched_tunable_scaling) { 167 case SCHED_TUNABLESCALING_NONE: 168 factor = 1; 169 break; 170 case SCHED_TUNABLESCALING_LINEAR: 171 factor = cpus; 172 break; 173 case SCHED_TUNABLESCALING_LOG: 174 default: 175 factor = 1 + ilog2(cpus); 176 break; 177 } 178 179 return factor; 180 } 181 182 static void update_sysctl(void) 183 { 184 unsigned int factor = get_update_sysctl_factor(); 185 186 #define SET_SYSCTL(name) \ 187 (sysctl_##name = (factor) * normalized_sysctl_##name) 188 SET_SYSCTL(sched_min_granularity); 189 SET_SYSCTL(sched_latency); 190 SET_SYSCTL(sched_wakeup_granularity); 191 #undef SET_SYSCTL 192 } 193 194 void sched_init_granularity(void) 195 { 196 update_sysctl(); 197 } 198 199 #define WMULT_CONST (~0U) 200 #define WMULT_SHIFT 32 201 202 static void __update_inv_weight(struct load_weight *lw) 203 { 204 unsigned long w; 205 206 if (likely(lw->inv_weight)) 207 return; 208 209 w = scale_load_down(lw->weight); 210 211 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 212 lw->inv_weight = 1; 213 else if (unlikely(!w)) 214 lw->inv_weight = WMULT_CONST; 215 else 216 lw->inv_weight = WMULT_CONST / w; 217 } 218 219 /* 220 * delta_exec * weight / lw.weight 221 * OR 222 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT 223 * 224 * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case 225 * we're guaranteed shift stays positive because inv_weight is guaranteed to 226 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. 227 * 228 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus 229 * weight/lw.weight <= 1, and therefore our shift will also be positive. 230 */ 231 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 232 { 233 u64 fact = scale_load_down(weight); 234 int shift = WMULT_SHIFT; 235 236 __update_inv_weight(lw); 237 238 if (unlikely(fact >> 32)) { 239 while (fact >> 32) { 240 fact >>= 1; 241 shift--; 242 } 243 } 244 245 fact = mul_u32_u32(fact, lw->inv_weight); 246 247 while (fact >> 32) { 248 fact >>= 1; 249 shift--; 250 } 251 252 return mul_u64_u32_shr(delta_exec, fact, shift); 253 } 254 255 256 const struct sched_class fair_sched_class; 257 258 /************************************************************** 259 * CFS operations on generic schedulable entities: 260 */ 261 262 #ifdef CONFIG_FAIR_GROUP_SCHED 263 static inline struct task_struct *task_of(struct sched_entity *se) 264 { 265 SCHED_WARN_ON(!entity_is_task(se)); 266 return container_of(se, struct task_struct, se); 267 } 268 269 /* Walk up scheduling entities hierarchy */ 270 #define for_each_sched_entity(se) \ 271 for (; se; se = se->parent) 272 273 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) 274 { 275 return p->se.cfs_rq; 276 } 277 278 /* runqueue on which this entity is (to be) queued */ 279 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) 280 { 281 return se->cfs_rq; 282 } 283 284 /* runqueue "owned" by this group */ 285 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) 286 { 287 return grp->my_q; 288 } 289 290 static inline void cfs_rq_tg_path(struct cfs_rq *cfs_rq, char *path, int len) 291 { 292 if (!path) 293 return; 294 295 if (cfs_rq && task_group_is_autogroup(cfs_rq->tg)) 296 autogroup_path(cfs_rq->tg, path, len); 297 else if (cfs_rq && cfs_rq->tg->css.cgroup) 298 cgroup_path(cfs_rq->tg->css.cgroup, path, len); 299 else 300 strlcpy(path, "(null)", len); 301 } 302 303 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 304 { 305 struct rq *rq = rq_of(cfs_rq); 306 int cpu = cpu_of(rq); 307 308 if (cfs_rq->on_list) 309 return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list; 310 311 cfs_rq->on_list = 1; 312 313 /* 314 * Ensure we either appear before our parent (if already 315 * enqueued) or force our parent to appear after us when it is 316 * enqueued. The fact that we always enqueue bottom-up 317 * reduces this to two cases and a special case for the root 318 * cfs_rq. Furthermore, it also means that we will always reset 319 * tmp_alone_branch either when the branch is connected 320 * to a tree or when we reach the top of the tree 321 */ 322 if (cfs_rq->tg->parent && 323 cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { 324 /* 325 * If parent is already on the list, we add the child 326 * just before. Thanks to circular linked property of 327 * the list, this means to put the child at the tail 328 * of the list that starts by parent. 329 */ 330 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 331 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); 332 /* 333 * The branch is now connected to its tree so we can 334 * reset tmp_alone_branch to the beginning of the 335 * list. 336 */ 337 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 338 return true; 339 } 340 341 if (!cfs_rq->tg->parent) { 342 /* 343 * cfs rq without parent should be put 344 * at the tail of the list. 345 */ 346 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 347 &rq->leaf_cfs_rq_list); 348 /* 349 * We have reach the top of a tree so we can reset 350 * tmp_alone_branch to the beginning of the list. 351 */ 352 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 353 return true; 354 } 355 356 /* 357 * The parent has not already been added so we want to 358 * make sure that it will be put after us. 359 * tmp_alone_branch points to the begin of the branch 360 * where we will add parent. 361 */ 362 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); 363 /* 364 * update tmp_alone_branch to points to the new begin 365 * of the branch 366 */ 367 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 368 return false; 369 } 370 371 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 372 { 373 if (cfs_rq->on_list) { 374 struct rq *rq = rq_of(cfs_rq); 375 376 /* 377 * With cfs_rq being unthrottled/throttled during an enqueue, 378 * it can happen the tmp_alone_branch points the a leaf that 379 * we finally want to del. In this case, tmp_alone_branch moves 380 * to the prev element but it will point to rq->leaf_cfs_rq_list 381 * at the end of the enqueue. 382 */ 383 if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) 384 rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev; 385 386 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 387 cfs_rq->on_list = 0; 388 } 389 } 390 391 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 392 { 393 SCHED_WARN_ON(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list); 394 } 395 396 /* Iterate thr' all leaf cfs_rq's on a runqueue */ 397 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 398 list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \ 399 leaf_cfs_rq_list) 400 401 /* Do the two (enqueued) entities belong to the same group ? */ 402 static inline struct cfs_rq * 403 is_same_group(struct sched_entity *se, struct sched_entity *pse) 404 { 405 if (se->cfs_rq == pse->cfs_rq) 406 return se->cfs_rq; 407 408 return NULL; 409 } 410 411 static inline struct sched_entity *parent_entity(struct sched_entity *se) 412 { 413 return se->parent; 414 } 415 416 static void 417 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 418 { 419 int se_depth, pse_depth; 420 421 /* 422 * preemption test can be made between sibling entities who are in the 423 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of 424 * both tasks until we find their ancestors who are siblings of common 425 * parent. 426 */ 427 428 /* First walk up until both entities are at same depth */ 429 se_depth = (*se)->depth; 430 pse_depth = (*pse)->depth; 431 432 while (se_depth > pse_depth) { 433 se_depth--; 434 *se = parent_entity(*se); 435 } 436 437 while (pse_depth > se_depth) { 438 pse_depth--; 439 *pse = parent_entity(*pse); 440 } 441 442 while (!is_same_group(*se, *pse)) { 443 *se = parent_entity(*se); 444 *pse = parent_entity(*pse); 445 } 446 } 447 448 #else /* !CONFIG_FAIR_GROUP_SCHED */ 449 450 static inline struct task_struct *task_of(struct sched_entity *se) 451 { 452 return container_of(se, struct task_struct, se); 453 } 454 455 #define for_each_sched_entity(se) \ 456 for (; se; se = NULL) 457 458 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) 459 { 460 return &task_rq(p)->cfs; 461 } 462 463 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) 464 { 465 struct task_struct *p = task_of(se); 466 struct rq *rq = task_rq(p); 467 468 return &rq->cfs; 469 } 470 471 /* runqueue "owned" by this group */ 472 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) 473 { 474 return NULL; 475 } 476 477 static inline void cfs_rq_tg_path(struct cfs_rq *cfs_rq, char *path, int len) 478 { 479 if (path) 480 strlcpy(path, "(null)", len); 481 } 482 483 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 484 { 485 return true; 486 } 487 488 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 489 { 490 } 491 492 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 493 { 494 } 495 496 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 497 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos) 498 499 static inline struct sched_entity *parent_entity(struct sched_entity *se) 500 { 501 return NULL; 502 } 503 504 static inline void 505 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 506 { 507 } 508 509 #endif /* CONFIG_FAIR_GROUP_SCHED */ 510 511 static __always_inline 512 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); 513 514 /************************************************************** 515 * Scheduling class tree data structure manipulation methods: 516 */ 517 518 static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) 519 { 520 s64 delta = (s64)(vruntime - max_vruntime); 521 if (delta > 0) 522 max_vruntime = vruntime; 523 524 return max_vruntime; 525 } 526 527 static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) 528 { 529 s64 delta = (s64)(vruntime - min_vruntime); 530 if (delta < 0) 531 min_vruntime = vruntime; 532 533 return min_vruntime; 534 } 535 536 static inline int entity_before(struct sched_entity *a, 537 struct sched_entity *b) 538 { 539 return (s64)(a->vruntime - b->vruntime) < 0; 540 } 541 542 static void update_min_vruntime(struct cfs_rq *cfs_rq) 543 { 544 struct sched_entity *curr = cfs_rq->curr; 545 struct rb_node *leftmost = rb_first_cached(&cfs_rq->tasks_timeline); 546 547 u64 vruntime = cfs_rq->min_vruntime; 548 549 if (curr) { 550 if (curr->on_rq) 551 vruntime = curr->vruntime; 552 else 553 curr = NULL; 554 } 555 556 if (leftmost) { /* non-empty tree */ 557 struct sched_entity *se; 558 se = rb_entry(leftmost, struct sched_entity, run_node); 559 560 if (!curr) 561 vruntime = se->vruntime; 562 else 563 vruntime = min_vruntime(vruntime, se->vruntime); 564 } 565 566 /* ensure we never gain time by being placed backwards. */ 567 cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime); 568 #ifndef CONFIG_64BIT 569 smp_wmb(); 570 cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; 571 #endif 572 } 573 574 /* 575 * Enqueue an entity into the rb-tree: 576 */ 577 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 578 { 579 struct rb_node **link = &cfs_rq->tasks_timeline.rb_root.rb_node; 580 struct rb_node *parent = NULL; 581 struct sched_entity *entry; 582 bool leftmost = true; 583 584 /* 585 * Find the right place in the rbtree: 586 */ 587 while (*link) { 588 parent = *link; 589 entry = rb_entry(parent, struct sched_entity, run_node); 590 /* 591 * We dont care about collisions. Nodes with 592 * the same key stay together. 593 */ 594 if (entity_before(se, entry)) { 595 link = &parent->rb_left; 596 } else { 597 link = &parent->rb_right; 598 leftmost = false; 599 } 600 } 601 602 rb_link_node(&se->run_node, parent, link); 603 rb_insert_color_cached(&se->run_node, 604 &cfs_rq->tasks_timeline, leftmost); 605 } 606 607 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 608 { 609 rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline); 610 } 611 612 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 613 { 614 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 615 616 if (!left) 617 return NULL; 618 619 return rb_entry(left, struct sched_entity, run_node); 620 } 621 622 static struct sched_entity *__pick_next_entity(struct sched_entity *se) 623 { 624 struct rb_node *next = rb_next(&se->run_node); 625 626 if (!next) 627 return NULL; 628 629 return rb_entry(next, struct sched_entity, run_node); 630 } 631 632 #ifdef CONFIG_SCHED_DEBUG 633 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) 634 { 635 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); 636 637 if (!last) 638 return NULL; 639 640 return rb_entry(last, struct sched_entity, run_node); 641 } 642 643 /************************************************************** 644 * Scheduling class statistics methods: 645 */ 646 647 int sched_proc_update_handler(struct ctl_table *table, int write, 648 void __user *buffer, size_t *lenp, 649 loff_t *ppos) 650 { 651 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 652 unsigned int factor = get_update_sysctl_factor(); 653 654 if (ret || !write) 655 return ret; 656 657 sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency, 658 sysctl_sched_min_granularity); 659 660 #define WRT_SYSCTL(name) \ 661 (normalized_sysctl_##name = sysctl_##name / (factor)) 662 WRT_SYSCTL(sched_min_granularity); 663 WRT_SYSCTL(sched_latency); 664 WRT_SYSCTL(sched_wakeup_granularity); 665 #undef WRT_SYSCTL 666 667 return 0; 668 } 669 #endif 670 671 /* 672 * delta /= w 673 */ 674 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) 675 { 676 if (unlikely(se->load.weight != NICE_0_LOAD)) 677 delta = __calc_delta(delta, NICE_0_LOAD, &se->load); 678 679 return delta; 680 } 681 682 /* 683 * The idea is to set a period in which each task runs once. 684 * 685 * When there are too many tasks (sched_nr_latency) we have to stretch 686 * this period because otherwise the slices get too small. 687 * 688 * p = (nr <= nl) ? l : l*nr/nl 689 */ 690 static u64 __sched_period(unsigned long nr_running) 691 { 692 if (unlikely(nr_running > sched_nr_latency)) 693 return nr_running * sysctl_sched_min_granularity; 694 else 695 return sysctl_sched_latency; 696 } 697 698 /* 699 * We calculate the wall-time slice from the period by taking a part 700 * proportional to the weight. 701 * 702 * s = p*P[w/rw] 703 */ 704 static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 705 { 706 u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq); 707 708 for_each_sched_entity(se) { 709 struct load_weight *load; 710 struct load_weight lw; 711 712 cfs_rq = cfs_rq_of(se); 713 load = &cfs_rq->load; 714 715 if (unlikely(!se->on_rq)) { 716 lw = cfs_rq->load; 717 718 update_load_add(&lw, se->load.weight); 719 load = &lw; 720 } 721 slice = __calc_delta(slice, se->load.weight, load); 722 } 723 return slice; 724 } 725 726 /* 727 * We calculate the vruntime slice of a to-be-inserted task. 728 * 729 * vs = s/w 730 */ 731 static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se) 732 { 733 return calc_delta_fair(sched_slice(cfs_rq, se), se); 734 } 735 736 #include "pelt.h" 737 #ifdef CONFIG_SMP 738 739 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); 740 static unsigned long task_h_load(struct task_struct *p); 741 static unsigned long capacity_of(int cpu); 742 743 /* Give new sched_entity start runnable values to heavy its load in infant time */ 744 void init_entity_runnable_average(struct sched_entity *se) 745 { 746 struct sched_avg *sa = &se->avg; 747 748 memset(sa, 0, sizeof(*sa)); 749 750 /* 751 * Tasks are initialized with full load to be seen as heavy tasks until 752 * they get a chance to stabilize to their real load level. 753 * Group entities are initialized with zero load to reflect the fact that 754 * nothing has been attached to the task group yet. 755 */ 756 if (entity_is_task(se)) 757 sa->load_avg = scale_load_down(se->load.weight); 758 759 /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ 760 } 761 762 static void attach_entity_cfs_rq(struct sched_entity *se); 763 764 /* 765 * With new tasks being created, their initial util_avgs are extrapolated 766 * based on the cfs_rq's current util_avg: 767 * 768 * util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight 769 * 770 * However, in many cases, the above util_avg does not give a desired 771 * value. Moreover, the sum of the util_avgs may be divergent, such 772 * as when the series is a harmonic series. 773 * 774 * To solve this problem, we also cap the util_avg of successive tasks to 775 * only 1/2 of the left utilization budget: 776 * 777 * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n 778 * 779 * where n denotes the nth task and cpu_scale the CPU capacity. 780 * 781 * For example, for a CPU with 1024 of capacity, a simplest series from 782 * the beginning would be like: 783 * 784 * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... 785 * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... 786 * 787 * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) 788 * if util_avg > util_avg_cap. 789 */ 790 void post_init_entity_util_avg(struct task_struct *p) 791 { 792 struct sched_entity *se = &p->se; 793 struct cfs_rq *cfs_rq = cfs_rq_of(se); 794 struct sched_avg *sa = &se->avg; 795 long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq))); 796 long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; 797 798 if (cap > 0) { 799 if (cfs_rq->avg.util_avg != 0) { 800 sa->util_avg = cfs_rq->avg.util_avg * se->load.weight; 801 sa->util_avg /= (cfs_rq->avg.load_avg + 1); 802 803 if (sa->util_avg > cap) 804 sa->util_avg = cap; 805 } else { 806 sa->util_avg = cap; 807 } 808 } 809 810 sa->runnable_avg = cpu_scale; 811 812 if (p->sched_class != &fair_sched_class) { 813 /* 814 * For !fair tasks do: 815 * 816 update_cfs_rq_load_avg(now, cfs_rq); 817 attach_entity_load_avg(cfs_rq, se); 818 switched_from_fair(rq, p); 819 * 820 * such that the next switched_to_fair() has the 821 * expected state. 822 */ 823 se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq); 824 return; 825 } 826 827 attach_entity_cfs_rq(se); 828 } 829 830 #else /* !CONFIG_SMP */ 831 void init_entity_runnable_average(struct sched_entity *se) 832 { 833 } 834 void post_init_entity_util_avg(struct task_struct *p) 835 { 836 } 837 static void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) 838 { 839 } 840 #endif /* CONFIG_SMP */ 841 842 /* 843 * Update the current task's runtime statistics. 844 */ 845 static void update_curr(struct cfs_rq *cfs_rq) 846 { 847 struct sched_entity *curr = cfs_rq->curr; 848 u64 now = rq_clock_task(rq_of(cfs_rq)); 849 u64 delta_exec; 850 851 if (unlikely(!curr)) 852 return; 853 854 delta_exec = now - curr->exec_start; 855 if (unlikely((s64)delta_exec <= 0)) 856 return; 857 858 curr->exec_start = now; 859 860 schedstat_set(curr->statistics.exec_max, 861 max(delta_exec, curr->statistics.exec_max)); 862 863 curr->sum_exec_runtime += delta_exec; 864 schedstat_add(cfs_rq->exec_clock, delta_exec); 865 866 curr->vruntime += calc_delta_fair(delta_exec, curr); 867 update_min_vruntime(cfs_rq); 868 869 if (entity_is_task(curr)) { 870 struct task_struct *curtask = task_of(curr); 871 872 trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); 873 cgroup_account_cputime(curtask, delta_exec); 874 account_group_exec_runtime(curtask, delta_exec); 875 } 876 877 account_cfs_rq_runtime(cfs_rq, delta_exec); 878 } 879 880 static void update_curr_fair(struct rq *rq) 881 { 882 update_curr(cfs_rq_of(&rq->curr->se)); 883 } 884 885 static inline void 886 update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 887 { 888 u64 wait_start, prev_wait_start; 889 890 if (!schedstat_enabled()) 891 return; 892 893 wait_start = rq_clock(rq_of(cfs_rq)); 894 prev_wait_start = schedstat_val(se->statistics.wait_start); 895 896 if (entity_is_task(se) && task_on_rq_migrating(task_of(se)) && 897 likely(wait_start > prev_wait_start)) 898 wait_start -= prev_wait_start; 899 900 __schedstat_set(se->statistics.wait_start, wait_start); 901 } 902 903 static inline void 904 update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) 905 { 906 struct task_struct *p; 907 u64 delta; 908 909 if (!schedstat_enabled()) 910 return; 911 912 delta = rq_clock(rq_of(cfs_rq)) - schedstat_val(se->statistics.wait_start); 913 914 if (entity_is_task(se)) { 915 p = task_of(se); 916 if (task_on_rq_migrating(p)) { 917 /* 918 * Preserve migrating task's wait time so wait_start 919 * time stamp can be adjusted to accumulate wait time 920 * prior to migration. 921 */ 922 __schedstat_set(se->statistics.wait_start, delta); 923 return; 924 } 925 trace_sched_stat_wait(p, delta); 926 } 927 928 __schedstat_set(se->statistics.wait_max, 929 max(schedstat_val(se->statistics.wait_max), delta)); 930 __schedstat_inc(se->statistics.wait_count); 931 __schedstat_add(se->statistics.wait_sum, delta); 932 __schedstat_set(se->statistics.wait_start, 0); 933 } 934 935 static inline void 936 update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) 937 { 938 struct task_struct *tsk = NULL; 939 u64 sleep_start, block_start; 940 941 if (!schedstat_enabled()) 942 return; 943 944 sleep_start = schedstat_val(se->statistics.sleep_start); 945 block_start = schedstat_val(se->statistics.block_start); 946 947 if (entity_is_task(se)) 948 tsk = task_of(se); 949 950 if (sleep_start) { 951 u64 delta = rq_clock(rq_of(cfs_rq)) - sleep_start; 952 953 if ((s64)delta < 0) 954 delta = 0; 955 956 if (unlikely(delta > schedstat_val(se->statistics.sleep_max))) 957 __schedstat_set(se->statistics.sleep_max, delta); 958 959 __schedstat_set(se->statistics.sleep_start, 0); 960 __schedstat_add(se->statistics.sum_sleep_runtime, delta); 961 962 if (tsk) { 963 account_scheduler_latency(tsk, delta >> 10, 1); 964 trace_sched_stat_sleep(tsk, delta); 965 } 966 } 967 if (block_start) { 968 u64 delta = rq_clock(rq_of(cfs_rq)) - block_start; 969 970 if ((s64)delta < 0) 971 delta = 0; 972 973 if (unlikely(delta > schedstat_val(se->statistics.block_max))) 974 __schedstat_set(se->statistics.block_max, delta); 975 976 __schedstat_set(se->statistics.block_start, 0); 977 __schedstat_add(se->statistics.sum_sleep_runtime, delta); 978 979 if (tsk) { 980 if (tsk->in_iowait) { 981 __schedstat_add(se->statistics.iowait_sum, delta); 982 __schedstat_inc(se->statistics.iowait_count); 983 trace_sched_stat_iowait(tsk, delta); 984 } 985 986 trace_sched_stat_blocked(tsk, delta); 987 988 /* 989 * Blocking time is in units of nanosecs, so shift by 990 * 20 to get a milliseconds-range estimation of the 991 * amount of time that the task spent sleeping: 992 */ 993 if (unlikely(prof_on == SLEEP_PROFILING)) { 994 profile_hits(SLEEP_PROFILING, 995 (void *)get_wchan(tsk), 996 delta >> 20); 997 } 998 account_scheduler_latency(tsk, delta >> 10, 0); 999 } 1000 } 1001 } 1002 1003 /* 1004 * Task is being enqueued - update stats: 1005 */ 1006 static inline void 1007 update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1008 { 1009 if (!schedstat_enabled()) 1010 return; 1011 1012 /* 1013 * Are we enqueueing a waiting task? (for current tasks 1014 * a dequeue/enqueue event is a NOP) 1015 */ 1016 if (se != cfs_rq->curr) 1017 update_stats_wait_start(cfs_rq, se); 1018 1019 if (flags & ENQUEUE_WAKEUP) 1020 update_stats_enqueue_sleeper(cfs_rq, se); 1021 } 1022 1023 static inline void 1024 update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1025 { 1026 1027 if (!schedstat_enabled()) 1028 return; 1029 1030 /* 1031 * Mark the end of the wait period if dequeueing a 1032 * waiting task: 1033 */ 1034 if (se != cfs_rq->curr) 1035 update_stats_wait_end(cfs_rq, se); 1036 1037 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { 1038 struct task_struct *tsk = task_of(se); 1039 1040 if (tsk->state & TASK_INTERRUPTIBLE) 1041 __schedstat_set(se->statistics.sleep_start, 1042 rq_clock(rq_of(cfs_rq))); 1043 if (tsk->state & TASK_UNINTERRUPTIBLE) 1044 __schedstat_set(se->statistics.block_start, 1045 rq_clock(rq_of(cfs_rq))); 1046 } 1047 } 1048 1049 /* 1050 * We are picking a new current task - update its stats: 1051 */ 1052 static inline void 1053 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 1054 { 1055 /* 1056 * We are starting a new run period: 1057 */ 1058 se->exec_start = rq_clock_task(rq_of(cfs_rq)); 1059 } 1060 1061 /************************************************** 1062 * Scheduling class queueing methods: 1063 */ 1064 1065 #ifdef CONFIG_NUMA_BALANCING 1066 /* 1067 * Approximate time to scan a full NUMA task in ms. The task scan period is 1068 * calculated based on the tasks virtual memory size and 1069 * numa_balancing_scan_size. 1070 */ 1071 unsigned int sysctl_numa_balancing_scan_period_min = 1000; 1072 unsigned int sysctl_numa_balancing_scan_period_max = 60000; 1073 1074 /* Portion of address space to scan in MB */ 1075 unsigned int sysctl_numa_balancing_scan_size = 256; 1076 1077 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ 1078 unsigned int sysctl_numa_balancing_scan_delay = 1000; 1079 1080 struct numa_group { 1081 refcount_t refcount; 1082 1083 spinlock_t lock; /* nr_tasks, tasks */ 1084 int nr_tasks; 1085 pid_t gid; 1086 int active_nodes; 1087 1088 struct rcu_head rcu; 1089 unsigned long total_faults; 1090 unsigned long max_faults_cpu; 1091 /* 1092 * Faults_cpu is used to decide whether memory should move 1093 * towards the CPU. As a consequence, these stats are weighted 1094 * more by CPU use than by memory faults. 1095 */ 1096 unsigned long *faults_cpu; 1097 unsigned long faults[0]; 1098 }; 1099 1100 /* 1101 * For functions that can be called in multiple contexts that permit reading 1102 * ->numa_group (see struct task_struct for locking rules). 1103 */ 1104 static struct numa_group *deref_task_numa_group(struct task_struct *p) 1105 { 1106 return rcu_dereference_check(p->numa_group, p == current || 1107 (lockdep_is_held(&task_rq(p)->lock) && !READ_ONCE(p->on_cpu))); 1108 } 1109 1110 static struct numa_group *deref_curr_numa_group(struct task_struct *p) 1111 { 1112 return rcu_dereference_protected(p->numa_group, p == current); 1113 } 1114 1115 static inline unsigned long group_faults_priv(struct numa_group *ng); 1116 static inline unsigned long group_faults_shared(struct numa_group *ng); 1117 1118 static unsigned int task_nr_scan_windows(struct task_struct *p) 1119 { 1120 unsigned long rss = 0; 1121 unsigned long nr_scan_pages; 1122 1123 /* 1124 * Calculations based on RSS as non-present and empty pages are skipped 1125 * by the PTE scanner and NUMA hinting faults should be trapped based 1126 * on resident pages 1127 */ 1128 nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT); 1129 rss = get_mm_rss(p->mm); 1130 if (!rss) 1131 rss = nr_scan_pages; 1132 1133 rss = round_up(rss, nr_scan_pages); 1134 return rss / nr_scan_pages; 1135 } 1136 1137 /* For sanitys sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ 1138 #define MAX_SCAN_WINDOW 2560 1139 1140 static unsigned int task_scan_min(struct task_struct *p) 1141 { 1142 unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); 1143 unsigned int scan, floor; 1144 unsigned int windows = 1; 1145 1146 if (scan_size < MAX_SCAN_WINDOW) 1147 windows = MAX_SCAN_WINDOW / scan_size; 1148 floor = 1000 / windows; 1149 1150 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); 1151 return max_t(unsigned int, floor, scan); 1152 } 1153 1154 static unsigned int task_scan_start(struct task_struct *p) 1155 { 1156 unsigned long smin = task_scan_min(p); 1157 unsigned long period = smin; 1158 struct numa_group *ng; 1159 1160 /* Scale the maximum scan period with the amount of shared memory. */ 1161 rcu_read_lock(); 1162 ng = rcu_dereference(p->numa_group); 1163 if (ng) { 1164 unsigned long shared = group_faults_shared(ng); 1165 unsigned long private = group_faults_priv(ng); 1166 1167 period *= refcount_read(&ng->refcount); 1168 period *= shared + 1; 1169 period /= private + shared + 1; 1170 } 1171 rcu_read_unlock(); 1172 1173 return max(smin, period); 1174 } 1175 1176 static unsigned int task_scan_max(struct task_struct *p) 1177 { 1178 unsigned long smin = task_scan_min(p); 1179 unsigned long smax; 1180 struct numa_group *ng; 1181 1182 /* Watch for min being lower than max due to floor calculations */ 1183 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); 1184 1185 /* Scale the maximum scan period with the amount of shared memory. */ 1186 ng = deref_curr_numa_group(p); 1187 if (ng) { 1188 unsigned long shared = group_faults_shared(ng); 1189 unsigned long private = group_faults_priv(ng); 1190 unsigned long period = smax; 1191 1192 period *= refcount_read(&ng->refcount); 1193 period *= shared + 1; 1194 period /= private + shared + 1; 1195 1196 smax = max(smax, period); 1197 } 1198 1199 return max(smin, smax); 1200 } 1201 1202 static void account_numa_enqueue(struct rq *rq, struct task_struct *p) 1203 { 1204 rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE); 1205 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 1206 } 1207 1208 static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 1209 { 1210 rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE); 1211 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 1212 } 1213 1214 /* Shared or private faults. */ 1215 #define NR_NUMA_HINT_FAULT_TYPES 2 1216 1217 /* Memory and CPU locality */ 1218 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) 1219 1220 /* Averaged statistics, and temporary buffers. */ 1221 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) 1222 1223 pid_t task_numa_group_id(struct task_struct *p) 1224 { 1225 struct numa_group *ng; 1226 pid_t gid = 0; 1227 1228 rcu_read_lock(); 1229 ng = rcu_dereference(p->numa_group); 1230 if (ng) 1231 gid = ng->gid; 1232 rcu_read_unlock(); 1233 1234 return gid; 1235 } 1236 1237 /* 1238 * The averaged statistics, shared & private, memory & CPU, 1239 * occupy the first half of the array. The second half of the 1240 * array is for current counters, which are averaged into the 1241 * first set by task_numa_placement. 1242 */ 1243 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) 1244 { 1245 return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 1246 } 1247 1248 static inline unsigned long task_faults(struct task_struct *p, int nid) 1249 { 1250 if (!p->numa_faults) 1251 return 0; 1252 1253 return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1254 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1255 } 1256 1257 static inline unsigned long group_faults(struct task_struct *p, int nid) 1258 { 1259 struct numa_group *ng = deref_task_numa_group(p); 1260 1261 if (!ng) 1262 return 0; 1263 1264 return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1265 ng->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1266 } 1267 1268 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 1269 { 1270 return group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 0)] + 1271 group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 1)]; 1272 } 1273 1274 static inline unsigned long group_faults_priv(struct numa_group *ng) 1275 { 1276 unsigned long faults = 0; 1277 int node; 1278 1279 for_each_online_node(node) { 1280 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 1281 } 1282 1283 return faults; 1284 } 1285 1286 static inline unsigned long group_faults_shared(struct numa_group *ng) 1287 { 1288 unsigned long faults = 0; 1289 int node; 1290 1291 for_each_online_node(node) { 1292 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 1293 } 1294 1295 return faults; 1296 } 1297 1298 /* 1299 * A node triggering more than 1/3 as many NUMA faults as the maximum is 1300 * considered part of a numa group's pseudo-interleaving set. Migrations 1301 * between these nodes are slowed down, to allow things to settle down. 1302 */ 1303 #define ACTIVE_NODE_FRACTION 3 1304 1305 static bool numa_is_active_node(int nid, struct numa_group *ng) 1306 { 1307 return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; 1308 } 1309 1310 /* Handle placement on systems where not all nodes are directly connected. */ 1311 static unsigned long score_nearby_nodes(struct task_struct *p, int nid, 1312 int maxdist, bool task) 1313 { 1314 unsigned long score = 0; 1315 int node; 1316 1317 /* 1318 * All nodes are directly connected, and the same distance 1319 * from each other. No need for fancy placement algorithms. 1320 */ 1321 if (sched_numa_topology_type == NUMA_DIRECT) 1322 return 0; 1323 1324 /* 1325 * This code is called for each node, introducing N^2 complexity, 1326 * which should be ok given the number of nodes rarely exceeds 8. 1327 */ 1328 for_each_online_node(node) { 1329 unsigned long faults; 1330 int dist = node_distance(nid, node); 1331 1332 /* 1333 * The furthest away nodes in the system are not interesting 1334 * for placement; nid was already counted. 1335 */ 1336 if (dist == sched_max_numa_distance || node == nid) 1337 continue; 1338 1339 /* 1340 * On systems with a backplane NUMA topology, compare groups 1341 * of nodes, and move tasks towards the group with the most 1342 * memory accesses. When comparing two nodes at distance 1343 * "hoplimit", only nodes closer by than "hoplimit" are part 1344 * of each group. Skip other nodes. 1345 */ 1346 if (sched_numa_topology_type == NUMA_BACKPLANE && 1347 dist >= maxdist) 1348 continue; 1349 1350 /* Add up the faults from nearby nodes. */ 1351 if (task) 1352 faults = task_faults(p, node); 1353 else 1354 faults = group_faults(p, node); 1355 1356 /* 1357 * On systems with a glueless mesh NUMA topology, there are 1358 * no fixed "groups of nodes". Instead, nodes that are not 1359 * directly connected bounce traffic through intermediate 1360 * nodes; a numa_group can occupy any set of nodes. 1361 * The further away a node is, the less the faults count. 1362 * This seems to result in good task placement. 1363 */ 1364 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 1365 faults *= (sched_max_numa_distance - dist); 1366 faults /= (sched_max_numa_distance - LOCAL_DISTANCE); 1367 } 1368 1369 score += faults; 1370 } 1371 1372 return score; 1373 } 1374 1375 /* 1376 * These return the fraction of accesses done by a particular task, or 1377 * task group, on a particular numa node. The group weight is given a 1378 * larger multiplier, in order to group tasks together that are almost 1379 * evenly spread out between numa nodes. 1380 */ 1381 static inline unsigned long task_weight(struct task_struct *p, int nid, 1382 int dist) 1383 { 1384 unsigned long faults, total_faults; 1385 1386 if (!p->numa_faults) 1387 return 0; 1388 1389 total_faults = p->total_numa_faults; 1390 1391 if (!total_faults) 1392 return 0; 1393 1394 faults = task_faults(p, nid); 1395 faults += score_nearby_nodes(p, nid, dist, true); 1396 1397 return 1000 * faults / total_faults; 1398 } 1399 1400 static inline unsigned long group_weight(struct task_struct *p, int nid, 1401 int dist) 1402 { 1403 struct numa_group *ng = deref_task_numa_group(p); 1404 unsigned long faults, total_faults; 1405 1406 if (!ng) 1407 return 0; 1408 1409 total_faults = ng->total_faults; 1410 1411 if (!total_faults) 1412 return 0; 1413 1414 faults = group_faults(p, nid); 1415 faults += score_nearby_nodes(p, nid, dist, false); 1416 1417 return 1000 * faults / total_faults; 1418 } 1419 1420 bool should_numa_migrate_memory(struct task_struct *p, struct page * page, 1421 int src_nid, int dst_cpu) 1422 { 1423 struct numa_group *ng = deref_curr_numa_group(p); 1424 int dst_nid = cpu_to_node(dst_cpu); 1425 int last_cpupid, this_cpupid; 1426 1427 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); 1428 last_cpupid = page_cpupid_xchg_last(page, this_cpupid); 1429 1430 /* 1431 * Allow first faults or private faults to migrate immediately early in 1432 * the lifetime of a task. The magic number 4 is based on waiting for 1433 * two full passes of the "multi-stage node selection" test that is 1434 * executed below. 1435 */ 1436 if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) && 1437 (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid))) 1438 return true; 1439 1440 /* 1441 * Multi-stage node selection is used in conjunction with a periodic 1442 * migration fault to build a temporal task<->page relation. By using 1443 * a two-stage filter we remove short/unlikely relations. 1444 * 1445 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate 1446 * a task's usage of a particular page (n_p) per total usage of this 1447 * page (n_t) (in a given time-span) to a probability. 1448 * 1449 * Our periodic faults will sample this probability and getting the 1450 * same result twice in a row, given these samples are fully 1451 * independent, is then given by P(n)^2, provided our sample period 1452 * is sufficiently short compared to the usage pattern. 1453 * 1454 * This quadric squishes small probabilities, making it less likely we 1455 * act on an unlikely task<->page relation. 1456 */ 1457 if (!cpupid_pid_unset(last_cpupid) && 1458 cpupid_to_nid(last_cpupid) != dst_nid) 1459 return false; 1460 1461 /* Always allow migrate on private faults */ 1462 if (cpupid_match_pid(p, last_cpupid)) 1463 return true; 1464 1465 /* A shared fault, but p->numa_group has not been set up yet. */ 1466 if (!ng) 1467 return true; 1468 1469 /* 1470 * Destination node is much more heavily used than the source 1471 * node? Allow migration. 1472 */ 1473 if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * 1474 ACTIVE_NODE_FRACTION) 1475 return true; 1476 1477 /* 1478 * Distribute memory according to CPU & memory use on each node, 1479 * with 3/4 hysteresis to avoid unnecessary memory migrations: 1480 * 1481 * faults_cpu(dst) 3 faults_cpu(src) 1482 * --------------- * - > --------------- 1483 * faults_mem(dst) 4 faults_mem(src) 1484 */ 1485 return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > 1486 group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; 1487 } 1488 1489 /* 1490 * 'numa_type' describes the node at the moment of load balancing. 1491 */ 1492 enum numa_type { 1493 /* The node has spare capacity that can be used to run more tasks. */ 1494 node_has_spare = 0, 1495 /* 1496 * The node is fully used and the tasks don't compete for more CPU 1497 * cycles. Nevertheless, some tasks might wait before running. 1498 */ 1499 node_fully_busy, 1500 /* 1501 * The node is overloaded and can't provide expected CPU cycles to all 1502 * tasks. 1503 */ 1504 node_overloaded 1505 }; 1506 1507 /* Cached statistics for all CPUs within a node */ 1508 struct numa_stats { 1509 unsigned long load; 1510 unsigned long util; 1511 /* Total compute capacity of CPUs on a node */ 1512 unsigned long compute_capacity; 1513 unsigned int nr_running; 1514 unsigned int weight; 1515 enum numa_type node_type; 1516 int idle_cpu; 1517 }; 1518 1519 static inline bool is_core_idle(int cpu) 1520 { 1521 #ifdef CONFIG_SCHED_SMT 1522 int sibling; 1523 1524 for_each_cpu(sibling, cpu_smt_mask(cpu)) { 1525 if (cpu == sibling) 1526 continue; 1527 1528 if (!idle_cpu(cpu)) 1529 return false; 1530 } 1531 #endif 1532 1533 return true; 1534 } 1535 1536 struct task_numa_env { 1537 struct task_struct *p; 1538 1539 int src_cpu, src_nid; 1540 int dst_cpu, dst_nid; 1541 1542 struct numa_stats src_stats, dst_stats; 1543 1544 int imbalance_pct; 1545 int dist; 1546 1547 struct task_struct *best_task; 1548 long best_imp; 1549 int best_cpu; 1550 }; 1551 1552 static unsigned long cpu_load(struct rq *rq); 1553 static unsigned long cpu_util(int cpu); 1554 static inline long adjust_numa_imbalance(int imbalance, int src_nr_running); 1555 1556 static inline enum 1557 numa_type numa_classify(unsigned int imbalance_pct, 1558 struct numa_stats *ns) 1559 { 1560 if ((ns->nr_running > ns->weight) && 1561 ((ns->compute_capacity * 100) < (ns->util * imbalance_pct))) 1562 return node_overloaded; 1563 1564 if ((ns->nr_running < ns->weight) || 1565 ((ns->compute_capacity * 100) > (ns->util * imbalance_pct))) 1566 return node_has_spare; 1567 1568 return node_fully_busy; 1569 } 1570 1571 #ifdef CONFIG_SCHED_SMT 1572 /* Forward declarations of select_idle_sibling helpers */ 1573 static inline bool test_idle_cores(int cpu, bool def); 1574 static inline int numa_idle_core(int idle_core, int cpu) 1575 { 1576 if (!static_branch_likely(&sched_smt_present) || 1577 idle_core >= 0 || !test_idle_cores(cpu, false)) 1578 return idle_core; 1579 1580 /* 1581 * Prefer cores instead of packing HT siblings 1582 * and triggering future load balancing. 1583 */ 1584 if (is_core_idle(cpu)) 1585 idle_core = cpu; 1586 1587 return idle_core; 1588 } 1589 #else 1590 static inline int numa_idle_core(int idle_core, int cpu) 1591 { 1592 return idle_core; 1593 } 1594 #endif 1595 1596 /* 1597 * Gather all necessary information to make NUMA balancing placement 1598 * decisions that are compatible with standard load balancer. This 1599 * borrows code and logic from update_sg_lb_stats but sharing a 1600 * common implementation is impractical. 1601 */ 1602 static void update_numa_stats(struct task_numa_env *env, 1603 struct numa_stats *ns, int nid, 1604 bool find_idle) 1605 { 1606 int cpu, idle_core = -1; 1607 1608 memset(ns, 0, sizeof(*ns)); 1609 ns->idle_cpu = -1; 1610 1611 rcu_read_lock(); 1612 for_each_cpu(cpu, cpumask_of_node(nid)) { 1613 struct rq *rq = cpu_rq(cpu); 1614 1615 ns->load += cpu_load(rq); 1616 ns->util += cpu_util(cpu); 1617 ns->nr_running += rq->cfs.h_nr_running; 1618 ns->compute_capacity += capacity_of(cpu); 1619 1620 if (find_idle && !rq->nr_running && idle_cpu(cpu)) { 1621 if (READ_ONCE(rq->numa_migrate_on) || 1622 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) 1623 continue; 1624 1625 if (ns->idle_cpu == -1) 1626 ns->idle_cpu = cpu; 1627 1628 idle_core = numa_idle_core(idle_core, cpu); 1629 } 1630 } 1631 rcu_read_unlock(); 1632 1633 ns->weight = cpumask_weight(cpumask_of_node(nid)); 1634 1635 ns->node_type = numa_classify(env->imbalance_pct, ns); 1636 1637 if (idle_core >= 0) 1638 ns->idle_cpu = idle_core; 1639 } 1640 1641 static void task_numa_assign(struct task_numa_env *env, 1642 struct task_struct *p, long imp) 1643 { 1644 struct rq *rq = cpu_rq(env->dst_cpu); 1645 1646 /* Check if run-queue part of active NUMA balance. */ 1647 if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) { 1648 int cpu; 1649 int start = env->dst_cpu; 1650 1651 /* Find alternative idle CPU. */ 1652 for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start) { 1653 if (cpu == env->best_cpu || !idle_cpu(cpu) || 1654 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) { 1655 continue; 1656 } 1657 1658 env->dst_cpu = cpu; 1659 rq = cpu_rq(env->dst_cpu); 1660 if (!xchg(&rq->numa_migrate_on, 1)) 1661 goto assign; 1662 } 1663 1664 /* Failed to find an alternative idle CPU */ 1665 return; 1666 } 1667 1668 assign: 1669 /* 1670 * Clear previous best_cpu/rq numa-migrate flag, since task now 1671 * found a better CPU to move/swap. 1672 */ 1673 if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) { 1674 rq = cpu_rq(env->best_cpu); 1675 WRITE_ONCE(rq->numa_migrate_on, 0); 1676 } 1677 1678 if (env->best_task) 1679 put_task_struct(env->best_task); 1680 if (p) 1681 get_task_struct(p); 1682 1683 env->best_task = p; 1684 env->best_imp = imp; 1685 env->best_cpu = env->dst_cpu; 1686 } 1687 1688 static bool load_too_imbalanced(long src_load, long dst_load, 1689 struct task_numa_env *env) 1690 { 1691 long imb, old_imb; 1692 long orig_src_load, orig_dst_load; 1693 long src_capacity, dst_capacity; 1694 1695 /* 1696 * The load is corrected for the CPU capacity available on each node. 1697 * 1698 * src_load dst_load 1699 * ------------ vs --------- 1700 * src_capacity dst_capacity 1701 */ 1702 src_capacity = env->src_stats.compute_capacity; 1703 dst_capacity = env->dst_stats.compute_capacity; 1704 1705 imb = abs(dst_load * src_capacity - src_load * dst_capacity); 1706 1707 orig_src_load = env->src_stats.load; 1708 orig_dst_load = env->dst_stats.load; 1709 1710 old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity); 1711 1712 /* Would this change make things worse? */ 1713 return (imb > old_imb); 1714 } 1715 1716 /* 1717 * Maximum NUMA importance can be 1998 (2*999); 1718 * SMALLIMP @ 30 would be close to 1998/64. 1719 * Used to deter task migration. 1720 */ 1721 #define SMALLIMP 30 1722 1723 /* 1724 * This checks if the overall compute and NUMA accesses of the system would 1725 * be improved if the source tasks was migrated to the target dst_cpu taking 1726 * into account that it might be best if task running on the dst_cpu should 1727 * be exchanged with the source task 1728 */ 1729 static bool task_numa_compare(struct task_numa_env *env, 1730 long taskimp, long groupimp, bool maymove) 1731 { 1732 struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p); 1733 struct rq *dst_rq = cpu_rq(env->dst_cpu); 1734 long imp = p_ng ? groupimp : taskimp; 1735 struct task_struct *cur; 1736 long src_load, dst_load; 1737 int dist = env->dist; 1738 long moveimp = imp; 1739 long load; 1740 bool stopsearch = false; 1741 1742 if (READ_ONCE(dst_rq->numa_migrate_on)) 1743 return false; 1744 1745 rcu_read_lock(); 1746 cur = rcu_dereference(dst_rq->curr); 1747 if (cur && ((cur->flags & PF_EXITING) || is_idle_task(cur))) 1748 cur = NULL; 1749 1750 /* 1751 * Because we have preemption enabled we can get migrated around and 1752 * end try selecting ourselves (current == env->p) as a swap candidate. 1753 */ 1754 if (cur == env->p) { 1755 stopsearch = true; 1756 goto unlock; 1757 } 1758 1759 if (!cur) { 1760 if (maymove && moveimp >= env->best_imp) 1761 goto assign; 1762 else 1763 goto unlock; 1764 } 1765 1766 /* Skip this swap candidate if cannot move to the source cpu. */ 1767 if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr)) 1768 goto unlock; 1769 1770 /* 1771 * Skip this swap candidate if it is not moving to its preferred 1772 * node and the best task is. 1773 */ 1774 if (env->best_task && 1775 env->best_task->numa_preferred_nid == env->src_nid && 1776 cur->numa_preferred_nid != env->src_nid) { 1777 goto unlock; 1778 } 1779 1780 /* 1781 * "imp" is the fault differential for the source task between the 1782 * source and destination node. Calculate the total differential for 1783 * the source task and potential destination task. The more negative 1784 * the value is, the more remote accesses that would be expected to 1785 * be incurred if the tasks were swapped. 1786 * 1787 * If dst and source tasks are in the same NUMA group, or not 1788 * in any group then look only at task weights. 1789 */ 1790 cur_ng = rcu_dereference(cur->numa_group); 1791 if (cur_ng == p_ng) { 1792 imp = taskimp + task_weight(cur, env->src_nid, dist) - 1793 task_weight(cur, env->dst_nid, dist); 1794 /* 1795 * Add some hysteresis to prevent swapping the 1796 * tasks within a group over tiny differences. 1797 */ 1798 if (cur_ng) 1799 imp -= imp / 16; 1800 } else { 1801 /* 1802 * Compare the group weights. If a task is all by itself 1803 * (not part of a group), use the task weight instead. 1804 */ 1805 if (cur_ng && p_ng) 1806 imp += group_weight(cur, env->src_nid, dist) - 1807 group_weight(cur, env->dst_nid, dist); 1808 else 1809 imp += task_weight(cur, env->src_nid, dist) - 1810 task_weight(cur, env->dst_nid, dist); 1811 } 1812 1813 /* Discourage picking a task already on its preferred node */ 1814 if (cur->numa_preferred_nid == env->dst_nid) 1815 imp -= imp / 16; 1816 1817 /* 1818 * Encourage picking a task that moves to its preferred node. 1819 * This potentially makes imp larger than it's maximum of 1820 * 1998 (see SMALLIMP and task_weight for why) but in this 1821 * case, it does not matter. 1822 */ 1823 if (cur->numa_preferred_nid == env->src_nid) 1824 imp += imp / 8; 1825 1826 if (maymove && moveimp > imp && moveimp > env->best_imp) { 1827 imp = moveimp; 1828 cur = NULL; 1829 goto assign; 1830 } 1831 1832 /* 1833 * Prefer swapping with a task moving to its preferred node over a 1834 * task that is not. 1835 */ 1836 if (env->best_task && cur->numa_preferred_nid == env->src_nid && 1837 env->best_task->numa_preferred_nid != env->src_nid) { 1838 goto assign; 1839 } 1840 1841 /* 1842 * If the NUMA importance is less than SMALLIMP, 1843 * task migration might only result in ping pong 1844 * of tasks and also hurt performance due to cache 1845 * misses. 1846 */ 1847 if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2) 1848 goto unlock; 1849 1850 /* 1851 * In the overloaded case, try and keep the load balanced. 1852 */ 1853 load = task_h_load(env->p) - task_h_load(cur); 1854 if (!load) 1855 goto assign; 1856 1857 dst_load = env->dst_stats.load + load; 1858 src_load = env->src_stats.load - load; 1859 1860 if (load_too_imbalanced(src_load, dst_load, env)) 1861 goto unlock; 1862 1863 assign: 1864 /* Evaluate an idle CPU for a task numa move. */ 1865 if (!cur) { 1866 int cpu = env->dst_stats.idle_cpu; 1867 1868 /* Nothing cached so current CPU went idle since the search. */ 1869 if (cpu < 0) 1870 cpu = env->dst_cpu; 1871 1872 /* 1873 * If the CPU is no longer truly idle and the previous best CPU 1874 * is, keep using it. 1875 */ 1876 if (!idle_cpu(cpu) && env->best_cpu >= 0 && 1877 idle_cpu(env->best_cpu)) { 1878 cpu = env->best_cpu; 1879 } 1880 1881 env->dst_cpu = cpu; 1882 } 1883 1884 task_numa_assign(env, cur, imp); 1885 1886 /* 1887 * If a move to idle is allowed because there is capacity or load 1888 * balance improves then stop the search. While a better swap 1889 * candidate may exist, a search is not free. 1890 */ 1891 if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu)) 1892 stopsearch = true; 1893 1894 /* 1895 * If a swap candidate must be identified and the current best task 1896 * moves its preferred node then stop the search. 1897 */ 1898 if (!maymove && env->best_task && 1899 env->best_task->numa_preferred_nid == env->src_nid) { 1900 stopsearch = true; 1901 } 1902 unlock: 1903 rcu_read_unlock(); 1904 1905 return stopsearch; 1906 } 1907 1908 static void task_numa_find_cpu(struct task_numa_env *env, 1909 long taskimp, long groupimp) 1910 { 1911 bool maymove = false; 1912 int cpu; 1913 1914 /* 1915 * If dst node has spare capacity, then check if there is an 1916 * imbalance that would be overruled by the load balancer. 1917 */ 1918 if (env->dst_stats.node_type == node_has_spare) { 1919 unsigned int imbalance; 1920 int src_running, dst_running; 1921 1922 /* 1923 * Would movement cause an imbalance? Note that if src has 1924 * more running tasks that the imbalance is ignored as the 1925 * move improves the imbalance from the perspective of the 1926 * CPU load balancer. 1927 * */ 1928 src_running = env->src_stats.nr_running - 1; 1929 dst_running = env->dst_stats.nr_running + 1; 1930 imbalance = max(0, dst_running - src_running); 1931 imbalance = adjust_numa_imbalance(imbalance, src_running); 1932 1933 /* Use idle CPU if there is no imbalance */ 1934 if (!imbalance) { 1935 maymove = true; 1936 if (env->dst_stats.idle_cpu >= 0) { 1937 env->dst_cpu = env->dst_stats.idle_cpu; 1938 task_numa_assign(env, NULL, 0); 1939 return; 1940 } 1941 } 1942 } else { 1943 long src_load, dst_load, load; 1944 /* 1945 * If the improvement from just moving env->p direction is better 1946 * than swapping tasks around, check if a move is possible. 1947 */ 1948 load = task_h_load(env->p); 1949 dst_load = env->dst_stats.load + load; 1950 src_load = env->src_stats.load - load; 1951 maymove = !load_too_imbalanced(src_load, dst_load, env); 1952 } 1953 1954 for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { 1955 /* Skip this CPU if the source task cannot migrate */ 1956 if (!cpumask_test_cpu(cpu, env->p->cpus_ptr)) 1957 continue; 1958 1959 env->dst_cpu = cpu; 1960 if (task_numa_compare(env, taskimp, groupimp, maymove)) 1961 break; 1962 } 1963 } 1964 1965 static int task_numa_migrate(struct task_struct *p) 1966 { 1967 struct task_numa_env env = { 1968 .p = p, 1969 1970 .src_cpu = task_cpu(p), 1971 .src_nid = task_node(p), 1972 1973 .imbalance_pct = 112, 1974 1975 .best_task = NULL, 1976 .best_imp = 0, 1977 .best_cpu = -1, 1978 }; 1979 unsigned long taskweight, groupweight; 1980 struct sched_domain *sd; 1981 long taskimp, groupimp; 1982 struct numa_group *ng; 1983 struct rq *best_rq; 1984 int nid, ret, dist; 1985 1986 /* 1987 * Pick the lowest SD_NUMA domain, as that would have the smallest 1988 * imbalance and would be the first to start moving tasks about. 1989 * 1990 * And we want to avoid any moving of tasks about, as that would create 1991 * random movement of tasks -- counter the numa conditions we're trying 1992 * to satisfy here. 1993 */ 1994 rcu_read_lock(); 1995 sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); 1996 if (sd) 1997 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; 1998 rcu_read_unlock(); 1999 2000 /* 2001 * Cpusets can break the scheduler domain tree into smaller 2002 * balance domains, some of which do not cross NUMA boundaries. 2003 * Tasks that are "trapped" in such domains cannot be migrated 2004 * elsewhere, so there is no point in (re)trying. 2005 */ 2006 if (unlikely(!sd)) { 2007 sched_setnuma(p, task_node(p)); 2008 return -EINVAL; 2009 } 2010 2011 env.dst_nid = p->numa_preferred_nid; 2012 dist = env.dist = node_distance(env.src_nid, env.dst_nid); 2013 taskweight = task_weight(p, env.src_nid, dist); 2014 groupweight = group_weight(p, env.src_nid, dist); 2015 update_numa_stats(&env, &env.src_stats, env.src_nid, false); 2016 taskimp = task_weight(p, env.dst_nid, dist) - taskweight; 2017 groupimp = group_weight(p, env.dst_nid, dist) - groupweight; 2018 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2019 2020 /* Try to find a spot on the preferred nid. */ 2021 task_numa_find_cpu(&env, taskimp, groupimp); 2022 2023 /* 2024 * Look at other nodes in these cases: 2025 * - there is no space available on the preferred_nid 2026 * - the task is part of a numa_group that is interleaved across 2027 * multiple NUMA nodes; in order to better consolidate the group, 2028 * we need to check other locations. 2029 */ 2030 ng = deref_curr_numa_group(p); 2031 if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) { 2032 for_each_online_node(nid) { 2033 if (nid == env.src_nid || nid == p->numa_preferred_nid) 2034 continue; 2035 2036 dist = node_distance(env.src_nid, env.dst_nid); 2037 if (sched_numa_topology_type == NUMA_BACKPLANE && 2038 dist != env.dist) { 2039 taskweight = task_weight(p, env.src_nid, dist); 2040 groupweight = group_weight(p, env.src_nid, dist); 2041 } 2042 2043 /* Only consider nodes where both task and groups benefit */ 2044 taskimp = task_weight(p, nid, dist) - taskweight; 2045 groupimp = group_weight(p, nid, dist) - groupweight; 2046 if (taskimp < 0 && groupimp < 0) 2047 continue; 2048 2049 env.dist = dist; 2050 env.dst_nid = nid; 2051 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2052 task_numa_find_cpu(&env, taskimp, groupimp); 2053 } 2054 } 2055 2056 /* 2057 * If the task is part of a workload that spans multiple NUMA nodes, 2058 * and is migrating into one of the workload's active nodes, remember 2059 * this node as the task's preferred numa node, so the workload can 2060 * settle down. 2061 * A task that migrated to a second choice node will be better off 2062 * trying for a better one later. Do not set the preferred node here. 2063 */ 2064 if (ng) { 2065 if (env.best_cpu == -1) 2066 nid = env.src_nid; 2067 else 2068 nid = cpu_to_node(env.best_cpu); 2069 2070 if (nid != p->numa_preferred_nid) 2071 sched_setnuma(p, nid); 2072 } 2073 2074 /* No better CPU than the current one was found. */ 2075 if (env.best_cpu == -1) { 2076 trace_sched_stick_numa(p, env.src_cpu, NULL, -1); 2077 return -EAGAIN; 2078 } 2079 2080 best_rq = cpu_rq(env.best_cpu); 2081 if (env.best_task == NULL) { 2082 ret = migrate_task_to(p, env.best_cpu); 2083 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2084 if (ret != 0) 2085 trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu); 2086 return ret; 2087 } 2088 2089 ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu); 2090 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2091 2092 if (ret != 0) 2093 trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu); 2094 put_task_struct(env.best_task); 2095 return ret; 2096 } 2097 2098 /* Attempt to migrate a task to a CPU on the preferred node. */ 2099 static void numa_migrate_preferred(struct task_struct *p) 2100 { 2101 unsigned long interval = HZ; 2102 2103 /* This task has no NUMA fault statistics yet */ 2104 if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults)) 2105 return; 2106 2107 /* Periodically retry migrating the task to the preferred node */ 2108 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); 2109 p->numa_migrate_retry = jiffies + interval; 2110 2111 /* Success if task is already running on preferred CPU */ 2112 if (task_node(p) == p->numa_preferred_nid) 2113 return; 2114 2115 /* Otherwise, try migrate to a CPU on the preferred node */ 2116 task_numa_migrate(p); 2117 } 2118 2119 /* 2120 * Find out how many nodes on the workload is actively running on. Do this by 2121 * tracking the nodes from which NUMA hinting faults are triggered. This can 2122 * be different from the set of nodes where the workload's memory is currently 2123 * located. 2124 */ 2125 static void numa_group_count_active_nodes(struct numa_group *numa_group) 2126 { 2127 unsigned long faults, max_faults = 0; 2128 int nid, active_nodes = 0; 2129 2130 for_each_online_node(nid) { 2131 faults = group_faults_cpu(numa_group, nid); 2132 if (faults > max_faults) 2133 max_faults = faults; 2134 } 2135 2136 for_each_online_node(nid) { 2137 faults = group_faults_cpu(numa_group, nid); 2138 if (faults * ACTIVE_NODE_FRACTION > max_faults) 2139 active_nodes++; 2140 } 2141 2142 numa_group->max_faults_cpu = max_faults; 2143 numa_group->active_nodes = active_nodes; 2144 } 2145 2146 /* 2147 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS 2148 * increments. The more local the fault statistics are, the higher the scan 2149 * period will be for the next scan window. If local/(local+remote) ratio is 2150 * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) 2151 * the scan period will decrease. Aim for 70% local accesses. 2152 */ 2153 #define NUMA_PERIOD_SLOTS 10 2154 #define NUMA_PERIOD_THRESHOLD 7 2155 2156 /* 2157 * Increase the scan period (slow down scanning) if the majority of 2158 * our memory is already on our local node, or if the majority of 2159 * the page accesses are shared with other processes. 2160 * Otherwise, decrease the scan period. 2161 */ 2162 static void update_task_scan_period(struct task_struct *p, 2163 unsigned long shared, unsigned long private) 2164 { 2165 unsigned int period_slot; 2166 int lr_ratio, ps_ratio; 2167 int diff; 2168 2169 unsigned long remote = p->numa_faults_locality[0]; 2170 unsigned long local = p->numa_faults_locality[1]; 2171 2172 /* 2173 * If there were no record hinting faults then either the task is 2174 * completely idle or all activity is areas that are not of interest 2175 * to automatic numa balancing. Related to that, if there were failed 2176 * migration then it implies we are migrating too quickly or the local 2177 * node is overloaded. In either case, scan slower 2178 */ 2179 if (local + shared == 0 || p->numa_faults_locality[2]) { 2180 p->numa_scan_period = min(p->numa_scan_period_max, 2181 p->numa_scan_period << 1); 2182 2183 p->mm->numa_next_scan = jiffies + 2184 msecs_to_jiffies(p->numa_scan_period); 2185 2186 return; 2187 } 2188 2189 /* 2190 * Prepare to scale scan period relative to the current period. 2191 * == NUMA_PERIOD_THRESHOLD scan period stays the same 2192 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) 2193 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) 2194 */ 2195 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); 2196 lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); 2197 ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); 2198 2199 if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { 2200 /* 2201 * Most memory accesses are local. There is no need to 2202 * do fast NUMA scanning, since memory is already local. 2203 */ 2204 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; 2205 if (!slot) 2206 slot = 1; 2207 diff = slot * period_slot; 2208 } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { 2209 /* 2210 * Most memory accesses are shared with other tasks. 2211 * There is no point in continuing fast NUMA scanning, 2212 * since other tasks may just move the memory elsewhere. 2213 */ 2214 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; 2215 if (!slot) 2216 slot = 1; 2217 diff = slot * period_slot; 2218 } else { 2219 /* 2220 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, 2221 * yet they are not on the local NUMA node. Speed up 2222 * NUMA scanning to get the memory moved over. 2223 */ 2224 int ratio = max(lr_ratio, ps_ratio); 2225 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; 2226 } 2227 2228 p->numa_scan_period = clamp(p->numa_scan_period + diff, 2229 task_scan_min(p), task_scan_max(p)); 2230 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2231 } 2232 2233 /* 2234 * Get the fraction of time the task has been running since the last 2235 * NUMA placement cycle. The scheduler keeps similar statistics, but 2236 * decays those on a 32ms period, which is orders of magnitude off 2237 * from the dozens-of-seconds NUMA balancing period. Use the scheduler 2238 * stats only if the task is so new there are no NUMA statistics yet. 2239 */ 2240 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) 2241 { 2242 u64 runtime, delta, now; 2243 /* Use the start of this time slice to avoid calculations. */ 2244 now = p->se.exec_start; 2245 runtime = p->se.sum_exec_runtime; 2246 2247 if (p->last_task_numa_placement) { 2248 delta = runtime - p->last_sum_exec_runtime; 2249 *period = now - p->last_task_numa_placement; 2250 2251 /* Avoid time going backwards, prevent potential divide error: */ 2252 if (unlikely((s64)*period < 0)) 2253 *period = 0; 2254 } else { 2255 delta = p->se.avg.load_sum; 2256 *period = LOAD_AVG_MAX; 2257 } 2258 2259 p->last_sum_exec_runtime = runtime; 2260 p->last_task_numa_placement = now; 2261 2262 return delta; 2263 } 2264 2265 /* 2266 * Determine the preferred nid for a task in a numa_group. This needs to 2267 * be done in a way that produces consistent results with group_weight, 2268 * otherwise workloads might not converge. 2269 */ 2270 static int preferred_group_nid(struct task_struct *p, int nid) 2271 { 2272 nodemask_t nodes; 2273 int dist; 2274 2275 /* Direct connections between all NUMA nodes. */ 2276 if (sched_numa_topology_type == NUMA_DIRECT) 2277 return nid; 2278 2279 /* 2280 * On a system with glueless mesh NUMA topology, group_weight 2281 * scores nodes according to the number of NUMA hinting faults on 2282 * both the node itself, and on nearby nodes. 2283 */ 2284 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 2285 unsigned long score, max_score = 0; 2286 int node, max_node = nid; 2287 2288 dist = sched_max_numa_distance; 2289 2290 for_each_online_node(node) { 2291 score = group_weight(p, node, dist); 2292 if (score > max_score) { 2293 max_score = score; 2294 max_node = node; 2295 } 2296 } 2297 return max_node; 2298 } 2299 2300 /* 2301 * Finding the preferred nid in a system with NUMA backplane 2302 * interconnect topology is more involved. The goal is to locate 2303 * tasks from numa_groups near each other in the system, and 2304 * untangle workloads from different sides of the system. This requires 2305 * searching down the hierarchy of node groups, recursively searching 2306 * inside the highest scoring group of nodes. The nodemask tricks 2307 * keep the complexity of the search down. 2308 */ 2309 nodes = node_online_map; 2310 for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { 2311 unsigned long max_faults = 0; 2312 nodemask_t max_group = NODE_MASK_NONE; 2313 int a, b; 2314 2315 /* Are there nodes at this distance from each other? */ 2316 if (!find_numa_distance(dist)) 2317 continue; 2318 2319 for_each_node_mask(a, nodes) { 2320 unsigned long faults = 0; 2321 nodemask_t this_group; 2322 nodes_clear(this_group); 2323 2324 /* Sum group's NUMA faults; includes a==b case. */ 2325 for_each_node_mask(b, nodes) { 2326 if (node_distance(a, b) < dist) { 2327 faults += group_faults(p, b); 2328 node_set(b, this_group); 2329 node_clear(b, nodes); 2330 } 2331 } 2332 2333 /* Remember the top group. */ 2334 if (faults > max_faults) { 2335 max_faults = faults; 2336 max_group = this_group; 2337 /* 2338 * subtle: at the smallest distance there is 2339 * just one node left in each "group", the 2340 * winner is the preferred nid. 2341 */ 2342 nid = a; 2343 } 2344 } 2345 /* Next round, evaluate the nodes within max_group. */ 2346 if (!max_faults) 2347 break; 2348 nodes = max_group; 2349 } 2350 return nid; 2351 } 2352 2353 static void task_numa_placement(struct task_struct *p) 2354 { 2355 int seq, nid, max_nid = NUMA_NO_NODE; 2356 unsigned long max_faults = 0; 2357 unsigned long fault_types[2] = { 0, 0 }; 2358 unsigned long total_faults; 2359 u64 runtime, period; 2360 spinlock_t *group_lock = NULL; 2361 struct numa_group *ng; 2362 2363 /* 2364 * The p->mm->numa_scan_seq field gets updated without 2365 * exclusive access. Use READ_ONCE() here to ensure 2366 * that the field is read in a single access: 2367 */ 2368 seq = READ_ONCE(p->mm->numa_scan_seq); 2369 if (p->numa_scan_seq == seq) 2370 return; 2371 p->numa_scan_seq = seq; 2372 p->numa_scan_period_max = task_scan_max(p); 2373 2374 total_faults = p->numa_faults_locality[0] + 2375 p->numa_faults_locality[1]; 2376 runtime = numa_get_avg_runtime(p, &period); 2377 2378 /* If the task is part of a group prevent parallel updates to group stats */ 2379 ng = deref_curr_numa_group(p); 2380 if (ng) { 2381 group_lock = &ng->lock; 2382 spin_lock_irq(group_lock); 2383 } 2384 2385 /* Find the node with the highest number of faults */ 2386 for_each_online_node(nid) { 2387 /* Keep track of the offsets in numa_faults array */ 2388 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; 2389 unsigned long faults = 0, group_faults = 0; 2390 int priv; 2391 2392 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { 2393 long diff, f_diff, f_weight; 2394 2395 mem_idx = task_faults_idx(NUMA_MEM, nid, priv); 2396 membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); 2397 cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); 2398 cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); 2399 2400 /* Decay existing window, copy faults since last scan */ 2401 diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; 2402 fault_types[priv] += p->numa_faults[membuf_idx]; 2403 p->numa_faults[membuf_idx] = 0; 2404 2405 /* 2406 * Normalize the faults_from, so all tasks in a group 2407 * count according to CPU use, instead of by the raw 2408 * number of faults. Tasks with little runtime have 2409 * little over-all impact on throughput, and thus their 2410 * faults are less important. 2411 */ 2412 f_weight = div64_u64(runtime << 16, period + 1); 2413 f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / 2414 (total_faults + 1); 2415 f_diff = f_weight - p->numa_faults[cpu_idx] / 2; 2416 p->numa_faults[cpubuf_idx] = 0; 2417 2418 p->numa_faults[mem_idx] += diff; 2419 p->numa_faults[cpu_idx] += f_diff; 2420 faults += p->numa_faults[mem_idx]; 2421 p->total_numa_faults += diff; 2422 if (ng) { 2423 /* 2424 * safe because we can only change our own group 2425 * 2426 * mem_idx represents the offset for a given 2427 * nid and priv in a specific region because it 2428 * is at the beginning of the numa_faults array. 2429 */ 2430 ng->faults[mem_idx] += diff; 2431 ng->faults_cpu[mem_idx] += f_diff; 2432 ng->total_faults += diff; 2433 group_faults += ng->faults[mem_idx]; 2434 } 2435 } 2436 2437 if (!ng) { 2438 if (faults > max_faults) { 2439 max_faults = faults; 2440 max_nid = nid; 2441 } 2442 } else if (group_faults > max_faults) { 2443 max_faults = group_faults; 2444 max_nid = nid; 2445 } 2446 } 2447 2448 if (ng) { 2449 numa_group_count_active_nodes(ng); 2450 spin_unlock_irq(group_lock); 2451 max_nid = preferred_group_nid(p, max_nid); 2452 } 2453 2454 if (max_faults) { 2455 /* Set the new preferred node */ 2456 if (max_nid != p->numa_preferred_nid) 2457 sched_setnuma(p, max_nid); 2458 } 2459 2460 update_task_scan_period(p, fault_types[0], fault_types[1]); 2461 } 2462 2463 static inline int get_numa_group(struct numa_group *grp) 2464 { 2465 return refcount_inc_not_zero(&grp->refcount); 2466 } 2467 2468 static inline void put_numa_group(struct numa_group *grp) 2469 { 2470 if (refcount_dec_and_test(&grp->refcount)) 2471 kfree_rcu(grp, rcu); 2472 } 2473 2474 static void task_numa_group(struct task_struct *p, int cpupid, int flags, 2475 int *priv) 2476 { 2477 struct numa_group *grp, *my_grp; 2478 struct task_struct *tsk; 2479 bool join = false; 2480 int cpu = cpupid_to_cpu(cpupid); 2481 int i; 2482 2483 if (unlikely(!deref_curr_numa_group(p))) { 2484 unsigned int size = sizeof(struct numa_group) + 2485 4*nr_node_ids*sizeof(unsigned long); 2486 2487 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 2488 if (!grp) 2489 return; 2490 2491 refcount_set(&grp->refcount, 1); 2492 grp->active_nodes = 1; 2493 grp->max_faults_cpu = 0; 2494 spin_lock_init(&grp->lock); 2495 grp->gid = p->pid; 2496 /* Second half of the array tracks nids where faults happen */ 2497 grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES * 2498 nr_node_ids; 2499 2500 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2501 grp->faults[i] = p->numa_faults[i]; 2502 2503 grp->total_faults = p->total_numa_faults; 2504 2505 grp->nr_tasks++; 2506 rcu_assign_pointer(p->numa_group, grp); 2507 } 2508 2509 rcu_read_lock(); 2510 tsk = READ_ONCE(cpu_rq(cpu)->curr); 2511 2512 if (!cpupid_match_pid(tsk, cpupid)) 2513 goto no_join; 2514 2515 grp = rcu_dereference(tsk->numa_group); 2516 if (!grp) 2517 goto no_join; 2518 2519 my_grp = deref_curr_numa_group(p); 2520 if (grp == my_grp) 2521 goto no_join; 2522 2523 /* 2524 * Only join the other group if its bigger; if we're the bigger group, 2525 * the other task will join us. 2526 */ 2527 if (my_grp->nr_tasks > grp->nr_tasks) 2528 goto no_join; 2529 2530 /* 2531 * Tie-break on the grp address. 2532 */ 2533 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) 2534 goto no_join; 2535 2536 /* Always join threads in the same process. */ 2537 if (tsk->mm == current->mm) 2538 join = true; 2539 2540 /* Simple filter to avoid false positives due to PID collisions */ 2541 if (flags & TNF_SHARED) 2542 join = true; 2543 2544 /* Update priv based on whether false sharing was detected */ 2545 *priv = !join; 2546 2547 if (join && !get_numa_group(grp)) 2548 goto no_join; 2549 2550 rcu_read_unlock(); 2551 2552 if (!join) 2553 return; 2554 2555 BUG_ON(irqs_disabled()); 2556 double_lock_irq(&my_grp->lock, &grp->lock); 2557 2558 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { 2559 my_grp->faults[i] -= p->numa_faults[i]; 2560 grp->faults[i] += p->numa_faults[i]; 2561 } 2562 my_grp->total_faults -= p->total_numa_faults; 2563 grp->total_faults += p->total_numa_faults; 2564 2565 my_grp->nr_tasks--; 2566 grp->nr_tasks++; 2567 2568 spin_unlock(&my_grp->lock); 2569 spin_unlock_irq(&grp->lock); 2570 2571 rcu_assign_pointer(p->numa_group, grp); 2572 2573 put_numa_group(my_grp); 2574 return; 2575 2576 no_join: 2577 rcu_read_unlock(); 2578 return; 2579 } 2580 2581 /* 2582 * Get rid of NUMA staticstics associated with a task (either current or dead). 2583 * If @final is set, the task is dead and has reached refcount zero, so we can 2584 * safely free all relevant data structures. Otherwise, there might be 2585 * concurrent reads from places like load balancing and procfs, and we should 2586 * reset the data back to default state without freeing ->numa_faults. 2587 */ 2588 void task_numa_free(struct task_struct *p, bool final) 2589 { 2590 /* safe: p either is current or is being freed by current */ 2591 struct numa_group *grp = rcu_dereference_raw(p->numa_group); 2592 unsigned long *numa_faults = p->numa_faults; 2593 unsigned long flags; 2594 int i; 2595 2596 if (!numa_faults) 2597 return; 2598 2599 if (grp) { 2600 spin_lock_irqsave(&grp->lock, flags); 2601 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2602 grp->faults[i] -= p->numa_faults[i]; 2603 grp->total_faults -= p->total_numa_faults; 2604 2605 grp->nr_tasks--; 2606 spin_unlock_irqrestore(&grp->lock, flags); 2607 RCU_INIT_POINTER(p->numa_group, NULL); 2608 put_numa_group(grp); 2609 } 2610 2611 if (final) { 2612 p->numa_faults = NULL; 2613 kfree(numa_faults); 2614 } else { 2615 p->total_numa_faults = 0; 2616 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2617 numa_faults[i] = 0; 2618 } 2619 } 2620 2621 /* 2622 * Got a PROT_NONE fault for a page on @node. 2623 */ 2624 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) 2625 { 2626 struct task_struct *p = current; 2627 bool migrated = flags & TNF_MIGRATED; 2628 int cpu_node = task_node(current); 2629 int local = !!(flags & TNF_FAULT_LOCAL); 2630 struct numa_group *ng; 2631 int priv; 2632 2633 if (!static_branch_likely(&sched_numa_balancing)) 2634 return; 2635 2636 /* for example, ksmd faulting in a user's mm */ 2637 if (!p->mm) 2638 return; 2639 2640 /* Allocate buffer to track faults on a per-node basis */ 2641 if (unlikely(!p->numa_faults)) { 2642 int size = sizeof(*p->numa_faults) * 2643 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; 2644 2645 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); 2646 if (!p->numa_faults) 2647 return; 2648 2649 p->total_numa_faults = 0; 2650 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2651 } 2652 2653 /* 2654 * First accesses are treated as private, otherwise consider accesses 2655 * to be private if the accessing pid has not changed 2656 */ 2657 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { 2658 priv = 1; 2659 } else { 2660 priv = cpupid_match_pid(p, last_cpupid); 2661 if (!priv && !(flags & TNF_NO_GROUP)) 2662 task_numa_group(p, last_cpupid, flags, &priv); 2663 } 2664 2665 /* 2666 * If a workload spans multiple NUMA nodes, a shared fault that 2667 * occurs wholly within the set of nodes that the workload is 2668 * actively using should be counted as local. This allows the 2669 * scan rate to slow down when a workload has settled down. 2670 */ 2671 ng = deref_curr_numa_group(p); 2672 if (!priv && !local && ng && ng->active_nodes > 1 && 2673 numa_is_active_node(cpu_node, ng) && 2674 numa_is_active_node(mem_node, ng)) 2675 local = 1; 2676 2677 /* 2678 * Retry to migrate task to preferred node periodically, in case it 2679 * previously failed, or the scheduler moved us. 2680 */ 2681 if (time_after(jiffies, p->numa_migrate_retry)) { 2682 task_numa_placement(p); 2683 numa_migrate_preferred(p); 2684 } 2685 2686 if (migrated) 2687 p->numa_pages_migrated += pages; 2688 if (flags & TNF_MIGRATE_FAIL) 2689 p->numa_faults_locality[2] += pages; 2690 2691 p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; 2692 p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; 2693 p->numa_faults_locality[local] += pages; 2694 } 2695 2696 static void reset_ptenuma_scan(struct task_struct *p) 2697 { 2698 /* 2699 * We only did a read acquisition of the mmap sem, so 2700 * p->mm->numa_scan_seq is written to without exclusive access 2701 * and the update is not guaranteed to be atomic. That's not 2702 * much of an issue though, since this is just used for 2703 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not 2704 * expensive, to avoid any form of compiler optimizations: 2705 */ 2706 WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 2707 p->mm->numa_scan_offset = 0; 2708 } 2709 2710 /* 2711 * The expensive part of numa migration is done from task_work context. 2712 * Triggered from task_tick_numa(). 2713 */ 2714 static void task_numa_work(struct callback_head *work) 2715 { 2716 unsigned long migrate, next_scan, now = jiffies; 2717 struct task_struct *p = current; 2718 struct mm_struct *mm = p->mm; 2719 u64 runtime = p->se.sum_exec_runtime; 2720 struct vm_area_struct *vma; 2721 unsigned long start, end; 2722 unsigned long nr_pte_updates = 0; 2723 long pages, virtpages; 2724 2725 SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work)); 2726 2727 work->next = work; 2728 /* 2729 * Who cares about NUMA placement when they're dying. 2730 * 2731 * NOTE: make sure not to dereference p->mm before this check, 2732 * exit_task_work() happens _after_ exit_mm() so we could be called 2733 * without p->mm even though we still had it when we enqueued this 2734 * work. 2735 */ 2736 if (p->flags & PF_EXITING) 2737 return; 2738 2739 if (!mm->numa_next_scan) { 2740 mm->numa_next_scan = now + 2741 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 2742 } 2743 2744 /* 2745 * Enforce maximal scan/migration frequency.. 2746 */ 2747 migrate = mm->numa_next_scan; 2748 if (time_before(now, migrate)) 2749 return; 2750 2751 if (p->numa_scan_period == 0) { 2752 p->numa_scan_period_max = task_scan_max(p); 2753 p->numa_scan_period = task_scan_start(p); 2754 } 2755 2756 next_scan = now + msecs_to_jiffies(p->numa_scan_period); 2757 if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) 2758 return; 2759 2760 /* 2761 * Delay this task enough that another task of this mm will likely win 2762 * the next time around. 2763 */ 2764 p->node_stamp += 2 * TICK_NSEC; 2765 2766 start = mm->numa_scan_offset; 2767 pages = sysctl_numa_balancing_scan_size; 2768 pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 2769 virtpages = pages * 8; /* Scan up to this much virtual space */ 2770 if (!pages) 2771 return; 2772 2773 2774 if (!down_read_trylock(&mm->mmap_sem)) 2775 return; 2776 vma = find_vma(mm, start); 2777 if (!vma) { 2778 reset_ptenuma_scan(p); 2779 start = 0; 2780 vma = mm->mmap; 2781 } 2782 for (; vma; vma = vma->vm_next) { 2783 if (!vma_migratable(vma) || !vma_policy_mof(vma) || 2784 is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { 2785 continue; 2786 } 2787 2788 /* 2789 * Shared library pages mapped by multiple processes are not 2790 * migrated as it is expected they are cache replicated. Avoid 2791 * hinting faults in read-only file-backed mappings or the vdso 2792 * as migrating the pages will be of marginal benefit. 2793 */ 2794 if (!vma->vm_mm || 2795 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) 2796 continue; 2797 2798 /* 2799 * Skip inaccessible VMAs to avoid any confusion between 2800 * PROT_NONE and NUMA hinting ptes 2801 */ 2802 if (!vma_is_accessible(vma)) 2803 continue; 2804 2805 do { 2806 start = max(start, vma->vm_start); 2807 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 2808 end = min(end, vma->vm_end); 2809 nr_pte_updates = change_prot_numa(vma, start, end); 2810 2811 /* 2812 * Try to scan sysctl_numa_balancing_size worth of 2813 * hpages that have at least one present PTE that 2814 * is not already pte-numa. If the VMA contains 2815 * areas that are unused or already full of prot_numa 2816 * PTEs, scan up to virtpages, to skip through those 2817 * areas faster. 2818 */ 2819 if (nr_pte_updates) 2820 pages -= (end - start) >> PAGE_SHIFT; 2821 virtpages -= (end - start) >> PAGE_SHIFT; 2822 2823 start = end; 2824 if (pages <= 0 || virtpages <= 0) 2825 goto out; 2826 2827 cond_resched(); 2828 } while (end != vma->vm_end); 2829 } 2830 2831 out: 2832 /* 2833 * It is possible to reach the end of the VMA list but the last few 2834 * VMAs are not guaranteed to the vma_migratable. If they are not, we 2835 * would find the !migratable VMA on the next scan but not reset the 2836 * scanner to the start so check it now. 2837 */ 2838 if (vma) 2839 mm->numa_scan_offset = start; 2840 else 2841 reset_ptenuma_scan(p); 2842 up_read(&mm->mmap_sem); 2843 2844 /* 2845 * Make sure tasks use at least 32x as much time to run other code 2846 * than they used here, to limit NUMA PTE scanning overhead to 3% max. 2847 * Usually update_task_scan_period slows down scanning enough; on an 2848 * overloaded system we need to limit overhead on a per task basis. 2849 */ 2850 if (unlikely(p->se.sum_exec_runtime != runtime)) { 2851 u64 diff = p->se.sum_exec_runtime - runtime; 2852 p->node_stamp += 32 * diff; 2853 } 2854 } 2855 2856 void init_numa_balancing(unsigned long clone_flags, struct task_struct *p) 2857 { 2858 int mm_users = 0; 2859 struct mm_struct *mm = p->mm; 2860 2861 if (mm) { 2862 mm_users = atomic_read(&mm->mm_users); 2863 if (mm_users == 1) { 2864 mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 2865 mm->numa_scan_seq = 0; 2866 } 2867 } 2868 p->node_stamp = 0; 2869 p->numa_scan_seq = mm ? mm->numa_scan_seq : 0; 2870 p->numa_scan_period = sysctl_numa_balancing_scan_delay; 2871 /* Protect against double add, see task_tick_numa and task_numa_work */ 2872 p->numa_work.next = &p->numa_work; 2873 p->numa_faults = NULL; 2874 RCU_INIT_POINTER(p->numa_group, NULL); 2875 p->last_task_numa_placement = 0; 2876 p->last_sum_exec_runtime = 0; 2877 2878 init_task_work(&p->numa_work, task_numa_work); 2879 2880 /* New address space, reset the preferred nid */ 2881 if (!(clone_flags & CLONE_VM)) { 2882 p->numa_preferred_nid = NUMA_NO_NODE; 2883 return; 2884 } 2885 2886 /* 2887 * New thread, keep existing numa_preferred_nid which should be copied 2888 * already by arch_dup_task_struct but stagger when scans start. 2889 */ 2890 if (mm) { 2891 unsigned int delay; 2892 2893 delay = min_t(unsigned int, task_scan_max(current), 2894 current->numa_scan_period * mm_users * NSEC_PER_MSEC); 2895 delay += 2 * TICK_NSEC; 2896 p->node_stamp = delay; 2897 } 2898 } 2899 2900 /* 2901 * Drive the periodic memory faults.. 2902 */ 2903 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 2904 { 2905 struct callback_head *work = &curr->numa_work; 2906 u64 period, now; 2907 2908 /* 2909 * We don't care about NUMA placement if we don't have memory. 2910 */ 2911 if ((curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work) 2912 return; 2913 2914 /* 2915 * Using runtime rather than walltime has the dual advantage that 2916 * we (mostly) drive the selection from busy threads and that the 2917 * task needs to have done some actual work before we bother with 2918 * NUMA placement. 2919 */ 2920 now = curr->se.sum_exec_runtime; 2921 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; 2922 2923 if (now > curr->node_stamp + period) { 2924 if (!curr->node_stamp) 2925 curr->numa_scan_period = task_scan_start(curr); 2926 curr->node_stamp += period; 2927 2928 if (!time_before(jiffies, curr->mm->numa_next_scan)) 2929 task_work_add(curr, work, true); 2930 } 2931 } 2932 2933 static void update_scan_period(struct task_struct *p, int new_cpu) 2934 { 2935 int src_nid = cpu_to_node(task_cpu(p)); 2936 int dst_nid = cpu_to_node(new_cpu); 2937 2938 if (!static_branch_likely(&sched_numa_balancing)) 2939 return; 2940 2941 if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING)) 2942 return; 2943 2944 if (src_nid == dst_nid) 2945 return; 2946 2947 /* 2948 * Allow resets if faults have been trapped before one scan 2949 * has completed. This is most likely due to a new task that 2950 * is pulled cross-node due to wakeups or load balancing. 2951 */ 2952 if (p->numa_scan_seq) { 2953 /* 2954 * Avoid scan adjustments if moving to the preferred 2955 * node or if the task was not previously running on 2956 * the preferred node. 2957 */ 2958 if (dst_nid == p->numa_preferred_nid || 2959 (p->numa_preferred_nid != NUMA_NO_NODE && 2960 src_nid != p->numa_preferred_nid)) 2961 return; 2962 } 2963 2964 p->numa_scan_period = task_scan_start(p); 2965 } 2966 2967 #else 2968 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 2969 { 2970 } 2971 2972 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) 2973 { 2974 } 2975 2976 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) 2977 { 2978 } 2979 2980 static inline void update_scan_period(struct task_struct *p, int new_cpu) 2981 { 2982 } 2983 2984 #endif /* CONFIG_NUMA_BALANCING */ 2985 2986 static void 2987 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 2988 { 2989 update_load_add(&cfs_rq->load, se->load.weight); 2990 #ifdef CONFIG_SMP 2991 if (entity_is_task(se)) { 2992 struct rq *rq = rq_of(cfs_rq); 2993 2994 account_numa_enqueue(rq, task_of(se)); 2995 list_add(&se->group_node, &rq->cfs_tasks); 2996 } 2997 #endif 2998 cfs_rq->nr_running++; 2999 } 3000 3001 static void 3002 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3003 { 3004 update_load_sub(&cfs_rq->load, se->load.weight); 3005 #ifdef CONFIG_SMP 3006 if (entity_is_task(se)) { 3007 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); 3008 list_del_init(&se->group_node); 3009 } 3010 #endif 3011 cfs_rq->nr_running--; 3012 } 3013 3014 /* 3015 * Signed add and clamp on underflow. 3016 * 3017 * Explicitly do a load-store to ensure the intermediate value never hits 3018 * memory. This allows lockless observations without ever seeing the negative 3019 * values. 3020 */ 3021 #define add_positive(_ptr, _val) do { \ 3022 typeof(_ptr) ptr = (_ptr); \ 3023 typeof(_val) val = (_val); \ 3024 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3025 \ 3026 res = var + val; \ 3027 \ 3028 if (val < 0 && res > var) \ 3029 res = 0; \ 3030 \ 3031 WRITE_ONCE(*ptr, res); \ 3032 } while (0) 3033 3034 /* 3035 * Unsigned subtract and clamp on underflow. 3036 * 3037 * Explicitly do a load-store to ensure the intermediate value never hits 3038 * memory. This allows lockless observations without ever seeing the negative 3039 * values. 3040 */ 3041 #define sub_positive(_ptr, _val) do { \ 3042 typeof(_ptr) ptr = (_ptr); \ 3043 typeof(*ptr) val = (_val); \ 3044 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3045 res = var - val; \ 3046 if (res > var) \ 3047 res = 0; \ 3048 WRITE_ONCE(*ptr, res); \ 3049 } while (0) 3050 3051 /* 3052 * Remove and clamp on negative, from a local variable. 3053 * 3054 * A variant of sub_positive(), which does not use explicit load-store 3055 * and is thus optimized for local variable updates. 3056 */ 3057 #define lsub_positive(_ptr, _val) do { \ 3058 typeof(_ptr) ptr = (_ptr); \ 3059 *ptr -= min_t(typeof(*ptr), *ptr, _val); \ 3060 } while (0) 3061 3062 #ifdef CONFIG_SMP 3063 static inline void 3064 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3065 { 3066 cfs_rq->avg.load_avg += se->avg.load_avg; 3067 cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; 3068 } 3069 3070 static inline void 3071 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3072 { 3073 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); 3074 sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); 3075 } 3076 #else 3077 static inline void 3078 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 3079 static inline void 3080 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 3081 #endif 3082 3083 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 3084 unsigned long weight) 3085 { 3086 if (se->on_rq) { 3087 /* commit outstanding execution time */ 3088 if (cfs_rq->curr == se) 3089 update_curr(cfs_rq); 3090 account_entity_dequeue(cfs_rq, se); 3091 } 3092 dequeue_load_avg(cfs_rq, se); 3093 3094 update_load_set(&se->load, weight); 3095 3096 #ifdef CONFIG_SMP 3097 do { 3098 u32 divider = LOAD_AVG_MAX - 1024 + se->avg.period_contrib; 3099 3100 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 3101 } while (0); 3102 #endif 3103 3104 enqueue_load_avg(cfs_rq, se); 3105 if (se->on_rq) 3106 account_entity_enqueue(cfs_rq, se); 3107 3108 } 3109 3110 void reweight_task(struct task_struct *p, int prio) 3111 { 3112 struct sched_entity *se = &p->se; 3113 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3114 struct load_weight *load = &se->load; 3115 unsigned long weight = scale_load(sched_prio_to_weight[prio]); 3116 3117 reweight_entity(cfs_rq, se, weight); 3118 load->inv_weight = sched_prio_to_wmult[prio]; 3119 } 3120 3121 #ifdef CONFIG_FAIR_GROUP_SCHED 3122 #ifdef CONFIG_SMP 3123 /* 3124 * All this does is approximate the hierarchical proportion which includes that 3125 * global sum we all love to hate. 3126 * 3127 * That is, the weight of a group entity, is the proportional share of the 3128 * group weight based on the group runqueue weights. That is: 3129 * 3130 * tg->weight * grq->load.weight 3131 * ge->load.weight = ----------------------------- (1) 3132 * \Sum grq->load.weight 3133 * 3134 * Now, because computing that sum is prohibitively expensive to compute (been 3135 * there, done that) we approximate it with this average stuff. The average 3136 * moves slower and therefore the approximation is cheaper and more stable. 3137 * 3138 * So instead of the above, we substitute: 3139 * 3140 * grq->load.weight -> grq->avg.load_avg (2) 3141 * 3142 * which yields the following: 3143 * 3144 * tg->weight * grq->avg.load_avg 3145 * ge->load.weight = ------------------------------ (3) 3146 * tg->load_avg 3147 * 3148 * Where: tg->load_avg ~= \Sum grq->avg.load_avg 3149 * 3150 * That is shares_avg, and it is right (given the approximation (2)). 3151 * 3152 * The problem with it is that because the average is slow -- it was designed 3153 * to be exactly that of course -- this leads to transients in boundary 3154 * conditions. In specific, the case where the group was idle and we start the 3155 * one task. It takes time for our CPU's grq->avg.load_avg to build up, 3156 * yielding bad latency etc.. 3157 * 3158 * Now, in that special case (1) reduces to: 3159 * 3160 * tg->weight * grq->load.weight 3161 * ge->load.weight = ----------------------------- = tg->weight (4) 3162 * grp->load.weight 3163 * 3164 * That is, the sum collapses because all other CPUs are idle; the UP scenario. 3165 * 3166 * So what we do is modify our approximation (3) to approach (4) in the (near) 3167 * UP case, like: 3168 * 3169 * ge->load.weight = 3170 * 3171 * tg->weight * grq->load.weight 3172 * --------------------------------------------------- (5) 3173 * tg->load_avg - grq->avg.load_avg + grq->load.weight 3174 * 3175 * But because grq->load.weight can drop to 0, resulting in a divide by zero, 3176 * we need to use grq->avg.load_avg as its lower bound, which then gives: 3177 * 3178 * 3179 * tg->weight * grq->load.weight 3180 * ge->load.weight = ----------------------------- (6) 3181 * tg_load_avg' 3182 * 3183 * Where: 3184 * 3185 * tg_load_avg' = tg->load_avg - grq->avg.load_avg + 3186 * max(grq->load.weight, grq->avg.load_avg) 3187 * 3188 * And that is shares_weight and is icky. In the (near) UP case it approaches 3189 * (4) while in the normal case it approaches (3). It consistently 3190 * overestimates the ge->load.weight and therefore: 3191 * 3192 * \Sum ge->load.weight >= tg->weight 3193 * 3194 * hence icky! 3195 */ 3196 static long calc_group_shares(struct cfs_rq *cfs_rq) 3197 { 3198 long tg_weight, tg_shares, load, shares; 3199 struct task_group *tg = cfs_rq->tg; 3200 3201 tg_shares = READ_ONCE(tg->shares); 3202 3203 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 3204 3205 tg_weight = atomic_long_read(&tg->load_avg); 3206 3207 /* Ensure tg_weight >= load */ 3208 tg_weight -= cfs_rq->tg_load_avg_contrib; 3209 tg_weight += load; 3210 3211 shares = (tg_shares * load); 3212 if (tg_weight) 3213 shares /= tg_weight; 3214 3215 /* 3216 * MIN_SHARES has to be unscaled here to support per-CPU partitioning 3217 * of a group with small tg->shares value. It is a floor value which is 3218 * assigned as a minimum load.weight to the sched_entity representing 3219 * the group on a CPU. 3220 * 3221 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 3222 * on an 8-core system with 8 tasks each runnable on one CPU shares has 3223 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In 3224 * case no task is runnable on a CPU MIN_SHARES=2 should be returned 3225 * instead of 0. 3226 */ 3227 return clamp_t(long, shares, MIN_SHARES, tg_shares); 3228 } 3229 #endif /* CONFIG_SMP */ 3230 3231 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); 3232 3233 /* 3234 * Recomputes the group entity based on the current state of its group 3235 * runqueue. 3236 */ 3237 static void update_cfs_group(struct sched_entity *se) 3238 { 3239 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 3240 long shares; 3241 3242 if (!gcfs_rq) 3243 return; 3244 3245 if (throttled_hierarchy(gcfs_rq)) 3246 return; 3247 3248 #ifndef CONFIG_SMP 3249 shares = READ_ONCE(gcfs_rq->tg->shares); 3250 3251 if (likely(se->load.weight == shares)) 3252 return; 3253 #else 3254 shares = calc_group_shares(gcfs_rq); 3255 #endif 3256 3257 reweight_entity(cfs_rq_of(se), se, shares); 3258 } 3259 3260 #else /* CONFIG_FAIR_GROUP_SCHED */ 3261 static inline void update_cfs_group(struct sched_entity *se) 3262 { 3263 } 3264 #endif /* CONFIG_FAIR_GROUP_SCHED */ 3265 3266 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) 3267 { 3268 struct rq *rq = rq_of(cfs_rq); 3269 3270 if (&rq->cfs == cfs_rq) { 3271 /* 3272 * There are a few boundary cases this might miss but it should 3273 * get called often enough that that should (hopefully) not be 3274 * a real problem. 3275 * 3276 * It will not get called when we go idle, because the idle 3277 * thread is a different class (!fair), nor will the utilization 3278 * number include things like RT tasks. 3279 * 3280 * As is, the util number is not freq-invariant (we'd have to 3281 * implement arch_scale_freq_capacity() for that). 3282 * 3283 * See cpu_util(). 3284 */ 3285 cpufreq_update_util(rq, flags); 3286 } 3287 } 3288 3289 #ifdef CONFIG_SMP 3290 #ifdef CONFIG_FAIR_GROUP_SCHED 3291 /** 3292 * update_tg_load_avg - update the tg's load avg 3293 * @cfs_rq: the cfs_rq whose avg changed 3294 * @force: update regardless of how small the difference 3295 * 3296 * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. 3297 * However, because tg->load_avg is a global value there are performance 3298 * considerations. 3299 * 3300 * In order to avoid having to look at the other cfs_rq's, we use a 3301 * differential update where we store the last value we propagated. This in 3302 * turn allows skipping updates if the differential is 'small'. 3303 * 3304 * Updating tg's load_avg is necessary before update_cfs_share(). 3305 */ 3306 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) 3307 { 3308 long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 3309 3310 /* 3311 * No need to update load_avg for root_task_group as it is not used. 3312 */ 3313 if (cfs_rq->tg == &root_task_group) 3314 return; 3315 3316 if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 3317 atomic_long_add(delta, &cfs_rq->tg->load_avg); 3318 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 3319 } 3320 } 3321 3322 /* 3323 * Called within set_task_rq() right before setting a task's CPU. The 3324 * caller only guarantees p->pi_lock is held; no other assumptions, 3325 * including the state of rq->lock, should be made. 3326 */ 3327 void set_task_rq_fair(struct sched_entity *se, 3328 struct cfs_rq *prev, struct cfs_rq *next) 3329 { 3330 u64 p_last_update_time; 3331 u64 n_last_update_time; 3332 3333 if (!sched_feat(ATTACH_AGE_LOAD)) 3334 return; 3335 3336 /* 3337 * We are supposed to update the task to "current" time, then its up to 3338 * date and ready to go to new CPU/cfs_rq. But we have difficulty in 3339 * getting what current time is, so simply throw away the out-of-date 3340 * time. This will result in the wakee task is less decayed, but giving 3341 * the wakee more load sounds not bad. 3342 */ 3343 if (!(se->avg.last_update_time && prev)) 3344 return; 3345 3346 #ifndef CONFIG_64BIT 3347 { 3348 u64 p_last_update_time_copy; 3349 u64 n_last_update_time_copy; 3350 3351 do { 3352 p_last_update_time_copy = prev->load_last_update_time_copy; 3353 n_last_update_time_copy = next->load_last_update_time_copy; 3354 3355 smp_rmb(); 3356 3357 p_last_update_time = prev->avg.last_update_time; 3358 n_last_update_time = next->avg.last_update_time; 3359 3360 } while (p_last_update_time != p_last_update_time_copy || 3361 n_last_update_time != n_last_update_time_copy); 3362 } 3363 #else 3364 p_last_update_time = prev->avg.last_update_time; 3365 n_last_update_time = next->avg.last_update_time; 3366 #endif 3367 __update_load_avg_blocked_se(p_last_update_time, se); 3368 se->avg.last_update_time = n_last_update_time; 3369 } 3370 3371 3372 /* 3373 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to 3374 * propagate its contribution. The key to this propagation is the invariant 3375 * that for each group: 3376 * 3377 * ge->avg == grq->avg (1) 3378 * 3379 * _IFF_ we look at the pure running and runnable sums. Because they 3380 * represent the very same entity, just at different points in the hierarchy. 3381 * 3382 * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial 3383 * and simply copies the running/runnable sum over (but still wrong, because 3384 * the group entity and group rq do not have their PELT windows aligned). 3385 * 3386 * However, update_tg_cfs_load() is more complex. So we have: 3387 * 3388 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2) 3389 * 3390 * And since, like util, the runnable part should be directly transferable, 3391 * the following would _appear_ to be the straight forward approach: 3392 * 3393 * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3) 3394 * 3395 * And per (1) we have: 3396 * 3397 * ge->avg.runnable_avg == grq->avg.runnable_avg 3398 * 3399 * Which gives: 3400 * 3401 * ge->load.weight * grq->avg.load_avg 3402 * ge->avg.load_avg = ----------------------------------- (4) 3403 * grq->load.weight 3404 * 3405 * Except that is wrong! 3406 * 3407 * Because while for entities historical weight is not important and we 3408 * really only care about our future and therefore can consider a pure 3409 * runnable sum, runqueues can NOT do this. 3410 * 3411 * We specifically want runqueues to have a load_avg that includes 3412 * historical weights. Those represent the blocked load, the load we expect 3413 * to (shortly) return to us. This only works by keeping the weights as 3414 * integral part of the sum. We therefore cannot decompose as per (3). 3415 * 3416 * Another reason this doesn't work is that runnable isn't a 0-sum entity. 3417 * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the 3418 * rq itself is runnable anywhere between 2/3 and 1 depending on how the 3419 * runnable section of these tasks overlap (or not). If they were to perfectly 3420 * align the rq as a whole would be runnable 2/3 of the time. If however we 3421 * always have at least 1 runnable task, the rq as a whole is always runnable. 3422 * 3423 * So we'll have to approximate.. :/ 3424 * 3425 * Given the constraint: 3426 * 3427 * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX 3428 * 3429 * We can construct a rule that adds runnable to a rq by assuming minimal 3430 * overlap. 3431 * 3432 * On removal, we'll assume each task is equally runnable; which yields: 3433 * 3434 * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight 3435 * 3436 * XXX: only do this for the part of runnable > running ? 3437 * 3438 */ 3439 3440 static inline void 3441 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 3442 { 3443 long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; 3444 3445 /* Nothing to update */ 3446 if (!delta) 3447 return; 3448 3449 /* 3450 * The relation between sum and avg is: 3451 * 3452 * LOAD_AVG_MAX - 1024 + sa->period_contrib 3453 * 3454 * however, the PELT windows are not aligned between grq and gse. 3455 */ 3456 3457 /* Set new sched_entity's utilization */ 3458 se->avg.util_avg = gcfs_rq->avg.util_avg; 3459 se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 3460 3461 /* Update parent cfs_rq utilization */ 3462 add_positive(&cfs_rq->avg.util_avg, delta); 3463 cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; 3464 } 3465 3466 static inline void 3467 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 3468 { 3469 long delta = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; 3470 3471 /* Nothing to update */ 3472 if (!delta) 3473 return; 3474 3475 /* 3476 * The relation between sum and avg is: 3477 * 3478 * LOAD_AVG_MAX - 1024 + sa->period_contrib 3479 * 3480 * however, the PELT windows are not aligned between grq and gse. 3481 */ 3482 3483 /* Set new sched_entity's runnable */ 3484 se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; 3485 se->avg.runnable_sum = se->avg.runnable_avg * LOAD_AVG_MAX; 3486 3487 /* Update parent cfs_rq runnable */ 3488 add_positive(&cfs_rq->avg.runnable_avg, delta); 3489 cfs_rq->avg.runnable_sum = cfs_rq->avg.runnable_avg * LOAD_AVG_MAX; 3490 } 3491 3492 static inline void 3493 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 3494 { 3495 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 3496 unsigned long load_avg; 3497 u64 load_sum = 0; 3498 s64 delta_sum; 3499 3500 if (!runnable_sum) 3501 return; 3502 3503 gcfs_rq->prop_runnable_sum = 0; 3504 3505 if (runnable_sum >= 0) { 3506 /* 3507 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until 3508 * the CPU is saturated running == runnable. 3509 */ 3510 runnable_sum += se->avg.load_sum; 3511 runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 3512 } else { 3513 /* 3514 * Estimate the new unweighted runnable_sum of the gcfs_rq by 3515 * assuming all tasks are equally runnable. 3516 */ 3517 if (scale_load_down(gcfs_rq->load.weight)) { 3518 load_sum = div_s64(gcfs_rq->avg.load_sum, 3519 scale_load_down(gcfs_rq->load.weight)); 3520 } 3521 3522 /* But make sure to not inflate se's runnable */ 3523 runnable_sum = min(se->avg.load_sum, load_sum); 3524 } 3525 3526 /* 3527 * runnable_sum can't be lower than running_sum 3528 * Rescale running sum to be in the same range as runnable sum 3529 * running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT] 3530 * runnable_sum is in [0 : LOAD_AVG_MAX] 3531 */ 3532 running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 3533 runnable_sum = max(runnable_sum, running_sum); 3534 3535 load_sum = (s64)se_weight(se) * runnable_sum; 3536 load_avg = div_s64(load_sum, LOAD_AVG_MAX); 3537 3538 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 3539 delta_avg = load_avg - se->avg.load_avg; 3540 3541 se->avg.load_sum = runnable_sum; 3542 se->avg.load_avg = load_avg; 3543 add_positive(&cfs_rq->avg.load_avg, delta_avg); 3544 add_positive(&cfs_rq->avg.load_sum, delta_sum); 3545 } 3546 3547 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) 3548 { 3549 cfs_rq->propagate = 1; 3550 cfs_rq->prop_runnable_sum += runnable_sum; 3551 } 3552 3553 /* Update task and its cfs_rq load average */ 3554 static inline int propagate_entity_load_avg(struct sched_entity *se) 3555 { 3556 struct cfs_rq *cfs_rq, *gcfs_rq; 3557 3558 if (entity_is_task(se)) 3559 return 0; 3560 3561 gcfs_rq = group_cfs_rq(se); 3562 if (!gcfs_rq->propagate) 3563 return 0; 3564 3565 gcfs_rq->propagate = 0; 3566 3567 cfs_rq = cfs_rq_of(se); 3568 3569 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 3570 3571 update_tg_cfs_util(cfs_rq, se, gcfs_rq); 3572 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); 3573 update_tg_cfs_load(cfs_rq, se, gcfs_rq); 3574 3575 trace_pelt_cfs_tp(cfs_rq); 3576 trace_pelt_se_tp(se); 3577 3578 return 1; 3579 } 3580 3581 /* 3582 * Check if we need to update the load and the utilization of a blocked 3583 * group_entity: 3584 */ 3585 static inline bool skip_blocked_update(struct sched_entity *se) 3586 { 3587 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 3588 3589 /* 3590 * If sched_entity still have not zero load or utilization, we have to 3591 * decay it: 3592 */ 3593 if (se->avg.load_avg || se->avg.util_avg) 3594 return false; 3595 3596 /* 3597 * If there is a pending propagation, we have to update the load and 3598 * the utilization of the sched_entity: 3599 */ 3600 if (gcfs_rq->propagate) 3601 return false; 3602 3603 /* 3604 * Otherwise, the load and the utilization of the sched_entity is 3605 * already zero and there is no pending propagation, so it will be a 3606 * waste of time to try to decay it: 3607 */ 3608 return true; 3609 } 3610 3611 #else /* CONFIG_FAIR_GROUP_SCHED */ 3612 3613 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {} 3614 3615 static inline int propagate_entity_load_avg(struct sched_entity *se) 3616 { 3617 return 0; 3618 } 3619 3620 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} 3621 3622 #endif /* CONFIG_FAIR_GROUP_SCHED */ 3623 3624 /** 3625 * update_cfs_rq_load_avg - update the cfs_rq's load/util averages 3626 * @now: current time, as per cfs_rq_clock_pelt() 3627 * @cfs_rq: cfs_rq to update 3628 * 3629 * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) 3630 * avg. The immediate corollary is that all (fair) tasks must be attached, see 3631 * post_init_entity_util_avg(). 3632 * 3633 * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. 3634 * 3635 * Returns true if the load decayed or we removed load. 3636 * 3637 * Since both these conditions indicate a changed cfs_rq->avg.load we should 3638 * call update_tg_load_avg() when this function returns true. 3639 */ 3640 static inline int 3641 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 3642 { 3643 unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0; 3644 struct sched_avg *sa = &cfs_rq->avg; 3645 int decayed = 0; 3646 3647 if (cfs_rq->removed.nr) { 3648 unsigned long r; 3649 u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 3650 3651 raw_spin_lock(&cfs_rq->removed.lock); 3652 swap(cfs_rq->removed.util_avg, removed_util); 3653 swap(cfs_rq->removed.load_avg, removed_load); 3654 swap(cfs_rq->removed.runnable_avg, removed_runnable); 3655 cfs_rq->removed.nr = 0; 3656 raw_spin_unlock(&cfs_rq->removed.lock); 3657 3658 r = removed_load; 3659 sub_positive(&sa->load_avg, r); 3660 sub_positive(&sa->load_sum, r * divider); 3661 3662 r = removed_util; 3663 sub_positive(&sa->util_avg, r); 3664 sub_positive(&sa->util_sum, r * divider); 3665 3666 r = removed_runnable; 3667 sub_positive(&sa->runnable_avg, r); 3668 sub_positive(&sa->runnable_sum, r * divider); 3669 3670 /* 3671 * removed_runnable is the unweighted version of removed_load so we 3672 * can use it to estimate removed_load_sum. 3673 */ 3674 add_tg_cfs_propagate(cfs_rq, 3675 -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT); 3676 3677 decayed = 1; 3678 } 3679 3680 decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 3681 3682 #ifndef CONFIG_64BIT 3683 smp_wmb(); 3684 cfs_rq->load_last_update_time_copy = sa->last_update_time; 3685 #endif 3686 3687 return decayed; 3688 } 3689 3690 /** 3691 * attach_entity_load_avg - attach this entity to its cfs_rq load avg 3692 * @cfs_rq: cfs_rq to attach to 3693 * @se: sched_entity to attach 3694 * 3695 * Must call update_cfs_rq_load_avg() before this, since we rely on 3696 * cfs_rq->avg.last_update_time being current. 3697 */ 3698 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3699 { 3700 u32 divider = LOAD_AVG_MAX - 1024 + cfs_rq->avg.period_contrib; 3701 3702 /* 3703 * When we attach the @se to the @cfs_rq, we must align the decay 3704 * window because without that, really weird and wonderful things can 3705 * happen. 3706 * 3707 * XXX illustrate 3708 */ 3709 se->avg.last_update_time = cfs_rq->avg.last_update_time; 3710 se->avg.period_contrib = cfs_rq->avg.period_contrib; 3711 3712 /* 3713 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new 3714 * period_contrib. This isn't strictly correct, but since we're 3715 * entirely outside of the PELT hierarchy, nobody cares if we truncate 3716 * _sum a little. 3717 */ 3718 se->avg.util_sum = se->avg.util_avg * divider; 3719 3720 se->avg.runnable_sum = se->avg.runnable_avg * divider; 3721 3722 se->avg.load_sum = divider; 3723 if (se_weight(se)) { 3724 se->avg.load_sum = 3725 div_u64(se->avg.load_avg * se->avg.load_sum, se_weight(se)); 3726 } 3727 3728 enqueue_load_avg(cfs_rq, se); 3729 cfs_rq->avg.util_avg += se->avg.util_avg; 3730 cfs_rq->avg.util_sum += se->avg.util_sum; 3731 cfs_rq->avg.runnable_avg += se->avg.runnable_avg; 3732 cfs_rq->avg.runnable_sum += se->avg.runnable_sum; 3733 3734 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); 3735 3736 cfs_rq_util_change(cfs_rq, 0); 3737 3738 trace_pelt_cfs_tp(cfs_rq); 3739 } 3740 3741 /** 3742 * detach_entity_load_avg - detach this entity from its cfs_rq load avg 3743 * @cfs_rq: cfs_rq to detach from 3744 * @se: sched_entity to detach 3745 * 3746 * Must call update_cfs_rq_load_avg() before this, since we rely on 3747 * cfs_rq->avg.last_update_time being current. 3748 */ 3749 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3750 { 3751 dequeue_load_avg(cfs_rq, se); 3752 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); 3753 sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); 3754 sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg); 3755 sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum); 3756 3757 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); 3758 3759 cfs_rq_util_change(cfs_rq, 0); 3760 3761 trace_pelt_cfs_tp(cfs_rq); 3762 } 3763 3764 /* 3765 * Optional action to be done while updating the load average 3766 */ 3767 #define UPDATE_TG 0x1 3768 #define SKIP_AGE_LOAD 0x2 3769 #define DO_ATTACH 0x4 3770 3771 /* Update task and its cfs_rq load average */ 3772 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 3773 { 3774 u64 now = cfs_rq_clock_pelt(cfs_rq); 3775 int decayed; 3776 3777 /* 3778 * Track task load average for carrying it to new CPU after migrated, and 3779 * track group sched_entity load average for task_h_load calc in migration 3780 */ 3781 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 3782 __update_load_avg_se(now, cfs_rq, se); 3783 3784 decayed = update_cfs_rq_load_avg(now, cfs_rq); 3785 decayed |= propagate_entity_load_avg(se); 3786 3787 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 3788 3789 /* 3790 * DO_ATTACH means we're here from enqueue_entity(). 3791 * !last_update_time means we've passed through 3792 * migrate_task_rq_fair() indicating we migrated. 3793 * 3794 * IOW we're enqueueing a task on a new CPU. 3795 */ 3796 attach_entity_load_avg(cfs_rq, se); 3797 update_tg_load_avg(cfs_rq, 0); 3798 3799 } else if (decayed) { 3800 cfs_rq_util_change(cfs_rq, 0); 3801 3802 if (flags & UPDATE_TG) 3803 update_tg_load_avg(cfs_rq, 0); 3804 } 3805 } 3806 3807 #ifndef CONFIG_64BIT 3808 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 3809 { 3810 u64 last_update_time_copy; 3811 u64 last_update_time; 3812 3813 do { 3814 last_update_time_copy = cfs_rq->load_last_update_time_copy; 3815 smp_rmb(); 3816 last_update_time = cfs_rq->avg.last_update_time; 3817 } while (last_update_time != last_update_time_copy); 3818 3819 return last_update_time; 3820 } 3821 #else 3822 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 3823 { 3824 return cfs_rq->avg.last_update_time; 3825 } 3826 #endif 3827 3828 /* 3829 * Synchronize entity load avg of dequeued entity without locking 3830 * the previous rq. 3831 */ 3832 static void sync_entity_load_avg(struct sched_entity *se) 3833 { 3834 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3835 u64 last_update_time; 3836 3837 last_update_time = cfs_rq_last_update_time(cfs_rq); 3838 __update_load_avg_blocked_se(last_update_time, se); 3839 } 3840 3841 /* 3842 * Task first catches up with cfs_rq, and then subtract 3843 * itself from the cfs_rq (task must be off the queue now). 3844 */ 3845 static void remove_entity_load_avg(struct sched_entity *se) 3846 { 3847 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3848 unsigned long flags; 3849 3850 /* 3851 * tasks cannot exit without having gone through wake_up_new_task() -> 3852 * post_init_entity_util_avg() which will have added things to the 3853 * cfs_rq, so we can remove unconditionally. 3854 */ 3855 3856 sync_entity_load_avg(se); 3857 3858 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); 3859 ++cfs_rq->removed.nr; 3860 cfs_rq->removed.util_avg += se->avg.util_avg; 3861 cfs_rq->removed.load_avg += se->avg.load_avg; 3862 cfs_rq->removed.runnable_avg += se->avg.runnable_avg; 3863 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); 3864 } 3865 3866 static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq) 3867 { 3868 return cfs_rq->avg.runnable_avg; 3869 } 3870 3871 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 3872 { 3873 return cfs_rq->avg.load_avg; 3874 } 3875 3876 static inline unsigned long task_util(struct task_struct *p) 3877 { 3878 return READ_ONCE(p->se.avg.util_avg); 3879 } 3880 3881 static inline unsigned long _task_util_est(struct task_struct *p) 3882 { 3883 struct util_est ue = READ_ONCE(p->se.avg.util_est); 3884 3885 return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 3886 } 3887 3888 static inline unsigned long task_util_est(struct task_struct *p) 3889 { 3890 return max(task_util(p), _task_util_est(p)); 3891 } 3892 3893 #ifdef CONFIG_UCLAMP_TASK 3894 static inline unsigned long uclamp_task_util(struct task_struct *p) 3895 { 3896 return clamp(task_util_est(p), 3897 uclamp_eff_value(p, UCLAMP_MIN), 3898 uclamp_eff_value(p, UCLAMP_MAX)); 3899 } 3900 #else 3901 static inline unsigned long uclamp_task_util(struct task_struct *p) 3902 { 3903 return task_util_est(p); 3904 } 3905 #endif 3906 3907 static inline void util_est_enqueue(struct cfs_rq *cfs_rq, 3908 struct task_struct *p) 3909 { 3910 unsigned int enqueued; 3911 3912 if (!sched_feat(UTIL_EST)) 3913 return; 3914 3915 /* Update root cfs_rq's estimated utilization */ 3916 enqueued = cfs_rq->avg.util_est.enqueued; 3917 enqueued += _task_util_est(p); 3918 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued); 3919 } 3920 3921 /* 3922 * Check if a (signed) value is within a specified (unsigned) margin, 3923 * based on the observation that: 3924 * 3925 * abs(x) < y := (unsigned)(x + y - 1) < (2 * y - 1) 3926 * 3927 * NOTE: this only works when value + maring < INT_MAX. 3928 */ 3929 static inline bool within_margin(int value, int margin) 3930 { 3931 return ((unsigned int)(value + margin - 1) < (2 * margin - 1)); 3932 } 3933 3934 static void 3935 util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p, bool task_sleep) 3936 { 3937 long last_ewma_diff; 3938 struct util_est ue; 3939 int cpu; 3940 3941 if (!sched_feat(UTIL_EST)) 3942 return; 3943 3944 /* Update root cfs_rq's estimated utilization */ 3945 ue.enqueued = cfs_rq->avg.util_est.enqueued; 3946 ue.enqueued -= min_t(unsigned int, ue.enqueued, _task_util_est(p)); 3947 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, ue.enqueued); 3948 3949 /* 3950 * Skip update of task's estimated utilization when the task has not 3951 * yet completed an activation, e.g. being migrated. 3952 */ 3953 if (!task_sleep) 3954 return; 3955 3956 /* 3957 * If the PELT values haven't changed since enqueue time, 3958 * skip the util_est update. 3959 */ 3960 ue = p->se.avg.util_est; 3961 if (ue.enqueued & UTIL_AVG_UNCHANGED) 3962 return; 3963 3964 /* 3965 * Reset EWMA on utilization increases, the moving average is used only 3966 * to smooth utilization decreases. 3967 */ 3968 ue.enqueued = (task_util(p) | UTIL_AVG_UNCHANGED); 3969 if (sched_feat(UTIL_EST_FASTUP)) { 3970 if (ue.ewma < ue.enqueued) { 3971 ue.ewma = ue.enqueued; 3972 goto done; 3973 } 3974 } 3975 3976 /* 3977 * Skip update of task's estimated utilization when its EWMA is 3978 * already ~1% close to its last activation value. 3979 */ 3980 last_ewma_diff = ue.enqueued - ue.ewma; 3981 if (within_margin(last_ewma_diff, (SCHED_CAPACITY_SCALE / 100))) 3982 return; 3983 3984 /* 3985 * To avoid overestimation of actual task utilization, skip updates if 3986 * we cannot grant there is idle time in this CPU. 3987 */ 3988 cpu = cpu_of(rq_of(cfs_rq)); 3989 if (task_util(p) > capacity_orig_of(cpu)) 3990 return; 3991 3992 /* 3993 * Update Task's estimated utilization 3994 * 3995 * When *p completes an activation we can consolidate another sample 3996 * of the task size. This is done by storing the current PELT value 3997 * as ue.enqueued and by using this value to update the Exponential 3998 * Weighted Moving Average (EWMA): 3999 * 4000 * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) 4001 * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) 4002 * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) 4003 * = w * ( last_ewma_diff ) + ewma(t-1) 4004 * = w * (last_ewma_diff + ewma(t-1) / w) 4005 * 4006 * Where 'w' is the weight of new samples, which is configured to be 4007 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) 4008 */ 4009 ue.ewma <<= UTIL_EST_WEIGHT_SHIFT; 4010 ue.ewma += last_ewma_diff; 4011 ue.ewma >>= UTIL_EST_WEIGHT_SHIFT; 4012 done: 4013 WRITE_ONCE(p->se.avg.util_est, ue); 4014 } 4015 4016 static inline int task_fits_capacity(struct task_struct *p, long capacity) 4017 { 4018 return fits_capacity(uclamp_task_util(p), capacity); 4019 } 4020 4021 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) 4022 { 4023 if (!static_branch_unlikely(&sched_asym_cpucapacity)) 4024 return; 4025 4026 if (!p) { 4027 rq->misfit_task_load = 0; 4028 return; 4029 } 4030 4031 if (task_fits_capacity(p, capacity_of(cpu_of(rq)))) { 4032 rq->misfit_task_load = 0; 4033 return; 4034 } 4035 4036 rq->misfit_task_load = task_h_load(p); 4037 } 4038 4039 #else /* CONFIG_SMP */ 4040 4041 #define UPDATE_TG 0x0 4042 #define SKIP_AGE_LOAD 0x0 4043 #define DO_ATTACH 0x0 4044 4045 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1) 4046 { 4047 cfs_rq_util_change(cfs_rq, 0); 4048 } 4049 4050 static inline void remove_entity_load_avg(struct sched_entity *se) {} 4051 4052 static inline void 4053 attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} 4054 static inline void 4055 detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} 4056 4057 static inline int idle_balance(struct rq *rq, struct rq_flags *rf) 4058 { 4059 return 0; 4060 } 4061 4062 static inline void 4063 util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {} 4064 4065 static inline void 4066 util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p, 4067 bool task_sleep) {} 4068 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) {} 4069 4070 #endif /* CONFIG_SMP */ 4071 4072 static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se) 4073 { 4074 #ifdef CONFIG_SCHED_DEBUG 4075 s64 d = se->vruntime - cfs_rq->min_vruntime; 4076 4077 if (d < 0) 4078 d = -d; 4079 4080 if (d > 3*sysctl_sched_latency) 4081 schedstat_inc(cfs_rq->nr_spread_over); 4082 #endif 4083 } 4084 4085 static void 4086 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) 4087 { 4088 u64 vruntime = cfs_rq->min_vruntime; 4089 4090 /* 4091 * The 'current' period is already promised to the current tasks, 4092 * however the extra weight of the new task will slow them down a 4093 * little, place the new task so that it fits in the slot that 4094 * stays open at the end. 4095 */ 4096 if (initial && sched_feat(START_DEBIT)) 4097 vruntime += sched_vslice(cfs_rq, se); 4098 4099 /* sleeps up to a single latency don't count. */ 4100 if (!initial) { 4101 unsigned long thresh = sysctl_sched_latency; 4102 4103 /* 4104 * Halve their sleep time's effect, to allow 4105 * for a gentler effect of sleepers: 4106 */ 4107 if (sched_feat(GENTLE_FAIR_SLEEPERS)) 4108 thresh >>= 1; 4109 4110 vruntime -= thresh; 4111 } 4112 4113 /* ensure we never gain time by being placed backwards. */ 4114 se->vruntime = max_vruntime(se->vruntime, vruntime); 4115 } 4116 4117 static void check_enqueue_throttle(struct cfs_rq *cfs_rq); 4118 4119 static inline void check_schedstat_required(void) 4120 { 4121 #ifdef CONFIG_SCHEDSTATS 4122 if (schedstat_enabled()) 4123 return; 4124 4125 /* Force schedstat enabled if a dependent tracepoint is active */ 4126 if (trace_sched_stat_wait_enabled() || 4127 trace_sched_stat_sleep_enabled() || 4128 trace_sched_stat_iowait_enabled() || 4129 trace_sched_stat_blocked_enabled() || 4130 trace_sched_stat_runtime_enabled()) { 4131 printk_deferred_once("Scheduler tracepoints stat_sleep, stat_iowait, " 4132 "stat_blocked and stat_runtime require the " 4133 "kernel parameter schedstats=enable or " 4134 "kernel.sched_schedstats=1\n"); 4135 } 4136 #endif 4137 } 4138 4139 static inline bool cfs_bandwidth_used(void); 4140 4141 /* 4142 * MIGRATION 4143 * 4144 * dequeue 4145 * update_curr() 4146 * update_min_vruntime() 4147 * vruntime -= min_vruntime 4148 * 4149 * enqueue 4150 * update_curr() 4151 * update_min_vruntime() 4152 * vruntime += min_vruntime 4153 * 4154 * this way the vruntime transition between RQs is done when both 4155 * min_vruntime are up-to-date. 4156 * 4157 * WAKEUP (remote) 4158 * 4159 * ->migrate_task_rq_fair() (p->state == TASK_WAKING) 4160 * vruntime -= min_vruntime 4161 * 4162 * enqueue 4163 * update_curr() 4164 * update_min_vruntime() 4165 * vruntime += min_vruntime 4166 * 4167 * this way we don't have the most up-to-date min_vruntime on the originating 4168 * CPU and an up-to-date min_vruntime on the destination CPU. 4169 */ 4170 4171 static void 4172 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4173 { 4174 bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED); 4175 bool curr = cfs_rq->curr == se; 4176 4177 /* 4178 * If we're the current task, we must renormalise before calling 4179 * update_curr(). 4180 */ 4181 if (renorm && curr) 4182 se->vruntime += cfs_rq->min_vruntime; 4183 4184 update_curr(cfs_rq); 4185 4186 /* 4187 * Otherwise, renormalise after, such that we're placed at the current 4188 * moment in time, instead of some random moment in the past. Being 4189 * placed in the past could significantly boost this task to the 4190 * fairness detriment of existing tasks. 4191 */ 4192 if (renorm && !curr) 4193 se->vruntime += cfs_rq->min_vruntime; 4194 4195 /* 4196 * When enqueuing a sched_entity, we must: 4197 * - Update loads to have both entity and cfs_rq synced with now. 4198 * - Add its load to cfs_rq->runnable_avg 4199 * - For group_entity, update its weight to reflect the new share of 4200 * its group cfs_rq 4201 * - Add its new weight to cfs_rq->load.weight 4202 */ 4203 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); 4204 se_update_runnable(se); 4205 update_cfs_group(se); 4206 account_entity_enqueue(cfs_rq, se); 4207 4208 if (flags & ENQUEUE_WAKEUP) 4209 place_entity(cfs_rq, se, 0); 4210 4211 check_schedstat_required(); 4212 update_stats_enqueue(cfs_rq, se, flags); 4213 check_spread(cfs_rq, se); 4214 if (!curr) 4215 __enqueue_entity(cfs_rq, se); 4216 se->on_rq = 1; 4217 4218 /* 4219 * When bandwidth control is enabled, cfs might have been removed 4220 * because of a parent been throttled but cfs->nr_running > 1. Try to 4221 * add it unconditionnally. 4222 */ 4223 if (cfs_rq->nr_running == 1 || cfs_bandwidth_used()) 4224 list_add_leaf_cfs_rq(cfs_rq); 4225 4226 if (cfs_rq->nr_running == 1) 4227 check_enqueue_throttle(cfs_rq); 4228 } 4229 4230 static void __clear_buddies_last(struct sched_entity *se) 4231 { 4232 for_each_sched_entity(se) { 4233 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4234 if (cfs_rq->last != se) 4235 break; 4236 4237 cfs_rq->last = NULL; 4238 } 4239 } 4240 4241 static void __clear_buddies_next(struct sched_entity *se) 4242 { 4243 for_each_sched_entity(se) { 4244 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4245 if (cfs_rq->next != se) 4246 break; 4247 4248 cfs_rq->next = NULL; 4249 } 4250 } 4251 4252 static void __clear_buddies_skip(struct sched_entity *se) 4253 { 4254 for_each_sched_entity(se) { 4255 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4256 if (cfs_rq->skip != se) 4257 break; 4258 4259 cfs_rq->skip = NULL; 4260 } 4261 } 4262 4263 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 4264 { 4265 if (cfs_rq->last == se) 4266 __clear_buddies_last(se); 4267 4268 if (cfs_rq->next == se) 4269 __clear_buddies_next(se); 4270 4271 if (cfs_rq->skip == se) 4272 __clear_buddies_skip(se); 4273 } 4274 4275 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 4276 4277 static void 4278 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4279 { 4280 /* 4281 * Update run-time statistics of the 'current'. 4282 */ 4283 update_curr(cfs_rq); 4284 4285 /* 4286 * When dequeuing a sched_entity, we must: 4287 * - Update loads to have both entity and cfs_rq synced with now. 4288 * - Subtract its load from the cfs_rq->runnable_avg. 4289 * - Subtract its previous weight from cfs_rq->load.weight. 4290 * - For group entity, update its weight to reflect the new share 4291 * of its group cfs_rq. 4292 */ 4293 update_load_avg(cfs_rq, se, UPDATE_TG); 4294 se_update_runnable(se); 4295 4296 update_stats_dequeue(cfs_rq, se, flags); 4297 4298 clear_buddies(cfs_rq, se); 4299 4300 if (se != cfs_rq->curr) 4301 __dequeue_entity(cfs_rq, se); 4302 se->on_rq = 0; 4303 account_entity_dequeue(cfs_rq, se); 4304 4305 /* 4306 * Normalize after update_curr(); which will also have moved 4307 * min_vruntime if @se is the one holding it back. But before doing 4308 * update_min_vruntime() again, which will discount @se's position and 4309 * can move min_vruntime forward still more. 4310 */ 4311 if (!(flags & DEQUEUE_SLEEP)) 4312 se->vruntime -= cfs_rq->min_vruntime; 4313 4314 /* return excess runtime on last dequeue */ 4315 return_cfs_rq_runtime(cfs_rq); 4316 4317 update_cfs_group(se); 4318 4319 /* 4320 * Now advance min_vruntime if @se was the entity holding it back, 4321 * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be 4322 * put back on, and if we advance min_vruntime, we'll be placed back 4323 * further than we started -- ie. we'll be penalized. 4324 */ 4325 if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE) 4326 update_min_vruntime(cfs_rq); 4327 } 4328 4329 /* 4330 * Preempt the current task with a newly woken task if needed: 4331 */ 4332 static void 4333 check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) 4334 { 4335 unsigned long ideal_runtime, delta_exec; 4336 struct sched_entity *se; 4337 s64 delta; 4338 4339 ideal_runtime = sched_slice(cfs_rq, curr); 4340 delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; 4341 if (delta_exec > ideal_runtime) { 4342 resched_curr(rq_of(cfs_rq)); 4343 /* 4344 * The current task ran long enough, ensure it doesn't get 4345 * re-elected due to buddy favours. 4346 */ 4347 clear_buddies(cfs_rq, curr); 4348 return; 4349 } 4350 4351 /* 4352 * Ensure that a task that missed wakeup preemption by a 4353 * narrow margin doesn't have to wait for a full slice. 4354 * This also mitigates buddy induced latencies under load. 4355 */ 4356 if (delta_exec < sysctl_sched_min_granularity) 4357 return; 4358 4359 se = __pick_first_entity(cfs_rq); 4360 delta = curr->vruntime - se->vruntime; 4361 4362 if (delta < 0) 4363 return; 4364 4365 if (delta > ideal_runtime) 4366 resched_curr(rq_of(cfs_rq)); 4367 } 4368 4369 static void 4370 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 4371 { 4372 /* 'current' is not kept within the tree. */ 4373 if (se->on_rq) { 4374 /* 4375 * Any task has to be enqueued before it get to execute on 4376 * a CPU. So account for the time it spent waiting on the 4377 * runqueue. 4378 */ 4379 update_stats_wait_end(cfs_rq, se); 4380 __dequeue_entity(cfs_rq, se); 4381 update_load_avg(cfs_rq, se, UPDATE_TG); 4382 } 4383 4384 update_stats_curr_start(cfs_rq, se); 4385 cfs_rq->curr = se; 4386 4387 /* 4388 * Track our maximum slice length, if the CPU's load is at 4389 * least twice that of our own weight (i.e. dont track it 4390 * when there are only lesser-weight tasks around): 4391 */ 4392 if (schedstat_enabled() && 4393 rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) { 4394 schedstat_set(se->statistics.slice_max, 4395 max((u64)schedstat_val(se->statistics.slice_max), 4396 se->sum_exec_runtime - se->prev_sum_exec_runtime)); 4397 } 4398 4399 se->prev_sum_exec_runtime = se->sum_exec_runtime; 4400 } 4401 4402 static int 4403 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); 4404 4405 /* 4406 * Pick the next process, keeping these things in mind, in this order: 4407 * 1) keep things fair between processes/task groups 4408 * 2) pick the "next" process, since someone really wants that to run 4409 * 3) pick the "last" process, for cache locality 4410 * 4) do not run the "skip" process, if something else is available 4411 */ 4412 static struct sched_entity * 4413 pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) 4414 { 4415 struct sched_entity *left = __pick_first_entity(cfs_rq); 4416 struct sched_entity *se; 4417 4418 /* 4419 * If curr is set we have to see if its left of the leftmost entity 4420 * still in the tree, provided there was anything in the tree at all. 4421 */ 4422 if (!left || (curr && entity_before(curr, left))) 4423 left = curr; 4424 4425 se = left; /* ideally we run the leftmost entity */ 4426 4427 /* 4428 * Avoid running the skip buddy, if running something else can 4429 * be done without getting too unfair. 4430 */ 4431 if (cfs_rq->skip == se) { 4432 struct sched_entity *second; 4433 4434 if (se == curr) { 4435 second = __pick_first_entity(cfs_rq); 4436 } else { 4437 second = __pick_next_entity(se); 4438 if (!second || (curr && entity_before(curr, second))) 4439 second = curr; 4440 } 4441 4442 if (second && wakeup_preempt_entity(second, left) < 1) 4443 se = second; 4444 } 4445 4446 /* 4447 * Prefer last buddy, try to return the CPU to a preempted task. 4448 */ 4449 if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) 4450 se = cfs_rq->last; 4451 4452 /* 4453 * Someone really wants this to run. If it's not unfair, run it. 4454 */ 4455 if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) 4456 se = cfs_rq->next; 4457 4458 clear_buddies(cfs_rq, se); 4459 4460 return se; 4461 } 4462 4463 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); 4464 4465 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) 4466 { 4467 /* 4468 * If still on the runqueue then deactivate_task() 4469 * was not called and update_curr() has to be done: 4470 */ 4471 if (prev->on_rq) 4472 update_curr(cfs_rq); 4473 4474 /* throttle cfs_rqs exceeding runtime */ 4475 check_cfs_rq_runtime(cfs_rq); 4476 4477 check_spread(cfs_rq, prev); 4478 4479 if (prev->on_rq) { 4480 update_stats_wait_start(cfs_rq, prev); 4481 /* Put 'current' back into the tree. */ 4482 __enqueue_entity(cfs_rq, prev); 4483 /* in !on_rq case, update occurred at dequeue */ 4484 update_load_avg(cfs_rq, prev, 0); 4485 } 4486 cfs_rq->curr = NULL; 4487 } 4488 4489 static void 4490 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 4491 { 4492 /* 4493 * Update run-time statistics of the 'current'. 4494 */ 4495 update_curr(cfs_rq); 4496 4497 /* 4498 * Ensure that runnable average is periodically updated. 4499 */ 4500 update_load_avg(cfs_rq, curr, UPDATE_TG); 4501 update_cfs_group(curr); 4502 4503 #ifdef CONFIG_SCHED_HRTICK 4504 /* 4505 * queued ticks are scheduled to match the slice, so don't bother 4506 * validating it and just reschedule. 4507 */ 4508 if (queued) { 4509 resched_curr(rq_of(cfs_rq)); 4510 return; 4511 } 4512 /* 4513 * don't let the period tick interfere with the hrtick preemption 4514 */ 4515 if (!sched_feat(DOUBLE_TICK) && 4516 hrtimer_active(&rq_of(cfs_rq)->hrtick_timer)) 4517 return; 4518 #endif 4519 4520 if (cfs_rq->nr_running > 1) 4521 check_preempt_tick(cfs_rq, curr); 4522 } 4523 4524 4525 /************************************************** 4526 * CFS bandwidth control machinery 4527 */ 4528 4529 #ifdef CONFIG_CFS_BANDWIDTH 4530 4531 #ifdef CONFIG_JUMP_LABEL 4532 static struct static_key __cfs_bandwidth_used; 4533 4534 static inline bool cfs_bandwidth_used(void) 4535 { 4536 return static_key_false(&__cfs_bandwidth_used); 4537 } 4538 4539 void cfs_bandwidth_usage_inc(void) 4540 { 4541 static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used); 4542 } 4543 4544 void cfs_bandwidth_usage_dec(void) 4545 { 4546 static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); 4547 } 4548 #else /* CONFIG_JUMP_LABEL */ 4549 static bool cfs_bandwidth_used(void) 4550 { 4551 return true; 4552 } 4553 4554 void cfs_bandwidth_usage_inc(void) {} 4555 void cfs_bandwidth_usage_dec(void) {} 4556 #endif /* CONFIG_JUMP_LABEL */ 4557 4558 /* 4559 * default period for cfs group bandwidth. 4560 * default: 0.1s, units: nanoseconds 4561 */ 4562 static inline u64 default_cfs_period(void) 4563 { 4564 return 100000000ULL; 4565 } 4566 4567 static inline u64 sched_cfs_bandwidth_slice(void) 4568 { 4569 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; 4570 } 4571 4572 /* 4573 * Replenish runtime according to assigned quota. We use sched_clock_cpu 4574 * directly instead of rq->clock to avoid adding additional synchronization 4575 * around rq->lock. 4576 * 4577 * requires cfs_b->lock 4578 */ 4579 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) 4580 { 4581 if (cfs_b->quota != RUNTIME_INF) 4582 cfs_b->runtime = cfs_b->quota; 4583 } 4584 4585 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 4586 { 4587 return &tg->cfs_bandwidth; 4588 } 4589 4590 /* returns 0 on failure to allocate runtime */ 4591 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) 4592 { 4593 struct task_group *tg = cfs_rq->tg; 4594 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); 4595 u64 amount = 0, min_amount; 4596 4597 /* note: this is a positive sum as runtime_remaining <= 0 */ 4598 min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining; 4599 4600 raw_spin_lock(&cfs_b->lock); 4601 if (cfs_b->quota == RUNTIME_INF) 4602 amount = min_amount; 4603 else { 4604 start_cfs_bandwidth(cfs_b); 4605 4606 if (cfs_b->runtime > 0) { 4607 amount = min(cfs_b->runtime, min_amount); 4608 cfs_b->runtime -= amount; 4609 cfs_b->idle = 0; 4610 } 4611 } 4612 raw_spin_unlock(&cfs_b->lock); 4613 4614 cfs_rq->runtime_remaining += amount; 4615 4616 return cfs_rq->runtime_remaining > 0; 4617 } 4618 4619 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 4620 { 4621 /* dock delta_exec before expiring quota (as it could span periods) */ 4622 cfs_rq->runtime_remaining -= delta_exec; 4623 4624 if (likely(cfs_rq->runtime_remaining > 0)) 4625 return; 4626 4627 if (cfs_rq->throttled) 4628 return; 4629 /* 4630 * if we're unable to extend our runtime we resched so that the active 4631 * hierarchy can be throttled 4632 */ 4633 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) 4634 resched_curr(rq_of(cfs_rq)); 4635 } 4636 4637 static __always_inline 4638 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 4639 { 4640 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) 4641 return; 4642 4643 __account_cfs_rq_runtime(cfs_rq, delta_exec); 4644 } 4645 4646 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 4647 { 4648 return cfs_bandwidth_used() && cfs_rq->throttled; 4649 } 4650 4651 /* check whether cfs_rq, or any parent, is throttled */ 4652 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 4653 { 4654 return cfs_bandwidth_used() && cfs_rq->throttle_count; 4655 } 4656 4657 /* 4658 * Ensure that neither of the group entities corresponding to src_cpu or 4659 * dest_cpu are members of a throttled hierarchy when performing group 4660 * load-balance operations. 4661 */ 4662 static inline int throttled_lb_pair(struct task_group *tg, 4663 int src_cpu, int dest_cpu) 4664 { 4665 struct cfs_rq *src_cfs_rq, *dest_cfs_rq; 4666 4667 src_cfs_rq = tg->cfs_rq[src_cpu]; 4668 dest_cfs_rq = tg->cfs_rq[dest_cpu]; 4669 4670 return throttled_hierarchy(src_cfs_rq) || 4671 throttled_hierarchy(dest_cfs_rq); 4672 } 4673 4674 static int tg_unthrottle_up(struct task_group *tg, void *data) 4675 { 4676 struct rq *rq = data; 4677 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4678 4679 cfs_rq->throttle_count--; 4680 if (!cfs_rq->throttle_count) { 4681 cfs_rq->throttled_clock_task_time += rq_clock_task(rq) - 4682 cfs_rq->throttled_clock_task; 4683 4684 /* Add cfs_rq with already running entity in the list */ 4685 if (cfs_rq->nr_running >= 1) 4686 list_add_leaf_cfs_rq(cfs_rq); 4687 } 4688 4689 return 0; 4690 } 4691 4692 static int tg_throttle_down(struct task_group *tg, void *data) 4693 { 4694 struct rq *rq = data; 4695 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4696 4697 /* group is entering throttled state, stop time */ 4698 if (!cfs_rq->throttle_count) { 4699 cfs_rq->throttled_clock_task = rq_clock_task(rq); 4700 list_del_leaf_cfs_rq(cfs_rq); 4701 } 4702 cfs_rq->throttle_count++; 4703 4704 return 0; 4705 } 4706 4707 static void throttle_cfs_rq(struct cfs_rq *cfs_rq) 4708 { 4709 struct rq *rq = rq_of(cfs_rq); 4710 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 4711 struct sched_entity *se; 4712 long task_delta, idle_task_delta, dequeue = 1; 4713 bool empty; 4714 4715 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; 4716 4717 /* freeze hierarchy runnable averages while throttled */ 4718 rcu_read_lock(); 4719 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); 4720 rcu_read_unlock(); 4721 4722 task_delta = cfs_rq->h_nr_running; 4723 idle_task_delta = cfs_rq->idle_h_nr_running; 4724 for_each_sched_entity(se) { 4725 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 4726 /* throttled entity or throttle-on-deactivate */ 4727 if (!se->on_rq) 4728 break; 4729 4730 if (dequeue) { 4731 dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); 4732 } else { 4733 update_load_avg(qcfs_rq, se, 0); 4734 se_update_runnable(se); 4735 } 4736 4737 qcfs_rq->h_nr_running -= task_delta; 4738 qcfs_rq->idle_h_nr_running -= idle_task_delta; 4739 4740 if (qcfs_rq->load.weight) 4741 dequeue = 0; 4742 } 4743 4744 if (!se) 4745 sub_nr_running(rq, task_delta); 4746 4747 cfs_rq->throttled = 1; 4748 cfs_rq->throttled_clock = rq_clock(rq); 4749 raw_spin_lock(&cfs_b->lock); 4750 empty = list_empty(&cfs_b->throttled_cfs_rq); 4751 4752 /* 4753 * Add to the _head_ of the list, so that an already-started 4754 * distribute_cfs_runtime will not see us. If disribute_cfs_runtime is 4755 * not running add to the tail so that later runqueues don't get starved. 4756 */ 4757 if (cfs_b->distribute_running) 4758 list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); 4759 else 4760 list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); 4761 4762 /* 4763 * If we're the first throttled task, make sure the bandwidth 4764 * timer is running. 4765 */ 4766 if (empty) 4767 start_cfs_bandwidth(cfs_b); 4768 4769 raw_spin_unlock(&cfs_b->lock); 4770 } 4771 4772 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) 4773 { 4774 struct rq *rq = rq_of(cfs_rq); 4775 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 4776 struct sched_entity *se; 4777 long task_delta, idle_task_delta; 4778 4779 se = cfs_rq->tg->se[cpu_of(rq)]; 4780 4781 cfs_rq->throttled = 0; 4782 4783 update_rq_clock(rq); 4784 4785 raw_spin_lock(&cfs_b->lock); 4786 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; 4787 list_del_rcu(&cfs_rq->throttled_list); 4788 raw_spin_unlock(&cfs_b->lock); 4789 4790 /* update hierarchical throttle state */ 4791 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); 4792 4793 if (!cfs_rq->load.weight) 4794 return; 4795 4796 task_delta = cfs_rq->h_nr_running; 4797 idle_task_delta = cfs_rq->idle_h_nr_running; 4798 for_each_sched_entity(se) { 4799 if (se->on_rq) 4800 break; 4801 cfs_rq = cfs_rq_of(se); 4802 enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); 4803 4804 cfs_rq->h_nr_running += task_delta; 4805 cfs_rq->idle_h_nr_running += idle_task_delta; 4806 4807 /* end evaluation on encountering a throttled cfs_rq */ 4808 if (cfs_rq_throttled(cfs_rq)) 4809 goto unthrottle_throttle; 4810 } 4811 4812 for_each_sched_entity(se) { 4813 cfs_rq = cfs_rq_of(se); 4814 4815 update_load_avg(cfs_rq, se, UPDATE_TG); 4816 se_update_runnable(se); 4817 4818 cfs_rq->h_nr_running += task_delta; 4819 cfs_rq->idle_h_nr_running += idle_task_delta; 4820 4821 4822 /* end evaluation on encountering a throttled cfs_rq */ 4823 if (cfs_rq_throttled(cfs_rq)) 4824 goto unthrottle_throttle; 4825 4826 /* 4827 * One parent has been throttled and cfs_rq removed from the 4828 * list. Add it back to not break the leaf list. 4829 */ 4830 if (throttled_hierarchy(cfs_rq)) 4831 list_add_leaf_cfs_rq(cfs_rq); 4832 } 4833 4834 /* At this point se is NULL and we are at root level*/ 4835 add_nr_running(rq, task_delta); 4836 4837 unthrottle_throttle: 4838 /* 4839 * The cfs_rq_throttled() breaks in the above iteration can result in 4840 * incomplete leaf list maintenance, resulting in triggering the 4841 * assertion below. 4842 */ 4843 for_each_sched_entity(se) { 4844 cfs_rq = cfs_rq_of(se); 4845 4846 if (list_add_leaf_cfs_rq(cfs_rq)) 4847 break; 4848 } 4849 4850 assert_list_leaf_cfs_rq(rq); 4851 4852 /* Determine whether we need to wake up potentially idle CPU: */ 4853 if (rq->curr == rq->idle && rq->cfs.nr_running) 4854 resched_curr(rq); 4855 } 4856 4857 static void distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) 4858 { 4859 struct cfs_rq *cfs_rq; 4860 u64 runtime, remaining = 1; 4861 4862 rcu_read_lock(); 4863 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, 4864 throttled_list) { 4865 struct rq *rq = rq_of(cfs_rq); 4866 struct rq_flags rf; 4867 4868 rq_lock_irqsave(rq, &rf); 4869 if (!cfs_rq_throttled(cfs_rq)) 4870 goto next; 4871 4872 /* By the above check, this should never be true */ 4873 SCHED_WARN_ON(cfs_rq->runtime_remaining > 0); 4874 4875 raw_spin_lock(&cfs_b->lock); 4876 runtime = -cfs_rq->runtime_remaining + 1; 4877 if (runtime > cfs_b->runtime) 4878 runtime = cfs_b->runtime; 4879 cfs_b->runtime -= runtime; 4880 remaining = cfs_b->runtime; 4881 raw_spin_unlock(&cfs_b->lock); 4882 4883 cfs_rq->runtime_remaining += runtime; 4884 4885 /* we check whether we're throttled above */ 4886 if (cfs_rq->runtime_remaining > 0) 4887 unthrottle_cfs_rq(cfs_rq); 4888 4889 next: 4890 rq_unlock_irqrestore(rq, &rf); 4891 4892 if (!remaining) 4893 break; 4894 } 4895 rcu_read_unlock(); 4896 } 4897 4898 /* 4899 * Responsible for refilling a task_group's bandwidth and unthrottling its 4900 * cfs_rqs as appropriate. If there has been no activity within the last 4901 * period the timer is deactivated until scheduling resumes; cfs_b->idle is 4902 * used to track this state. 4903 */ 4904 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) 4905 { 4906 int throttled; 4907 4908 /* no need to continue the timer with no bandwidth constraint */ 4909 if (cfs_b->quota == RUNTIME_INF) 4910 goto out_deactivate; 4911 4912 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 4913 cfs_b->nr_periods += overrun; 4914 4915 /* 4916 * idle depends on !throttled (for the case of a large deficit), and if 4917 * we're going inactive then everything else can be deferred 4918 */ 4919 if (cfs_b->idle && !throttled) 4920 goto out_deactivate; 4921 4922 __refill_cfs_bandwidth_runtime(cfs_b); 4923 4924 if (!throttled) { 4925 /* mark as potentially idle for the upcoming period */ 4926 cfs_b->idle = 1; 4927 return 0; 4928 } 4929 4930 /* account preceding periods in which throttling occurred */ 4931 cfs_b->nr_throttled += overrun; 4932 4933 /* 4934 * This check is repeated as we release cfs_b->lock while we unthrottle. 4935 */ 4936 while (throttled && cfs_b->runtime > 0 && !cfs_b->distribute_running) { 4937 cfs_b->distribute_running = 1; 4938 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 4939 /* we can't nest cfs_b->lock while distributing bandwidth */ 4940 distribute_cfs_runtime(cfs_b); 4941 raw_spin_lock_irqsave(&cfs_b->lock, flags); 4942 4943 cfs_b->distribute_running = 0; 4944 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 4945 } 4946 4947 /* 4948 * While we are ensured activity in the period following an 4949 * unthrottle, this also covers the case in which the new bandwidth is 4950 * insufficient to cover the existing bandwidth deficit. (Forcing the 4951 * timer to remain active while there are any throttled entities.) 4952 */ 4953 cfs_b->idle = 0; 4954 4955 return 0; 4956 4957 out_deactivate: 4958 return 1; 4959 } 4960 4961 /* a cfs_rq won't donate quota below this amount */ 4962 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; 4963 /* minimum remaining period time to redistribute slack quota */ 4964 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; 4965 /* how long we wait to gather additional slack before distributing */ 4966 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; 4967 4968 /* 4969 * Are we near the end of the current quota period? 4970 * 4971 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the 4972 * hrtimer base being cleared by hrtimer_start. In the case of 4973 * migrate_hrtimers, base is never cleared, so we are fine. 4974 */ 4975 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) 4976 { 4977 struct hrtimer *refresh_timer = &cfs_b->period_timer; 4978 u64 remaining; 4979 4980 /* if the call-back is running a quota refresh is already occurring */ 4981 if (hrtimer_callback_running(refresh_timer)) 4982 return 1; 4983 4984 /* is a quota refresh about to occur? */ 4985 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); 4986 if (remaining < min_expire) 4987 return 1; 4988 4989 return 0; 4990 } 4991 4992 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) 4993 { 4994 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; 4995 4996 /* if there's a quota refresh soon don't bother with slack */ 4997 if (runtime_refresh_within(cfs_b, min_left)) 4998 return; 4999 5000 /* don't push forwards an existing deferred unthrottle */ 5001 if (cfs_b->slack_started) 5002 return; 5003 cfs_b->slack_started = true; 5004 5005 hrtimer_start(&cfs_b->slack_timer, 5006 ns_to_ktime(cfs_bandwidth_slack_period), 5007 HRTIMER_MODE_REL); 5008 } 5009 5010 /* we know any runtime found here is valid as update_curr() precedes return */ 5011 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5012 { 5013 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5014 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; 5015 5016 if (slack_runtime <= 0) 5017 return; 5018 5019 raw_spin_lock(&cfs_b->lock); 5020 if (cfs_b->quota != RUNTIME_INF) { 5021 cfs_b->runtime += slack_runtime; 5022 5023 /* we are under rq->lock, defer unthrottling using a timer */ 5024 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && 5025 !list_empty(&cfs_b->throttled_cfs_rq)) 5026 start_cfs_slack_bandwidth(cfs_b); 5027 } 5028 raw_spin_unlock(&cfs_b->lock); 5029 5030 /* even if it's not valid for return we don't want to try again */ 5031 cfs_rq->runtime_remaining -= slack_runtime; 5032 } 5033 5034 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5035 { 5036 if (!cfs_bandwidth_used()) 5037 return; 5038 5039 if (!cfs_rq->runtime_enabled || cfs_rq->nr_running) 5040 return; 5041 5042 __return_cfs_rq_runtime(cfs_rq); 5043 } 5044 5045 /* 5046 * This is done with a timer (instead of inline with bandwidth return) since 5047 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. 5048 */ 5049 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 5050 { 5051 u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); 5052 unsigned long flags; 5053 5054 /* confirm we're still not at a refresh boundary */ 5055 raw_spin_lock_irqsave(&cfs_b->lock, flags); 5056 cfs_b->slack_started = false; 5057 if (cfs_b->distribute_running) { 5058 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 5059 return; 5060 } 5061 5062 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { 5063 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 5064 return; 5065 } 5066 5067 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) 5068 runtime = cfs_b->runtime; 5069 5070 if (runtime) 5071 cfs_b->distribute_running = 1; 5072 5073 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 5074 5075 if (!runtime) 5076 return; 5077 5078 distribute_cfs_runtime(cfs_b); 5079 5080 raw_spin_lock_irqsave(&cfs_b->lock, flags); 5081 cfs_b->distribute_running = 0; 5082 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 5083 } 5084 5085 /* 5086 * When a group wakes up we want to make sure that its quota is not already 5087 * expired/exceeded, otherwise it may be allowed to steal additional ticks of 5088 * runtime as update_curr() throttling can not not trigger until it's on-rq. 5089 */ 5090 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) 5091 { 5092 if (!cfs_bandwidth_used()) 5093 return; 5094 5095 /* an active group must be handled by the update_curr()->put() path */ 5096 if (!cfs_rq->runtime_enabled || cfs_rq->curr) 5097 return; 5098 5099 /* ensure the group is not already throttled */ 5100 if (cfs_rq_throttled(cfs_rq)) 5101 return; 5102 5103 /* update runtime allocation */ 5104 account_cfs_rq_runtime(cfs_rq, 0); 5105 if (cfs_rq->runtime_remaining <= 0) 5106 throttle_cfs_rq(cfs_rq); 5107 } 5108 5109 static void sync_throttle(struct task_group *tg, int cpu) 5110 { 5111 struct cfs_rq *pcfs_rq, *cfs_rq; 5112 5113 if (!cfs_bandwidth_used()) 5114 return; 5115 5116 if (!tg->parent) 5117 return; 5118 5119 cfs_rq = tg->cfs_rq[cpu]; 5120 pcfs_rq = tg->parent->cfs_rq[cpu]; 5121 5122 cfs_rq->throttle_count = pcfs_rq->throttle_count; 5123 cfs_rq->throttled_clock_task = rq_clock_task(cpu_rq(cpu)); 5124 } 5125 5126 /* conditionally throttle active cfs_rq's from put_prev_entity() */ 5127 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5128 { 5129 if (!cfs_bandwidth_used()) 5130 return false; 5131 5132 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) 5133 return false; 5134 5135 /* 5136 * it's possible for a throttled entity to be forced into a running 5137 * state (e.g. set_curr_task), in this case we're finished. 5138 */ 5139 if (cfs_rq_throttled(cfs_rq)) 5140 return true; 5141 5142 throttle_cfs_rq(cfs_rq); 5143 return true; 5144 } 5145 5146 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) 5147 { 5148 struct cfs_bandwidth *cfs_b = 5149 container_of(timer, struct cfs_bandwidth, slack_timer); 5150 5151 do_sched_cfs_slack_timer(cfs_b); 5152 5153 return HRTIMER_NORESTART; 5154 } 5155 5156 extern const u64 max_cfs_quota_period; 5157 5158 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) 5159 { 5160 struct cfs_bandwidth *cfs_b = 5161 container_of(timer, struct cfs_bandwidth, period_timer); 5162 unsigned long flags; 5163 int overrun; 5164 int idle = 0; 5165 int count = 0; 5166 5167 raw_spin_lock_irqsave(&cfs_b->lock, flags); 5168 for (;;) { 5169 overrun = hrtimer_forward_now(timer, cfs_b->period); 5170 if (!overrun) 5171 break; 5172 5173 if (++count > 3) { 5174 u64 new, old = ktime_to_ns(cfs_b->period); 5175 5176 /* 5177 * Grow period by a factor of 2 to avoid losing precision. 5178 * Precision loss in the quota/period ratio can cause __cfs_schedulable 5179 * to fail. 5180 */ 5181 new = old * 2; 5182 if (new < max_cfs_quota_period) { 5183 cfs_b->period = ns_to_ktime(new); 5184 cfs_b->quota *= 2; 5185 5186 pr_warn_ratelimited( 5187 "cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n", 5188 smp_processor_id(), 5189 div_u64(new, NSEC_PER_USEC), 5190 div_u64(cfs_b->quota, NSEC_PER_USEC)); 5191 } else { 5192 pr_warn_ratelimited( 5193 "cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n", 5194 smp_processor_id(), 5195 div_u64(old, NSEC_PER_USEC), 5196 div_u64(cfs_b->quota, NSEC_PER_USEC)); 5197 } 5198 5199 /* reset count so we don't come right back in here */ 5200 count = 0; 5201 } 5202 5203 idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); 5204 } 5205 if (idle) 5206 cfs_b->period_active = 0; 5207 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 5208 5209 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 5210 } 5211 5212 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 5213 { 5214 raw_spin_lock_init(&cfs_b->lock); 5215 cfs_b->runtime = 0; 5216 cfs_b->quota = RUNTIME_INF; 5217 cfs_b->period = ns_to_ktime(default_cfs_period()); 5218 5219 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); 5220 hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 5221 cfs_b->period_timer.function = sched_cfs_period_timer; 5222 hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 5223 cfs_b->slack_timer.function = sched_cfs_slack_timer; 5224 cfs_b->distribute_running = 0; 5225 cfs_b->slack_started = false; 5226 } 5227 5228 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5229 { 5230 cfs_rq->runtime_enabled = 0; 5231 INIT_LIST_HEAD(&cfs_rq->throttled_list); 5232 } 5233 5234 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 5235 { 5236 lockdep_assert_held(&cfs_b->lock); 5237 5238 if (cfs_b->period_active) 5239 return; 5240 5241 cfs_b->period_active = 1; 5242 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); 5243 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); 5244 } 5245 5246 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 5247 { 5248 /* init_cfs_bandwidth() was not called */ 5249 if (!cfs_b->throttled_cfs_rq.next) 5250 return; 5251 5252 hrtimer_cancel(&cfs_b->period_timer); 5253 hrtimer_cancel(&cfs_b->slack_timer); 5254 } 5255 5256 /* 5257 * Both these CPU hotplug callbacks race against unregister_fair_sched_group() 5258 * 5259 * The race is harmless, since modifying bandwidth settings of unhooked group 5260 * bits doesn't do much. 5261 */ 5262 5263 /* cpu online calback */ 5264 static void __maybe_unused update_runtime_enabled(struct rq *rq) 5265 { 5266 struct task_group *tg; 5267 5268 lockdep_assert_held(&rq->lock); 5269 5270 rcu_read_lock(); 5271 list_for_each_entry_rcu(tg, &task_groups, list) { 5272 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 5273 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5274 5275 raw_spin_lock(&cfs_b->lock); 5276 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; 5277 raw_spin_unlock(&cfs_b->lock); 5278 } 5279 rcu_read_unlock(); 5280 } 5281 5282 /* cpu offline callback */ 5283 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) 5284 { 5285 struct task_group *tg; 5286 5287 lockdep_assert_held(&rq->lock); 5288 5289 rcu_read_lock(); 5290 list_for_each_entry_rcu(tg, &task_groups, list) { 5291 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5292 5293 if (!cfs_rq->runtime_enabled) 5294 continue; 5295 5296 /* 5297 * clock_task is not advancing so we just need to make sure 5298 * there's some valid quota amount 5299 */ 5300 cfs_rq->runtime_remaining = 1; 5301 /* 5302 * Offline rq is schedulable till CPU is completely disabled 5303 * in take_cpu_down(), so we prevent new cfs throttling here. 5304 */ 5305 cfs_rq->runtime_enabled = 0; 5306 5307 if (cfs_rq_throttled(cfs_rq)) 5308 unthrottle_cfs_rq(cfs_rq); 5309 } 5310 rcu_read_unlock(); 5311 } 5312 5313 #else /* CONFIG_CFS_BANDWIDTH */ 5314 5315 static inline bool cfs_bandwidth_used(void) 5316 { 5317 return false; 5318 } 5319 5320 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} 5321 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } 5322 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 5323 static inline void sync_throttle(struct task_group *tg, int cpu) {} 5324 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 5325 5326 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 5327 { 5328 return 0; 5329 } 5330 5331 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 5332 { 5333 return 0; 5334 } 5335 5336 static inline int throttled_lb_pair(struct task_group *tg, 5337 int src_cpu, int dest_cpu) 5338 { 5339 return 0; 5340 } 5341 5342 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 5343 5344 #ifdef CONFIG_FAIR_GROUP_SCHED 5345 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 5346 #endif 5347 5348 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 5349 { 5350 return NULL; 5351 } 5352 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 5353 static inline void update_runtime_enabled(struct rq *rq) {} 5354 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} 5355 5356 #endif /* CONFIG_CFS_BANDWIDTH */ 5357 5358 /************************************************** 5359 * CFS operations on tasks: 5360 */ 5361 5362 #ifdef CONFIG_SCHED_HRTICK 5363 static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 5364 { 5365 struct sched_entity *se = &p->se; 5366 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5367 5368 SCHED_WARN_ON(task_rq(p) != rq); 5369 5370 if (rq->cfs.h_nr_running > 1) { 5371 u64 slice = sched_slice(cfs_rq, se); 5372 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; 5373 s64 delta = slice - ran; 5374 5375 if (delta < 0) { 5376 if (rq->curr == p) 5377 resched_curr(rq); 5378 return; 5379 } 5380 hrtick_start(rq, delta); 5381 } 5382 } 5383 5384 /* 5385 * called from enqueue/dequeue and updates the hrtick when the 5386 * current task is from our class and nr_running is low enough 5387 * to matter. 5388 */ 5389 static void hrtick_update(struct rq *rq) 5390 { 5391 struct task_struct *curr = rq->curr; 5392 5393 if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class) 5394 return; 5395 5396 if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency) 5397 hrtick_start_fair(rq, curr); 5398 } 5399 #else /* !CONFIG_SCHED_HRTICK */ 5400 static inline void 5401 hrtick_start_fair(struct rq *rq, struct task_struct *p) 5402 { 5403 } 5404 5405 static inline void hrtick_update(struct rq *rq) 5406 { 5407 } 5408 #endif 5409 5410 #ifdef CONFIG_SMP 5411 static inline unsigned long cpu_util(int cpu); 5412 5413 static inline bool cpu_overutilized(int cpu) 5414 { 5415 return !fits_capacity(cpu_util(cpu), capacity_of(cpu)); 5416 } 5417 5418 static inline void update_overutilized_status(struct rq *rq) 5419 { 5420 if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) { 5421 WRITE_ONCE(rq->rd->overutilized, SG_OVERUTILIZED); 5422 trace_sched_overutilized_tp(rq->rd, SG_OVERUTILIZED); 5423 } 5424 } 5425 #else 5426 static inline void update_overutilized_status(struct rq *rq) { } 5427 #endif 5428 5429 /* Runqueue only has SCHED_IDLE tasks enqueued */ 5430 static int sched_idle_rq(struct rq *rq) 5431 { 5432 return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 5433 rq->nr_running); 5434 } 5435 5436 #ifdef CONFIG_SMP 5437 static int sched_idle_cpu(int cpu) 5438 { 5439 return sched_idle_rq(cpu_rq(cpu)); 5440 } 5441 #endif 5442 5443 /* 5444 * The enqueue_task method is called before nr_running is 5445 * increased. Here we update the fair scheduling stats and 5446 * then put the task into the rbtree: 5447 */ 5448 static void 5449 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 5450 { 5451 struct cfs_rq *cfs_rq; 5452 struct sched_entity *se = &p->se; 5453 int idle_h_nr_running = task_has_idle_policy(p); 5454 5455 /* 5456 * The code below (indirectly) updates schedutil which looks at 5457 * the cfs_rq utilization to select a frequency. 5458 * Let's add the task's estimated utilization to the cfs_rq's 5459 * estimated utilization, before we update schedutil. 5460 */ 5461 util_est_enqueue(&rq->cfs, p); 5462 5463 /* 5464 * If in_iowait is set, the code below may not trigger any cpufreq 5465 * utilization updates, so do it here explicitly with the IOWAIT flag 5466 * passed. 5467 */ 5468 if (p->in_iowait) 5469 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); 5470 5471 for_each_sched_entity(se) { 5472 if (se->on_rq) 5473 break; 5474 cfs_rq = cfs_rq_of(se); 5475 enqueue_entity(cfs_rq, se, flags); 5476 5477 cfs_rq->h_nr_running++; 5478 cfs_rq->idle_h_nr_running += idle_h_nr_running; 5479 5480 /* end evaluation on encountering a throttled cfs_rq */ 5481 if (cfs_rq_throttled(cfs_rq)) 5482 goto enqueue_throttle; 5483 5484 flags = ENQUEUE_WAKEUP; 5485 } 5486 5487 for_each_sched_entity(se) { 5488 cfs_rq = cfs_rq_of(se); 5489 5490 update_load_avg(cfs_rq, se, UPDATE_TG); 5491 se_update_runnable(se); 5492 update_cfs_group(se); 5493 5494 cfs_rq->h_nr_running++; 5495 cfs_rq->idle_h_nr_running += idle_h_nr_running; 5496 5497 /* end evaluation on encountering a throttled cfs_rq */ 5498 if (cfs_rq_throttled(cfs_rq)) 5499 goto enqueue_throttle; 5500 5501 /* 5502 * One parent has been throttled and cfs_rq removed from the 5503 * list. Add it back to not break the leaf list. 5504 */ 5505 if (throttled_hierarchy(cfs_rq)) 5506 list_add_leaf_cfs_rq(cfs_rq); 5507 } 5508 5509 enqueue_throttle: 5510 if (!se) { 5511 add_nr_running(rq, 1); 5512 /* 5513 * Since new tasks are assigned an initial util_avg equal to 5514 * half of the spare capacity of their CPU, tiny tasks have the 5515 * ability to cross the overutilized threshold, which will 5516 * result in the load balancer ruining all the task placement 5517 * done by EAS. As a way to mitigate that effect, do not account 5518 * for the first enqueue operation of new tasks during the 5519 * overutilized flag detection. 5520 * 5521 * A better way of solving this problem would be to wait for 5522 * the PELT signals of tasks to converge before taking them 5523 * into account, but that is not straightforward to implement, 5524 * and the following generally works well enough in practice. 5525 */ 5526 if (flags & ENQUEUE_WAKEUP) 5527 update_overutilized_status(rq); 5528 5529 } 5530 5531 if (cfs_bandwidth_used()) { 5532 /* 5533 * When bandwidth control is enabled; the cfs_rq_throttled() 5534 * breaks in the above iteration can result in incomplete 5535 * leaf list maintenance, resulting in triggering the assertion 5536 * below. 5537 */ 5538 for_each_sched_entity(se) { 5539 cfs_rq = cfs_rq_of(se); 5540 5541 if (list_add_leaf_cfs_rq(cfs_rq)) 5542 break; 5543 } 5544 } 5545 5546 assert_list_leaf_cfs_rq(rq); 5547 5548 hrtick_update(rq); 5549 } 5550 5551 static void set_next_buddy(struct sched_entity *se); 5552 5553 /* 5554 * The dequeue_task method is called before nr_running is 5555 * decreased. We remove the task from the rbtree and 5556 * update the fair scheduling stats: 5557 */ 5558 static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 5559 { 5560 struct cfs_rq *cfs_rq; 5561 struct sched_entity *se = &p->se; 5562 int task_sleep = flags & DEQUEUE_SLEEP; 5563 int idle_h_nr_running = task_has_idle_policy(p); 5564 bool was_sched_idle = sched_idle_rq(rq); 5565 5566 for_each_sched_entity(se) { 5567 cfs_rq = cfs_rq_of(se); 5568 dequeue_entity(cfs_rq, se, flags); 5569 5570 cfs_rq->h_nr_running--; 5571 cfs_rq->idle_h_nr_running -= idle_h_nr_running; 5572 5573 /* end evaluation on encountering a throttled cfs_rq */ 5574 if (cfs_rq_throttled(cfs_rq)) 5575 goto dequeue_throttle; 5576 5577 /* Don't dequeue parent if it has other entities besides us */ 5578 if (cfs_rq->load.weight) { 5579 /* Avoid re-evaluating load for this entity: */ 5580 se = parent_entity(se); 5581 /* 5582 * Bias pick_next to pick a task from this cfs_rq, as 5583 * p is sleeping when it is within its sched_slice. 5584 */ 5585 if (task_sleep && se && !throttled_hierarchy(cfs_rq)) 5586 set_next_buddy(se); 5587 break; 5588 } 5589 flags |= DEQUEUE_SLEEP; 5590 } 5591 5592 for_each_sched_entity(se) { 5593 cfs_rq = cfs_rq_of(se); 5594 5595 update_load_avg(cfs_rq, se, UPDATE_TG); 5596 se_update_runnable(se); 5597 update_cfs_group(se); 5598 5599 cfs_rq->h_nr_running--; 5600 cfs_rq->idle_h_nr_running -= idle_h_nr_running; 5601 5602 /* end evaluation on encountering a throttled cfs_rq */ 5603 if (cfs_rq_throttled(cfs_rq)) 5604 goto dequeue_throttle; 5605 5606 } 5607 5608 dequeue_throttle: 5609 if (!se) 5610 sub_nr_running(rq, 1); 5611 5612 /* balance early to pull high priority tasks */ 5613 if (unlikely(!was_sched_idle && sched_idle_rq(rq))) 5614 rq->next_balance = jiffies; 5615 5616 util_est_dequeue(&rq->cfs, p, task_sleep); 5617 hrtick_update(rq); 5618 } 5619 5620 #ifdef CONFIG_SMP 5621 5622 /* Working cpumask for: load_balance, load_balance_newidle. */ 5623 DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); 5624 DEFINE_PER_CPU(cpumask_var_t, select_idle_mask); 5625 5626 #ifdef CONFIG_NO_HZ_COMMON 5627 5628 static struct { 5629 cpumask_var_t idle_cpus_mask; 5630 atomic_t nr_cpus; 5631 int has_blocked; /* Idle CPUS has blocked load */ 5632 unsigned long next_balance; /* in jiffy units */ 5633 unsigned long next_blocked; /* Next update of blocked load in jiffies */ 5634 } nohz ____cacheline_aligned; 5635 5636 #endif /* CONFIG_NO_HZ_COMMON */ 5637 5638 static unsigned long cpu_load(struct rq *rq) 5639 { 5640 return cfs_rq_load_avg(&rq->cfs); 5641 } 5642 5643 /* 5644 * cpu_load_without - compute CPU load without any contributions from *p 5645 * @cpu: the CPU which load is requested 5646 * @p: the task which load should be discounted 5647 * 5648 * The load of a CPU is defined by the load of tasks currently enqueued on that 5649 * CPU as well as tasks which are currently sleeping after an execution on that 5650 * CPU. 5651 * 5652 * This method returns the load of the specified CPU by discounting the load of 5653 * the specified task, whenever the task is currently contributing to the CPU 5654 * load. 5655 */ 5656 static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p) 5657 { 5658 struct cfs_rq *cfs_rq; 5659 unsigned int load; 5660 5661 /* Task has no contribution or is new */ 5662 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 5663 return cpu_load(rq); 5664 5665 cfs_rq = &rq->cfs; 5666 load = READ_ONCE(cfs_rq->avg.load_avg); 5667 5668 /* Discount task's util from CPU's util */ 5669 lsub_positive(&load, task_h_load(p)); 5670 5671 return load; 5672 } 5673 5674 static unsigned long cpu_runnable(struct rq *rq) 5675 { 5676 return cfs_rq_runnable_avg(&rq->cfs); 5677 } 5678 5679 static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p) 5680 { 5681 struct cfs_rq *cfs_rq; 5682 unsigned int runnable; 5683 5684 /* Task has no contribution or is new */ 5685 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 5686 return cpu_runnable(rq); 5687 5688 cfs_rq = &rq->cfs; 5689 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 5690 5691 /* Discount task's runnable from CPU's runnable */ 5692 lsub_positive(&runnable, p->se.avg.runnable_avg); 5693 5694 return runnable; 5695 } 5696 5697 static unsigned long capacity_of(int cpu) 5698 { 5699 return cpu_rq(cpu)->cpu_capacity; 5700 } 5701 5702 static void record_wakee(struct task_struct *p) 5703 { 5704 /* 5705 * Only decay a single time; tasks that have less then 1 wakeup per 5706 * jiffy will not have built up many flips. 5707 */ 5708 if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 5709 current->wakee_flips >>= 1; 5710 current->wakee_flip_decay_ts = jiffies; 5711 } 5712 5713 if (current->last_wakee != p) { 5714 current->last_wakee = p; 5715 current->wakee_flips++; 5716 } 5717 } 5718 5719 /* 5720 * Detect M:N waker/wakee relationships via a switching-frequency heuristic. 5721 * 5722 * A waker of many should wake a different task than the one last awakened 5723 * at a frequency roughly N times higher than one of its wakees. 5724 * 5725 * In order to determine whether we should let the load spread vs consolidating 5726 * to shared cache, we look for a minimum 'flip' frequency of llc_size in one 5727 * partner, and a factor of lls_size higher frequency in the other. 5728 * 5729 * With both conditions met, we can be relatively sure that the relationship is 5730 * non-monogamous, with partner count exceeding socket size. 5731 * 5732 * Waker/wakee being client/server, worker/dispatcher, interrupt source or 5733 * whatever is irrelevant, spread criteria is apparent partner count exceeds 5734 * socket size. 5735 */ 5736 static int wake_wide(struct task_struct *p) 5737 { 5738 unsigned int master = current->wakee_flips; 5739 unsigned int slave = p->wakee_flips; 5740 int factor = this_cpu_read(sd_llc_size); 5741 5742 if (master < slave) 5743 swap(master, slave); 5744 if (slave < factor || master < slave * factor) 5745 return 0; 5746 return 1; 5747 } 5748 5749 /* 5750 * The purpose of wake_affine() is to quickly determine on which CPU we can run 5751 * soonest. For the purpose of speed we only consider the waking and previous 5752 * CPU. 5753 * 5754 * wake_affine_idle() - only considers 'now', it check if the waking CPU is 5755 * cache-affine and is (or will be) idle. 5756 * 5757 * wake_affine_weight() - considers the weight to reflect the average 5758 * scheduling latency of the CPUs. This seems to work 5759 * for the overloaded case. 5760 */ 5761 static int 5762 wake_affine_idle(int this_cpu, int prev_cpu, int sync) 5763 { 5764 /* 5765 * If this_cpu is idle, it implies the wakeup is from interrupt 5766 * context. Only allow the move if cache is shared. Otherwise an 5767 * interrupt intensive workload could force all tasks onto one 5768 * node depending on the IO topology or IRQ affinity settings. 5769 * 5770 * If the prev_cpu is idle and cache affine then avoid a migration. 5771 * There is no guarantee that the cache hot data from an interrupt 5772 * is more important than cache hot data on the prev_cpu and from 5773 * a cpufreq perspective, it's better to have higher utilisation 5774 * on one CPU. 5775 */ 5776 if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 5777 return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 5778 5779 if (sync && cpu_rq(this_cpu)->nr_running == 1) 5780 return this_cpu; 5781 5782 return nr_cpumask_bits; 5783 } 5784 5785 static int 5786 wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 5787 int this_cpu, int prev_cpu, int sync) 5788 { 5789 s64 this_eff_load, prev_eff_load; 5790 unsigned long task_load; 5791 5792 this_eff_load = cpu_load(cpu_rq(this_cpu)); 5793 5794 if (sync) { 5795 unsigned long current_load = task_h_load(current); 5796 5797 if (current_load > this_eff_load) 5798 return this_cpu; 5799 5800 this_eff_load -= current_load; 5801 } 5802 5803 task_load = task_h_load(p); 5804 5805 this_eff_load += task_load; 5806 if (sched_feat(WA_BIAS)) 5807 this_eff_load *= 100; 5808 this_eff_load *= capacity_of(prev_cpu); 5809 5810 prev_eff_load = cpu_load(cpu_rq(prev_cpu)); 5811 prev_eff_load -= task_load; 5812 if (sched_feat(WA_BIAS)) 5813 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 5814 prev_eff_load *= capacity_of(this_cpu); 5815 5816 /* 5817 * If sync, adjust the weight of prev_eff_load such that if 5818 * prev_eff == this_eff that select_idle_sibling() will consider 5819 * stacking the wakee on top of the waker if no other CPU is 5820 * idle. 5821 */ 5822 if (sync) 5823 prev_eff_load += 1; 5824 5825 return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 5826 } 5827 5828 static int wake_affine(struct sched_domain *sd, struct task_struct *p, 5829 int this_cpu, int prev_cpu, int sync) 5830 { 5831 int target = nr_cpumask_bits; 5832 5833 if (sched_feat(WA_IDLE)) 5834 target = wake_affine_idle(this_cpu, prev_cpu, sync); 5835 5836 if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 5837 target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync); 5838 5839 schedstat_inc(p->se.statistics.nr_wakeups_affine_attempts); 5840 if (target == nr_cpumask_bits) 5841 return prev_cpu; 5842 5843 schedstat_inc(sd->ttwu_move_affine); 5844 schedstat_inc(p->se.statistics.nr_wakeups_affine); 5845 return target; 5846 } 5847 5848 static struct sched_group * 5849 find_idlest_group(struct sched_domain *sd, struct task_struct *p, 5850 int this_cpu, int sd_flag); 5851 5852 /* 5853 * find_idlest_group_cpu - find the idlest CPU among the CPUs in the group. 5854 */ 5855 static int 5856 find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 5857 { 5858 unsigned long load, min_load = ULONG_MAX; 5859 unsigned int min_exit_latency = UINT_MAX; 5860 u64 latest_idle_timestamp = 0; 5861 int least_loaded_cpu = this_cpu; 5862 int shallowest_idle_cpu = -1; 5863 int i; 5864 5865 /* Check if we have any choice: */ 5866 if (group->group_weight == 1) 5867 return cpumask_first(sched_group_span(group)); 5868 5869 /* Traverse only the allowed CPUs */ 5870 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 5871 if (sched_idle_cpu(i)) 5872 return i; 5873 5874 if (available_idle_cpu(i)) { 5875 struct rq *rq = cpu_rq(i); 5876 struct cpuidle_state *idle = idle_get_state(rq); 5877 if (idle && idle->exit_latency < min_exit_latency) { 5878 /* 5879 * We give priority to a CPU whose idle state 5880 * has the smallest exit latency irrespective 5881 * of any idle timestamp. 5882 */ 5883 min_exit_latency = idle->exit_latency; 5884 latest_idle_timestamp = rq->idle_stamp; 5885 shallowest_idle_cpu = i; 5886 } else if ((!idle || idle->exit_latency == min_exit_latency) && 5887 rq->idle_stamp > latest_idle_timestamp) { 5888 /* 5889 * If equal or no active idle state, then 5890 * the most recently idled CPU might have 5891 * a warmer cache. 5892 */ 5893 latest_idle_timestamp = rq->idle_stamp; 5894 shallowest_idle_cpu = i; 5895 } 5896 } else if (shallowest_idle_cpu == -1) { 5897 load = cpu_load(cpu_rq(i)); 5898 if (load < min_load) { 5899 min_load = load; 5900 least_loaded_cpu = i; 5901 } 5902 } 5903 } 5904 5905 return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 5906 } 5907 5908 static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p, 5909 int cpu, int prev_cpu, int sd_flag) 5910 { 5911 int new_cpu = cpu; 5912 5913 if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr)) 5914 return prev_cpu; 5915 5916 /* 5917 * We need task's util for cpu_util_without, sync it up to 5918 * prev_cpu's last_update_time. 5919 */ 5920 if (!(sd_flag & SD_BALANCE_FORK)) 5921 sync_entity_load_avg(&p->se); 5922 5923 while (sd) { 5924 struct sched_group *group; 5925 struct sched_domain *tmp; 5926 int weight; 5927 5928 if (!(sd->flags & sd_flag)) { 5929 sd = sd->child; 5930 continue; 5931 } 5932 5933 group = find_idlest_group(sd, p, cpu, sd_flag); 5934 if (!group) { 5935 sd = sd->child; 5936 continue; 5937 } 5938 5939 new_cpu = find_idlest_group_cpu(group, p, cpu); 5940 if (new_cpu == cpu) { 5941 /* Now try balancing at a lower domain level of 'cpu': */ 5942 sd = sd->child; 5943 continue; 5944 } 5945 5946 /* Now try balancing at a lower domain level of 'new_cpu': */ 5947 cpu = new_cpu; 5948 weight = sd->span_weight; 5949 sd = NULL; 5950 for_each_domain(cpu, tmp) { 5951 if (weight <= tmp->span_weight) 5952 break; 5953 if (tmp->flags & sd_flag) 5954 sd = tmp; 5955 } 5956 } 5957 5958 return new_cpu; 5959 } 5960 5961 #ifdef CONFIG_SCHED_SMT 5962 DEFINE_STATIC_KEY_FALSE(sched_smt_present); 5963 EXPORT_SYMBOL_GPL(sched_smt_present); 5964 5965 static inline void set_idle_cores(int cpu, int val) 5966 { 5967 struct sched_domain_shared *sds; 5968 5969 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 5970 if (sds) 5971 WRITE_ONCE(sds->has_idle_cores, val); 5972 } 5973 5974 static inline bool test_idle_cores(int cpu, bool def) 5975 { 5976 struct sched_domain_shared *sds; 5977 5978 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 5979 if (sds) 5980 return READ_ONCE(sds->has_idle_cores); 5981 5982 return def; 5983 } 5984 5985 /* 5986 * Scans the local SMT mask to see if the entire core is idle, and records this 5987 * information in sd_llc_shared->has_idle_cores. 5988 * 5989 * Since SMT siblings share all cache levels, inspecting this limited remote 5990 * state should be fairly cheap. 5991 */ 5992 void __update_idle_core(struct rq *rq) 5993 { 5994 int core = cpu_of(rq); 5995 int cpu; 5996 5997 rcu_read_lock(); 5998 if (test_idle_cores(core, true)) 5999 goto unlock; 6000 6001 for_each_cpu(cpu, cpu_smt_mask(core)) { 6002 if (cpu == core) 6003 continue; 6004 6005 if (!available_idle_cpu(cpu)) 6006 goto unlock; 6007 } 6008 6009 set_idle_cores(core, 1); 6010 unlock: 6011 rcu_read_unlock(); 6012 } 6013 6014 /* 6015 * Scan the entire LLC domain for idle cores; this dynamically switches off if 6016 * there are no idle cores left in the system; tracked through 6017 * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. 6018 */ 6019 static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) 6020 { 6021 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); 6022 int core, cpu; 6023 6024 if (!static_branch_likely(&sched_smt_present)) 6025 return -1; 6026 6027 if (!test_idle_cores(target, false)) 6028 return -1; 6029 6030 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 6031 6032 for_each_cpu_wrap(core, cpus, target) { 6033 bool idle = true; 6034 6035 for_each_cpu(cpu, cpu_smt_mask(core)) { 6036 if (!available_idle_cpu(cpu)) { 6037 idle = false; 6038 break; 6039 } 6040 } 6041 cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); 6042 6043 if (idle) 6044 return core; 6045 } 6046 6047 /* 6048 * Failed to find an idle core; stop looking for one. 6049 */ 6050 set_idle_cores(target, 0); 6051 6052 return -1; 6053 } 6054 6055 /* 6056 * Scan the local SMT mask for idle CPUs. 6057 */ 6058 static int select_idle_smt(struct task_struct *p, int target) 6059 { 6060 int cpu; 6061 6062 if (!static_branch_likely(&sched_smt_present)) 6063 return -1; 6064 6065 for_each_cpu(cpu, cpu_smt_mask(target)) { 6066 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 6067 continue; 6068 if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 6069 return cpu; 6070 } 6071 6072 return -1; 6073 } 6074 6075 #else /* CONFIG_SCHED_SMT */ 6076 6077 static inline int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) 6078 { 6079 return -1; 6080 } 6081 6082 static inline int select_idle_smt(struct task_struct *p, int target) 6083 { 6084 return -1; 6085 } 6086 6087 #endif /* CONFIG_SCHED_SMT */ 6088 6089 /* 6090 * Scan the LLC domain for idle CPUs; this is dynamically regulated by 6091 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the 6092 * average idle time for this rq (as found in rq->avg_idle). 6093 */ 6094 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int target) 6095 { 6096 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); 6097 struct sched_domain *this_sd; 6098 u64 avg_cost, avg_idle; 6099 u64 time; 6100 int this = smp_processor_id(); 6101 int cpu, nr = INT_MAX; 6102 6103 this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); 6104 if (!this_sd) 6105 return -1; 6106 6107 /* 6108 * Due to large variance we need a large fuzz factor; hackbench in 6109 * particularly is sensitive here. 6110 */ 6111 avg_idle = this_rq()->avg_idle / 512; 6112 avg_cost = this_sd->avg_scan_cost + 1; 6113 6114 if (sched_feat(SIS_AVG_CPU) && avg_idle < avg_cost) 6115 return -1; 6116 6117 if (sched_feat(SIS_PROP)) { 6118 u64 span_avg = sd->span_weight * avg_idle; 6119 if (span_avg > 4*avg_cost) 6120 nr = div_u64(span_avg, avg_cost); 6121 else 6122 nr = 4; 6123 } 6124 6125 time = cpu_clock(this); 6126 6127 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 6128 6129 for_each_cpu_wrap(cpu, cpus, target) { 6130 if (!--nr) 6131 return -1; 6132 if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 6133 break; 6134 } 6135 6136 time = cpu_clock(this) - time; 6137 update_avg(&this_sd->avg_scan_cost, time); 6138 6139 return cpu; 6140 } 6141 6142 /* 6143 * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which 6144 * the task fits. If no CPU is big enough, but there are idle ones, try to 6145 * maximize capacity. 6146 */ 6147 static int 6148 select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) 6149 { 6150 unsigned long best_cap = 0; 6151 int cpu, best_cpu = -1; 6152 struct cpumask *cpus; 6153 6154 sync_entity_load_avg(&p->se); 6155 6156 cpus = this_cpu_cpumask_var_ptr(select_idle_mask); 6157 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 6158 6159 for_each_cpu_wrap(cpu, cpus, target) { 6160 unsigned long cpu_cap = capacity_of(cpu); 6161 6162 if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) 6163 continue; 6164 if (task_fits_capacity(p, cpu_cap)) 6165 return cpu; 6166 6167 if (cpu_cap > best_cap) { 6168 best_cap = cpu_cap; 6169 best_cpu = cpu; 6170 } 6171 } 6172 6173 return best_cpu; 6174 } 6175 6176 /* 6177 * Try and locate an idle core/thread in the LLC cache domain. 6178 */ 6179 static int select_idle_sibling(struct task_struct *p, int prev, int target) 6180 { 6181 struct sched_domain *sd; 6182 int i, recent_used_cpu; 6183 6184 /* 6185 * For asymmetric CPU capacity systems, our domain of interest is 6186 * sd_asym_cpucapacity rather than sd_llc. 6187 */ 6188 if (static_branch_unlikely(&sched_asym_cpucapacity)) { 6189 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target)); 6190 /* 6191 * On an asymmetric CPU capacity system where an exclusive 6192 * cpuset defines a symmetric island (i.e. one unique 6193 * capacity_orig value through the cpuset), the key will be set 6194 * but the CPUs within that cpuset will not have a domain with 6195 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric 6196 * capacity path. 6197 */ 6198 if (!sd) 6199 goto symmetric; 6200 6201 i = select_idle_capacity(p, sd, target); 6202 return ((unsigned)i < nr_cpumask_bits) ? i : target; 6203 } 6204 6205 symmetric: 6206 if (available_idle_cpu(target) || sched_idle_cpu(target)) 6207 return target; 6208 6209 /* 6210 * If the previous CPU is cache affine and idle, don't be stupid: 6211 */ 6212 if (prev != target && cpus_share_cache(prev, target) && 6213 (available_idle_cpu(prev) || sched_idle_cpu(prev))) 6214 return prev; 6215 6216 /* 6217 * Allow a per-cpu kthread to stack with the wakee if the 6218 * kworker thread and the tasks previous CPUs are the same. 6219 * The assumption is that the wakee queued work for the 6220 * per-cpu kthread that is now complete and the wakeup is 6221 * essentially a sync wakeup. An obvious example of this 6222 * pattern is IO completions. 6223 */ 6224 if (is_per_cpu_kthread(current) && 6225 prev == smp_processor_id() && 6226 this_rq()->nr_running <= 1) { 6227 return prev; 6228 } 6229 6230 /* Check a recently used CPU as a potential idle candidate: */ 6231 recent_used_cpu = p->recent_used_cpu; 6232 if (recent_used_cpu != prev && 6233 recent_used_cpu != target && 6234 cpus_share_cache(recent_used_cpu, target) && 6235 (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) && 6236 cpumask_test_cpu(p->recent_used_cpu, p->cpus_ptr)) { 6237 /* 6238 * Replace recent_used_cpu with prev as it is a potential 6239 * candidate for the next wake: 6240 */ 6241 p->recent_used_cpu = prev; 6242 return recent_used_cpu; 6243 } 6244 6245 sd = rcu_dereference(per_cpu(sd_llc, target)); 6246 if (!sd) 6247 return target; 6248 6249 i = select_idle_core(p, sd, target); 6250 if ((unsigned)i < nr_cpumask_bits) 6251 return i; 6252 6253 i = select_idle_cpu(p, sd, target); 6254 if ((unsigned)i < nr_cpumask_bits) 6255 return i; 6256 6257 i = select_idle_smt(p, target); 6258 if ((unsigned)i < nr_cpumask_bits) 6259 return i; 6260 6261 return target; 6262 } 6263 6264 /** 6265 * Amount of capacity of a CPU that is (estimated to be) used by CFS tasks 6266 * @cpu: the CPU to get the utilization of 6267 * 6268 * The unit of the return value must be the one of capacity so we can compare 6269 * the utilization with the capacity of the CPU that is available for CFS task 6270 * (ie cpu_capacity). 6271 * 6272 * cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the 6273 * recent utilization of currently non-runnable tasks on a CPU. It represents 6274 * the amount of utilization of a CPU in the range [0..capacity_orig] where 6275 * capacity_orig is the cpu_capacity available at the highest frequency 6276 * (arch_scale_freq_capacity()). 6277 * The utilization of a CPU converges towards a sum equal to or less than the 6278 * current capacity (capacity_curr <= capacity_orig) of the CPU because it is 6279 * the running time on this CPU scaled by capacity_curr. 6280 * 6281 * The estimated utilization of a CPU is defined to be the maximum between its 6282 * cfs_rq.avg.util_avg and the sum of the estimated utilization of the tasks 6283 * currently RUNNABLE on that CPU. 6284 * This allows to properly represent the expected utilization of a CPU which 6285 * has just got a big task running since a long sleep period. At the same time 6286 * however it preserves the benefits of the "blocked utilization" in 6287 * describing the potential for other tasks waking up on the same CPU. 6288 * 6289 * Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even 6290 * higher than capacity_orig because of unfortunate rounding in 6291 * cfs.avg.util_avg or just after migrating tasks and new task wakeups until 6292 * the average stabilizes with the new running time. We need to check that the 6293 * utilization stays within the range of [0..capacity_orig] and cap it if 6294 * necessary. Without utilization capping, a group could be seen as overloaded 6295 * (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of 6296 * available capacity. We allow utilization to overshoot capacity_curr (but not 6297 * capacity_orig) as it useful for predicting the capacity required after task 6298 * migrations (scheduler-driven DVFS). 6299 * 6300 * Return: the (estimated) utilization for the specified CPU 6301 */ 6302 static inline unsigned long cpu_util(int cpu) 6303 { 6304 struct cfs_rq *cfs_rq; 6305 unsigned int util; 6306 6307 cfs_rq = &cpu_rq(cpu)->cfs; 6308 util = READ_ONCE(cfs_rq->avg.util_avg); 6309 6310 if (sched_feat(UTIL_EST)) 6311 util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued)); 6312 6313 return min_t(unsigned long, util, capacity_orig_of(cpu)); 6314 } 6315 6316 /* 6317 * cpu_util_without: compute cpu utilization without any contributions from *p 6318 * @cpu: the CPU which utilization is requested 6319 * @p: the task which utilization should be discounted 6320 * 6321 * The utilization of a CPU is defined by the utilization of tasks currently 6322 * enqueued on that CPU as well as tasks which are currently sleeping after an 6323 * execution on that CPU. 6324 * 6325 * This method returns the utilization of the specified CPU by discounting the 6326 * utilization of the specified task, whenever the task is currently 6327 * contributing to the CPU utilization. 6328 */ 6329 static unsigned long cpu_util_without(int cpu, struct task_struct *p) 6330 { 6331 struct cfs_rq *cfs_rq; 6332 unsigned int util; 6333 6334 /* Task has no contribution or is new */ 6335 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 6336 return cpu_util(cpu); 6337 6338 cfs_rq = &cpu_rq(cpu)->cfs; 6339 util = READ_ONCE(cfs_rq->avg.util_avg); 6340 6341 /* Discount task's util from CPU's util */ 6342 lsub_positive(&util, task_util(p)); 6343 6344 /* 6345 * Covered cases: 6346 * 6347 * a) if *p is the only task sleeping on this CPU, then: 6348 * cpu_util (== task_util) > util_est (== 0) 6349 * and thus we return: 6350 * cpu_util_without = (cpu_util - task_util) = 0 6351 * 6352 * b) if other tasks are SLEEPING on this CPU, which is now exiting 6353 * IDLE, then: 6354 * cpu_util >= task_util 6355 * cpu_util > util_est (== 0) 6356 * and thus we discount *p's blocked utilization to return: 6357 * cpu_util_without = (cpu_util - task_util) >= 0 6358 * 6359 * c) if other tasks are RUNNABLE on that CPU and 6360 * util_est > cpu_util 6361 * then we use util_est since it returns a more restrictive 6362 * estimation of the spare capacity on that CPU, by just 6363 * considering the expected utilization of tasks already 6364 * runnable on that CPU. 6365 * 6366 * Cases a) and b) are covered by the above code, while case c) is 6367 * covered by the following code when estimated utilization is 6368 * enabled. 6369 */ 6370 if (sched_feat(UTIL_EST)) { 6371 unsigned int estimated = 6372 READ_ONCE(cfs_rq->avg.util_est.enqueued); 6373 6374 /* 6375 * Despite the following checks we still have a small window 6376 * for a possible race, when an execl's select_task_rq_fair() 6377 * races with LB's detach_task(): 6378 * 6379 * detach_task() 6380 * p->on_rq = TASK_ON_RQ_MIGRATING; 6381 * ---------------------------------- A 6382 * deactivate_task() \ 6383 * dequeue_task() + RaceTime 6384 * util_est_dequeue() / 6385 * ---------------------------------- B 6386 * 6387 * The additional check on "current == p" it's required to 6388 * properly fix the execl regression and it helps in further 6389 * reducing the chances for the above race. 6390 */ 6391 if (unlikely(task_on_rq_queued(p) || current == p)) 6392 lsub_positive(&estimated, _task_util_est(p)); 6393 6394 util = max(util, estimated); 6395 } 6396 6397 /* 6398 * Utilization (estimated) can exceed the CPU capacity, thus let's 6399 * clamp to the maximum CPU capacity to ensure consistency with 6400 * the cpu_util call. 6401 */ 6402 return min_t(unsigned long, util, capacity_orig_of(cpu)); 6403 } 6404 6405 /* 6406 * Predicts what cpu_util(@cpu) would return if @p was migrated (and enqueued) 6407 * to @dst_cpu. 6408 */ 6409 static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu) 6410 { 6411 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; 6412 unsigned long util_est, util = READ_ONCE(cfs_rq->avg.util_avg); 6413 6414 /* 6415 * If @p migrates from @cpu to another, remove its contribution. Or, 6416 * if @p migrates from another CPU to @cpu, add its contribution. In 6417 * the other cases, @cpu is not impacted by the migration, so the 6418 * util_avg should already be correct. 6419 */ 6420 if (task_cpu(p) == cpu && dst_cpu != cpu) 6421 sub_positive(&util, task_util(p)); 6422 else if (task_cpu(p) != cpu && dst_cpu == cpu) 6423 util += task_util(p); 6424 6425 if (sched_feat(UTIL_EST)) { 6426 util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued); 6427 6428 /* 6429 * During wake-up, the task isn't enqueued yet and doesn't 6430 * appear in the cfs_rq->avg.util_est.enqueued of any rq, 6431 * so just add it (if needed) to "simulate" what will be 6432 * cpu_util() after the task has been enqueued. 6433 */ 6434 if (dst_cpu == cpu) 6435 util_est += _task_util_est(p); 6436 6437 util = max(util, util_est); 6438 } 6439 6440 return min(util, capacity_orig_of(cpu)); 6441 } 6442 6443 /* 6444 * compute_energy(): Estimates the energy that @pd would consume if @p was 6445 * migrated to @dst_cpu. compute_energy() predicts what will be the utilization 6446 * landscape of @pd's CPUs after the task migration, and uses the Energy Model 6447 * to compute what would be the energy if we decided to actually migrate that 6448 * task. 6449 */ 6450 static long 6451 compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) 6452 { 6453 struct cpumask *pd_mask = perf_domain_span(pd); 6454 unsigned long cpu_cap = arch_scale_cpu_capacity(cpumask_first(pd_mask)); 6455 unsigned long max_util = 0, sum_util = 0; 6456 int cpu; 6457 6458 /* 6459 * The capacity state of CPUs of the current rd can be driven by CPUs 6460 * of another rd if they belong to the same pd. So, account for the 6461 * utilization of these CPUs too by masking pd with cpu_online_mask 6462 * instead of the rd span. 6463 * 6464 * If an entire pd is outside of the current rd, it will not appear in 6465 * its pd list and will not be accounted by compute_energy(). 6466 */ 6467 for_each_cpu_and(cpu, pd_mask, cpu_online_mask) { 6468 unsigned long cpu_util, util_cfs = cpu_util_next(cpu, p, dst_cpu); 6469 struct task_struct *tsk = cpu == dst_cpu ? p : NULL; 6470 6471 /* 6472 * Busy time computation: utilization clamping is not 6473 * required since the ratio (sum_util / cpu_capacity) 6474 * is already enough to scale the EM reported power 6475 * consumption at the (eventually clamped) cpu_capacity. 6476 */ 6477 sum_util += schedutil_cpu_util(cpu, util_cfs, cpu_cap, 6478 ENERGY_UTIL, NULL); 6479 6480 /* 6481 * Performance domain frequency: utilization clamping 6482 * must be considered since it affects the selection 6483 * of the performance domain frequency. 6484 * NOTE: in case RT tasks are running, by default the 6485 * FREQUENCY_UTIL's utilization can be max OPP. 6486 */ 6487 cpu_util = schedutil_cpu_util(cpu, util_cfs, cpu_cap, 6488 FREQUENCY_UTIL, tsk); 6489 max_util = max(max_util, cpu_util); 6490 } 6491 6492 return em_pd_energy(pd->em_pd, max_util, sum_util); 6493 } 6494 6495 /* 6496 * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the 6497 * waking task. find_energy_efficient_cpu() looks for the CPU with maximum 6498 * spare capacity in each performance domain and uses it as a potential 6499 * candidate to execute the task. Then, it uses the Energy Model to figure 6500 * out which of the CPU candidates is the most energy-efficient. 6501 * 6502 * The rationale for this heuristic is as follows. In a performance domain, 6503 * all the most energy efficient CPU candidates (according to the Energy 6504 * Model) are those for which we'll request a low frequency. When there are 6505 * several CPUs for which the frequency request will be the same, we don't 6506 * have enough data to break the tie between them, because the Energy Model 6507 * only includes active power costs. With this model, if we assume that 6508 * frequency requests follow utilization (e.g. using schedutil), the CPU with 6509 * the maximum spare capacity in a performance domain is guaranteed to be among 6510 * the best candidates of the performance domain. 6511 * 6512 * In practice, it could be preferable from an energy standpoint to pack 6513 * small tasks on a CPU in order to let other CPUs go in deeper idle states, 6514 * but that could also hurt our chances to go cluster idle, and we have no 6515 * ways to tell with the current Energy Model if this is actually a good 6516 * idea or not. So, find_energy_efficient_cpu() basically favors 6517 * cluster-packing, and spreading inside a cluster. That should at least be 6518 * a good thing for latency, and this is consistent with the idea that most 6519 * of the energy savings of EAS come from the asymmetry of the system, and 6520 * not so much from breaking the tie between identical CPUs. That's also the 6521 * reason why EAS is enabled in the topology code only for systems where 6522 * SD_ASYM_CPUCAPACITY is set. 6523 * 6524 * NOTE: Forkees are not accepted in the energy-aware wake-up path because 6525 * they don't have any useful utilization data yet and it's not possible to 6526 * forecast their impact on energy consumption. Consequently, they will be 6527 * placed by find_idlest_cpu() on the least loaded CPU, which might turn out 6528 * to be energy-inefficient in some use-cases. The alternative would be to 6529 * bias new tasks towards specific types of CPUs first, or to try to infer 6530 * their util_avg from the parent task, but those heuristics could hurt 6531 * other use-cases too. So, until someone finds a better way to solve this, 6532 * let's keep things simple by re-using the existing slow path. 6533 */ 6534 static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 6535 { 6536 unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; 6537 struct root_domain *rd = cpu_rq(smp_processor_id())->rd; 6538 unsigned long cpu_cap, util, base_energy = 0; 6539 int cpu, best_energy_cpu = prev_cpu; 6540 struct sched_domain *sd; 6541 struct perf_domain *pd; 6542 6543 rcu_read_lock(); 6544 pd = rcu_dereference(rd->pd); 6545 if (!pd || READ_ONCE(rd->overutilized)) 6546 goto fail; 6547 6548 /* 6549 * Energy-aware wake-up happens on the lowest sched_domain starting 6550 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu. 6551 */ 6552 sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity)); 6553 while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 6554 sd = sd->parent; 6555 if (!sd) 6556 goto fail; 6557 6558 sync_entity_load_avg(&p->se); 6559 if (!task_util_est(p)) 6560 goto unlock; 6561 6562 for (; pd; pd = pd->next) { 6563 unsigned long cur_delta, spare_cap, max_spare_cap = 0; 6564 unsigned long base_energy_pd; 6565 int max_spare_cap_cpu = -1; 6566 6567 /* Compute the 'base' energy of the pd, without @p */ 6568 base_energy_pd = compute_energy(p, -1, pd); 6569 base_energy += base_energy_pd; 6570 6571 for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) { 6572 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 6573 continue; 6574 6575 util = cpu_util_next(cpu, p, cpu); 6576 cpu_cap = capacity_of(cpu); 6577 spare_cap = cpu_cap - util; 6578 6579 /* 6580 * Skip CPUs that cannot satisfy the capacity request. 6581 * IOW, placing the task there would make the CPU 6582 * overutilized. Take uclamp into account to see how 6583 * much capacity we can get out of the CPU; this is 6584 * aligned with schedutil_cpu_util(). 6585 */ 6586 util = uclamp_rq_util_with(cpu_rq(cpu), util, p); 6587 if (!fits_capacity(util, cpu_cap)) 6588 continue; 6589 6590 /* Always use prev_cpu as a candidate. */ 6591 if (cpu == prev_cpu) { 6592 prev_delta = compute_energy(p, prev_cpu, pd); 6593 prev_delta -= base_energy_pd; 6594 best_delta = min(best_delta, prev_delta); 6595 } 6596 6597 /* 6598 * Find the CPU with the maximum spare capacity in 6599 * the performance domain 6600 */ 6601 if (spare_cap > max_spare_cap) { 6602 max_spare_cap = spare_cap; 6603 max_spare_cap_cpu = cpu; 6604 } 6605 } 6606 6607 /* Evaluate the energy impact of using this CPU. */ 6608 if (max_spare_cap_cpu >= 0 && max_spare_cap_cpu != prev_cpu) { 6609 cur_delta = compute_energy(p, max_spare_cap_cpu, pd); 6610 cur_delta -= base_energy_pd; 6611 if (cur_delta < best_delta) { 6612 best_delta = cur_delta; 6613 best_energy_cpu = max_spare_cap_cpu; 6614 } 6615 } 6616 } 6617 unlock: 6618 rcu_read_unlock(); 6619 6620 /* 6621 * Pick the best CPU if prev_cpu cannot be used, or if it saves at 6622 * least 6% of the energy used by prev_cpu. 6623 */ 6624 if (prev_delta == ULONG_MAX) 6625 return best_energy_cpu; 6626 6627 if ((prev_delta - best_delta) > ((prev_delta + base_energy) >> 4)) 6628 return best_energy_cpu; 6629 6630 return prev_cpu; 6631 6632 fail: 6633 rcu_read_unlock(); 6634 6635 return -1; 6636 } 6637 6638 /* 6639 * select_task_rq_fair: Select target runqueue for the waking task in domains 6640 * that have the 'sd_flag' flag set. In practice, this is SD_BALANCE_WAKE, 6641 * SD_BALANCE_FORK, or SD_BALANCE_EXEC. 6642 * 6643 * Balances load by selecting the idlest CPU in the idlest group, or under 6644 * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set. 6645 * 6646 * Returns the target CPU number. 6647 * 6648 * preempt must be disabled. 6649 */ 6650 static int 6651 select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags) 6652 { 6653 struct sched_domain *tmp, *sd = NULL; 6654 int cpu = smp_processor_id(); 6655 int new_cpu = prev_cpu; 6656 int want_affine = 0; 6657 int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 6658 6659 if (sd_flag & SD_BALANCE_WAKE) { 6660 record_wakee(p); 6661 6662 if (sched_energy_enabled()) { 6663 new_cpu = find_energy_efficient_cpu(p, prev_cpu); 6664 if (new_cpu >= 0) 6665 return new_cpu; 6666 new_cpu = prev_cpu; 6667 } 6668 6669 want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr); 6670 } 6671 6672 rcu_read_lock(); 6673 for_each_domain(cpu, tmp) { 6674 if (!(tmp->flags & SD_LOAD_BALANCE)) 6675 break; 6676 6677 /* 6678 * If both 'cpu' and 'prev_cpu' are part of this domain, 6679 * cpu is a valid SD_WAKE_AFFINE target. 6680 */ 6681 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && 6682 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { 6683 if (cpu != prev_cpu) 6684 new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync); 6685 6686 sd = NULL; /* Prefer wake_affine over balance flags */ 6687 break; 6688 } 6689 6690 if (tmp->flags & sd_flag) 6691 sd = tmp; 6692 else if (!want_affine) 6693 break; 6694 } 6695 6696 if (unlikely(sd)) { 6697 /* Slow path */ 6698 new_cpu = find_idlest_cpu(sd, p, cpu, prev_cpu, sd_flag); 6699 } else if (sd_flag & SD_BALANCE_WAKE) { /* XXX always ? */ 6700 /* Fast path */ 6701 6702 new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); 6703 6704 if (want_affine) 6705 current->recent_used_cpu = cpu; 6706 } 6707 rcu_read_unlock(); 6708 6709 return new_cpu; 6710 } 6711 6712 static void detach_entity_cfs_rq(struct sched_entity *se); 6713 6714 /* 6715 * Called immediately before a task is migrated to a new CPU; task_cpu(p) and 6716 * cfs_rq_of(p) references at time of call are still valid and identify the 6717 * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held. 6718 */ 6719 static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) 6720 { 6721 /* 6722 * As blocked tasks retain absolute vruntime the migration needs to 6723 * deal with this by subtracting the old and adding the new 6724 * min_vruntime -- the latter is done by enqueue_entity() when placing 6725 * the task on the new runqueue. 6726 */ 6727 if (p->state == TASK_WAKING) { 6728 struct sched_entity *se = &p->se; 6729 struct cfs_rq *cfs_rq = cfs_rq_of(se); 6730 u64 min_vruntime; 6731 6732 #ifndef CONFIG_64BIT 6733 u64 min_vruntime_copy; 6734 6735 do { 6736 min_vruntime_copy = cfs_rq->min_vruntime_copy; 6737 smp_rmb(); 6738 min_vruntime = cfs_rq->min_vruntime; 6739 } while (min_vruntime != min_vruntime_copy); 6740 #else 6741 min_vruntime = cfs_rq->min_vruntime; 6742 #endif 6743 6744 se->vruntime -= min_vruntime; 6745 } 6746 6747 if (p->on_rq == TASK_ON_RQ_MIGRATING) { 6748 /* 6749 * In case of TASK_ON_RQ_MIGRATING we in fact hold the 'old' 6750 * rq->lock and can modify state directly. 6751 */ 6752 lockdep_assert_held(&task_rq(p)->lock); 6753 detach_entity_cfs_rq(&p->se); 6754 6755 } else { 6756 /* 6757 * We are supposed to update the task to "current" time, then 6758 * its up to date and ready to go to new CPU/cfs_rq. But we 6759 * have difficulty in getting what current time is, so simply 6760 * throw away the out-of-date time. This will result in the 6761 * wakee task is less decayed, but giving the wakee more load 6762 * sounds not bad. 6763 */ 6764 remove_entity_load_avg(&p->se); 6765 } 6766 6767 /* Tell new CPU we are migrated */ 6768 p->se.avg.last_update_time = 0; 6769 6770 /* We have migrated, no longer consider this task hot */ 6771 p->se.exec_start = 0; 6772 6773 update_scan_period(p, new_cpu); 6774 } 6775 6776 static void task_dead_fair(struct task_struct *p) 6777 { 6778 remove_entity_load_avg(&p->se); 6779 } 6780 6781 static int 6782 balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6783 { 6784 if (rq->nr_running) 6785 return 1; 6786 6787 return newidle_balance(rq, rf) != 0; 6788 } 6789 #endif /* CONFIG_SMP */ 6790 6791 static unsigned long wakeup_gran(struct sched_entity *se) 6792 { 6793 unsigned long gran = sysctl_sched_wakeup_granularity; 6794 6795 /* 6796 * Since its curr running now, convert the gran from real-time 6797 * to virtual-time in his units. 6798 * 6799 * By using 'se' instead of 'curr' we penalize light tasks, so 6800 * they get preempted easier. That is, if 'se' < 'curr' then 6801 * the resulting gran will be larger, therefore penalizing the 6802 * lighter, if otoh 'se' > 'curr' then the resulting gran will 6803 * be smaller, again penalizing the lighter task. 6804 * 6805 * This is especially important for buddies when the leftmost 6806 * task is higher priority than the buddy. 6807 */ 6808 return calc_delta_fair(gran, se); 6809 } 6810 6811 /* 6812 * Should 'se' preempt 'curr'. 6813 * 6814 * |s1 6815 * |s2 6816 * |s3 6817 * g 6818 * |<--->|c 6819 * 6820 * w(c, s1) = -1 6821 * w(c, s2) = 0 6822 * w(c, s3) = 1 6823 * 6824 */ 6825 static int 6826 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se) 6827 { 6828 s64 gran, vdiff = curr->vruntime - se->vruntime; 6829 6830 if (vdiff <= 0) 6831 return -1; 6832 6833 gran = wakeup_gran(se); 6834 if (vdiff > gran) 6835 return 1; 6836 6837 return 0; 6838 } 6839 6840 static void set_last_buddy(struct sched_entity *se) 6841 { 6842 if (entity_is_task(se) && unlikely(task_has_idle_policy(task_of(se)))) 6843 return; 6844 6845 for_each_sched_entity(se) { 6846 if (SCHED_WARN_ON(!se->on_rq)) 6847 return; 6848 cfs_rq_of(se)->last = se; 6849 } 6850 } 6851 6852 static void set_next_buddy(struct sched_entity *se) 6853 { 6854 if (entity_is_task(se) && unlikely(task_has_idle_policy(task_of(se)))) 6855 return; 6856 6857 for_each_sched_entity(se) { 6858 if (SCHED_WARN_ON(!se->on_rq)) 6859 return; 6860 cfs_rq_of(se)->next = se; 6861 } 6862 } 6863 6864 static void set_skip_buddy(struct sched_entity *se) 6865 { 6866 for_each_sched_entity(se) 6867 cfs_rq_of(se)->skip = se; 6868 } 6869 6870 /* 6871 * Preempt the current task with a newly woken task if needed: 6872 */ 6873 static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) 6874 { 6875 struct task_struct *curr = rq->curr; 6876 struct sched_entity *se = &curr->se, *pse = &p->se; 6877 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 6878 int scale = cfs_rq->nr_running >= sched_nr_latency; 6879 int next_buddy_marked = 0; 6880 6881 if (unlikely(se == pse)) 6882 return; 6883 6884 /* 6885 * This is possible from callers such as attach_tasks(), in which we 6886 * unconditionally check_prempt_curr() after an enqueue (which may have 6887 * lead to a throttle). This both saves work and prevents false 6888 * next-buddy nomination below. 6889 */ 6890 if (unlikely(throttled_hierarchy(cfs_rq_of(pse)))) 6891 return; 6892 6893 if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) { 6894 set_next_buddy(pse); 6895 next_buddy_marked = 1; 6896 } 6897 6898 /* 6899 * We can come here with TIF_NEED_RESCHED already set from new task 6900 * wake up path. 6901 * 6902 * Note: this also catches the edge-case of curr being in a throttled 6903 * group (e.g. via set_curr_task), since update_curr() (in the 6904 * enqueue of curr) will have resulted in resched being set. This 6905 * prevents us from potentially nominating it as a false LAST_BUDDY 6906 * below. 6907 */ 6908 if (test_tsk_need_resched(curr)) 6909 return; 6910 6911 /* Idle tasks are by definition preempted by non-idle tasks. */ 6912 if (unlikely(task_has_idle_policy(curr)) && 6913 likely(!task_has_idle_policy(p))) 6914 goto preempt; 6915 6916 /* 6917 * Batch and idle tasks do not preempt non-idle tasks (their preemption 6918 * is driven by the tick): 6919 */ 6920 if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION)) 6921 return; 6922 6923 find_matching_se(&se, &pse); 6924 update_curr(cfs_rq_of(se)); 6925 BUG_ON(!pse); 6926 if (wakeup_preempt_entity(se, pse) == 1) { 6927 /* 6928 * Bias pick_next to pick the sched entity that is 6929 * triggering this preemption. 6930 */ 6931 if (!next_buddy_marked) 6932 set_next_buddy(pse); 6933 goto preempt; 6934 } 6935 6936 return; 6937 6938 preempt: 6939 resched_curr(rq); 6940 /* 6941 * Only set the backward buddy when the current task is still 6942 * on the rq. This can happen when a wakeup gets interleaved 6943 * with schedule on the ->pre_schedule() or idle_balance() 6944 * point, either of which can * drop the rq lock. 6945 * 6946 * Also, during early boot the idle thread is in the fair class, 6947 * for obvious reasons its a bad idea to schedule back to it. 6948 */ 6949 if (unlikely(!se->on_rq || curr == rq->idle)) 6950 return; 6951 6952 if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se)) 6953 set_last_buddy(se); 6954 } 6955 6956 struct task_struct * 6957 pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6958 { 6959 struct cfs_rq *cfs_rq = &rq->cfs; 6960 struct sched_entity *se; 6961 struct task_struct *p; 6962 int new_tasks; 6963 6964 again: 6965 if (!sched_fair_runnable(rq)) 6966 goto idle; 6967 6968 #ifdef CONFIG_FAIR_GROUP_SCHED 6969 if (!prev || prev->sched_class != &fair_sched_class) 6970 goto simple; 6971 6972 /* 6973 * Because of the set_next_buddy() in dequeue_task_fair() it is rather 6974 * likely that a next task is from the same cgroup as the current. 6975 * 6976 * Therefore attempt to avoid putting and setting the entire cgroup 6977 * hierarchy, only change the part that actually changes. 6978 */ 6979 6980 do { 6981 struct sched_entity *curr = cfs_rq->curr; 6982 6983 /* 6984 * Since we got here without doing put_prev_entity() we also 6985 * have to consider cfs_rq->curr. If it is still a runnable 6986 * entity, update_curr() will update its vruntime, otherwise 6987 * forget we've ever seen it. 6988 */ 6989 if (curr) { 6990 if (curr->on_rq) 6991 update_curr(cfs_rq); 6992 else 6993 curr = NULL; 6994 6995 /* 6996 * This call to check_cfs_rq_runtime() will do the 6997 * throttle and dequeue its entity in the parent(s). 6998 * Therefore the nr_running test will indeed 6999 * be correct. 7000 */ 7001 if (unlikely(check_cfs_rq_runtime(cfs_rq))) { 7002 cfs_rq = &rq->cfs; 7003 7004 if (!cfs_rq->nr_running) 7005 goto idle; 7006 7007 goto simple; 7008 } 7009 } 7010 7011 se = pick_next_entity(cfs_rq, curr); 7012 cfs_rq = group_cfs_rq(se); 7013 } while (cfs_rq); 7014 7015 p = task_of(se); 7016 7017 /* 7018 * Since we haven't yet done put_prev_entity and if the selected task 7019 * is a different task than we started out with, try and touch the 7020 * least amount of cfs_rqs. 7021 */ 7022 if (prev != p) { 7023 struct sched_entity *pse = &prev->se; 7024 7025 while (!(cfs_rq = is_same_group(se, pse))) { 7026 int se_depth = se->depth; 7027 int pse_depth = pse->depth; 7028 7029 if (se_depth <= pse_depth) { 7030 put_prev_entity(cfs_rq_of(pse), pse); 7031 pse = parent_entity(pse); 7032 } 7033 if (se_depth >= pse_depth) { 7034 set_next_entity(cfs_rq_of(se), se); 7035 se = parent_entity(se); 7036 } 7037 } 7038 7039 put_prev_entity(cfs_rq, pse); 7040 set_next_entity(cfs_rq, se); 7041 } 7042 7043 goto done; 7044 simple: 7045 #endif 7046 if (prev) 7047 put_prev_task(rq, prev); 7048 7049 do { 7050 se = pick_next_entity(cfs_rq, NULL); 7051 set_next_entity(cfs_rq, se); 7052 cfs_rq = group_cfs_rq(se); 7053 } while (cfs_rq); 7054 7055 p = task_of(se); 7056 7057 done: __maybe_unused; 7058 #ifdef CONFIG_SMP 7059 /* 7060 * Move the next running task to the front of 7061 * the list, so our cfs_tasks list becomes MRU 7062 * one. 7063 */ 7064 list_move(&p->se.group_node, &rq->cfs_tasks); 7065 #endif 7066 7067 if (hrtick_enabled(rq)) 7068 hrtick_start_fair(rq, p); 7069 7070 update_misfit_status(p, rq); 7071 7072 return p; 7073 7074 idle: 7075 if (!rf) 7076 return NULL; 7077 7078 new_tasks = newidle_balance(rq, rf); 7079 7080 /* 7081 * Because newidle_balance() releases (and re-acquires) rq->lock, it is 7082 * possible for any higher priority task to appear. In that case we 7083 * must re-start the pick_next_entity() loop. 7084 */ 7085 if (new_tasks < 0) 7086 return RETRY_TASK; 7087 7088 if (new_tasks > 0) 7089 goto again; 7090 7091 /* 7092 * rq is about to be idle, check if we need to update the 7093 * lost_idle_time of clock_pelt 7094 */ 7095 update_idle_rq_clock_pelt(rq); 7096 7097 return NULL; 7098 } 7099 7100 static struct task_struct *__pick_next_task_fair(struct rq *rq) 7101 { 7102 return pick_next_task_fair(rq, NULL, NULL); 7103 } 7104 7105 /* 7106 * Account for a descheduled task: 7107 */ 7108 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) 7109 { 7110 struct sched_entity *se = &prev->se; 7111 struct cfs_rq *cfs_rq; 7112 7113 for_each_sched_entity(se) { 7114 cfs_rq = cfs_rq_of(se); 7115 put_prev_entity(cfs_rq, se); 7116 } 7117 } 7118 7119 /* 7120 * sched_yield() is very simple 7121 * 7122 * The magic of dealing with the ->skip buddy is in pick_next_entity. 7123 */ 7124 static void yield_task_fair(struct rq *rq) 7125 { 7126 struct task_struct *curr = rq->curr; 7127 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 7128 struct sched_entity *se = &curr->se; 7129 7130 /* 7131 * Are we the only task in the tree? 7132 */ 7133 if (unlikely(rq->nr_running == 1)) 7134 return; 7135 7136 clear_buddies(cfs_rq, se); 7137 7138 if (curr->policy != SCHED_BATCH) { 7139 update_rq_clock(rq); 7140 /* 7141 * Update run-time statistics of the 'current'. 7142 */ 7143 update_curr(cfs_rq); 7144 /* 7145 * Tell update_rq_clock() that we've just updated, 7146 * so we don't do microscopic update in schedule() 7147 * and double the fastpath cost. 7148 */ 7149 rq_clock_skip_update(rq); 7150 } 7151 7152 set_skip_buddy(se); 7153 } 7154 7155 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preempt) 7156 { 7157 struct sched_entity *se = &p->se; 7158 7159 /* throttled hierarchies are not runnable */ 7160 if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se))) 7161 return false; 7162 7163 /* Tell the scheduler that we'd really like pse to run next. */ 7164 set_next_buddy(se); 7165 7166 yield_task_fair(rq); 7167 7168 return true; 7169 } 7170 7171 #ifdef CONFIG_SMP 7172 /************************************************** 7173 * Fair scheduling class load-balancing methods. 7174 * 7175 * BASICS 7176 * 7177 * The purpose of load-balancing is to achieve the same basic fairness the 7178 * per-CPU scheduler provides, namely provide a proportional amount of compute 7179 * time to each task. This is expressed in the following equation: 7180 * 7181 * W_i,n/P_i == W_j,n/P_j for all i,j (1) 7182 * 7183 * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight 7184 * W_i,0 is defined as: 7185 * 7186 * W_i,0 = \Sum_j w_i,j (2) 7187 * 7188 * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight 7189 * is derived from the nice value as per sched_prio_to_weight[]. 7190 * 7191 * The weight average is an exponential decay average of the instantaneous 7192 * weight: 7193 * 7194 * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) 7195 * 7196 * C_i is the compute capacity of CPU i, typically it is the 7197 * fraction of 'recent' time available for SCHED_OTHER task execution. But it 7198 * can also include other factors [XXX]. 7199 * 7200 * To achieve this balance we define a measure of imbalance which follows 7201 * directly from (1): 7202 * 7203 * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) 7204 * 7205 * We them move tasks around to minimize the imbalance. In the continuous 7206 * function space it is obvious this converges, in the discrete case we get 7207 * a few fun cases generally called infeasible weight scenarios. 7208 * 7209 * [XXX expand on: 7210 * - infeasible weights; 7211 * - local vs global optima in the discrete case. ] 7212 * 7213 * 7214 * SCHED DOMAINS 7215 * 7216 * In order to solve the imbalance equation (4), and avoid the obvious O(n^2) 7217 * for all i,j solution, we create a tree of CPUs that follows the hardware 7218 * topology where each level pairs two lower groups (or better). This results 7219 * in O(log n) layers. Furthermore we reduce the number of CPUs going up the 7220 * tree to only the first of the previous level and we decrease the frequency 7221 * of load-balance at each level inv. proportional to the number of CPUs in 7222 * the groups. 7223 * 7224 * This yields: 7225 * 7226 * log_2 n 1 n 7227 * \Sum { --- * --- * 2^i } = O(n) (5) 7228 * i = 0 2^i 2^i 7229 * `- size of each group 7230 * | | `- number of CPUs doing load-balance 7231 * | `- freq 7232 * `- sum over all levels 7233 * 7234 * Coupled with a limit on how many tasks we can migrate every balance pass, 7235 * this makes (5) the runtime complexity of the balancer. 7236 * 7237 * An important property here is that each CPU is still (indirectly) connected 7238 * to every other CPU in at most O(log n) steps: 7239 * 7240 * The adjacency matrix of the resulting graph is given by: 7241 * 7242 * log_2 n 7243 * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) 7244 * k = 0 7245 * 7246 * And you'll find that: 7247 * 7248 * A^(log_2 n)_i,j != 0 for all i,j (7) 7249 * 7250 * Showing there's indeed a path between every CPU in at most O(log n) steps. 7251 * The task movement gives a factor of O(m), giving a convergence complexity 7252 * of: 7253 * 7254 * O(nm log n), n := nr_cpus, m := nr_tasks (8) 7255 * 7256 * 7257 * WORK CONSERVING 7258 * 7259 * In order to avoid CPUs going idle while there's still work to do, new idle 7260 * balancing is more aggressive and has the newly idle CPU iterate up the domain 7261 * tree itself instead of relying on other CPUs to bring it work. 7262 * 7263 * This adds some complexity to both (5) and (8) but it reduces the total idle 7264 * time. 7265 * 7266 * [XXX more?] 7267 * 7268 * 7269 * CGROUPS 7270 * 7271 * Cgroups make a horror show out of (2), instead of a simple sum we get: 7272 * 7273 * s_k,i 7274 * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) 7275 * S_k 7276 * 7277 * Where 7278 * 7279 * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) 7280 * 7281 * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i. 7282 * 7283 * The big problem is S_k, its a global sum needed to compute a local (W_i) 7284 * property. 7285 * 7286 * [XXX write more on how we solve this.. _after_ merging pjt's patches that 7287 * rewrite all of this once again.] 7288 */ 7289 7290 static unsigned long __read_mostly max_load_balance_interval = HZ/10; 7291 7292 enum fbq_type { regular, remote, all }; 7293 7294 /* 7295 * 'group_type' describes the group of CPUs at the moment of load balancing. 7296 * 7297 * The enum is ordered by pulling priority, with the group with lowest priority 7298 * first so the group_type can simply be compared when selecting the busiest 7299 * group. See update_sd_pick_busiest(). 7300 */ 7301 enum group_type { 7302 /* The group has spare capacity that can be used to run more tasks. */ 7303 group_has_spare = 0, 7304 /* 7305 * The group is fully used and the tasks don't compete for more CPU 7306 * cycles. Nevertheless, some tasks might wait before running. 7307 */ 7308 group_fully_busy, 7309 /* 7310 * SD_ASYM_CPUCAPACITY only: One task doesn't fit with CPU's capacity 7311 * and must be migrated to a more powerful CPU. 7312 */ 7313 group_misfit_task, 7314 /* 7315 * SD_ASYM_PACKING only: One local CPU with higher capacity is available, 7316 * and the task should be migrated to it instead of running on the 7317 * current CPU. 7318 */ 7319 group_asym_packing, 7320 /* 7321 * The tasks' affinity constraints previously prevented the scheduler 7322 * from balancing the load across the system. 7323 */ 7324 group_imbalanced, 7325 /* 7326 * The CPU is overloaded and can't provide expected CPU cycles to all 7327 * tasks. 7328 */ 7329 group_overloaded 7330 }; 7331 7332 enum migration_type { 7333 migrate_load = 0, 7334 migrate_util, 7335 migrate_task, 7336 migrate_misfit 7337 }; 7338 7339 #define LBF_ALL_PINNED 0x01 7340 #define LBF_NEED_BREAK 0x02 7341 #define LBF_DST_PINNED 0x04 7342 #define LBF_SOME_PINNED 0x08 7343 #define LBF_NOHZ_STATS 0x10 7344 #define LBF_NOHZ_AGAIN 0x20 7345 7346 struct lb_env { 7347 struct sched_domain *sd; 7348 7349 struct rq *src_rq; 7350 int src_cpu; 7351 7352 int dst_cpu; 7353 struct rq *dst_rq; 7354 7355 struct cpumask *dst_grpmask; 7356 int new_dst_cpu; 7357 enum cpu_idle_type idle; 7358 long imbalance; 7359 /* The set of CPUs under consideration for load-balancing */ 7360 struct cpumask *cpus; 7361 7362 unsigned int flags; 7363 7364 unsigned int loop; 7365 unsigned int loop_break; 7366 unsigned int loop_max; 7367 7368 enum fbq_type fbq_type; 7369 enum migration_type migration_type; 7370 struct list_head tasks; 7371 }; 7372 7373 /* 7374 * Is this task likely cache-hot: 7375 */ 7376 static int task_hot(struct task_struct *p, struct lb_env *env) 7377 { 7378 s64 delta; 7379 7380 lockdep_assert_held(&env->src_rq->lock); 7381 7382 if (p->sched_class != &fair_sched_class) 7383 return 0; 7384 7385 if (unlikely(task_has_idle_policy(p))) 7386 return 0; 7387 7388 /* 7389 * Buddy candidates are cache hot: 7390 */ 7391 if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 7392 (&p->se == cfs_rq_of(&p->se)->next || 7393 &p->se == cfs_rq_of(&p->se)->last)) 7394 return 1; 7395 7396 if (sysctl_sched_migration_cost == -1) 7397 return 1; 7398 if (sysctl_sched_migration_cost == 0) 7399 return 0; 7400 7401 delta = rq_clock_task(env->src_rq) - p->se.exec_start; 7402 7403 return delta < (s64)sysctl_sched_migration_cost; 7404 } 7405 7406 #ifdef CONFIG_NUMA_BALANCING 7407 /* 7408 * Returns 1, if task migration degrades locality 7409 * Returns 0, if task migration improves locality i.e migration preferred. 7410 * Returns -1, if task migration is not affected by locality. 7411 */ 7412 static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env) 7413 { 7414 struct numa_group *numa_group = rcu_dereference(p->numa_group); 7415 unsigned long src_weight, dst_weight; 7416 int src_nid, dst_nid, dist; 7417 7418 if (!static_branch_likely(&sched_numa_balancing)) 7419 return -1; 7420 7421 if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) 7422 return -1; 7423 7424 src_nid = cpu_to_node(env->src_cpu); 7425 dst_nid = cpu_to_node(env->dst_cpu); 7426 7427 if (src_nid == dst_nid) 7428 return -1; 7429 7430 /* Migrating away from the preferred node is always bad. */ 7431 if (src_nid == p->numa_preferred_nid) { 7432 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) 7433 return 1; 7434 else 7435 return -1; 7436 } 7437 7438 /* Encourage migration to the preferred node. */ 7439 if (dst_nid == p->numa_preferred_nid) 7440 return 0; 7441 7442 /* Leaving a core idle is often worse than degrading locality. */ 7443 if (env->idle == CPU_IDLE) 7444 return -1; 7445 7446 dist = node_distance(src_nid, dst_nid); 7447 if (numa_group) { 7448 src_weight = group_weight(p, src_nid, dist); 7449 dst_weight = group_weight(p, dst_nid, dist); 7450 } else { 7451 src_weight = task_weight(p, src_nid, dist); 7452 dst_weight = task_weight(p, dst_nid, dist); 7453 } 7454 7455 return dst_weight < src_weight; 7456 } 7457 7458 #else 7459 static inline int migrate_degrades_locality(struct task_struct *p, 7460 struct lb_env *env) 7461 { 7462 return -1; 7463 } 7464 #endif 7465 7466 /* 7467 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? 7468 */ 7469 static 7470 int can_migrate_task(struct task_struct *p, struct lb_env *env) 7471 { 7472 int tsk_cache_hot; 7473 7474 lockdep_assert_held(&env->src_rq->lock); 7475 7476 /* 7477 * We do not migrate tasks that are: 7478 * 1) throttled_lb_pair, or 7479 * 2) cannot be migrated to this CPU due to cpus_ptr, or 7480 * 3) running (obviously), or 7481 * 4) are cache-hot on their current CPU. 7482 */ 7483 if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) 7484 return 0; 7485 7486 if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { 7487 int cpu; 7488 7489 schedstat_inc(p->se.statistics.nr_failed_migrations_affine); 7490 7491 env->flags |= LBF_SOME_PINNED; 7492 7493 /* 7494 * Remember if this task can be migrated to any other CPU in 7495 * our sched_group. We may want to revisit it if we couldn't 7496 * meet load balance goals by pulling other tasks on src_cpu. 7497 * 7498 * Avoid computing new_dst_cpu for NEWLY_IDLE or if we have 7499 * already computed one in current iteration. 7500 */ 7501 if (env->idle == CPU_NEWLY_IDLE || (env->flags & LBF_DST_PINNED)) 7502 return 0; 7503 7504 /* Prevent to re-select dst_cpu via env's CPUs: */ 7505 for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { 7506 if (cpumask_test_cpu(cpu, p->cpus_ptr)) { 7507 env->flags |= LBF_DST_PINNED; 7508 env->new_dst_cpu = cpu; 7509 break; 7510 } 7511 } 7512 7513 return 0; 7514 } 7515 7516 /* Record that we found atleast one task that could run on dst_cpu */ 7517 env->flags &= ~LBF_ALL_PINNED; 7518 7519 if (task_running(env->src_rq, p)) { 7520 schedstat_inc(p->se.statistics.nr_failed_migrations_running); 7521 return 0; 7522 } 7523 7524 /* 7525 * Aggressive migration if: 7526 * 1) destination numa is preferred 7527 * 2) task is cache cold, or 7528 * 3) too many balance attempts have failed. 7529 */ 7530 tsk_cache_hot = migrate_degrades_locality(p, env); 7531 if (tsk_cache_hot == -1) 7532 tsk_cache_hot = task_hot(p, env); 7533 7534 if (tsk_cache_hot <= 0 || 7535 env->sd->nr_balance_failed > env->sd->cache_nice_tries) { 7536 if (tsk_cache_hot == 1) { 7537 schedstat_inc(env->sd->lb_hot_gained[env->idle]); 7538 schedstat_inc(p->se.statistics.nr_forced_migrations); 7539 } 7540 return 1; 7541 } 7542 7543 schedstat_inc(p->se.statistics.nr_failed_migrations_hot); 7544 return 0; 7545 } 7546 7547 /* 7548 * detach_task() -- detach the task for the migration specified in env 7549 */ 7550 static void detach_task(struct task_struct *p, struct lb_env *env) 7551 { 7552 lockdep_assert_held(&env->src_rq->lock); 7553 7554 deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 7555 set_task_cpu(p, env->dst_cpu); 7556 } 7557 7558 /* 7559 * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as 7560 * part of active balancing operations within "domain". 7561 * 7562 * Returns a task if successful and NULL otherwise. 7563 */ 7564 static struct task_struct *detach_one_task(struct lb_env *env) 7565 { 7566 struct task_struct *p; 7567 7568 lockdep_assert_held(&env->src_rq->lock); 7569 7570 list_for_each_entry_reverse(p, 7571 &env->src_rq->cfs_tasks, se.group_node) { 7572 if (!can_migrate_task(p, env)) 7573 continue; 7574 7575 detach_task(p, env); 7576 7577 /* 7578 * Right now, this is only the second place where 7579 * lb_gained[env->idle] is updated (other is detach_tasks) 7580 * so we can safely collect stats here rather than 7581 * inside detach_tasks(). 7582 */ 7583 schedstat_inc(env->sd->lb_gained[env->idle]); 7584 return p; 7585 } 7586 return NULL; 7587 } 7588 7589 static const unsigned int sched_nr_migrate_break = 32; 7590 7591 /* 7592 * detach_tasks() -- tries to detach up to imbalance load/util/tasks from 7593 * busiest_rq, as part of a balancing operation within domain "sd". 7594 * 7595 * Returns number of detached tasks if successful and 0 otherwise. 7596 */ 7597 static int detach_tasks(struct lb_env *env) 7598 { 7599 struct list_head *tasks = &env->src_rq->cfs_tasks; 7600 unsigned long util, load; 7601 struct task_struct *p; 7602 int detached = 0; 7603 7604 lockdep_assert_held(&env->src_rq->lock); 7605 7606 if (env->imbalance <= 0) 7607 return 0; 7608 7609 while (!list_empty(tasks)) { 7610 /* 7611 * We don't want to steal all, otherwise we may be treated likewise, 7612 * which could at worst lead to a livelock crash. 7613 */ 7614 if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) 7615 break; 7616 7617 p = list_last_entry(tasks, struct task_struct, se.group_node); 7618 7619 env->loop++; 7620 /* We've more or less seen every task there is, call it quits */ 7621 if (env->loop > env->loop_max) 7622 break; 7623 7624 /* take a breather every nr_migrate tasks */ 7625 if (env->loop > env->loop_break) { 7626 env->loop_break += sched_nr_migrate_break; 7627 env->flags |= LBF_NEED_BREAK; 7628 break; 7629 } 7630 7631 if (!can_migrate_task(p, env)) 7632 goto next; 7633 7634 switch (env->migration_type) { 7635 case migrate_load: 7636 load = task_h_load(p); 7637 7638 if (sched_feat(LB_MIN) && 7639 load < 16 && !env->sd->nr_balance_failed) 7640 goto next; 7641 7642 /* 7643 * Make sure that we don't migrate too much load. 7644 * Nevertheless, let relax the constraint if 7645 * scheduler fails to find a good waiting task to 7646 * migrate. 7647 */ 7648 if (load/2 > env->imbalance && 7649 env->sd->nr_balance_failed <= env->sd->cache_nice_tries) 7650 goto next; 7651 7652 env->imbalance -= load; 7653 break; 7654 7655 case migrate_util: 7656 util = task_util_est(p); 7657 7658 if (util > env->imbalance) 7659 goto next; 7660 7661 env->imbalance -= util; 7662 break; 7663 7664 case migrate_task: 7665 env->imbalance--; 7666 break; 7667 7668 case migrate_misfit: 7669 /* This is not a misfit task */ 7670 if (task_fits_capacity(p, capacity_of(env->src_cpu))) 7671 goto next; 7672 7673 env->imbalance = 0; 7674 break; 7675 } 7676 7677 detach_task(p, env); 7678 list_add(&p->se.group_node, &env->tasks); 7679 7680 detached++; 7681 7682 #ifdef CONFIG_PREEMPTION 7683 /* 7684 * NEWIDLE balancing is a source of latency, so preemptible 7685 * kernels will stop after the first task is detached to minimize 7686 * the critical section. 7687 */ 7688 if (env->idle == CPU_NEWLY_IDLE) 7689 break; 7690 #endif 7691 7692 /* 7693 * We only want to steal up to the prescribed amount of 7694 * load/util/tasks. 7695 */ 7696 if (env->imbalance <= 0) 7697 break; 7698 7699 continue; 7700 next: 7701 list_move(&p->se.group_node, tasks); 7702 } 7703 7704 /* 7705 * Right now, this is one of only two places we collect this stat 7706 * so we can safely collect detach_one_task() stats here rather 7707 * than inside detach_one_task(). 7708 */ 7709 schedstat_add(env->sd->lb_gained[env->idle], detached); 7710 7711 return detached; 7712 } 7713 7714 /* 7715 * attach_task() -- attach the task detached by detach_task() to its new rq. 7716 */ 7717 static void attach_task(struct rq *rq, struct task_struct *p) 7718 { 7719 lockdep_assert_held(&rq->lock); 7720 7721 BUG_ON(task_rq(p) != rq); 7722 activate_task(rq, p, ENQUEUE_NOCLOCK); 7723 check_preempt_curr(rq, p, 0); 7724 } 7725 7726 /* 7727 * attach_one_task() -- attaches the task returned from detach_one_task() to 7728 * its new rq. 7729 */ 7730 static void attach_one_task(struct rq *rq, struct task_struct *p) 7731 { 7732 struct rq_flags rf; 7733 7734 rq_lock(rq, &rf); 7735 update_rq_clock(rq); 7736 attach_task(rq, p); 7737 rq_unlock(rq, &rf); 7738 } 7739 7740 /* 7741 * attach_tasks() -- attaches all tasks detached by detach_tasks() to their 7742 * new rq. 7743 */ 7744 static void attach_tasks(struct lb_env *env) 7745 { 7746 struct list_head *tasks = &env->tasks; 7747 struct task_struct *p; 7748 struct rq_flags rf; 7749 7750 rq_lock(env->dst_rq, &rf); 7751 update_rq_clock(env->dst_rq); 7752 7753 while (!list_empty(tasks)) { 7754 p = list_first_entry(tasks, struct task_struct, se.group_node); 7755 list_del_init(&p->se.group_node); 7756 7757 attach_task(env->dst_rq, p); 7758 } 7759 7760 rq_unlock(env->dst_rq, &rf); 7761 } 7762 7763 #ifdef CONFIG_NO_HZ_COMMON 7764 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) 7765 { 7766 if (cfs_rq->avg.load_avg) 7767 return true; 7768 7769 if (cfs_rq->avg.util_avg) 7770 return true; 7771 7772 return false; 7773 } 7774 7775 static inline bool others_have_blocked(struct rq *rq) 7776 { 7777 if (READ_ONCE(rq->avg_rt.util_avg)) 7778 return true; 7779 7780 if (READ_ONCE(rq->avg_dl.util_avg)) 7781 return true; 7782 7783 if (thermal_load_avg(rq)) 7784 return true; 7785 7786 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 7787 if (READ_ONCE(rq->avg_irq.util_avg)) 7788 return true; 7789 #endif 7790 7791 return false; 7792 } 7793 7794 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) 7795 { 7796 rq->last_blocked_load_update_tick = jiffies; 7797 7798 if (!has_blocked) 7799 rq->has_blocked_load = 0; 7800 } 7801 #else 7802 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; } 7803 static inline bool others_have_blocked(struct rq *rq) { return false; } 7804 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {} 7805 #endif 7806 7807 static bool __update_blocked_others(struct rq *rq, bool *done) 7808 { 7809 const struct sched_class *curr_class; 7810 u64 now = rq_clock_pelt(rq); 7811 unsigned long thermal_pressure; 7812 bool decayed; 7813 7814 /* 7815 * update_load_avg() can call cpufreq_update_util(). Make sure that RT, 7816 * DL and IRQ signals have been updated before updating CFS. 7817 */ 7818 curr_class = rq->curr->sched_class; 7819 7820 thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); 7821 7822 decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) | 7823 update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) | 7824 update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure) | 7825 update_irq_load_avg(rq, 0); 7826 7827 if (others_have_blocked(rq)) 7828 *done = false; 7829 7830 return decayed; 7831 } 7832 7833 #ifdef CONFIG_FAIR_GROUP_SCHED 7834 7835 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 7836 { 7837 if (cfs_rq->load.weight) 7838 return false; 7839 7840 if (cfs_rq->avg.load_sum) 7841 return false; 7842 7843 if (cfs_rq->avg.util_sum) 7844 return false; 7845 7846 if (cfs_rq->avg.runnable_sum) 7847 return false; 7848 7849 return true; 7850 } 7851 7852 static bool __update_blocked_fair(struct rq *rq, bool *done) 7853 { 7854 struct cfs_rq *cfs_rq, *pos; 7855 bool decayed = false; 7856 int cpu = cpu_of(rq); 7857 7858 /* 7859 * Iterates the task_group tree in a bottom up fashion, see 7860 * list_add_leaf_cfs_rq() for details. 7861 */ 7862 for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 7863 struct sched_entity *se; 7864 7865 if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { 7866 update_tg_load_avg(cfs_rq, 0); 7867 7868 if (cfs_rq == &rq->cfs) 7869 decayed = true; 7870 } 7871 7872 /* Propagate pending load changes to the parent, if any: */ 7873 se = cfs_rq->tg->se[cpu]; 7874 if (se && !skip_blocked_update(se)) 7875 update_load_avg(cfs_rq_of(se), se, 0); 7876 7877 /* 7878 * There can be a lot of idle CPU cgroups. Don't let fully 7879 * decayed cfs_rqs linger on the list. 7880 */ 7881 if (cfs_rq_is_decayed(cfs_rq)) 7882 list_del_leaf_cfs_rq(cfs_rq); 7883 7884 /* Don't need periodic decay once load/util_avg are null */ 7885 if (cfs_rq_has_blocked(cfs_rq)) 7886 *done = false; 7887 } 7888 7889 return decayed; 7890 } 7891 7892 /* 7893 * Compute the hierarchical load factor for cfs_rq and all its ascendants. 7894 * This needs to be done in a top-down fashion because the load of a child 7895 * group is a fraction of its parents load. 7896 */ 7897 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) 7898 { 7899 struct rq *rq = rq_of(cfs_rq); 7900 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 7901 unsigned long now = jiffies; 7902 unsigned long load; 7903 7904 if (cfs_rq->last_h_load_update == now) 7905 return; 7906 7907 WRITE_ONCE(cfs_rq->h_load_next, NULL); 7908 for_each_sched_entity(se) { 7909 cfs_rq = cfs_rq_of(se); 7910 WRITE_ONCE(cfs_rq->h_load_next, se); 7911 if (cfs_rq->last_h_load_update == now) 7912 break; 7913 } 7914 7915 if (!se) { 7916 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); 7917 cfs_rq->last_h_load_update = now; 7918 } 7919 7920 while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { 7921 load = cfs_rq->h_load; 7922 load = div64_ul(load * se->avg.load_avg, 7923 cfs_rq_load_avg(cfs_rq) + 1); 7924 cfs_rq = group_cfs_rq(se); 7925 cfs_rq->h_load = load; 7926 cfs_rq->last_h_load_update = now; 7927 } 7928 } 7929 7930 static unsigned long task_h_load(struct task_struct *p) 7931 { 7932 struct cfs_rq *cfs_rq = task_cfs_rq(p); 7933 7934 update_cfs_rq_h_load(cfs_rq); 7935 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 7936 cfs_rq_load_avg(cfs_rq) + 1); 7937 } 7938 #else 7939 static bool __update_blocked_fair(struct rq *rq, bool *done) 7940 { 7941 struct cfs_rq *cfs_rq = &rq->cfs; 7942 bool decayed; 7943 7944 decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq); 7945 if (cfs_rq_has_blocked(cfs_rq)) 7946 *done = false; 7947 7948 return decayed; 7949 } 7950 7951 static unsigned long task_h_load(struct task_struct *p) 7952 { 7953 return p->se.avg.load_avg; 7954 } 7955 #endif 7956 7957 static void update_blocked_averages(int cpu) 7958 { 7959 bool decayed = false, done = true; 7960 struct rq *rq = cpu_rq(cpu); 7961 struct rq_flags rf; 7962 7963 rq_lock_irqsave(rq, &rf); 7964 update_rq_clock(rq); 7965 7966 decayed |= __update_blocked_others(rq, &done); 7967 decayed |= __update_blocked_fair(rq, &done); 7968 7969 update_blocked_load_status(rq, !done); 7970 if (decayed) 7971 cpufreq_update_util(rq, 0); 7972 rq_unlock_irqrestore(rq, &rf); 7973 } 7974 7975 /********** Helpers for find_busiest_group ************************/ 7976 7977 /* 7978 * sg_lb_stats - stats of a sched_group required for load_balancing 7979 */ 7980 struct sg_lb_stats { 7981 unsigned long avg_load; /*Avg load across the CPUs of the group */ 7982 unsigned long group_load; /* Total load over the CPUs of the group */ 7983 unsigned long group_capacity; 7984 unsigned long group_util; /* Total utilization over the CPUs of the group */ 7985 unsigned long group_runnable; /* Total runnable time over the CPUs of the group */ 7986 unsigned int sum_nr_running; /* Nr of tasks running in the group */ 7987 unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */ 7988 unsigned int idle_cpus; 7989 unsigned int group_weight; 7990 enum group_type group_type; 7991 unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */ 7992 unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ 7993 #ifdef CONFIG_NUMA_BALANCING 7994 unsigned int nr_numa_running; 7995 unsigned int nr_preferred_running; 7996 #endif 7997 }; 7998 7999 /* 8000 * sd_lb_stats - Structure to store the statistics of a sched_domain 8001 * during load balancing. 8002 */ 8003 struct sd_lb_stats { 8004 struct sched_group *busiest; /* Busiest group in this sd */ 8005 struct sched_group *local; /* Local group in this sd */ 8006 unsigned long total_load; /* Total load of all groups in sd */ 8007 unsigned long total_capacity; /* Total capacity of all groups in sd */ 8008 unsigned long avg_load; /* Average load across all groups in sd */ 8009 unsigned int prefer_sibling; /* tasks should go to sibling first */ 8010 8011 struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */ 8012 struct sg_lb_stats local_stat; /* Statistics of the local group */ 8013 }; 8014 8015 static inline void init_sd_lb_stats(struct sd_lb_stats *sds) 8016 { 8017 /* 8018 * Skimp on the clearing to avoid duplicate work. We can avoid clearing 8019 * local_stat because update_sg_lb_stats() does a full clear/assignment. 8020 * We must however set busiest_stat::group_type and 8021 * busiest_stat::idle_cpus to the worst busiest group because 8022 * update_sd_pick_busiest() reads these before assignment. 8023 */ 8024 *sds = (struct sd_lb_stats){ 8025 .busiest = NULL, 8026 .local = NULL, 8027 .total_load = 0UL, 8028 .total_capacity = 0UL, 8029 .busiest_stat = { 8030 .idle_cpus = UINT_MAX, 8031 .group_type = group_has_spare, 8032 }, 8033 }; 8034 } 8035 8036 static unsigned long scale_rt_capacity(struct sched_domain *sd, int cpu) 8037 { 8038 struct rq *rq = cpu_rq(cpu); 8039 unsigned long max = arch_scale_cpu_capacity(cpu); 8040 unsigned long used, free; 8041 unsigned long irq; 8042 8043 irq = cpu_util_irq(rq); 8044 8045 if (unlikely(irq >= max)) 8046 return 1; 8047 8048 /* 8049 * avg_rt.util_avg and avg_dl.util_avg track binary signals 8050 * (running and not running) with weights 0 and 1024 respectively. 8051 * avg_thermal.load_avg tracks thermal pressure and the weighted 8052 * average uses the actual delta max capacity(load). 8053 */ 8054 used = READ_ONCE(rq->avg_rt.util_avg); 8055 used += READ_ONCE(rq->avg_dl.util_avg); 8056 used += thermal_load_avg(rq); 8057 8058 if (unlikely(used >= max)) 8059 return 1; 8060 8061 free = max - used; 8062 8063 return scale_irq_capacity(free, irq, max); 8064 } 8065 8066 static void update_cpu_capacity(struct sched_domain *sd, int cpu) 8067 { 8068 unsigned long capacity = scale_rt_capacity(sd, cpu); 8069 struct sched_group *sdg = sd->groups; 8070 8071 cpu_rq(cpu)->cpu_capacity_orig = arch_scale_cpu_capacity(cpu); 8072 8073 if (!capacity) 8074 capacity = 1; 8075 8076 cpu_rq(cpu)->cpu_capacity = capacity; 8077 sdg->sgc->capacity = capacity; 8078 sdg->sgc->min_capacity = capacity; 8079 sdg->sgc->max_capacity = capacity; 8080 } 8081 8082 void update_group_capacity(struct sched_domain *sd, int cpu) 8083 { 8084 struct sched_domain *child = sd->child; 8085 struct sched_group *group, *sdg = sd->groups; 8086 unsigned long capacity, min_capacity, max_capacity; 8087 unsigned long interval; 8088 8089 interval = msecs_to_jiffies(sd->balance_interval); 8090 interval = clamp(interval, 1UL, max_load_balance_interval); 8091 sdg->sgc->next_update = jiffies + interval; 8092 8093 if (!child) { 8094 update_cpu_capacity(sd, cpu); 8095 return; 8096 } 8097 8098 capacity = 0; 8099 min_capacity = ULONG_MAX; 8100 max_capacity = 0; 8101 8102 if (child->flags & SD_OVERLAP) { 8103 /* 8104 * SD_OVERLAP domains cannot assume that child groups 8105 * span the current group. 8106 */ 8107 8108 for_each_cpu(cpu, sched_group_span(sdg)) { 8109 unsigned long cpu_cap = capacity_of(cpu); 8110 8111 capacity += cpu_cap; 8112 min_capacity = min(cpu_cap, min_capacity); 8113 max_capacity = max(cpu_cap, max_capacity); 8114 } 8115 } else { 8116 /* 8117 * !SD_OVERLAP domains can assume that child groups 8118 * span the current group. 8119 */ 8120 8121 group = child->groups; 8122 do { 8123 struct sched_group_capacity *sgc = group->sgc; 8124 8125 capacity += sgc->capacity; 8126 min_capacity = min(sgc->min_capacity, min_capacity); 8127 max_capacity = max(sgc->max_capacity, max_capacity); 8128 group = group->next; 8129 } while (group != child->groups); 8130 } 8131 8132 sdg->sgc->capacity = capacity; 8133 sdg->sgc->min_capacity = min_capacity; 8134 sdg->sgc->max_capacity = max_capacity; 8135 } 8136 8137 /* 8138 * Check whether the capacity of the rq has been noticeably reduced by side 8139 * activity. The imbalance_pct is used for the threshold. 8140 * Return true is the capacity is reduced 8141 */ 8142 static inline int 8143 check_cpu_capacity(struct rq *rq, struct sched_domain *sd) 8144 { 8145 return ((rq->cpu_capacity * sd->imbalance_pct) < 8146 (rq->cpu_capacity_orig * 100)); 8147 } 8148 8149 /* 8150 * Check whether a rq has a misfit task and if it looks like we can actually 8151 * help that task: we can migrate the task to a CPU of higher capacity, or 8152 * the task's current CPU is heavily pressured. 8153 */ 8154 static inline int check_misfit_status(struct rq *rq, struct sched_domain *sd) 8155 { 8156 return rq->misfit_task_load && 8157 (rq->cpu_capacity_orig < rq->rd->max_cpu_capacity || 8158 check_cpu_capacity(rq, sd)); 8159 } 8160 8161 /* 8162 * Group imbalance indicates (and tries to solve) the problem where balancing 8163 * groups is inadequate due to ->cpus_ptr constraints. 8164 * 8165 * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a 8166 * cpumask covering 1 CPU of the first group and 3 CPUs of the second group. 8167 * Something like: 8168 * 8169 * { 0 1 2 3 } { 4 5 6 7 } 8170 * * * * * 8171 * 8172 * If we were to balance group-wise we'd place two tasks in the first group and 8173 * two tasks in the second group. Clearly this is undesired as it will overload 8174 * cpu 3 and leave one of the CPUs in the second group unused. 8175 * 8176 * The current solution to this issue is detecting the skew in the first group 8177 * by noticing the lower domain failed to reach balance and had difficulty 8178 * moving tasks due to affinity constraints. 8179 * 8180 * When this is so detected; this group becomes a candidate for busiest; see 8181 * update_sd_pick_busiest(). And calculate_imbalance() and 8182 * find_busiest_group() avoid some of the usual balance conditions to allow it 8183 * to create an effective group imbalance. 8184 * 8185 * This is a somewhat tricky proposition since the next run might not find the 8186 * group imbalance and decide the groups need to be balanced again. A most 8187 * subtle and fragile situation. 8188 */ 8189 8190 static inline int sg_imbalanced(struct sched_group *group) 8191 { 8192 return group->sgc->imbalance; 8193 } 8194 8195 /* 8196 * group_has_capacity returns true if the group has spare capacity that could 8197 * be used by some tasks. 8198 * We consider that a group has spare capacity if the * number of task is 8199 * smaller than the number of CPUs or if the utilization is lower than the 8200 * available capacity for CFS tasks. 8201 * For the latter, we use a threshold to stabilize the state, to take into 8202 * account the variance of the tasks' load and to return true if the available 8203 * capacity in meaningful for the load balancer. 8204 * As an example, an available capacity of 1% can appear but it doesn't make 8205 * any benefit for the load balance. 8206 */ 8207 static inline bool 8208 group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 8209 { 8210 if (sgs->sum_nr_running < sgs->group_weight) 8211 return true; 8212 8213 if ((sgs->group_capacity * imbalance_pct) < 8214 (sgs->group_runnable * 100)) 8215 return false; 8216 8217 if ((sgs->group_capacity * 100) > 8218 (sgs->group_util * imbalance_pct)) 8219 return true; 8220 8221 return false; 8222 } 8223 8224 /* 8225 * group_is_overloaded returns true if the group has more tasks than it can 8226 * handle. 8227 * group_is_overloaded is not equals to !group_has_capacity because a group 8228 * with the exact right number of tasks, has no more spare capacity but is not 8229 * overloaded so both group_has_capacity and group_is_overloaded return 8230 * false. 8231 */ 8232 static inline bool 8233 group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 8234 { 8235 if (sgs->sum_nr_running <= sgs->group_weight) 8236 return false; 8237 8238 if ((sgs->group_capacity * 100) < 8239 (sgs->group_util * imbalance_pct)) 8240 return true; 8241 8242 if ((sgs->group_capacity * imbalance_pct) < 8243 (sgs->group_runnable * 100)) 8244 return true; 8245 8246 return false; 8247 } 8248 8249 /* 8250 * group_smaller_min_cpu_capacity: Returns true if sched_group sg has smaller 8251 * per-CPU capacity than sched_group ref. 8252 */ 8253 static inline bool 8254 group_smaller_min_cpu_capacity(struct sched_group *sg, struct sched_group *ref) 8255 { 8256 return fits_capacity(sg->sgc->min_capacity, ref->sgc->min_capacity); 8257 } 8258 8259 /* 8260 * group_smaller_max_cpu_capacity: Returns true if sched_group sg has smaller 8261 * per-CPU capacity_orig than sched_group ref. 8262 */ 8263 static inline bool 8264 group_smaller_max_cpu_capacity(struct sched_group *sg, struct sched_group *ref) 8265 { 8266 return fits_capacity(sg->sgc->max_capacity, ref->sgc->max_capacity); 8267 } 8268 8269 static inline enum 8270 group_type group_classify(unsigned int imbalance_pct, 8271 struct sched_group *group, 8272 struct sg_lb_stats *sgs) 8273 { 8274 if (group_is_overloaded(imbalance_pct, sgs)) 8275 return group_overloaded; 8276 8277 if (sg_imbalanced(group)) 8278 return group_imbalanced; 8279 8280 if (sgs->group_asym_packing) 8281 return group_asym_packing; 8282 8283 if (sgs->group_misfit_task_load) 8284 return group_misfit_task; 8285 8286 if (!group_has_capacity(imbalance_pct, sgs)) 8287 return group_fully_busy; 8288 8289 return group_has_spare; 8290 } 8291 8292 static bool update_nohz_stats(struct rq *rq, bool force) 8293 { 8294 #ifdef CONFIG_NO_HZ_COMMON 8295 unsigned int cpu = rq->cpu; 8296 8297 if (!rq->has_blocked_load) 8298 return false; 8299 8300 if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask)) 8301 return false; 8302 8303 if (!force && !time_after(jiffies, rq->last_blocked_load_update_tick)) 8304 return true; 8305 8306 update_blocked_averages(cpu); 8307 8308 return rq->has_blocked_load; 8309 #else 8310 return false; 8311 #endif 8312 } 8313 8314 /** 8315 * update_sg_lb_stats - Update sched_group's statistics for load balancing. 8316 * @env: The load balancing environment. 8317 * @group: sched_group whose statistics are to be updated. 8318 * @sgs: variable to hold the statistics for this group. 8319 * @sg_status: Holds flag indicating the status of the sched_group 8320 */ 8321 static inline void update_sg_lb_stats(struct lb_env *env, 8322 struct sched_group *group, 8323 struct sg_lb_stats *sgs, 8324 int *sg_status) 8325 { 8326 int i, nr_running, local_group; 8327 8328 memset(sgs, 0, sizeof(*sgs)); 8329 8330 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(group)); 8331 8332 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 8333 struct rq *rq = cpu_rq(i); 8334 8335 if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false)) 8336 env->flags |= LBF_NOHZ_AGAIN; 8337 8338 sgs->group_load += cpu_load(rq); 8339 sgs->group_util += cpu_util(i); 8340 sgs->group_runnable += cpu_runnable(rq); 8341 sgs->sum_h_nr_running += rq->cfs.h_nr_running; 8342 8343 nr_running = rq->nr_running; 8344 sgs->sum_nr_running += nr_running; 8345 8346 if (nr_running > 1) 8347 *sg_status |= SG_OVERLOAD; 8348 8349 if (cpu_overutilized(i)) 8350 *sg_status |= SG_OVERUTILIZED; 8351 8352 #ifdef CONFIG_NUMA_BALANCING 8353 sgs->nr_numa_running += rq->nr_numa_running; 8354 sgs->nr_preferred_running += rq->nr_preferred_running; 8355 #endif 8356 /* 8357 * No need to call idle_cpu() if nr_running is not 0 8358 */ 8359 if (!nr_running && idle_cpu(i)) { 8360 sgs->idle_cpus++; 8361 /* Idle cpu can't have misfit task */ 8362 continue; 8363 } 8364 8365 if (local_group) 8366 continue; 8367 8368 /* Check for a misfit task on the cpu */ 8369 if (env->sd->flags & SD_ASYM_CPUCAPACITY && 8370 sgs->group_misfit_task_load < rq->misfit_task_load) { 8371 sgs->group_misfit_task_load = rq->misfit_task_load; 8372 *sg_status |= SG_OVERLOAD; 8373 } 8374 } 8375 8376 /* Check if dst CPU is idle and preferred to this group */ 8377 if (env->sd->flags & SD_ASYM_PACKING && 8378 env->idle != CPU_NOT_IDLE && 8379 sgs->sum_h_nr_running && 8380 sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu)) { 8381 sgs->group_asym_packing = 1; 8382 } 8383 8384 sgs->group_capacity = group->sgc->capacity; 8385 8386 sgs->group_weight = group->group_weight; 8387 8388 sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 8389 8390 /* Computing avg_load makes sense only when group is overloaded */ 8391 if (sgs->group_type == group_overloaded) 8392 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 8393 sgs->group_capacity; 8394 } 8395 8396 /** 8397 * update_sd_pick_busiest - return 1 on busiest group 8398 * @env: The load balancing environment. 8399 * @sds: sched_domain statistics 8400 * @sg: sched_group candidate to be checked for being the busiest 8401 * @sgs: sched_group statistics 8402 * 8403 * Determine if @sg is a busier group than the previously selected 8404 * busiest group. 8405 * 8406 * Return: %true if @sg is a busier group than the previously selected 8407 * busiest group. %false otherwise. 8408 */ 8409 static bool update_sd_pick_busiest(struct lb_env *env, 8410 struct sd_lb_stats *sds, 8411 struct sched_group *sg, 8412 struct sg_lb_stats *sgs) 8413 { 8414 struct sg_lb_stats *busiest = &sds->busiest_stat; 8415 8416 /* Make sure that there is at least one task to pull */ 8417 if (!sgs->sum_h_nr_running) 8418 return false; 8419 8420 /* 8421 * Don't try to pull misfit tasks we can't help. 8422 * We can use max_capacity here as reduction in capacity on some 8423 * CPUs in the group should either be possible to resolve 8424 * internally or be covered by avg_load imbalance (eventually). 8425 */ 8426 if (sgs->group_type == group_misfit_task && 8427 (!group_smaller_max_cpu_capacity(sg, sds->local) || 8428 sds->local_stat.group_type != group_has_spare)) 8429 return false; 8430 8431 if (sgs->group_type > busiest->group_type) 8432 return true; 8433 8434 if (sgs->group_type < busiest->group_type) 8435 return false; 8436 8437 /* 8438 * The candidate and the current busiest group are the same type of 8439 * group. Let check which one is the busiest according to the type. 8440 */ 8441 8442 switch (sgs->group_type) { 8443 case group_overloaded: 8444 /* Select the overloaded group with highest avg_load. */ 8445 if (sgs->avg_load <= busiest->avg_load) 8446 return false; 8447 break; 8448 8449 case group_imbalanced: 8450 /* 8451 * Select the 1st imbalanced group as we don't have any way to 8452 * choose one more than another. 8453 */ 8454 return false; 8455 8456 case group_asym_packing: 8457 /* Prefer to move from lowest priority CPU's work */ 8458 if (sched_asym_prefer(sg->asym_prefer_cpu, sds->busiest->asym_prefer_cpu)) 8459 return false; 8460 break; 8461 8462 case group_misfit_task: 8463 /* 8464 * If we have more than one misfit sg go with the biggest 8465 * misfit. 8466 */ 8467 if (sgs->group_misfit_task_load < busiest->group_misfit_task_load) 8468 return false; 8469 break; 8470 8471 case group_fully_busy: 8472 /* 8473 * Select the fully busy group with highest avg_load. In 8474 * theory, there is no need to pull task from such kind of 8475 * group because tasks have all compute capacity that they need 8476 * but we can still improve the overall throughput by reducing 8477 * contention when accessing shared HW resources. 8478 * 8479 * XXX for now avg_load is not computed and always 0 so we 8480 * select the 1st one. 8481 */ 8482 if (sgs->avg_load <= busiest->avg_load) 8483 return false; 8484 break; 8485 8486 case group_has_spare: 8487 /* 8488 * Select not overloaded group with lowest number of idle cpus 8489 * and highest number of running tasks. We could also compare 8490 * the spare capacity which is more stable but it can end up 8491 * that the group has less spare capacity but finally more idle 8492 * CPUs which means less opportunity to pull tasks. 8493 */ 8494 if (sgs->idle_cpus > busiest->idle_cpus) 8495 return false; 8496 else if ((sgs->idle_cpus == busiest->idle_cpus) && 8497 (sgs->sum_nr_running <= busiest->sum_nr_running)) 8498 return false; 8499 8500 break; 8501 } 8502 8503 /* 8504 * Candidate sg has no more than one task per CPU and has higher 8505 * per-CPU capacity. Migrating tasks to less capable CPUs may harm 8506 * throughput. Maximize throughput, power/energy consequences are not 8507 * considered. 8508 */ 8509 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 8510 (sgs->group_type <= group_fully_busy) && 8511 (group_smaller_min_cpu_capacity(sds->local, sg))) 8512 return false; 8513 8514 return true; 8515 } 8516 8517 #ifdef CONFIG_NUMA_BALANCING 8518 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 8519 { 8520 if (sgs->sum_h_nr_running > sgs->nr_numa_running) 8521 return regular; 8522 if (sgs->sum_h_nr_running > sgs->nr_preferred_running) 8523 return remote; 8524 return all; 8525 } 8526 8527 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 8528 { 8529 if (rq->nr_running > rq->nr_numa_running) 8530 return regular; 8531 if (rq->nr_running > rq->nr_preferred_running) 8532 return remote; 8533 return all; 8534 } 8535 #else 8536 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 8537 { 8538 return all; 8539 } 8540 8541 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 8542 { 8543 return regular; 8544 } 8545 #endif /* CONFIG_NUMA_BALANCING */ 8546 8547 8548 struct sg_lb_stats; 8549 8550 /* 8551 * task_running_on_cpu - return 1 if @p is running on @cpu. 8552 */ 8553 8554 static unsigned int task_running_on_cpu(int cpu, struct task_struct *p) 8555 { 8556 /* Task has no contribution or is new */ 8557 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 8558 return 0; 8559 8560 if (task_on_rq_queued(p)) 8561 return 1; 8562 8563 return 0; 8564 } 8565 8566 /** 8567 * idle_cpu_without - would a given CPU be idle without p ? 8568 * @cpu: the processor on which idleness is tested. 8569 * @p: task which should be ignored. 8570 * 8571 * Return: 1 if the CPU would be idle. 0 otherwise. 8572 */ 8573 static int idle_cpu_without(int cpu, struct task_struct *p) 8574 { 8575 struct rq *rq = cpu_rq(cpu); 8576 8577 if (rq->curr != rq->idle && rq->curr != p) 8578 return 0; 8579 8580 /* 8581 * rq->nr_running can't be used but an updated version without the 8582 * impact of p on cpu must be used instead. The updated nr_running 8583 * be computed and tested before calling idle_cpu_without(). 8584 */ 8585 8586 #ifdef CONFIG_SMP 8587 if (!llist_empty(&rq->wake_list)) 8588 return 0; 8589 #endif 8590 8591 return 1; 8592 } 8593 8594 /* 8595 * update_sg_wakeup_stats - Update sched_group's statistics for wakeup. 8596 * @sd: The sched_domain level to look for idlest group. 8597 * @group: sched_group whose statistics are to be updated. 8598 * @sgs: variable to hold the statistics for this group. 8599 * @p: The task for which we look for the idlest group/CPU. 8600 */ 8601 static inline void update_sg_wakeup_stats(struct sched_domain *sd, 8602 struct sched_group *group, 8603 struct sg_lb_stats *sgs, 8604 struct task_struct *p) 8605 { 8606 int i, nr_running; 8607 8608 memset(sgs, 0, sizeof(*sgs)); 8609 8610 for_each_cpu(i, sched_group_span(group)) { 8611 struct rq *rq = cpu_rq(i); 8612 unsigned int local; 8613 8614 sgs->group_load += cpu_load_without(rq, p); 8615 sgs->group_util += cpu_util_without(i, p); 8616 sgs->group_runnable += cpu_runnable_without(rq, p); 8617 local = task_running_on_cpu(i, p); 8618 sgs->sum_h_nr_running += rq->cfs.h_nr_running - local; 8619 8620 nr_running = rq->nr_running - local; 8621 sgs->sum_nr_running += nr_running; 8622 8623 /* 8624 * No need to call idle_cpu_without() if nr_running is not 0 8625 */ 8626 if (!nr_running && idle_cpu_without(i, p)) 8627 sgs->idle_cpus++; 8628 8629 } 8630 8631 /* Check if task fits in the group */ 8632 if (sd->flags & SD_ASYM_CPUCAPACITY && 8633 !task_fits_capacity(p, group->sgc->max_capacity)) { 8634 sgs->group_misfit_task_load = 1; 8635 } 8636 8637 sgs->group_capacity = group->sgc->capacity; 8638 8639 sgs->group_weight = group->group_weight; 8640 8641 sgs->group_type = group_classify(sd->imbalance_pct, group, sgs); 8642 8643 /* 8644 * Computing avg_load makes sense only when group is fully busy or 8645 * overloaded 8646 */ 8647 if (sgs->group_type == group_fully_busy || 8648 sgs->group_type == group_overloaded) 8649 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 8650 sgs->group_capacity; 8651 } 8652 8653 static bool update_pick_idlest(struct sched_group *idlest, 8654 struct sg_lb_stats *idlest_sgs, 8655 struct sched_group *group, 8656 struct sg_lb_stats *sgs) 8657 { 8658 if (sgs->group_type < idlest_sgs->group_type) 8659 return true; 8660 8661 if (sgs->group_type > idlest_sgs->group_type) 8662 return false; 8663 8664 /* 8665 * The candidate and the current idlest group are the same type of 8666 * group. Let check which one is the idlest according to the type. 8667 */ 8668 8669 switch (sgs->group_type) { 8670 case group_overloaded: 8671 case group_fully_busy: 8672 /* Select the group with lowest avg_load. */ 8673 if (idlest_sgs->avg_load <= sgs->avg_load) 8674 return false; 8675 break; 8676 8677 case group_imbalanced: 8678 case group_asym_packing: 8679 /* Those types are not used in the slow wakeup path */ 8680 return false; 8681 8682 case group_misfit_task: 8683 /* Select group with the highest max capacity */ 8684 if (idlest->sgc->max_capacity >= group->sgc->max_capacity) 8685 return false; 8686 break; 8687 8688 case group_has_spare: 8689 /* Select group with most idle CPUs */ 8690 if (idlest_sgs->idle_cpus >= sgs->idle_cpus) 8691 return false; 8692 break; 8693 } 8694 8695 return true; 8696 } 8697 8698 /* 8699 * find_idlest_group() finds and returns the least busy CPU group within the 8700 * domain. 8701 * 8702 * Assumes p is allowed on at least one CPU in sd. 8703 */ 8704 static struct sched_group * 8705 find_idlest_group(struct sched_domain *sd, struct task_struct *p, 8706 int this_cpu, int sd_flag) 8707 { 8708 struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups; 8709 struct sg_lb_stats local_sgs, tmp_sgs; 8710 struct sg_lb_stats *sgs; 8711 unsigned long imbalance; 8712 struct sg_lb_stats idlest_sgs = { 8713 .avg_load = UINT_MAX, 8714 .group_type = group_overloaded, 8715 }; 8716 8717 imbalance = scale_load_down(NICE_0_LOAD) * 8718 (sd->imbalance_pct-100) / 100; 8719 8720 do { 8721 int local_group; 8722 8723 /* Skip over this group if it has no CPUs allowed */ 8724 if (!cpumask_intersects(sched_group_span(group), 8725 p->cpus_ptr)) 8726 continue; 8727 8728 local_group = cpumask_test_cpu(this_cpu, 8729 sched_group_span(group)); 8730 8731 if (local_group) { 8732 sgs = &local_sgs; 8733 local = group; 8734 } else { 8735 sgs = &tmp_sgs; 8736 } 8737 8738 update_sg_wakeup_stats(sd, group, sgs, p); 8739 8740 if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) { 8741 idlest = group; 8742 idlest_sgs = *sgs; 8743 } 8744 8745 } while (group = group->next, group != sd->groups); 8746 8747 8748 /* There is no idlest group to push tasks to */ 8749 if (!idlest) 8750 return NULL; 8751 8752 /* The local group has been skipped because of CPU affinity */ 8753 if (!local) 8754 return idlest; 8755 8756 /* 8757 * If the local group is idler than the selected idlest group 8758 * don't try and push the task. 8759 */ 8760 if (local_sgs.group_type < idlest_sgs.group_type) 8761 return NULL; 8762 8763 /* 8764 * If the local group is busier than the selected idlest group 8765 * try and push the task. 8766 */ 8767 if (local_sgs.group_type > idlest_sgs.group_type) 8768 return idlest; 8769 8770 switch (local_sgs.group_type) { 8771 case group_overloaded: 8772 case group_fully_busy: 8773 /* 8774 * When comparing groups across NUMA domains, it's possible for 8775 * the local domain to be very lightly loaded relative to the 8776 * remote domains but "imbalance" skews the comparison making 8777 * remote CPUs look much more favourable. When considering 8778 * cross-domain, add imbalance to the load on the remote node 8779 * and consider staying local. 8780 */ 8781 8782 if ((sd->flags & SD_NUMA) && 8783 ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load)) 8784 return NULL; 8785 8786 /* 8787 * If the local group is less loaded than the selected 8788 * idlest group don't try and push any tasks. 8789 */ 8790 if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance)) 8791 return NULL; 8792 8793 if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load) 8794 return NULL; 8795 break; 8796 8797 case group_imbalanced: 8798 case group_asym_packing: 8799 /* Those type are not used in the slow wakeup path */ 8800 return NULL; 8801 8802 case group_misfit_task: 8803 /* Select group with the highest max capacity */ 8804 if (local->sgc->max_capacity >= idlest->sgc->max_capacity) 8805 return NULL; 8806 break; 8807 8808 case group_has_spare: 8809 if (sd->flags & SD_NUMA) { 8810 #ifdef CONFIG_NUMA_BALANCING 8811 int idlest_cpu; 8812 /* 8813 * If there is spare capacity at NUMA, try to select 8814 * the preferred node 8815 */ 8816 if (cpu_to_node(this_cpu) == p->numa_preferred_nid) 8817 return NULL; 8818 8819 idlest_cpu = cpumask_first(sched_group_span(idlest)); 8820 if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid) 8821 return idlest; 8822 #endif 8823 /* 8824 * Otherwise, keep the task on this node to stay close 8825 * its wakeup source and improve locality. If there is 8826 * a real need of migration, periodic load balance will 8827 * take care of it. 8828 */ 8829 if (local_sgs.idle_cpus) 8830 return NULL; 8831 } 8832 8833 /* 8834 * Select group with highest number of idle CPUs. We could also 8835 * compare the utilization which is more stable but it can end 8836 * up that the group has less spare capacity but finally more 8837 * idle CPUs which means more opportunity to run task. 8838 */ 8839 if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus) 8840 return NULL; 8841 break; 8842 } 8843 8844 return idlest; 8845 } 8846 8847 /** 8848 * update_sd_lb_stats - Update sched_domain's statistics for load balancing. 8849 * @env: The load balancing environment. 8850 * @sds: variable to hold the statistics for this sched_domain. 8851 */ 8852 8853 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) 8854 { 8855 struct sched_domain *child = env->sd->child; 8856 struct sched_group *sg = env->sd->groups; 8857 struct sg_lb_stats *local = &sds->local_stat; 8858 struct sg_lb_stats tmp_sgs; 8859 int sg_status = 0; 8860 8861 #ifdef CONFIG_NO_HZ_COMMON 8862 if (env->idle == CPU_NEWLY_IDLE && READ_ONCE(nohz.has_blocked)) 8863 env->flags |= LBF_NOHZ_STATS; 8864 #endif 8865 8866 do { 8867 struct sg_lb_stats *sgs = &tmp_sgs; 8868 int local_group; 8869 8870 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg)); 8871 if (local_group) { 8872 sds->local = sg; 8873 sgs = local; 8874 8875 if (env->idle != CPU_NEWLY_IDLE || 8876 time_after_eq(jiffies, sg->sgc->next_update)) 8877 update_group_capacity(env->sd, env->dst_cpu); 8878 } 8879 8880 update_sg_lb_stats(env, sg, sgs, &sg_status); 8881 8882 if (local_group) 8883 goto next_group; 8884 8885 8886 if (update_sd_pick_busiest(env, sds, sg, sgs)) { 8887 sds->busiest = sg; 8888 sds->busiest_stat = *sgs; 8889 } 8890 8891 next_group: 8892 /* Now, start updating sd_lb_stats */ 8893 sds->total_load += sgs->group_load; 8894 sds->total_capacity += sgs->group_capacity; 8895 8896 sg = sg->next; 8897 } while (sg != env->sd->groups); 8898 8899 /* Tag domain that child domain prefers tasks go to siblings first */ 8900 sds->prefer_sibling = child && child->flags & SD_PREFER_SIBLING; 8901 8902 #ifdef CONFIG_NO_HZ_COMMON 8903 if ((env->flags & LBF_NOHZ_AGAIN) && 8904 cpumask_subset(nohz.idle_cpus_mask, sched_domain_span(env->sd))) { 8905 8906 WRITE_ONCE(nohz.next_blocked, 8907 jiffies + msecs_to_jiffies(LOAD_AVG_PERIOD)); 8908 } 8909 #endif 8910 8911 if (env->sd->flags & SD_NUMA) 8912 env->fbq_type = fbq_classify_group(&sds->busiest_stat); 8913 8914 if (!env->sd->parent) { 8915 struct root_domain *rd = env->dst_rq->rd; 8916 8917 /* update overload indicator if we are at root domain */ 8918 WRITE_ONCE(rd->overload, sg_status & SG_OVERLOAD); 8919 8920 /* Update over-utilization (tipping point, U >= 0) indicator */ 8921 WRITE_ONCE(rd->overutilized, sg_status & SG_OVERUTILIZED); 8922 trace_sched_overutilized_tp(rd, sg_status & SG_OVERUTILIZED); 8923 } else if (sg_status & SG_OVERUTILIZED) { 8924 struct root_domain *rd = env->dst_rq->rd; 8925 8926 WRITE_ONCE(rd->overutilized, SG_OVERUTILIZED); 8927 trace_sched_overutilized_tp(rd, SG_OVERUTILIZED); 8928 } 8929 } 8930 8931 static inline long adjust_numa_imbalance(int imbalance, int src_nr_running) 8932 { 8933 unsigned int imbalance_min; 8934 8935 /* 8936 * Allow a small imbalance based on a simple pair of communicating 8937 * tasks that remain local when the source domain is almost idle. 8938 */ 8939 imbalance_min = 2; 8940 if (src_nr_running <= imbalance_min) 8941 return 0; 8942 8943 return imbalance; 8944 } 8945 8946 /** 8947 * calculate_imbalance - Calculate the amount of imbalance present within the 8948 * groups of a given sched_domain during load balance. 8949 * @env: load balance environment 8950 * @sds: statistics of the sched_domain whose imbalance is to be calculated. 8951 */ 8952 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) 8953 { 8954 struct sg_lb_stats *local, *busiest; 8955 8956 local = &sds->local_stat; 8957 busiest = &sds->busiest_stat; 8958 8959 if (busiest->group_type == group_misfit_task) { 8960 /* Set imbalance to allow misfit tasks to be balanced. */ 8961 env->migration_type = migrate_misfit; 8962 env->imbalance = 1; 8963 return; 8964 } 8965 8966 if (busiest->group_type == group_asym_packing) { 8967 /* 8968 * In case of asym capacity, we will try to migrate all load to 8969 * the preferred CPU. 8970 */ 8971 env->migration_type = migrate_task; 8972 env->imbalance = busiest->sum_h_nr_running; 8973 return; 8974 } 8975 8976 if (busiest->group_type == group_imbalanced) { 8977 /* 8978 * In the group_imb case we cannot rely on group-wide averages 8979 * to ensure CPU-load equilibrium, try to move any task to fix 8980 * the imbalance. The next load balance will take care of 8981 * balancing back the system. 8982 */ 8983 env->migration_type = migrate_task; 8984 env->imbalance = 1; 8985 return; 8986 } 8987 8988 /* 8989 * Try to use spare capacity of local group without overloading it or 8990 * emptying busiest. 8991 */ 8992 if (local->group_type == group_has_spare) { 8993 if (busiest->group_type > group_fully_busy) { 8994 /* 8995 * If busiest is overloaded, try to fill spare 8996 * capacity. This might end up creating spare capacity 8997 * in busiest or busiest still being overloaded but 8998 * there is no simple way to directly compute the 8999 * amount of load to migrate in order to balance the 9000 * system. 9001 */ 9002 env->migration_type = migrate_util; 9003 env->imbalance = max(local->group_capacity, local->group_util) - 9004 local->group_util; 9005 9006 /* 9007 * In some cases, the group's utilization is max or even 9008 * higher than capacity because of migrations but the 9009 * local CPU is (newly) idle. There is at least one 9010 * waiting task in this overloaded busiest group. Let's 9011 * try to pull it. 9012 */ 9013 if (env->idle != CPU_NOT_IDLE && env->imbalance == 0) { 9014 env->migration_type = migrate_task; 9015 env->imbalance = 1; 9016 } 9017 9018 return; 9019 } 9020 9021 if (busiest->group_weight == 1 || sds->prefer_sibling) { 9022 unsigned int nr_diff = busiest->sum_nr_running; 9023 /* 9024 * When prefer sibling, evenly spread running tasks on 9025 * groups. 9026 */ 9027 env->migration_type = migrate_task; 9028 lsub_positive(&nr_diff, local->sum_nr_running); 9029 env->imbalance = nr_diff >> 1; 9030 } else { 9031 9032 /* 9033 * If there is no overload, we just want to even the number of 9034 * idle cpus. 9035 */ 9036 env->migration_type = migrate_task; 9037 env->imbalance = max_t(long, 0, (local->idle_cpus - 9038 busiest->idle_cpus) >> 1); 9039 } 9040 9041 /* Consider allowing a small imbalance between NUMA groups */ 9042 if (env->sd->flags & SD_NUMA) 9043 env->imbalance = adjust_numa_imbalance(env->imbalance, 9044 busiest->sum_nr_running); 9045 9046 return; 9047 } 9048 9049 /* 9050 * Local is fully busy but has to take more load to relieve the 9051 * busiest group 9052 */ 9053 if (local->group_type < group_overloaded) { 9054 /* 9055 * Local will become overloaded so the avg_load metrics are 9056 * finally needed. 9057 */ 9058 9059 local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 9060 local->group_capacity; 9061 9062 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 9063 sds->total_capacity; 9064 /* 9065 * If the local group is more loaded than the selected 9066 * busiest group don't try to pull any tasks. 9067 */ 9068 if (local->avg_load >= busiest->avg_load) { 9069 env->imbalance = 0; 9070 return; 9071 } 9072 } 9073 9074 /* 9075 * Both group are or will become overloaded and we're trying to get all 9076 * the CPUs to the average_load, so we don't want to push ourselves 9077 * above the average load, nor do we wish to reduce the max loaded CPU 9078 * below the average load. At the same time, we also don't want to 9079 * reduce the group load below the group capacity. Thus we look for 9080 * the minimum possible imbalance. 9081 */ 9082 env->migration_type = migrate_load; 9083 env->imbalance = min( 9084 (busiest->avg_load - sds->avg_load) * busiest->group_capacity, 9085 (sds->avg_load - local->avg_load) * local->group_capacity 9086 ) / SCHED_CAPACITY_SCALE; 9087 } 9088 9089 /******* find_busiest_group() helpers end here *********************/ 9090 9091 /* 9092 * Decision matrix according to the local and busiest group type: 9093 * 9094 * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded 9095 * has_spare nr_idle balanced N/A N/A balanced balanced 9096 * fully_busy nr_idle nr_idle N/A N/A balanced balanced 9097 * misfit_task force N/A N/A N/A force force 9098 * asym_packing force force N/A N/A force force 9099 * imbalanced force force N/A N/A force force 9100 * overloaded force force N/A N/A force avg_load 9101 * 9102 * N/A : Not Applicable because already filtered while updating 9103 * statistics. 9104 * balanced : The system is balanced for these 2 groups. 9105 * force : Calculate the imbalance as load migration is probably needed. 9106 * avg_load : Only if imbalance is significant enough. 9107 * nr_idle : dst_cpu is not busy and the number of idle CPUs is quite 9108 * different in groups. 9109 */ 9110 9111 /** 9112 * find_busiest_group - Returns the busiest group within the sched_domain 9113 * if there is an imbalance. 9114 * 9115 * Also calculates the amount of runnable load which should be moved 9116 * to restore balance. 9117 * 9118 * @env: The load balancing environment. 9119 * 9120 * Return: - The busiest group if imbalance exists. 9121 */ 9122 static struct sched_group *find_busiest_group(struct lb_env *env) 9123 { 9124 struct sg_lb_stats *local, *busiest; 9125 struct sd_lb_stats sds; 9126 9127 init_sd_lb_stats(&sds); 9128 9129 /* 9130 * Compute the various statistics relevant for load balancing at 9131 * this level. 9132 */ 9133 update_sd_lb_stats(env, &sds); 9134 9135 if (sched_energy_enabled()) { 9136 struct root_domain *rd = env->dst_rq->rd; 9137 9138 if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized)) 9139 goto out_balanced; 9140 } 9141 9142 local = &sds.local_stat; 9143 busiest = &sds.busiest_stat; 9144 9145 /* There is no busy sibling group to pull tasks from */ 9146 if (!sds.busiest) 9147 goto out_balanced; 9148 9149 /* Misfit tasks should be dealt with regardless of the avg load */ 9150 if (busiest->group_type == group_misfit_task) 9151 goto force_balance; 9152 9153 /* ASYM feature bypasses nice load balance check */ 9154 if (busiest->group_type == group_asym_packing) 9155 goto force_balance; 9156 9157 /* 9158 * If the busiest group is imbalanced the below checks don't 9159 * work because they assume all things are equal, which typically 9160 * isn't true due to cpus_ptr constraints and the like. 9161 */ 9162 if (busiest->group_type == group_imbalanced) 9163 goto force_balance; 9164 9165 /* 9166 * If the local group is busier than the selected busiest group 9167 * don't try and pull any tasks. 9168 */ 9169 if (local->group_type > busiest->group_type) 9170 goto out_balanced; 9171 9172 /* 9173 * When groups are overloaded, use the avg_load to ensure fairness 9174 * between tasks. 9175 */ 9176 if (local->group_type == group_overloaded) { 9177 /* 9178 * If the local group is more loaded than the selected 9179 * busiest group don't try to pull any tasks. 9180 */ 9181 if (local->avg_load >= busiest->avg_load) 9182 goto out_balanced; 9183 9184 /* XXX broken for overlapping NUMA groups */ 9185 sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) / 9186 sds.total_capacity; 9187 9188 /* 9189 * Don't pull any tasks if this group is already above the 9190 * domain average load. 9191 */ 9192 if (local->avg_load >= sds.avg_load) 9193 goto out_balanced; 9194 9195 /* 9196 * If the busiest group is more loaded, use imbalance_pct to be 9197 * conservative. 9198 */ 9199 if (100 * busiest->avg_load <= 9200 env->sd->imbalance_pct * local->avg_load) 9201 goto out_balanced; 9202 } 9203 9204 /* Try to move all excess tasks to child's sibling domain */ 9205 if (sds.prefer_sibling && local->group_type == group_has_spare && 9206 busiest->sum_nr_running > local->sum_nr_running + 1) 9207 goto force_balance; 9208 9209 if (busiest->group_type != group_overloaded) { 9210 if (env->idle == CPU_NOT_IDLE) 9211 /* 9212 * If the busiest group is not overloaded (and as a 9213 * result the local one too) but this CPU is already 9214 * busy, let another idle CPU try to pull task. 9215 */ 9216 goto out_balanced; 9217 9218 if (busiest->group_weight > 1 && 9219 local->idle_cpus <= (busiest->idle_cpus + 1)) 9220 /* 9221 * If the busiest group is not overloaded 9222 * and there is no imbalance between this and busiest 9223 * group wrt idle CPUs, it is balanced. The imbalance 9224 * becomes significant if the diff is greater than 1 9225 * otherwise we might end up to just move the imbalance 9226 * on another group. Of course this applies only if 9227 * there is more than 1 CPU per group. 9228 */ 9229 goto out_balanced; 9230 9231 if (busiest->sum_h_nr_running == 1) 9232 /* 9233 * busiest doesn't have any tasks waiting to run 9234 */ 9235 goto out_balanced; 9236 } 9237 9238 force_balance: 9239 /* Looks like there is an imbalance. Compute it */ 9240 calculate_imbalance(env, &sds); 9241 return env->imbalance ? sds.busiest : NULL; 9242 9243 out_balanced: 9244 env->imbalance = 0; 9245 return NULL; 9246 } 9247 9248 /* 9249 * find_busiest_queue - find the busiest runqueue among the CPUs in the group. 9250 */ 9251 static struct rq *find_busiest_queue(struct lb_env *env, 9252 struct sched_group *group) 9253 { 9254 struct rq *busiest = NULL, *rq; 9255 unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; 9256 unsigned int busiest_nr = 0; 9257 int i; 9258 9259 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 9260 unsigned long capacity, load, util; 9261 unsigned int nr_running; 9262 enum fbq_type rt; 9263 9264 rq = cpu_rq(i); 9265 rt = fbq_classify_rq(rq); 9266 9267 /* 9268 * We classify groups/runqueues into three groups: 9269 * - regular: there are !numa tasks 9270 * - remote: there are numa tasks that run on the 'wrong' node 9271 * - all: there is no distinction 9272 * 9273 * In order to avoid migrating ideally placed numa tasks, 9274 * ignore those when there's better options. 9275 * 9276 * If we ignore the actual busiest queue to migrate another 9277 * task, the next balance pass can still reduce the busiest 9278 * queue by moving tasks around inside the node. 9279 * 9280 * If we cannot move enough load due to this classification 9281 * the next pass will adjust the group classification and 9282 * allow migration of more tasks. 9283 * 9284 * Both cases only affect the total convergence complexity. 9285 */ 9286 if (rt > env->fbq_type) 9287 continue; 9288 9289 capacity = capacity_of(i); 9290 nr_running = rq->cfs.h_nr_running; 9291 9292 /* 9293 * For ASYM_CPUCAPACITY domains, don't pick a CPU that could 9294 * eventually lead to active_balancing high->low capacity. 9295 * Higher per-CPU capacity is considered better than balancing 9296 * average load. 9297 */ 9298 if (env->sd->flags & SD_ASYM_CPUCAPACITY && 9299 capacity_of(env->dst_cpu) < capacity && 9300 nr_running == 1) 9301 continue; 9302 9303 switch (env->migration_type) { 9304 case migrate_load: 9305 /* 9306 * When comparing with load imbalance, use cpu_load() 9307 * which is not scaled with the CPU capacity. 9308 */ 9309 load = cpu_load(rq); 9310 9311 if (nr_running == 1 && load > env->imbalance && 9312 !check_cpu_capacity(rq, env->sd)) 9313 break; 9314 9315 /* 9316 * For the load comparisons with the other CPUs, 9317 * consider the cpu_load() scaled with the CPU 9318 * capacity, so that the load can be moved away 9319 * from the CPU that is potentially running at a 9320 * lower capacity. 9321 * 9322 * Thus we're looking for max(load_i / capacity_i), 9323 * crosswise multiplication to rid ourselves of the 9324 * division works out to: 9325 * load_i * capacity_j > load_j * capacity_i; 9326 * where j is our previous maximum. 9327 */ 9328 if (load * busiest_capacity > busiest_load * capacity) { 9329 busiest_load = load; 9330 busiest_capacity = capacity; 9331 busiest = rq; 9332 } 9333 break; 9334 9335 case migrate_util: 9336 util = cpu_util(cpu_of(rq)); 9337 9338 /* 9339 * Don't try to pull utilization from a CPU with one 9340 * running task. Whatever its utilization, we will fail 9341 * detach the task. 9342 */ 9343 if (nr_running <= 1) 9344 continue; 9345 9346 if (busiest_util < util) { 9347 busiest_util = util; 9348 busiest = rq; 9349 } 9350 break; 9351 9352 case migrate_task: 9353 if (busiest_nr < nr_running) { 9354 busiest_nr = nr_running; 9355 busiest = rq; 9356 } 9357 break; 9358 9359 case migrate_misfit: 9360 /* 9361 * For ASYM_CPUCAPACITY domains with misfit tasks we 9362 * simply seek the "biggest" misfit task. 9363 */ 9364 if (rq->misfit_task_load > busiest_load) { 9365 busiest_load = rq->misfit_task_load; 9366 busiest = rq; 9367 } 9368 9369 break; 9370 9371 } 9372 } 9373 9374 return busiest; 9375 } 9376 9377 /* 9378 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but 9379 * so long as it is large enough. 9380 */ 9381 #define MAX_PINNED_INTERVAL 512 9382 9383 static inline bool 9384 asym_active_balance(struct lb_env *env) 9385 { 9386 /* 9387 * ASYM_PACKING needs to force migrate tasks from busy but 9388 * lower priority CPUs in order to pack all tasks in the 9389 * highest priority CPUs. 9390 */ 9391 return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 9392 sched_asym_prefer(env->dst_cpu, env->src_cpu); 9393 } 9394 9395 static inline bool 9396 voluntary_active_balance(struct lb_env *env) 9397 { 9398 struct sched_domain *sd = env->sd; 9399 9400 if (asym_active_balance(env)) 9401 return 1; 9402 9403 /* 9404 * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. 9405 * It's worth migrating the task if the src_cpu's capacity is reduced 9406 * because of other sched_class or IRQs if more capacity stays 9407 * available on dst_cpu. 9408 */ 9409 if ((env->idle != CPU_NOT_IDLE) && 9410 (env->src_rq->cfs.h_nr_running == 1)) { 9411 if ((check_cpu_capacity(env->src_rq, sd)) && 9412 (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) 9413 return 1; 9414 } 9415 9416 if (env->migration_type == migrate_misfit) 9417 return 1; 9418 9419 return 0; 9420 } 9421 9422 static int need_active_balance(struct lb_env *env) 9423 { 9424 struct sched_domain *sd = env->sd; 9425 9426 if (voluntary_active_balance(env)) 9427 return 1; 9428 9429 return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); 9430 } 9431 9432 static int active_load_balance_cpu_stop(void *data); 9433 9434 static int should_we_balance(struct lb_env *env) 9435 { 9436 struct sched_group *sg = env->sd->groups; 9437 int cpu, balance_cpu = -1; 9438 9439 /* 9440 * Ensure the balancing environment is consistent; can happen 9441 * when the softirq triggers 'during' hotplug. 9442 */ 9443 if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 9444 return 0; 9445 9446 /* 9447 * In the newly idle case, we will allow all the CPUs 9448 * to do the newly idle load balance. 9449 */ 9450 if (env->idle == CPU_NEWLY_IDLE) 9451 return 1; 9452 9453 /* Try to find first idle CPU */ 9454 for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) { 9455 if (!idle_cpu(cpu)) 9456 continue; 9457 9458 balance_cpu = cpu; 9459 break; 9460 } 9461 9462 if (balance_cpu == -1) 9463 balance_cpu = group_balance_cpu(sg); 9464 9465 /* 9466 * First idle CPU or the first CPU(busiest) in this sched group 9467 * is eligible for doing load balancing at this and above domains. 9468 */ 9469 return balance_cpu == env->dst_cpu; 9470 } 9471 9472 /* 9473 * Check this_cpu to ensure it is balanced within domain. Attempt to move 9474 * tasks if there is an imbalance. 9475 */ 9476 static int load_balance(int this_cpu, struct rq *this_rq, 9477 struct sched_domain *sd, enum cpu_idle_type idle, 9478 int *continue_balancing) 9479 { 9480 int ld_moved, cur_ld_moved, active_balance = 0; 9481 struct sched_domain *sd_parent = sd->parent; 9482 struct sched_group *group; 9483 struct rq *busiest; 9484 struct rq_flags rf; 9485 struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 9486 9487 struct lb_env env = { 9488 .sd = sd, 9489 .dst_cpu = this_cpu, 9490 .dst_rq = this_rq, 9491 .dst_grpmask = sched_group_span(sd->groups), 9492 .idle = idle, 9493 .loop_break = sched_nr_migrate_break, 9494 .cpus = cpus, 9495 .fbq_type = all, 9496 .tasks = LIST_HEAD_INIT(env.tasks), 9497 }; 9498 9499 cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); 9500 9501 schedstat_inc(sd->lb_count[idle]); 9502 9503 redo: 9504 if (!should_we_balance(&env)) { 9505 *continue_balancing = 0; 9506 goto out_balanced; 9507 } 9508 9509 group = find_busiest_group(&env); 9510 if (!group) { 9511 schedstat_inc(sd->lb_nobusyg[idle]); 9512 goto out_balanced; 9513 } 9514 9515 busiest = find_busiest_queue(&env, group); 9516 if (!busiest) { 9517 schedstat_inc(sd->lb_nobusyq[idle]); 9518 goto out_balanced; 9519 } 9520 9521 BUG_ON(busiest == env.dst_rq); 9522 9523 schedstat_add(sd->lb_imbalance[idle], env.imbalance); 9524 9525 env.src_cpu = busiest->cpu; 9526 env.src_rq = busiest; 9527 9528 ld_moved = 0; 9529 if (busiest->nr_running > 1) { 9530 /* 9531 * Attempt to move tasks. If find_busiest_group has found 9532 * an imbalance but busiest->nr_running <= 1, the group is 9533 * still unbalanced. ld_moved simply stays zero, so it is 9534 * correctly treated as an imbalance. 9535 */ 9536 env.flags |= LBF_ALL_PINNED; 9537 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 9538 9539 more_balance: 9540 rq_lock_irqsave(busiest, &rf); 9541 update_rq_clock(busiest); 9542 9543 /* 9544 * cur_ld_moved - load moved in current iteration 9545 * ld_moved - cumulative load moved across iterations 9546 */ 9547 cur_ld_moved = detach_tasks(&env); 9548 9549 /* 9550 * We've detached some tasks from busiest_rq. Every 9551 * task is masked "TASK_ON_RQ_MIGRATING", so we can safely 9552 * unlock busiest->lock, and we are able to be sure 9553 * that nobody can manipulate the tasks in parallel. 9554 * See task_rq_lock() family for the details. 9555 */ 9556 9557 rq_unlock(busiest, &rf); 9558 9559 if (cur_ld_moved) { 9560 attach_tasks(&env); 9561 ld_moved += cur_ld_moved; 9562 } 9563 9564 local_irq_restore(rf.flags); 9565 9566 if (env.flags & LBF_NEED_BREAK) { 9567 env.flags &= ~LBF_NEED_BREAK; 9568 goto more_balance; 9569 } 9570 9571 /* 9572 * Revisit (affine) tasks on src_cpu that couldn't be moved to 9573 * us and move them to an alternate dst_cpu in our sched_group 9574 * where they can run. The upper limit on how many times we 9575 * iterate on same src_cpu is dependent on number of CPUs in our 9576 * sched_group. 9577 * 9578 * This changes load balance semantics a bit on who can move 9579 * load to a given_cpu. In addition to the given_cpu itself 9580 * (or a ilb_cpu acting on its behalf where given_cpu is 9581 * nohz-idle), we now have balance_cpu in a position to move 9582 * load to given_cpu. In rare situations, this may cause 9583 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding 9584 * _independently_ and at _same_ time to move some load to 9585 * given_cpu) causing exceess load to be moved to given_cpu. 9586 * This however should not happen so much in practice and 9587 * moreover subsequent load balance cycles should correct the 9588 * excess load moved. 9589 */ 9590 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { 9591 9592 /* Prevent to re-select dst_cpu via env's CPUs */ 9593 __cpumask_clear_cpu(env.dst_cpu, env.cpus); 9594 9595 env.dst_rq = cpu_rq(env.new_dst_cpu); 9596 env.dst_cpu = env.new_dst_cpu; 9597 env.flags &= ~LBF_DST_PINNED; 9598 env.loop = 0; 9599 env.loop_break = sched_nr_migrate_break; 9600 9601 /* 9602 * Go back to "more_balance" rather than "redo" since we 9603 * need to continue with same src_cpu. 9604 */ 9605 goto more_balance; 9606 } 9607 9608 /* 9609 * We failed to reach balance because of affinity. 9610 */ 9611 if (sd_parent) { 9612 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 9613 9614 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) 9615 *group_imbalance = 1; 9616 } 9617 9618 /* All tasks on this runqueue were pinned by CPU affinity */ 9619 if (unlikely(env.flags & LBF_ALL_PINNED)) { 9620 __cpumask_clear_cpu(cpu_of(busiest), cpus); 9621 /* 9622 * Attempting to continue load balancing at the current 9623 * sched_domain level only makes sense if there are 9624 * active CPUs remaining as possible busiest CPUs to 9625 * pull load from which are not contained within the 9626 * destination group that is receiving any migrated 9627 * load. 9628 */ 9629 if (!cpumask_subset(cpus, env.dst_grpmask)) { 9630 env.loop = 0; 9631 env.loop_break = sched_nr_migrate_break; 9632 goto redo; 9633 } 9634 goto out_all_pinned; 9635 } 9636 } 9637 9638 if (!ld_moved) { 9639 schedstat_inc(sd->lb_failed[idle]); 9640 /* 9641 * Increment the failure counter only on periodic balance. 9642 * We do not want newidle balance, which can be very 9643 * frequent, pollute the failure counter causing 9644 * excessive cache_hot migrations and active balances. 9645 */ 9646 if (idle != CPU_NEWLY_IDLE) 9647 sd->nr_balance_failed++; 9648 9649 if (need_active_balance(&env)) { 9650 unsigned long flags; 9651 9652 raw_spin_lock_irqsave(&busiest->lock, flags); 9653 9654 /* 9655 * Don't kick the active_load_balance_cpu_stop, 9656 * if the curr task on busiest CPU can't be 9657 * moved to this_cpu: 9658 */ 9659 if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { 9660 raw_spin_unlock_irqrestore(&busiest->lock, 9661 flags); 9662 env.flags |= LBF_ALL_PINNED; 9663 goto out_one_pinned; 9664 } 9665 9666 /* 9667 * ->active_balance synchronizes accesses to 9668 * ->active_balance_work. Once set, it's cleared 9669 * only after active load balance is finished. 9670 */ 9671 if (!busiest->active_balance) { 9672 busiest->active_balance = 1; 9673 busiest->push_cpu = this_cpu; 9674 active_balance = 1; 9675 } 9676 raw_spin_unlock_irqrestore(&busiest->lock, flags); 9677 9678 if (active_balance) { 9679 stop_one_cpu_nowait(cpu_of(busiest), 9680 active_load_balance_cpu_stop, busiest, 9681 &busiest->active_balance_work); 9682 } 9683 9684 /* We've kicked active balancing, force task migration. */ 9685 sd->nr_balance_failed = sd->cache_nice_tries+1; 9686 } 9687 } else 9688 sd->nr_balance_failed = 0; 9689 9690 if (likely(!active_balance) || voluntary_active_balance(&env)) { 9691 /* We were unbalanced, so reset the balancing interval */ 9692 sd->balance_interval = sd->min_interval; 9693 } else { 9694 /* 9695 * If we've begun active balancing, start to back off. This 9696 * case may not be covered by the all_pinned logic if there 9697 * is only 1 task on the busy runqueue (because we don't call 9698 * detach_tasks). 9699 */ 9700 if (sd->balance_interval < sd->max_interval) 9701 sd->balance_interval *= 2; 9702 } 9703 9704 goto out; 9705 9706 out_balanced: 9707 /* 9708 * We reach balance although we may have faced some affinity 9709 * constraints. Clear the imbalance flag only if other tasks got 9710 * a chance to move and fix the imbalance. 9711 */ 9712 if (sd_parent && !(env.flags & LBF_ALL_PINNED)) { 9713 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 9714 9715 if (*group_imbalance) 9716 *group_imbalance = 0; 9717 } 9718 9719 out_all_pinned: 9720 /* 9721 * We reach balance because all tasks are pinned at this level so 9722 * we can't migrate them. Let the imbalance flag set so parent level 9723 * can try to migrate them. 9724 */ 9725 schedstat_inc(sd->lb_balanced[idle]); 9726 9727 sd->nr_balance_failed = 0; 9728 9729 out_one_pinned: 9730 ld_moved = 0; 9731 9732 /* 9733 * newidle_balance() disregards balance intervals, so we could 9734 * repeatedly reach this code, which would lead to balance_interval 9735 * skyrocketting in a short amount of time. Skip the balance_interval 9736 * increase logic to avoid that. 9737 */ 9738 if (env.idle == CPU_NEWLY_IDLE) 9739 goto out; 9740 9741 /* tune up the balancing interval */ 9742 if ((env.flags & LBF_ALL_PINNED && 9743 sd->balance_interval < MAX_PINNED_INTERVAL) || 9744 sd->balance_interval < sd->max_interval) 9745 sd->balance_interval *= 2; 9746 out: 9747 return ld_moved; 9748 } 9749 9750 static inline unsigned long 9751 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) 9752 { 9753 unsigned long interval = sd->balance_interval; 9754 9755 if (cpu_busy) 9756 interval *= sd->busy_factor; 9757 9758 /* scale ms to jiffies */ 9759 interval = msecs_to_jiffies(interval); 9760 interval = clamp(interval, 1UL, max_load_balance_interval); 9761 9762 return interval; 9763 } 9764 9765 static inline void 9766 update_next_balance(struct sched_domain *sd, unsigned long *next_balance) 9767 { 9768 unsigned long interval, next; 9769 9770 /* used by idle balance, so cpu_busy = 0 */ 9771 interval = get_sd_balance_interval(sd, 0); 9772 next = sd->last_balance + interval; 9773 9774 if (time_after(*next_balance, next)) 9775 *next_balance = next; 9776 } 9777 9778 /* 9779 * active_load_balance_cpu_stop is run by the CPU stopper. It pushes 9780 * running tasks off the busiest CPU onto idle CPUs. It requires at 9781 * least 1 task to be running on each physical CPU where possible, and 9782 * avoids physical / logical imbalances. 9783 */ 9784 static int active_load_balance_cpu_stop(void *data) 9785 { 9786 struct rq *busiest_rq = data; 9787 int busiest_cpu = cpu_of(busiest_rq); 9788 int target_cpu = busiest_rq->push_cpu; 9789 struct rq *target_rq = cpu_rq(target_cpu); 9790 struct sched_domain *sd; 9791 struct task_struct *p = NULL; 9792 struct rq_flags rf; 9793 9794 rq_lock_irq(busiest_rq, &rf); 9795 /* 9796 * Between queueing the stop-work and running it is a hole in which 9797 * CPUs can become inactive. We should not move tasks from or to 9798 * inactive CPUs. 9799 */ 9800 if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu)) 9801 goto out_unlock; 9802 9803 /* Make sure the requested CPU hasn't gone down in the meantime: */ 9804 if (unlikely(busiest_cpu != smp_processor_id() || 9805 !busiest_rq->active_balance)) 9806 goto out_unlock; 9807 9808 /* Is there any task to move? */ 9809 if (busiest_rq->nr_running <= 1) 9810 goto out_unlock; 9811 9812 /* 9813 * This condition is "impossible", if it occurs 9814 * we need to fix it. Originally reported by 9815 * Bjorn Helgaas on a 128-CPU setup. 9816 */ 9817 BUG_ON(busiest_rq == target_rq); 9818 9819 /* Search for an sd spanning us and the target CPU. */ 9820 rcu_read_lock(); 9821 for_each_domain(target_cpu, sd) { 9822 if ((sd->flags & SD_LOAD_BALANCE) && 9823 cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) 9824 break; 9825 } 9826 9827 if (likely(sd)) { 9828 struct lb_env env = { 9829 .sd = sd, 9830 .dst_cpu = target_cpu, 9831 .dst_rq = target_rq, 9832 .src_cpu = busiest_rq->cpu, 9833 .src_rq = busiest_rq, 9834 .idle = CPU_IDLE, 9835 /* 9836 * can_migrate_task() doesn't need to compute new_dst_cpu 9837 * for active balancing. Since we have CPU_IDLE, but no 9838 * @dst_grpmask we need to make that test go away with lying 9839 * about DST_PINNED. 9840 */ 9841 .flags = LBF_DST_PINNED, 9842 }; 9843 9844 schedstat_inc(sd->alb_count); 9845 update_rq_clock(busiest_rq); 9846 9847 p = detach_one_task(&env); 9848 if (p) { 9849 schedstat_inc(sd->alb_pushed); 9850 /* Active balancing done, reset the failure counter. */ 9851 sd->nr_balance_failed = 0; 9852 } else { 9853 schedstat_inc(sd->alb_failed); 9854 } 9855 } 9856 rcu_read_unlock(); 9857 out_unlock: 9858 busiest_rq->active_balance = 0; 9859 rq_unlock(busiest_rq, &rf); 9860 9861 if (p) 9862 attach_one_task(target_rq, p); 9863 9864 local_irq_enable(); 9865 9866 return 0; 9867 } 9868 9869 static DEFINE_SPINLOCK(balancing); 9870 9871 /* 9872 * Scale the max load_balance interval with the number of CPUs in the system. 9873 * This trades load-balance latency on larger machines for less cross talk. 9874 */ 9875 void update_max_interval(void) 9876 { 9877 max_load_balance_interval = HZ*num_online_cpus()/10; 9878 } 9879 9880 /* 9881 * It checks each scheduling domain to see if it is due to be balanced, 9882 * and initiates a balancing operation if so. 9883 * 9884 * Balancing parameters are set up in init_sched_domains. 9885 */ 9886 static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) 9887 { 9888 int continue_balancing = 1; 9889 int cpu = rq->cpu; 9890 int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 9891 unsigned long interval; 9892 struct sched_domain *sd; 9893 /* Earliest time when we have to do rebalance again */ 9894 unsigned long next_balance = jiffies + 60*HZ; 9895 int update_next_balance = 0; 9896 int need_serialize, need_decay = 0; 9897 u64 max_cost = 0; 9898 9899 rcu_read_lock(); 9900 for_each_domain(cpu, sd) { 9901 /* 9902 * Decay the newidle max times here because this is a regular 9903 * visit to all the domains. Decay ~1% per second. 9904 */ 9905 if (time_after(jiffies, sd->next_decay_max_lb_cost)) { 9906 sd->max_newidle_lb_cost = 9907 (sd->max_newidle_lb_cost * 253) / 256; 9908 sd->next_decay_max_lb_cost = jiffies + HZ; 9909 need_decay = 1; 9910 } 9911 max_cost += sd->max_newidle_lb_cost; 9912 9913 if (!(sd->flags & SD_LOAD_BALANCE)) 9914 continue; 9915 9916 /* 9917 * Stop the load balance at this level. There is another 9918 * CPU in our sched group which is doing load balancing more 9919 * actively. 9920 */ 9921 if (!continue_balancing) { 9922 if (need_decay) 9923 continue; 9924 break; 9925 } 9926 9927 interval = get_sd_balance_interval(sd, busy); 9928 9929 need_serialize = sd->flags & SD_SERIALIZE; 9930 if (need_serialize) { 9931 if (!spin_trylock(&balancing)) 9932 goto out; 9933 } 9934 9935 if (time_after_eq(jiffies, sd->last_balance + interval)) { 9936 if (load_balance(cpu, rq, sd, idle, &continue_balancing)) { 9937 /* 9938 * The LBF_DST_PINNED logic could have changed 9939 * env->dst_cpu, so we can't know our idle 9940 * state even if we migrated tasks. Update it. 9941 */ 9942 idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; 9943 busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 9944 } 9945 sd->last_balance = jiffies; 9946 interval = get_sd_balance_interval(sd, busy); 9947 } 9948 if (need_serialize) 9949 spin_unlock(&balancing); 9950 out: 9951 if (time_after(next_balance, sd->last_balance + interval)) { 9952 next_balance = sd->last_balance + interval; 9953 update_next_balance = 1; 9954 } 9955 } 9956 if (need_decay) { 9957 /* 9958 * Ensure the rq-wide value also decays but keep it at a 9959 * reasonable floor to avoid funnies with rq->avg_idle. 9960 */ 9961 rq->max_idle_balance_cost = 9962 max((u64)sysctl_sched_migration_cost, max_cost); 9963 } 9964 rcu_read_unlock(); 9965 9966 /* 9967 * next_balance will be updated only when there is a need. 9968 * When the cpu is attached to null domain for ex, it will not be 9969 * updated. 9970 */ 9971 if (likely(update_next_balance)) { 9972 rq->next_balance = next_balance; 9973 9974 #ifdef CONFIG_NO_HZ_COMMON 9975 /* 9976 * If this CPU has been elected to perform the nohz idle 9977 * balance. Other idle CPUs have already rebalanced with 9978 * nohz_idle_balance() and nohz.next_balance has been 9979 * updated accordingly. This CPU is now running the idle load 9980 * balance for itself and we need to update the 9981 * nohz.next_balance accordingly. 9982 */ 9983 if ((idle == CPU_IDLE) && time_after(nohz.next_balance, rq->next_balance)) 9984 nohz.next_balance = rq->next_balance; 9985 #endif 9986 } 9987 } 9988 9989 static inline int on_null_domain(struct rq *rq) 9990 { 9991 return unlikely(!rcu_dereference_sched(rq->sd)); 9992 } 9993 9994 #ifdef CONFIG_NO_HZ_COMMON 9995 /* 9996 * idle load balancing details 9997 * - When one of the busy CPUs notice that there may be an idle rebalancing 9998 * needed, they will kick the idle load balancer, which then does idle 9999 * load balancing for all the idle CPUs. 10000 * - HK_FLAG_MISC CPUs are used for this task, because HK_FLAG_SCHED not set 10001 * anywhere yet. 10002 */ 10003 10004 static inline int find_new_ilb(void) 10005 { 10006 int ilb; 10007 10008 for_each_cpu_and(ilb, nohz.idle_cpus_mask, 10009 housekeeping_cpumask(HK_FLAG_MISC)) { 10010 if (idle_cpu(ilb)) 10011 return ilb; 10012 } 10013 10014 return nr_cpu_ids; 10015 } 10016 10017 /* 10018 * Kick a CPU to do the nohz balancing, if it is time for it. We pick any 10019 * idle CPU in the HK_FLAG_MISC housekeeping set (if there is one). 10020 */ 10021 static void kick_ilb(unsigned int flags) 10022 { 10023 int ilb_cpu; 10024 10025 nohz.next_balance++; 10026 10027 ilb_cpu = find_new_ilb(); 10028 10029 if (ilb_cpu >= nr_cpu_ids) 10030 return; 10031 10032 flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu)); 10033 if (flags & NOHZ_KICK_MASK) 10034 return; 10035 10036 /* 10037 * Use smp_send_reschedule() instead of resched_cpu(). 10038 * This way we generate a sched IPI on the target CPU which 10039 * is idle. And the softirq performing nohz idle load balance 10040 * will be run before returning from the IPI. 10041 */ 10042 smp_send_reschedule(ilb_cpu); 10043 } 10044 10045 /* 10046 * Current decision point for kicking the idle load balancer in the presence 10047 * of idle CPUs in the system. 10048 */ 10049 static void nohz_balancer_kick(struct rq *rq) 10050 { 10051 unsigned long now = jiffies; 10052 struct sched_domain_shared *sds; 10053 struct sched_domain *sd; 10054 int nr_busy, i, cpu = rq->cpu; 10055 unsigned int flags = 0; 10056 10057 if (unlikely(rq->idle_balance)) 10058 return; 10059 10060 /* 10061 * We may be recently in ticked or tickless idle mode. At the first 10062 * busy tick after returning from idle, we will update the busy stats. 10063 */ 10064 nohz_balance_exit_idle(rq); 10065 10066 /* 10067 * None are in tickless mode and hence no need for NOHZ idle load 10068 * balancing. 10069 */ 10070 if (likely(!atomic_read(&nohz.nr_cpus))) 10071 return; 10072 10073 if (READ_ONCE(nohz.has_blocked) && 10074 time_after(now, READ_ONCE(nohz.next_blocked))) 10075 flags = NOHZ_STATS_KICK; 10076 10077 if (time_before(now, nohz.next_balance)) 10078 goto out; 10079 10080 if (rq->nr_running >= 2) { 10081 flags = NOHZ_KICK_MASK; 10082 goto out; 10083 } 10084 10085 rcu_read_lock(); 10086 10087 sd = rcu_dereference(rq->sd); 10088 if (sd) { 10089 /* 10090 * If there's a CFS task and the current CPU has reduced 10091 * capacity; kick the ILB to see if there's a better CPU to run 10092 * on. 10093 */ 10094 if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) { 10095 flags = NOHZ_KICK_MASK; 10096 goto unlock; 10097 } 10098 } 10099 10100 sd = rcu_dereference(per_cpu(sd_asym_packing, cpu)); 10101 if (sd) { 10102 /* 10103 * When ASYM_PACKING; see if there's a more preferred CPU 10104 * currently idle; in which case, kick the ILB to move tasks 10105 * around. 10106 */ 10107 for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { 10108 if (sched_asym_prefer(i, cpu)) { 10109 flags = NOHZ_KICK_MASK; 10110 goto unlock; 10111 } 10112 } 10113 } 10114 10115 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu)); 10116 if (sd) { 10117 /* 10118 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU 10119 * to run the misfit task on. 10120 */ 10121 if (check_misfit_status(rq, sd)) { 10122 flags = NOHZ_KICK_MASK; 10123 goto unlock; 10124 } 10125 10126 /* 10127 * For asymmetric systems, we do not want to nicely balance 10128 * cache use, instead we want to embrace asymmetry and only 10129 * ensure tasks have enough CPU capacity. 10130 * 10131 * Skip the LLC logic because it's not relevant in that case. 10132 */ 10133 goto unlock; 10134 } 10135 10136 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 10137 if (sds) { 10138 /* 10139 * If there is an imbalance between LLC domains (IOW we could 10140 * increase the overall cache use), we need some less-loaded LLC 10141 * domain to pull some load. Likewise, we may need to spread 10142 * load within the current LLC domain (e.g. packed SMT cores but 10143 * other CPUs are idle). We can't really know from here how busy 10144 * the others are - so just get a nohz balance going if it looks 10145 * like this LLC domain has tasks we could move. 10146 */ 10147 nr_busy = atomic_read(&sds->nr_busy_cpus); 10148 if (nr_busy > 1) { 10149 flags = NOHZ_KICK_MASK; 10150 goto unlock; 10151 } 10152 } 10153 unlock: 10154 rcu_read_unlock(); 10155 out: 10156 if (flags) 10157 kick_ilb(flags); 10158 } 10159 10160 static void set_cpu_sd_state_busy(int cpu) 10161 { 10162 struct sched_domain *sd; 10163 10164 rcu_read_lock(); 10165 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 10166 10167 if (!sd || !sd->nohz_idle) 10168 goto unlock; 10169 sd->nohz_idle = 0; 10170 10171 atomic_inc(&sd->shared->nr_busy_cpus); 10172 unlock: 10173 rcu_read_unlock(); 10174 } 10175 10176 void nohz_balance_exit_idle(struct rq *rq) 10177 { 10178 SCHED_WARN_ON(rq != this_rq()); 10179 10180 if (likely(!rq->nohz_tick_stopped)) 10181 return; 10182 10183 rq->nohz_tick_stopped = 0; 10184 cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask); 10185 atomic_dec(&nohz.nr_cpus); 10186 10187 set_cpu_sd_state_busy(rq->cpu); 10188 } 10189 10190 static void set_cpu_sd_state_idle(int cpu) 10191 { 10192 struct sched_domain *sd; 10193 10194 rcu_read_lock(); 10195 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 10196 10197 if (!sd || sd->nohz_idle) 10198 goto unlock; 10199 sd->nohz_idle = 1; 10200 10201 atomic_dec(&sd->shared->nr_busy_cpus); 10202 unlock: 10203 rcu_read_unlock(); 10204 } 10205 10206 /* 10207 * This routine will record that the CPU is going idle with tick stopped. 10208 * This info will be used in performing idle load balancing in the future. 10209 */ 10210 void nohz_balance_enter_idle(int cpu) 10211 { 10212 struct rq *rq = cpu_rq(cpu); 10213 10214 SCHED_WARN_ON(cpu != smp_processor_id()); 10215 10216 /* If this CPU is going down, then nothing needs to be done: */ 10217 if (!cpu_active(cpu)) 10218 return; 10219 10220 /* Spare idle load balancing on CPUs that don't want to be disturbed: */ 10221 if (!housekeeping_cpu(cpu, HK_FLAG_SCHED)) 10222 return; 10223 10224 /* 10225 * Can be set safely without rq->lock held 10226 * If a clear happens, it will have evaluated last additions because 10227 * rq->lock is held during the check and the clear 10228 */ 10229 rq->has_blocked_load = 1; 10230 10231 /* 10232 * The tick is still stopped but load could have been added in the 10233 * meantime. We set the nohz.has_blocked flag to trig a check of the 10234 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear 10235 * of nohz.has_blocked can only happen after checking the new load 10236 */ 10237 if (rq->nohz_tick_stopped) 10238 goto out; 10239 10240 /* If we're a completely isolated CPU, we don't play: */ 10241 if (on_null_domain(rq)) 10242 return; 10243 10244 rq->nohz_tick_stopped = 1; 10245 10246 cpumask_set_cpu(cpu, nohz.idle_cpus_mask); 10247 atomic_inc(&nohz.nr_cpus); 10248 10249 /* 10250 * Ensures that if nohz_idle_balance() fails to observe our 10251 * @idle_cpus_mask store, it must observe the @has_blocked 10252 * store. 10253 */ 10254 smp_mb__after_atomic(); 10255 10256 set_cpu_sd_state_idle(cpu); 10257 10258 out: 10259 /* 10260 * Each time a cpu enter idle, we assume that it has blocked load and 10261 * enable the periodic update of the load of idle cpus 10262 */ 10263 WRITE_ONCE(nohz.has_blocked, 1); 10264 } 10265 10266 /* 10267 * Internal function that runs load balance for all idle cpus. The load balance 10268 * can be a simple update of blocked load or a complete load balance with 10269 * tasks movement depending of flags. 10270 * The function returns false if the loop has stopped before running 10271 * through all idle CPUs. 10272 */ 10273 static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags, 10274 enum cpu_idle_type idle) 10275 { 10276 /* Earliest time when we have to do rebalance again */ 10277 unsigned long now = jiffies; 10278 unsigned long next_balance = now + 60*HZ; 10279 bool has_blocked_load = false; 10280 int update_next_balance = 0; 10281 int this_cpu = this_rq->cpu; 10282 int balance_cpu; 10283 int ret = false; 10284 struct rq *rq; 10285 10286 SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); 10287 10288 /* 10289 * We assume there will be no idle load after this update and clear 10290 * the has_blocked flag. If a cpu enters idle in the mean time, it will 10291 * set the has_blocked flag and trig another update of idle load. 10292 * Because a cpu that becomes idle, is added to idle_cpus_mask before 10293 * setting the flag, we are sure to not clear the state and not 10294 * check the load of an idle cpu. 10295 */ 10296 WRITE_ONCE(nohz.has_blocked, 0); 10297 10298 /* 10299 * Ensures that if we miss the CPU, we must see the has_blocked 10300 * store from nohz_balance_enter_idle(). 10301 */ 10302 smp_mb(); 10303 10304 for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { 10305 if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) 10306 continue; 10307 10308 /* 10309 * If this CPU gets work to do, stop the load balancing 10310 * work being done for other CPUs. Next load 10311 * balancing owner will pick it up. 10312 */ 10313 if (need_resched()) { 10314 has_blocked_load = true; 10315 goto abort; 10316 } 10317 10318 rq = cpu_rq(balance_cpu); 10319 10320 has_blocked_load |= update_nohz_stats(rq, true); 10321 10322 /* 10323 * If time for next balance is due, 10324 * do the balance. 10325 */ 10326 if (time_after_eq(jiffies, rq->next_balance)) { 10327 struct rq_flags rf; 10328 10329 rq_lock_irqsave(rq, &rf); 10330 update_rq_clock(rq); 10331 rq_unlock_irqrestore(rq, &rf); 10332 10333 if (flags & NOHZ_BALANCE_KICK) 10334 rebalance_domains(rq, CPU_IDLE); 10335 } 10336 10337 if (time_after(next_balance, rq->next_balance)) { 10338 next_balance = rq->next_balance; 10339 update_next_balance = 1; 10340 } 10341 } 10342 10343 /* Newly idle CPU doesn't need an update */ 10344 if (idle != CPU_NEWLY_IDLE) { 10345 update_blocked_averages(this_cpu); 10346 has_blocked_load |= this_rq->has_blocked_load; 10347 } 10348 10349 if (flags & NOHZ_BALANCE_KICK) 10350 rebalance_domains(this_rq, CPU_IDLE); 10351 10352 WRITE_ONCE(nohz.next_blocked, 10353 now + msecs_to_jiffies(LOAD_AVG_PERIOD)); 10354 10355 /* The full idle balance loop has been done */ 10356 ret = true; 10357 10358 abort: 10359 /* There is still blocked load, enable periodic update */ 10360 if (has_blocked_load) 10361 WRITE_ONCE(nohz.has_blocked, 1); 10362 10363 /* 10364 * next_balance will be updated only when there is a need. 10365 * When the CPU is attached to null domain for ex, it will not be 10366 * updated. 10367 */ 10368 if (likely(update_next_balance)) 10369 nohz.next_balance = next_balance; 10370 10371 return ret; 10372 } 10373 10374 /* 10375 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the 10376 * rebalancing for all the cpus for whom scheduler ticks are stopped. 10377 */ 10378 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 10379 { 10380 int this_cpu = this_rq->cpu; 10381 unsigned int flags; 10382 10383 if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK)) 10384 return false; 10385 10386 if (idle != CPU_IDLE) { 10387 atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu)); 10388 return false; 10389 } 10390 10391 /* could be _relaxed() */ 10392 flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu)); 10393 if (!(flags & NOHZ_KICK_MASK)) 10394 return false; 10395 10396 _nohz_idle_balance(this_rq, flags, idle); 10397 10398 return true; 10399 } 10400 10401 static void nohz_newidle_balance(struct rq *this_rq) 10402 { 10403 int this_cpu = this_rq->cpu; 10404 10405 /* 10406 * This CPU doesn't want to be disturbed by scheduler 10407 * housekeeping 10408 */ 10409 if (!housekeeping_cpu(this_cpu, HK_FLAG_SCHED)) 10410 return; 10411 10412 /* Will wake up very soon. No time for doing anything else*/ 10413 if (this_rq->avg_idle < sysctl_sched_migration_cost) 10414 return; 10415 10416 /* Don't need to update blocked load of idle CPUs*/ 10417 if (!READ_ONCE(nohz.has_blocked) || 10418 time_before(jiffies, READ_ONCE(nohz.next_blocked))) 10419 return; 10420 10421 raw_spin_unlock(&this_rq->lock); 10422 /* 10423 * This CPU is going to be idle and blocked load of idle CPUs 10424 * need to be updated. Run the ilb locally as it is a good 10425 * candidate for ilb instead of waking up another idle CPU. 10426 * Kick an normal ilb if we failed to do the update. 10427 */ 10428 if (!_nohz_idle_balance(this_rq, NOHZ_STATS_KICK, CPU_NEWLY_IDLE)) 10429 kick_ilb(NOHZ_STATS_KICK); 10430 raw_spin_lock(&this_rq->lock); 10431 } 10432 10433 #else /* !CONFIG_NO_HZ_COMMON */ 10434 static inline void nohz_balancer_kick(struct rq *rq) { } 10435 10436 static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 10437 { 10438 return false; 10439 } 10440 10441 static inline void nohz_newidle_balance(struct rq *this_rq) { } 10442 #endif /* CONFIG_NO_HZ_COMMON */ 10443 10444 /* 10445 * idle_balance is called by schedule() if this_cpu is about to become 10446 * idle. Attempts to pull tasks from other CPUs. 10447 * 10448 * Returns: 10449 * < 0 - we released the lock and there are !fair tasks present 10450 * 0 - failed, no new tasks 10451 * > 0 - success, new (fair) tasks present 10452 */ 10453 int newidle_balance(struct rq *this_rq, struct rq_flags *rf) 10454 { 10455 unsigned long next_balance = jiffies + HZ; 10456 int this_cpu = this_rq->cpu; 10457 struct sched_domain *sd; 10458 int pulled_task = 0; 10459 u64 curr_cost = 0; 10460 10461 update_misfit_status(NULL, this_rq); 10462 /* 10463 * We must set idle_stamp _before_ calling idle_balance(), such that we 10464 * measure the duration of idle_balance() as idle time. 10465 */ 10466 this_rq->idle_stamp = rq_clock(this_rq); 10467 10468 /* 10469 * Do not pull tasks towards !active CPUs... 10470 */ 10471 if (!cpu_active(this_cpu)) 10472 return 0; 10473 10474 /* 10475 * This is OK, because current is on_cpu, which avoids it being picked 10476 * for load-balance and preemption/IRQs are still disabled avoiding 10477 * further scheduler activity on it and we're being very careful to 10478 * re-start the picking loop. 10479 */ 10480 rq_unpin_lock(this_rq, rf); 10481 10482 if (this_rq->avg_idle < sysctl_sched_migration_cost || 10483 !READ_ONCE(this_rq->rd->overload)) { 10484 10485 rcu_read_lock(); 10486 sd = rcu_dereference_check_sched_domain(this_rq->sd); 10487 if (sd) 10488 update_next_balance(sd, &next_balance); 10489 rcu_read_unlock(); 10490 10491 nohz_newidle_balance(this_rq); 10492 10493 goto out; 10494 } 10495 10496 raw_spin_unlock(&this_rq->lock); 10497 10498 update_blocked_averages(this_cpu); 10499 rcu_read_lock(); 10500 for_each_domain(this_cpu, sd) { 10501 int continue_balancing = 1; 10502 u64 t0, domain_cost; 10503 10504 if (!(sd->flags & SD_LOAD_BALANCE)) 10505 continue; 10506 10507 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) { 10508 update_next_balance(sd, &next_balance); 10509 break; 10510 } 10511 10512 if (sd->flags & SD_BALANCE_NEWIDLE) { 10513 t0 = sched_clock_cpu(this_cpu); 10514 10515 pulled_task = load_balance(this_cpu, this_rq, 10516 sd, CPU_NEWLY_IDLE, 10517 &continue_balancing); 10518 10519 domain_cost = sched_clock_cpu(this_cpu) - t0; 10520 if (domain_cost > sd->max_newidle_lb_cost) 10521 sd->max_newidle_lb_cost = domain_cost; 10522 10523 curr_cost += domain_cost; 10524 } 10525 10526 update_next_balance(sd, &next_balance); 10527 10528 /* 10529 * Stop searching for tasks to pull if there are 10530 * now runnable tasks on this rq. 10531 */ 10532 if (pulled_task || this_rq->nr_running > 0) 10533 break; 10534 } 10535 rcu_read_unlock(); 10536 10537 raw_spin_lock(&this_rq->lock); 10538 10539 if (curr_cost > this_rq->max_idle_balance_cost) 10540 this_rq->max_idle_balance_cost = curr_cost; 10541 10542 out: 10543 /* 10544 * While browsing the domains, we released the rq lock, a task could 10545 * have been enqueued in the meantime. Since we're not going idle, 10546 * pretend we pulled a task. 10547 */ 10548 if (this_rq->cfs.h_nr_running && !pulled_task) 10549 pulled_task = 1; 10550 10551 /* Move the next balance forward */ 10552 if (time_after(this_rq->next_balance, next_balance)) 10553 this_rq->next_balance = next_balance; 10554 10555 /* Is there a task of a high priority class? */ 10556 if (this_rq->nr_running != this_rq->cfs.h_nr_running) 10557 pulled_task = -1; 10558 10559 if (pulled_task) 10560 this_rq->idle_stamp = 0; 10561 10562 rq_repin_lock(this_rq, rf); 10563 10564 return pulled_task; 10565 } 10566 10567 /* 10568 * run_rebalance_domains is triggered when needed from the scheduler tick. 10569 * Also triggered for nohz idle balancing (with nohz_balancing_kick set). 10570 */ 10571 static __latent_entropy void run_rebalance_domains(struct softirq_action *h) 10572 { 10573 struct rq *this_rq = this_rq(); 10574 enum cpu_idle_type idle = this_rq->idle_balance ? 10575 CPU_IDLE : CPU_NOT_IDLE; 10576 10577 /* 10578 * If this CPU has a pending nohz_balance_kick, then do the 10579 * balancing on behalf of the other idle CPUs whose ticks are 10580 * stopped. Do nohz_idle_balance *before* rebalance_domains to 10581 * give the idle CPUs a chance to load balance. Else we may 10582 * load balance only within the local sched_domain hierarchy 10583 * and abort nohz_idle_balance altogether if we pull some load. 10584 */ 10585 if (nohz_idle_balance(this_rq, idle)) 10586 return; 10587 10588 /* normal load balance */ 10589 update_blocked_averages(this_rq->cpu); 10590 rebalance_domains(this_rq, idle); 10591 } 10592 10593 /* 10594 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. 10595 */ 10596 void trigger_load_balance(struct rq *rq) 10597 { 10598 /* Don't need to rebalance while attached to NULL domain */ 10599 if (unlikely(on_null_domain(rq))) 10600 return; 10601 10602 if (time_after_eq(jiffies, rq->next_balance)) 10603 raise_softirq(SCHED_SOFTIRQ); 10604 10605 nohz_balancer_kick(rq); 10606 } 10607 10608 static void rq_online_fair(struct rq *rq) 10609 { 10610 update_sysctl(); 10611 10612 update_runtime_enabled(rq); 10613 } 10614 10615 static void rq_offline_fair(struct rq *rq) 10616 { 10617 update_sysctl(); 10618 10619 /* Ensure any throttled groups are reachable by pick_next_task */ 10620 unthrottle_offline_cfs_rqs(rq); 10621 } 10622 10623 #endif /* CONFIG_SMP */ 10624 10625 /* 10626 * scheduler tick hitting a task of our scheduling class. 10627 * 10628 * NOTE: This function can be called remotely by the tick offload that 10629 * goes along full dynticks. Therefore no local assumption can be made 10630 * and everything must be accessed through the @rq and @curr passed in 10631 * parameters. 10632 */ 10633 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) 10634 { 10635 struct cfs_rq *cfs_rq; 10636 struct sched_entity *se = &curr->se; 10637 10638 for_each_sched_entity(se) { 10639 cfs_rq = cfs_rq_of(se); 10640 entity_tick(cfs_rq, se, queued); 10641 } 10642 10643 if (static_branch_unlikely(&sched_numa_balancing)) 10644 task_tick_numa(rq, curr); 10645 10646 update_misfit_status(curr, rq); 10647 update_overutilized_status(task_rq(curr)); 10648 } 10649 10650 /* 10651 * called on fork with the child task as argument from the parent's context 10652 * - child not yet on the tasklist 10653 * - preemption disabled 10654 */ 10655 static void task_fork_fair(struct task_struct *p) 10656 { 10657 struct cfs_rq *cfs_rq; 10658 struct sched_entity *se = &p->se, *curr; 10659 struct rq *rq = this_rq(); 10660 struct rq_flags rf; 10661 10662 rq_lock(rq, &rf); 10663 update_rq_clock(rq); 10664 10665 cfs_rq = task_cfs_rq(current); 10666 curr = cfs_rq->curr; 10667 if (curr) { 10668 update_curr(cfs_rq); 10669 se->vruntime = curr->vruntime; 10670 } 10671 place_entity(cfs_rq, se, 1); 10672 10673 if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) { 10674 /* 10675 * Upon rescheduling, sched_class::put_prev_task() will place 10676 * 'current' within the tree based on its new key value. 10677 */ 10678 swap(curr->vruntime, se->vruntime); 10679 resched_curr(rq); 10680 } 10681 10682 se->vruntime -= cfs_rq->min_vruntime; 10683 rq_unlock(rq, &rf); 10684 } 10685 10686 /* 10687 * Priority of the task has changed. Check to see if we preempt 10688 * the current task. 10689 */ 10690 static void 10691 prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) 10692 { 10693 if (!task_on_rq_queued(p)) 10694 return; 10695 10696 if (rq->cfs.nr_running == 1) 10697 return; 10698 10699 /* 10700 * Reschedule if we are currently running on this runqueue and 10701 * our priority decreased, or if we are not currently running on 10702 * this runqueue and our priority is higher than the current's 10703 */ 10704 if (rq->curr == p) { 10705 if (p->prio > oldprio) 10706 resched_curr(rq); 10707 } else 10708 check_preempt_curr(rq, p, 0); 10709 } 10710 10711 static inline bool vruntime_normalized(struct task_struct *p) 10712 { 10713 struct sched_entity *se = &p->se; 10714 10715 /* 10716 * In both the TASK_ON_RQ_QUEUED and TASK_ON_RQ_MIGRATING cases, 10717 * the dequeue_entity(.flags=0) will already have normalized the 10718 * vruntime. 10719 */ 10720 if (p->on_rq) 10721 return true; 10722 10723 /* 10724 * When !on_rq, vruntime of the task has usually NOT been normalized. 10725 * But there are some cases where it has already been normalized: 10726 * 10727 * - A forked child which is waiting for being woken up by 10728 * wake_up_new_task(). 10729 * - A task which has been woken up by try_to_wake_up() and 10730 * waiting for actually being woken up by sched_ttwu_pending(). 10731 */ 10732 if (!se->sum_exec_runtime || 10733 (p->state == TASK_WAKING && p->sched_remote_wakeup)) 10734 return true; 10735 10736 return false; 10737 } 10738 10739 #ifdef CONFIG_FAIR_GROUP_SCHED 10740 /* 10741 * Propagate the changes of the sched_entity across the tg tree to make it 10742 * visible to the root 10743 */ 10744 static void propagate_entity_cfs_rq(struct sched_entity *se) 10745 { 10746 struct cfs_rq *cfs_rq; 10747 10748 /* Start to propagate at parent */ 10749 se = se->parent; 10750 10751 for_each_sched_entity(se) { 10752 cfs_rq = cfs_rq_of(se); 10753 10754 if (cfs_rq_throttled(cfs_rq)) 10755 break; 10756 10757 update_load_avg(cfs_rq, se, UPDATE_TG); 10758 } 10759 } 10760 #else 10761 static void propagate_entity_cfs_rq(struct sched_entity *se) { } 10762 #endif 10763 10764 static void detach_entity_cfs_rq(struct sched_entity *se) 10765 { 10766 struct cfs_rq *cfs_rq = cfs_rq_of(se); 10767 10768 /* Catch up with the cfs_rq and remove our load when we leave */ 10769 update_load_avg(cfs_rq, se, 0); 10770 detach_entity_load_avg(cfs_rq, se); 10771 update_tg_load_avg(cfs_rq, false); 10772 propagate_entity_cfs_rq(se); 10773 } 10774 10775 static void attach_entity_cfs_rq(struct sched_entity *se) 10776 { 10777 struct cfs_rq *cfs_rq = cfs_rq_of(se); 10778 10779 #ifdef CONFIG_FAIR_GROUP_SCHED 10780 /* 10781 * Since the real-depth could have been changed (only FAIR 10782 * class maintain depth value), reset depth properly. 10783 */ 10784 se->depth = se->parent ? se->parent->depth + 1 : 0; 10785 #endif 10786 10787 /* Synchronize entity with its cfs_rq */ 10788 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); 10789 attach_entity_load_avg(cfs_rq, se); 10790 update_tg_load_avg(cfs_rq, false); 10791 propagate_entity_cfs_rq(se); 10792 } 10793 10794 static void detach_task_cfs_rq(struct task_struct *p) 10795 { 10796 struct sched_entity *se = &p->se; 10797 struct cfs_rq *cfs_rq = cfs_rq_of(se); 10798 10799 if (!vruntime_normalized(p)) { 10800 /* 10801 * Fix up our vruntime so that the current sleep doesn't 10802 * cause 'unlimited' sleep bonus. 10803 */ 10804 place_entity(cfs_rq, se, 0); 10805 se->vruntime -= cfs_rq->min_vruntime; 10806 } 10807 10808 detach_entity_cfs_rq(se); 10809 } 10810 10811 static void attach_task_cfs_rq(struct task_struct *p) 10812 { 10813 struct sched_entity *se = &p->se; 10814 struct cfs_rq *cfs_rq = cfs_rq_of(se); 10815 10816 attach_entity_cfs_rq(se); 10817 10818 if (!vruntime_normalized(p)) 10819 se->vruntime += cfs_rq->min_vruntime; 10820 } 10821 10822 static void switched_from_fair(struct rq *rq, struct task_struct *p) 10823 { 10824 detach_task_cfs_rq(p); 10825 } 10826 10827 static void switched_to_fair(struct rq *rq, struct task_struct *p) 10828 { 10829 attach_task_cfs_rq(p); 10830 10831 if (task_on_rq_queued(p)) { 10832 /* 10833 * We were most likely switched from sched_rt, so 10834 * kick off the schedule if running, otherwise just see 10835 * if we can still preempt the current task. 10836 */ 10837 if (rq->curr == p) 10838 resched_curr(rq); 10839 else 10840 check_preempt_curr(rq, p, 0); 10841 } 10842 } 10843 10844 /* Account for a task changing its policy or group. 10845 * 10846 * This routine is mostly called to set cfs_rq->curr field when a task 10847 * migrates between groups/classes. 10848 */ 10849 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 10850 { 10851 struct sched_entity *se = &p->se; 10852 10853 #ifdef CONFIG_SMP 10854 if (task_on_rq_queued(p)) { 10855 /* 10856 * Move the next running task to the front of the list, so our 10857 * cfs_tasks list becomes MRU one. 10858 */ 10859 list_move(&se->group_node, &rq->cfs_tasks); 10860 } 10861 #endif 10862 10863 for_each_sched_entity(se) { 10864 struct cfs_rq *cfs_rq = cfs_rq_of(se); 10865 10866 set_next_entity(cfs_rq, se); 10867 /* ensure bandwidth has been allocated on our new cfs_rq */ 10868 account_cfs_rq_runtime(cfs_rq, 0); 10869 } 10870 } 10871 10872 void init_cfs_rq(struct cfs_rq *cfs_rq) 10873 { 10874 cfs_rq->tasks_timeline = RB_ROOT_CACHED; 10875 cfs_rq->min_vruntime = (u64)(-(1LL << 20)); 10876 #ifndef CONFIG_64BIT 10877 cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; 10878 #endif 10879 #ifdef CONFIG_SMP 10880 raw_spin_lock_init(&cfs_rq->removed.lock); 10881 #endif 10882 } 10883 10884 #ifdef CONFIG_FAIR_GROUP_SCHED 10885 static void task_set_group_fair(struct task_struct *p) 10886 { 10887 struct sched_entity *se = &p->se; 10888 10889 set_task_rq(p, task_cpu(p)); 10890 se->depth = se->parent ? se->parent->depth + 1 : 0; 10891 } 10892 10893 static void task_move_group_fair(struct task_struct *p) 10894 { 10895 detach_task_cfs_rq(p); 10896 set_task_rq(p, task_cpu(p)); 10897 10898 #ifdef CONFIG_SMP 10899 /* Tell se's cfs_rq has been changed -- migrated */ 10900 p->se.avg.last_update_time = 0; 10901 #endif 10902 attach_task_cfs_rq(p); 10903 } 10904 10905 static void task_change_group_fair(struct task_struct *p, int type) 10906 { 10907 switch (type) { 10908 case TASK_SET_GROUP: 10909 task_set_group_fair(p); 10910 break; 10911 10912 case TASK_MOVE_GROUP: 10913 task_move_group_fair(p); 10914 break; 10915 } 10916 } 10917 10918 void free_fair_sched_group(struct task_group *tg) 10919 { 10920 int i; 10921 10922 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); 10923 10924 for_each_possible_cpu(i) { 10925 if (tg->cfs_rq) 10926 kfree(tg->cfs_rq[i]); 10927 if (tg->se) 10928 kfree(tg->se[i]); 10929 } 10930 10931 kfree(tg->cfs_rq); 10932 kfree(tg->se); 10933 } 10934 10935 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 10936 { 10937 struct sched_entity *se; 10938 struct cfs_rq *cfs_rq; 10939 int i; 10940 10941 tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL); 10942 if (!tg->cfs_rq) 10943 goto err; 10944 tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL); 10945 if (!tg->se) 10946 goto err; 10947 10948 tg->shares = NICE_0_LOAD; 10949 10950 init_cfs_bandwidth(tg_cfs_bandwidth(tg)); 10951 10952 for_each_possible_cpu(i) { 10953 cfs_rq = kzalloc_node(sizeof(struct cfs_rq), 10954 GFP_KERNEL, cpu_to_node(i)); 10955 if (!cfs_rq) 10956 goto err; 10957 10958 se = kzalloc_node(sizeof(struct sched_entity), 10959 GFP_KERNEL, cpu_to_node(i)); 10960 if (!se) 10961 goto err_free_rq; 10962 10963 init_cfs_rq(cfs_rq); 10964 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); 10965 init_entity_runnable_average(se); 10966 } 10967 10968 return 1; 10969 10970 err_free_rq: 10971 kfree(cfs_rq); 10972 err: 10973 return 0; 10974 } 10975 10976 void online_fair_sched_group(struct task_group *tg) 10977 { 10978 struct sched_entity *se; 10979 struct rq_flags rf; 10980 struct rq *rq; 10981 int i; 10982 10983 for_each_possible_cpu(i) { 10984 rq = cpu_rq(i); 10985 se = tg->se[i]; 10986 rq_lock_irq(rq, &rf); 10987 update_rq_clock(rq); 10988 attach_entity_cfs_rq(se); 10989 sync_throttle(tg, i); 10990 rq_unlock_irq(rq, &rf); 10991 } 10992 } 10993 10994 void unregister_fair_sched_group(struct task_group *tg) 10995 { 10996 unsigned long flags; 10997 struct rq *rq; 10998 int cpu; 10999 11000 for_each_possible_cpu(cpu) { 11001 if (tg->se[cpu]) 11002 remove_entity_load_avg(tg->se[cpu]); 11003 11004 /* 11005 * Only empty task groups can be destroyed; so we can speculatively 11006 * check on_list without danger of it being re-added. 11007 */ 11008 if (!tg->cfs_rq[cpu]->on_list) 11009 continue; 11010 11011 rq = cpu_rq(cpu); 11012 11013 raw_spin_lock_irqsave(&rq->lock, flags); 11014 list_del_leaf_cfs_rq(tg->cfs_rq[cpu]); 11015 raw_spin_unlock_irqrestore(&rq->lock, flags); 11016 } 11017 } 11018 11019 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, 11020 struct sched_entity *se, int cpu, 11021 struct sched_entity *parent) 11022 { 11023 struct rq *rq = cpu_rq(cpu); 11024 11025 cfs_rq->tg = tg; 11026 cfs_rq->rq = rq; 11027 init_cfs_rq_runtime(cfs_rq); 11028 11029 tg->cfs_rq[cpu] = cfs_rq; 11030 tg->se[cpu] = se; 11031 11032 /* se could be NULL for root_task_group */ 11033 if (!se) 11034 return; 11035 11036 if (!parent) { 11037 se->cfs_rq = &rq->cfs; 11038 se->depth = 0; 11039 } else { 11040 se->cfs_rq = parent->my_q; 11041 se->depth = parent->depth + 1; 11042 } 11043 11044 se->my_q = cfs_rq; 11045 /* guarantee group entities always have weight */ 11046 update_load_set(&se->load, NICE_0_LOAD); 11047 se->parent = parent; 11048 } 11049 11050 static DEFINE_MUTEX(shares_mutex); 11051 11052 int sched_group_set_shares(struct task_group *tg, unsigned long shares) 11053 { 11054 int i; 11055 11056 /* 11057 * We can't change the weight of the root cgroup. 11058 */ 11059 if (!tg->se[0]) 11060 return -EINVAL; 11061 11062 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); 11063 11064 mutex_lock(&shares_mutex); 11065 if (tg->shares == shares) 11066 goto done; 11067 11068 tg->shares = shares; 11069 for_each_possible_cpu(i) { 11070 struct rq *rq = cpu_rq(i); 11071 struct sched_entity *se = tg->se[i]; 11072 struct rq_flags rf; 11073 11074 /* Propagate contribution to hierarchy */ 11075 rq_lock_irqsave(rq, &rf); 11076 update_rq_clock(rq); 11077 for_each_sched_entity(se) { 11078 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 11079 update_cfs_group(se); 11080 } 11081 rq_unlock_irqrestore(rq, &rf); 11082 } 11083 11084 done: 11085 mutex_unlock(&shares_mutex); 11086 return 0; 11087 } 11088 #else /* CONFIG_FAIR_GROUP_SCHED */ 11089 11090 void free_fair_sched_group(struct task_group *tg) { } 11091 11092 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 11093 { 11094 return 1; 11095 } 11096 11097 void online_fair_sched_group(struct task_group *tg) { } 11098 11099 void unregister_fair_sched_group(struct task_group *tg) { } 11100 11101 #endif /* CONFIG_FAIR_GROUP_SCHED */ 11102 11103 11104 static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) 11105 { 11106 struct sched_entity *se = &task->se; 11107 unsigned int rr_interval = 0; 11108 11109 /* 11110 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise 11111 * idle runqueue: 11112 */ 11113 if (rq->cfs.load.weight) 11114 rr_interval = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se)); 11115 11116 return rr_interval; 11117 } 11118 11119 /* 11120 * All the scheduling class methods: 11121 */ 11122 const struct sched_class fair_sched_class = { 11123 .next = &idle_sched_class, 11124 .enqueue_task = enqueue_task_fair, 11125 .dequeue_task = dequeue_task_fair, 11126 .yield_task = yield_task_fair, 11127 .yield_to_task = yield_to_task_fair, 11128 11129 .check_preempt_curr = check_preempt_wakeup, 11130 11131 .pick_next_task = __pick_next_task_fair, 11132 .put_prev_task = put_prev_task_fair, 11133 .set_next_task = set_next_task_fair, 11134 11135 #ifdef CONFIG_SMP 11136 .balance = balance_fair, 11137 .select_task_rq = select_task_rq_fair, 11138 .migrate_task_rq = migrate_task_rq_fair, 11139 11140 .rq_online = rq_online_fair, 11141 .rq_offline = rq_offline_fair, 11142 11143 .task_dead = task_dead_fair, 11144 .set_cpus_allowed = set_cpus_allowed_common, 11145 #endif 11146 11147 .task_tick = task_tick_fair, 11148 .task_fork = task_fork_fair, 11149 11150 .prio_changed = prio_changed_fair, 11151 .switched_from = switched_from_fair, 11152 .switched_to = switched_to_fair, 11153 11154 .get_rr_interval = get_rr_interval_fair, 11155 11156 .update_curr = update_curr_fair, 11157 11158 #ifdef CONFIG_FAIR_GROUP_SCHED 11159 .task_change_group = task_change_group_fair, 11160 #endif 11161 11162 #ifdef CONFIG_UCLAMP_TASK 11163 .uclamp_enabled = 1, 11164 #endif 11165 }; 11166 11167 #ifdef CONFIG_SCHED_DEBUG 11168 void print_cfs_stats(struct seq_file *m, int cpu) 11169 { 11170 struct cfs_rq *cfs_rq, *pos; 11171 11172 rcu_read_lock(); 11173 for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos) 11174 print_cfs_rq(m, cpu, cfs_rq); 11175 rcu_read_unlock(); 11176 } 11177 11178 #ifdef CONFIG_NUMA_BALANCING 11179 void show_numa_stats(struct task_struct *p, struct seq_file *m) 11180 { 11181 int node; 11182 unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; 11183 struct numa_group *ng; 11184 11185 rcu_read_lock(); 11186 ng = rcu_dereference(p->numa_group); 11187 for_each_online_node(node) { 11188 if (p->numa_faults) { 11189 tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; 11190 tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; 11191 } 11192 if (ng) { 11193 gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)], 11194 gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 11195 } 11196 print_numa_stats(m, node, tsf, tpf, gsf, gpf); 11197 } 11198 rcu_read_unlock(); 11199 } 11200 #endif /* CONFIG_NUMA_BALANCING */ 11201 #endif /* CONFIG_SCHED_DEBUG */ 11202 11203 __init void init_sched_fair_class(void) 11204 { 11205 #ifdef CONFIG_SMP 11206 open_softirq(SCHED_SOFTIRQ, run_rebalance_domains); 11207 11208 #ifdef CONFIG_NO_HZ_COMMON 11209 nohz.next_balance = jiffies; 11210 nohz.next_blocked = jiffies; 11211 zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); 11212 #endif 11213 #endif /* SMP */ 11214 11215 } 11216 11217 /* 11218 * Helper functions to facilitate extracting info from tracepoints. 11219 */ 11220 11221 const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq) 11222 { 11223 #ifdef CONFIG_SMP 11224 return cfs_rq ? &cfs_rq->avg : NULL; 11225 #else 11226 return NULL; 11227 #endif 11228 } 11229 EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_avg); 11230 11231 char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len) 11232 { 11233 if (!cfs_rq) { 11234 if (str) 11235 strlcpy(str, "(null)", len); 11236 else 11237 return NULL; 11238 } 11239 11240 cfs_rq_tg_path(cfs_rq, str, len); 11241 return str; 11242 } 11243 EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_path); 11244 11245 int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq) 11246 { 11247 return cfs_rq ? cpu_of(rq_of(cfs_rq)) : -1; 11248 } 11249 EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_cpu); 11250 11251 const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq) 11252 { 11253 #ifdef CONFIG_SMP 11254 return rq ? &rq->avg_rt : NULL; 11255 #else 11256 return NULL; 11257 #endif 11258 } 11259 EXPORT_SYMBOL_GPL(sched_trace_rq_avg_rt); 11260 11261 const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq) 11262 { 11263 #ifdef CONFIG_SMP 11264 return rq ? &rq->avg_dl : NULL; 11265 #else 11266 return NULL; 11267 #endif 11268 } 11269 EXPORT_SYMBOL_GPL(sched_trace_rq_avg_dl); 11270 11271 const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq) 11272 { 11273 #if defined(CONFIG_SMP) && defined(CONFIG_HAVE_SCHED_AVG_IRQ) 11274 return rq ? &rq->avg_irq : NULL; 11275 #else 11276 return NULL; 11277 #endif 11278 } 11279 EXPORT_SYMBOL_GPL(sched_trace_rq_avg_irq); 11280 11281 int sched_trace_rq_cpu(struct rq *rq) 11282 { 11283 return rq ? cpu_of(rq) : -1; 11284 } 11285 EXPORT_SYMBOL_GPL(sched_trace_rq_cpu); 11286 11287 const struct cpumask *sched_trace_rd_span(struct root_domain *rd) 11288 { 11289 #ifdef CONFIG_SMP 11290 return rd ? rd->span : NULL; 11291 #else 11292 return NULL; 11293 #endif 11294 } 11295 EXPORT_SYMBOL_GPL(sched_trace_rd_span); 11296