1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/sched/core.c 4 * 5 * Core kernel CPU scheduler code 6 * 7 * Copyright (C) 1991-2002 Linus Torvalds 8 * Copyright (C) 1998-2024 Ingo Molnar, Red Hat 9 */ 10 #include <linux/highmem.h> 11 #include <linux/hrtimer_api.h> 12 #include <linux/ktime_api.h> 13 #include <linux/sched/signal.h> 14 #include <linux/syscalls_api.h> 15 #include <linux/debug_locks.h> 16 #include <linux/prefetch.h> 17 #include <linux/capability.h> 18 #include <linux/pgtable_api.h> 19 #include <linux/wait_bit.h> 20 #include <linux/jiffies.h> 21 #include <linux/spinlock_api.h> 22 #include <linux/cpumask_api.h> 23 #include <linux/lockdep_api.h> 24 #include <linux/hardirq.h> 25 #include <linux/softirq.h> 26 #include <linux/refcount_api.h> 27 #include <linux/topology.h> 28 #include <linux/sched/clock.h> 29 #include <linux/sched/cond_resched.h> 30 #include <linux/sched/cputime.h> 31 #include <linux/sched/debug.h> 32 #include <linux/sched/hotplug.h> 33 #include <linux/sched/init.h> 34 #include <linux/sched/isolation.h> 35 #include <linux/sched/loadavg.h> 36 #include <linux/sched/mm.h> 37 #include <linux/sched/nohz.h> 38 #include <linux/sched/rseq_api.h> 39 #include <linux/sched/rt.h> 40 41 #include <linux/blkdev.h> 42 #include <linux/context_tracking.h> 43 #include <linux/cpuset.h> 44 #include <linux/delayacct.h> 45 #include <linux/init_task.h> 46 #include <linux/interrupt.h> 47 #include <linux/ioprio.h> 48 #include <linux/kallsyms.h> 49 #include <linux/kcov.h> 50 #include <linux/kprobes.h> 51 #include <linux/llist_api.h> 52 #include <linux/mmu_context.h> 53 #include <linux/mmzone.h> 54 #include <linux/mutex_api.h> 55 #include <linux/nmi.h> 56 #include <linux/nospec.h> 57 #include <linux/perf_event_api.h> 58 #include <linux/profile.h> 59 #include <linux/psi.h> 60 #include <linux/rcuwait_api.h> 61 #include <linux/rseq.h> 62 #include <linux/sched/wake_q.h> 63 #include <linux/scs.h> 64 #include <linux/slab.h> 65 #include <linux/syscalls.h> 66 #include <linux/vtime.h> 67 #include <linux/wait_api.h> 68 #include <linux/workqueue_api.h> 69 70 #ifdef CONFIG_PREEMPT_DYNAMIC 71 # ifdef CONFIG_GENERIC_ENTRY 72 # include <linux/entry-common.h> 73 # endif 74 #endif 75 76 #include <uapi/linux/sched/types.h> 77 78 #include <asm/irq_regs.h> 79 #include <asm/switch_to.h> 80 #include <asm/tlb.h> 81 82 #define CREATE_TRACE_POINTS 83 #include <linux/sched/rseq_api.h> 84 #include <trace/events/sched.h> 85 #include <trace/events/ipi.h> 86 #undef CREATE_TRACE_POINTS 87 88 #include "sched.h" 89 #include "stats.h" 90 91 #include "autogroup.h" 92 #include "pelt.h" 93 #include "smp.h" 94 #include "stats.h" 95 96 #include "../workqueue_internal.h" 97 #include "../../io_uring/io-wq.h" 98 #include "../smpboot.h" 99 100 EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); 101 EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); 102 103 /* 104 * Export tracepoints that act as a bare tracehook (ie: have no trace event 105 * associated with them) to allow external modules to probe them. 106 */ 107 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp); 108 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp); 109 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp); 110 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp); 111 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp); 112 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_hw_tp); 113 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp); 114 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp); 115 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); 116 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); 117 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); 118 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_compute_energy_tp); 119 120 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 121 122 #ifdef CONFIG_SCHED_DEBUG 123 /* 124 * Debugging: various feature bits 125 * 126 * If SCHED_DEBUG is disabled, each compilation unit has its own copy of 127 * sysctl_sched_features, defined in sched.h, to allow constants propagation 128 * at compile time and compiler optimization based on features default. 129 */ 130 #define SCHED_FEAT(name, enabled) \ 131 (1UL << __SCHED_FEAT_##name) * enabled | 132 const_debug unsigned int sysctl_sched_features = 133 #include "features.h" 134 0; 135 #undef SCHED_FEAT 136 137 /* 138 * Print a warning if need_resched is set for the given duration (if 139 * LATENCY_WARN is enabled). 140 * 141 * If sysctl_resched_latency_warn_once is set, only one warning will be shown 142 * per boot. 143 */ 144 __read_mostly int sysctl_resched_latency_warn_ms = 100; 145 __read_mostly int sysctl_resched_latency_warn_once = 1; 146 #endif /* CONFIG_SCHED_DEBUG */ 147 148 /* 149 * Number of tasks to iterate in a single balance run. 150 * Limited because this is done with IRQs disabled. 151 */ 152 const_debug unsigned int sysctl_sched_nr_migrate = SCHED_NR_MIGRATE_BREAK; 153 154 __read_mostly int scheduler_running; 155 156 #ifdef CONFIG_SCHED_CORE 157 158 DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); 159 160 /* kernel prio, less is more */ 161 static inline int __task_prio(const struct task_struct *p) 162 { 163 if (p->sched_class == &stop_sched_class) /* trumps deadline */ 164 return -2; 165 166 if (p->dl_server) 167 return -1; /* deadline */ 168 169 if (rt_or_dl_prio(p->prio)) 170 return p->prio; /* [-1, 99] */ 171 172 if (p->sched_class == &idle_sched_class) 173 return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ 174 175 return MAX_RT_PRIO + MAX_NICE; /* 120, squash fair */ 176 } 177 178 /* 179 * l(a,b) 180 * le(a,b) := !l(b,a) 181 * g(a,b) := l(b,a) 182 * ge(a,b) := !l(a,b) 183 */ 184 185 /* real prio, less is less */ 186 static inline bool prio_less(const struct task_struct *a, 187 const struct task_struct *b, bool in_fi) 188 { 189 190 int pa = __task_prio(a), pb = __task_prio(b); 191 192 if (-pa < -pb) 193 return true; 194 195 if (-pb < -pa) 196 return false; 197 198 if (pa == -1) { /* dl_prio() doesn't work because of stop_class above */ 199 const struct sched_dl_entity *a_dl, *b_dl; 200 201 a_dl = &a->dl; 202 /* 203 * Since,'a' and 'b' can be CFS tasks served by DL server, 204 * __task_prio() can return -1 (for DL) even for those. In that 205 * case, get to the dl_server's DL entity. 206 */ 207 if (a->dl_server) 208 a_dl = a->dl_server; 209 210 b_dl = &b->dl; 211 if (b->dl_server) 212 b_dl = b->dl_server; 213 214 return !dl_time_before(a_dl->deadline, b_dl->deadline); 215 } 216 217 if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ 218 return cfs_prio_less(a, b, in_fi); 219 220 return false; 221 } 222 223 static inline bool __sched_core_less(const struct task_struct *a, 224 const struct task_struct *b) 225 { 226 if (a->core_cookie < b->core_cookie) 227 return true; 228 229 if (a->core_cookie > b->core_cookie) 230 return false; 231 232 /* flip prio, so high prio is leftmost */ 233 if (prio_less(b, a, !!task_rq(a)->core->core_forceidle_count)) 234 return true; 235 236 return false; 237 } 238 239 #define __node_2_sc(node) rb_entry((node), struct task_struct, core_node) 240 241 static inline bool rb_sched_core_less(struct rb_node *a, const struct rb_node *b) 242 { 243 return __sched_core_less(__node_2_sc(a), __node_2_sc(b)); 244 } 245 246 static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node) 247 { 248 const struct task_struct *p = __node_2_sc(node); 249 unsigned long cookie = (unsigned long)key; 250 251 if (cookie < p->core_cookie) 252 return -1; 253 254 if (cookie > p->core_cookie) 255 return 1; 256 257 return 0; 258 } 259 260 void sched_core_enqueue(struct rq *rq, struct task_struct *p) 261 { 262 if (p->se.sched_delayed) 263 return; 264 265 rq->core->core_task_seq++; 266 267 if (!p->core_cookie) 268 return; 269 270 rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less); 271 } 272 273 void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) 274 { 275 if (p->se.sched_delayed) 276 return; 277 278 rq->core->core_task_seq++; 279 280 if (sched_core_enqueued(p)) { 281 rb_erase(&p->core_node, &rq->core_tree); 282 RB_CLEAR_NODE(&p->core_node); 283 } 284 285 /* 286 * Migrating the last task off the cpu, with the cpu in forced idle 287 * state. Reschedule to create an accounting edge for forced idle, 288 * and re-examine whether the core is still in forced idle state. 289 */ 290 if (!(flags & DEQUEUE_SAVE) && rq->nr_running == 1 && 291 rq->core->core_forceidle_count && rq->curr == rq->idle) 292 resched_curr(rq); 293 } 294 295 static int sched_task_is_throttled(struct task_struct *p, int cpu) 296 { 297 if (p->sched_class->task_is_throttled) 298 return p->sched_class->task_is_throttled(p, cpu); 299 300 return 0; 301 } 302 303 static struct task_struct *sched_core_next(struct task_struct *p, unsigned long cookie) 304 { 305 struct rb_node *node = &p->core_node; 306 int cpu = task_cpu(p); 307 308 do { 309 node = rb_next(node); 310 if (!node) 311 return NULL; 312 313 p = __node_2_sc(node); 314 if (p->core_cookie != cookie) 315 return NULL; 316 317 } while (sched_task_is_throttled(p, cpu)); 318 319 return p; 320 } 321 322 /* 323 * Find left-most (aka, highest priority) and unthrottled task matching @cookie. 324 * If no suitable task is found, NULL will be returned. 325 */ 326 static struct task_struct *sched_core_find(struct rq *rq, unsigned long cookie) 327 { 328 struct task_struct *p; 329 struct rb_node *node; 330 331 node = rb_find_first((void *)cookie, &rq->core_tree, rb_sched_core_cmp); 332 if (!node) 333 return NULL; 334 335 p = __node_2_sc(node); 336 if (!sched_task_is_throttled(p, rq->cpu)) 337 return p; 338 339 return sched_core_next(p, cookie); 340 } 341 342 /* 343 * Magic required such that: 344 * 345 * raw_spin_rq_lock(rq); 346 * ... 347 * raw_spin_rq_unlock(rq); 348 * 349 * ends up locking and unlocking the _same_ lock, and all CPUs 350 * always agree on what rq has what lock. 351 * 352 * XXX entirely possible to selectively enable cores, don't bother for now. 353 */ 354 355 static DEFINE_MUTEX(sched_core_mutex); 356 static atomic_t sched_core_count; 357 static struct cpumask sched_core_mask; 358 359 static void sched_core_lock(int cpu, unsigned long *flags) 360 { 361 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 362 int t, i = 0; 363 364 local_irq_save(*flags); 365 for_each_cpu(t, smt_mask) 366 raw_spin_lock_nested(&cpu_rq(t)->__lock, i++); 367 } 368 369 static void sched_core_unlock(int cpu, unsigned long *flags) 370 { 371 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 372 int t; 373 374 for_each_cpu(t, smt_mask) 375 raw_spin_unlock(&cpu_rq(t)->__lock); 376 local_irq_restore(*flags); 377 } 378 379 static void __sched_core_flip(bool enabled) 380 { 381 unsigned long flags; 382 int cpu, t; 383 384 cpus_read_lock(); 385 386 /* 387 * Toggle the online cores, one by one. 388 */ 389 cpumask_copy(&sched_core_mask, cpu_online_mask); 390 for_each_cpu(cpu, &sched_core_mask) { 391 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 392 393 sched_core_lock(cpu, &flags); 394 395 for_each_cpu(t, smt_mask) 396 cpu_rq(t)->core_enabled = enabled; 397 398 cpu_rq(cpu)->core->core_forceidle_start = 0; 399 400 sched_core_unlock(cpu, &flags); 401 402 cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask); 403 } 404 405 /* 406 * Toggle the offline CPUs. 407 */ 408 for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask) 409 cpu_rq(cpu)->core_enabled = enabled; 410 411 cpus_read_unlock(); 412 } 413 414 static void sched_core_assert_empty(void) 415 { 416 int cpu; 417 418 for_each_possible_cpu(cpu) 419 WARN_ON_ONCE(!RB_EMPTY_ROOT(&cpu_rq(cpu)->core_tree)); 420 } 421 422 static void __sched_core_enable(void) 423 { 424 static_branch_enable(&__sched_core_enabled); 425 /* 426 * Ensure all previous instances of raw_spin_rq_*lock() have finished 427 * and future ones will observe !sched_core_disabled(). 428 */ 429 synchronize_rcu(); 430 __sched_core_flip(true); 431 sched_core_assert_empty(); 432 } 433 434 static void __sched_core_disable(void) 435 { 436 sched_core_assert_empty(); 437 __sched_core_flip(false); 438 static_branch_disable(&__sched_core_enabled); 439 } 440 441 void sched_core_get(void) 442 { 443 if (atomic_inc_not_zero(&sched_core_count)) 444 return; 445 446 mutex_lock(&sched_core_mutex); 447 if (!atomic_read(&sched_core_count)) 448 __sched_core_enable(); 449 450 smp_mb__before_atomic(); 451 atomic_inc(&sched_core_count); 452 mutex_unlock(&sched_core_mutex); 453 } 454 455 static void __sched_core_put(struct work_struct *work) 456 { 457 if (atomic_dec_and_mutex_lock(&sched_core_count, &sched_core_mutex)) { 458 __sched_core_disable(); 459 mutex_unlock(&sched_core_mutex); 460 } 461 } 462 463 void sched_core_put(void) 464 { 465 static DECLARE_WORK(_work, __sched_core_put); 466 467 /* 468 * "There can be only one" 469 * 470 * Either this is the last one, or we don't actually need to do any 471 * 'work'. If it is the last *again*, we rely on 472 * WORK_STRUCT_PENDING_BIT. 473 */ 474 if (!atomic_add_unless(&sched_core_count, -1, 1)) 475 schedule_work(&_work); 476 } 477 478 #else /* !CONFIG_SCHED_CORE */ 479 480 static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { } 481 static inline void 482 sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } 483 484 #endif /* CONFIG_SCHED_CORE */ 485 486 /* 487 * Serialization rules: 488 * 489 * Lock order: 490 * 491 * p->pi_lock 492 * rq->lock 493 * hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls) 494 * 495 * rq1->lock 496 * rq2->lock where: rq1 < rq2 497 * 498 * Regular state: 499 * 500 * Normal scheduling state is serialized by rq->lock. __schedule() takes the 501 * local CPU's rq->lock, it optionally removes the task from the runqueue and 502 * always looks at the local rq data structures to find the most eligible task 503 * to run next. 504 * 505 * Task enqueue is also under rq->lock, possibly taken from another CPU. 506 * Wakeups from another LLC domain might use an IPI to transfer the enqueue to 507 * the local CPU to avoid bouncing the runqueue state around [ see 508 * ttwu_queue_wakelist() ] 509 * 510 * Task wakeup, specifically wakeups that involve migration, are horribly 511 * complicated to avoid having to take two rq->locks. 512 * 513 * Special state: 514 * 515 * System-calls and anything external will use task_rq_lock() which acquires 516 * both p->pi_lock and rq->lock. As a consequence the state they change is 517 * stable while holding either lock: 518 * 519 * - sched_setaffinity()/ 520 * set_cpus_allowed_ptr(): p->cpus_ptr, p->nr_cpus_allowed 521 * - set_user_nice(): p->se.load, p->*prio 522 * - __sched_setscheduler(): p->sched_class, p->policy, p->*prio, 523 * p->se.load, p->rt_priority, 524 * p->dl.dl_{runtime, deadline, period, flags, bw, density} 525 * - sched_setnuma(): p->numa_preferred_nid 526 * - sched_move_task(): p->sched_task_group 527 * - uclamp_update_active() p->uclamp* 528 * 529 * p->state <- TASK_*: 530 * 531 * is changed locklessly using set_current_state(), __set_current_state() or 532 * set_special_state(), see their respective comments, or by 533 * try_to_wake_up(). This latter uses p->pi_lock to serialize against 534 * concurrent self. 535 * 536 * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }: 537 * 538 * is set by activate_task() and cleared by deactivate_task(), under 539 * rq->lock. Non-zero indicates the task is runnable, the special 540 * ON_RQ_MIGRATING state is used for migration without holding both 541 * rq->locks. It indicates task_cpu() is not stable, see task_rq_lock(). 542 * 543 * p->on_cpu <- { 0, 1 }: 544 * 545 * is set by prepare_task() and cleared by finish_task() such that it will be 546 * set before p is scheduled-in and cleared after p is scheduled-out, both 547 * under rq->lock. Non-zero indicates the task is running on its CPU. 548 * 549 * [ The astute reader will observe that it is possible for two tasks on one 550 * CPU to have ->on_cpu = 1 at the same time. ] 551 * 552 * task_cpu(p): is changed by set_task_cpu(), the rules are: 553 * 554 * - Don't call set_task_cpu() on a blocked task: 555 * 556 * We don't care what CPU we're not running on, this simplifies hotplug, 557 * the CPU assignment of blocked tasks isn't required to be valid. 558 * 559 * - for try_to_wake_up(), called under p->pi_lock: 560 * 561 * This allows try_to_wake_up() to only take one rq->lock, see its comment. 562 * 563 * - for migration called under rq->lock: 564 * [ see task_on_rq_migrating() in task_rq_lock() ] 565 * 566 * o move_queued_task() 567 * o detach_task() 568 * 569 * - for migration called under double_rq_lock(): 570 * 571 * o __migrate_swap_task() 572 * o push_rt_task() / pull_rt_task() 573 * o push_dl_task() / pull_dl_task() 574 * o dl_task_offline_migration() 575 * 576 */ 577 578 void raw_spin_rq_lock_nested(struct rq *rq, int subclass) 579 { 580 raw_spinlock_t *lock; 581 582 /* Matches synchronize_rcu() in __sched_core_enable() */ 583 preempt_disable(); 584 if (sched_core_disabled()) { 585 raw_spin_lock_nested(&rq->__lock, subclass); 586 /* preempt_count *MUST* be > 1 */ 587 preempt_enable_no_resched(); 588 return; 589 } 590 591 for (;;) { 592 lock = __rq_lockp(rq); 593 raw_spin_lock_nested(lock, subclass); 594 if (likely(lock == __rq_lockp(rq))) { 595 /* preempt_count *MUST* be > 1 */ 596 preempt_enable_no_resched(); 597 return; 598 } 599 raw_spin_unlock(lock); 600 } 601 } 602 603 bool raw_spin_rq_trylock(struct rq *rq) 604 { 605 raw_spinlock_t *lock; 606 bool ret; 607 608 /* Matches synchronize_rcu() in __sched_core_enable() */ 609 preempt_disable(); 610 if (sched_core_disabled()) { 611 ret = raw_spin_trylock(&rq->__lock); 612 preempt_enable(); 613 return ret; 614 } 615 616 for (;;) { 617 lock = __rq_lockp(rq); 618 ret = raw_spin_trylock(lock); 619 if (!ret || (likely(lock == __rq_lockp(rq)))) { 620 preempt_enable(); 621 return ret; 622 } 623 raw_spin_unlock(lock); 624 } 625 } 626 627 void raw_spin_rq_unlock(struct rq *rq) 628 { 629 raw_spin_unlock(rq_lockp(rq)); 630 } 631 632 #ifdef CONFIG_SMP 633 /* 634 * double_rq_lock - safely lock two runqueues 635 */ 636 void double_rq_lock(struct rq *rq1, struct rq *rq2) 637 { 638 lockdep_assert_irqs_disabled(); 639 640 if (rq_order_less(rq2, rq1)) 641 swap(rq1, rq2); 642 643 raw_spin_rq_lock(rq1); 644 if (__rq_lockp(rq1) != __rq_lockp(rq2)) 645 raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING); 646 647 double_rq_clock_clear_update(rq1, rq2); 648 } 649 #endif 650 651 /* 652 * __task_rq_lock - lock the rq @p resides on. 653 */ 654 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) 655 __acquires(rq->lock) 656 { 657 struct rq *rq; 658 659 lockdep_assert_held(&p->pi_lock); 660 661 for (;;) { 662 rq = task_rq(p); 663 raw_spin_rq_lock(rq); 664 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 665 rq_pin_lock(rq, rf); 666 return rq; 667 } 668 raw_spin_rq_unlock(rq); 669 670 while (unlikely(task_on_rq_migrating(p))) 671 cpu_relax(); 672 } 673 } 674 675 /* 676 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. 677 */ 678 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) 679 __acquires(p->pi_lock) 680 __acquires(rq->lock) 681 { 682 struct rq *rq; 683 684 for (;;) { 685 raw_spin_lock_irqsave(&p->pi_lock, rf->flags); 686 rq = task_rq(p); 687 raw_spin_rq_lock(rq); 688 /* 689 * move_queued_task() task_rq_lock() 690 * 691 * ACQUIRE (rq->lock) 692 * [S] ->on_rq = MIGRATING [L] rq = task_rq() 693 * WMB (__set_task_cpu()) ACQUIRE (rq->lock); 694 * [S] ->cpu = new_cpu [L] task_rq() 695 * [L] ->on_rq 696 * RELEASE (rq->lock) 697 * 698 * If we observe the old CPU in task_rq_lock(), the acquire of 699 * the old rq->lock will fully serialize against the stores. 700 * 701 * If we observe the new CPU in task_rq_lock(), the address 702 * dependency headed by '[L] rq = task_rq()' and the acquire 703 * will pair with the WMB to ensure we then also see migrating. 704 */ 705 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 706 rq_pin_lock(rq, rf); 707 return rq; 708 } 709 raw_spin_rq_unlock(rq); 710 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 711 712 while (unlikely(task_on_rq_migrating(p))) 713 cpu_relax(); 714 } 715 } 716 717 /* 718 * RQ-clock updating methods: 719 */ 720 721 static void update_rq_clock_task(struct rq *rq, s64 delta) 722 { 723 /* 724 * In theory, the compile should just see 0 here, and optimize out the call 725 * to sched_rt_avg_update. But I don't trust it... 726 */ 727 s64 __maybe_unused steal = 0, irq_delta = 0; 728 729 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 730 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 731 732 /* 733 * Since irq_time is only updated on {soft,}irq_exit, we might run into 734 * this case when a previous update_rq_clock() happened inside a 735 * {soft,}IRQ region. 736 * 737 * When this happens, we stop ->clock_task and only update the 738 * prev_irq_time stamp to account for the part that fit, so that a next 739 * update will consume the rest. This ensures ->clock_task is 740 * monotonic. 741 * 742 * It does however cause some slight miss-attribution of {soft,}IRQ 743 * time, a more accurate solution would be to update the irq_time using 744 * the current rq->clock timestamp, except that would require using 745 * atomic ops. 746 */ 747 if (irq_delta > delta) 748 irq_delta = delta; 749 750 rq->prev_irq_time += irq_delta; 751 delta -= irq_delta; 752 delayacct_irq(rq->curr, irq_delta); 753 #endif 754 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 755 if (static_key_false((¶virt_steal_rq_enabled))) { 756 steal = paravirt_steal_clock(cpu_of(rq)); 757 steal -= rq->prev_steal_time_rq; 758 759 if (unlikely(steal > delta)) 760 steal = delta; 761 762 rq->prev_steal_time_rq += steal; 763 delta -= steal; 764 } 765 #endif 766 767 rq->clock_task += delta; 768 769 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 770 if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) 771 update_irq_load_avg(rq, irq_delta + steal); 772 #endif 773 update_rq_clock_pelt(rq, delta); 774 } 775 776 void update_rq_clock(struct rq *rq) 777 { 778 s64 delta; 779 780 lockdep_assert_rq_held(rq); 781 782 if (rq->clock_update_flags & RQCF_ACT_SKIP) 783 return; 784 785 #ifdef CONFIG_SCHED_DEBUG 786 if (sched_feat(WARN_DOUBLE_CLOCK)) 787 SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED); 788 rq->clock_update_flags |= RQCF_UPDATED; 789 #endif 790 791 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; 792 if (delta < 0) 793 return; 794 rq->clock += delta; 795 update_rq_clock_task(rq, delta); 796 } 797 798 #ifdef CONFIG_SCHED_HRTICK 799 /* 800 * Use HR-timers to deliver accurate preemption points. 801 */ 802 803 static void hrtick_clear(struct rq *rq) 804 { 805 if (hrtimer_active(&rq->hrtick_timer)) 806 hrtimer_cancel(&rq->hrtick_timer); 807 } 808 809 /* 810 * High-resolution timer tick. 811 * Runs from hardirq context with interrupts disabled. 812 */ 813 static enum hrtimer_restart hrtick(struct hrtimer *timer) 814 { 815 struct rq *rq = container_of(timer, struct rq, hrtick_timer); 816 struct rq_flags rf; 817 818 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 819 820 rq_lock(rq, &rf); 821 update_rq_clock(rq); 822 rq->curr->sched_class->task_tick(rq, rq->curr, 1); 823 rq_unlock(rq, &rf); 824 825 return HRTIMER_NORESTART; 826 } 827 828 #ifdef CONFIG_SMP 829 830 static void __hrtick_restart(struct rq *rq) 831 { 832 struct hrtimer *timer = &rq->hrtick_timer; 833 ktime_t time = rq->hrtick_time; 834 835 hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD); 836 } 837 838 /* 839 * called from hardirq (IPI) context 840 */ 841 static void __hrtick_start(void *arg) 842 { 843 struct rq *rq = arg; 844 struct rq_flags rf; 845 846 rq_lock(rq, &rf); 847 __hrtick_restart(rq); 848 rq_unlock(rq, &rf); 849 } 850 851 /* 852 * Called to set the hrtick timer state. 853 * 854 * called with rq->lock held and IRQs disabled 855 */ 856 void hrtick_start(struct rq *rq, u64 delay) 857 { 858 struct hrtimer *timer = &rq->hrtick_timer; 859 s64 delta; 860 861 /* 862 * Don't schedule slices shorter than 10000ns, that just 863 * doesn't make sense and can cause timer DoS. 864 */ 865 delta = max_t(s64, delay, 10000LL); 866 rq->hrtick_time = ktime_add_ns(timer->base->get_time(), delta); 867 868 if (rq == this_rq()) 869 __hrtick_restart(rq); 870 else 871 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); 872 } 873 874 #else 875 /* 876 * Called to set the hrtick timer state. 877 * 878 * called with rq->lock held and IRQs disabled 879 */ 880 void hrtick_start(struct rq *rq, u64 delay) 881 { 882 /* 883 * Don't schedule slices shorter than 10000ns, that just 884 * doesn't make sense. Rely on vruntime for fairness. 885 */ 886 delay = max_t(u64, delay, 10000LL); 887 hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), 888 HRTIMER_MODE_REL_PINNED_HARD); 889 } 890 891 #endif /* CONFIG_SMP */ 892 893 static void hrtick_rq_init(struct rq *rq) 894 { 895 #ifdef CONFIG_SMP 896 INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq); 897 #endif 898 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 899 rq->hrtick_timer.function = hrtick; 900 } 901 #else /* CONFIG_SCHED_HRTICK */ 902 static inline void hrtick_clear(struct rq *rq) 903 { 904 } 905 906 static inline void hrtick_rq_init(struct rq *rq) 907 { 908 } 909 #endif /* CONFIG_SCHED_HRTICK */ 910 911 /* 912 * try_cmpxchg based fetch_or() macro so it works for different integer types: 913 */ 914 #define fetch_or(ptr, mask) \ 915 ({ \ 916 typeof(ptr) _ptr = (ptr); \ 917 typeof(mask) _mask = (mask); \ 918 typeof(*_ptr) _val = *_ptr; \ 919 \ 920 do { \ 921 } while (!try_cmpxchg(_ptr, &_val, _val | _mask)); \ 922 _val; \ 923 }) 924 925 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) 926 /* 927 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, 928 * this avoids any races wrt polling state changes and thereby avoids 929 * spurious IPIs. 930 */ 931 static inline bool set_nr_and_not_polling(struct task_struct *p) 932 { 933 struct thread_info *ti = task_thread_info(p); 934 return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); 935 } 936 937 /* 938 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. 939 * 940 * If this returns true, then the idle task promises to call 941 * sched_ttwu_pending() and reschedule soon. 942 */ 943 static bool set_nr_if_polling(struct task_struct *p) 944 { 945 struct thread_info *ti = task_thread_info(p); 946 typeof(ti->flags) val = READ_ONCE(ti->flags); 947 948 do { 949 if (!(val & _TIF_POLLING_NRFLAG)) 950 return false; 951 if (val & _TIF_NEED_RESCHED) 952 return true; 953 } while (!try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED)); 954 955 return true; 956 } 957 958 #else 959 static inline bool set_nr_and_not_polling(struct task_struct *p) 960 { 961 set_tsk_need_resched(p); 962 return true; 963 } 964 965 #ifdef CONFIG_SMP 966 static inline bool set_nr_if_polling(struct task_struct *p) 967 { 968 return false; 969 } 970 #endif 971 #endif 972 973 static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task) 974 { 975 struct wake_q_node *node = &task->wake_q; 976 977 /* 978 * Atomically grab the task, if ->wake_q is !nil already it means 979 * it's already queued (either by us or someone else) and will get the 980 * wakeup due to that. 981 * 982 * In order to ensure that a pending wakeup will observe our pending 983 * state, even in the failed case, an explicit smp_mb() must be used. 984 */ 985 smp_mb__before_atomic(); 986 if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL))) 987 return false; 988 989 /* 990 * The head is context local, there can be no concurrency. 991 */ 992 *head->lastp = node; 993 head->lastp = &node->next; 994 return true; 995 } 996 997 /** 998 * wake_q_add() - queue a wakeup for 'later' waking. 999 * @head: the wake_q_head to add @task to 1000 * @task: the task to queue for 'later' wakeup 1001 * 1002 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 1003 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 1004 * instantly. 1005 * 1006 * This function must be used as-if it were wake_up_process(); IOW the task 1007 * must be ready to be woken at this location. 1008 */ 1009 void wake_q_add(struct wake_q_head *head, struct task_struct *task) 1010 { 1011 if (__wake_q_add(head, task)) 1012 get_task_struct(task); 1013 } 1014 1015 /** 1016 * wake_q_add_safe() - safely queue a wakeup for 'later' waking. 1017 * @head: the wake_q_head to add @task to 1018 * @task: the task to queue for 'later' wakeup 1019 * 1020 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 1021 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 1022 * instantly. 1023 * 1024 * This function must be used as-if it were wake_up_process(); IOW the task 1025 * must be ready to be woken at this location. 1026 * 1027 * This function is essentially a task-safe equivalent to wake_q_add(). Callers 1028 * that already hold reference to @task can call the 'safe' version and trust 1029 * wake_q to do the right thing depending whether or not the @task is already 1030 * queued for wakeup. 1031 */ 1032 void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task) 1033 { 1034 if (!__wake_q_add(head, task)) 1035 put_task_struct(task); 1036 } 1037 1038 void wake_up_q(struct wake_q_head *head) 1039 { 1040 struct wake_q_node *node = head->first; 1041 1042 while (node != WAKE_Q_TAIL) { 1043 struct task_struct *task; 1044 1045 task = container_of(node, struct task_struct, wake_q); 1046 /* Task can safely be re-inserted now: */ 1047 node = node->next; 1048 task->wake_q.next = NULL; 1049 1050 /* 1051 * wake_up_process() executes a full barrier, which pairs with 1052 * the queueing in wake_q_add() so as not to miss wakeups. 1053 */ 1054 wake_up_process(task); 1055 put_task_struct(task); 1056 } 1057 } 1058 1059 /* 1060 * resched_curr - mark rq's current task 'to be rescheduled now'. 1061 * 1062 * On UP this means the setting of the need_resched flag, on SMP it 1063 * might also involve a cross-CPU call to trigger the scheduler on 1064 * the target CPU. 1065 */ 1066 void resched_curr(struct rq *rq) 1067 { 1068 struct task_struct *curr = rq->curr; 1069 int cpu; 1070 1071 lockdep_assert_rq_held(rq); 1072 1073 if (test_tsk_need_resched(curr)) 1074 return; 1075 1076 cpu = cpu_of(rq); 1077 1078 if (cpu == smp_processor_id()) { 1079 set_tsk_need_resched(curr); 1080 set_preempt_need_resched(); 1081 return; 1082 } 1083 1084 if (set_nr_and_not_polling(curr)) 1085 smp_send_reschedule(cpu); 1086 else 1087 trace_sched_wake_idle_without_ipi(cpu); 1088 } 1089 1090 void resched_cpu(int cpu) 1091 { 1092 struct rq *rq = cpu_rq(cpu); 1093 unsigned long flags; 1094 1095 raw_spin_rq_lock_irqsave(rq, flags); 1096 if (cpu_online(cpu) || cpu == smp_processor_id()) 1097 resched_curr(rq); 1098 raw_spin_rq_unlock_irqrestore(rq, flags); 1099 } 1100 1101 #ifdef CONFIG_SMP 1102 #ifdef CONFIG_NO_HZ_COMMON 1103 /* 1104 * In the semi idle case, use the nearest busy CPU for migrating timers 1105 * from an idle CPU. This is good for power-savings. 1106 * 1107 * We don't do similar optimization for completely idle system, as 1108 * selecting an idle CPU will add more delays to the timers than intended 1109 * (as that CPU's timer base may not be up to date wrt jiffies etc). 1110 */ 1111 int get_nohz_timer_target(void) 1112 { 1113 int i, cpu = smp_processor_id(), default_cpu = -1; 1114 struct sched_domain *sd; 1115 const struct cpumask *hk_mask; 1116 1117 if (housekeeping_cpu(cpu, HK_TYPE_TIMER)) { 1118 if (!idle_cpu(cpu)) 1119 return cpu; 1120 default_cpu = cpu; 1121 } 1122 1123 hk_mask = housekeeping_cpumask(HK_TYPE_TIMER); 1124 1125 guard(rcu)(); 1126 1127 for_each_domain(cpu, sd) { 1128 for_each_cpu_and(i, sched_domain_span(sd), hk_mask) { 1129 if (cpu == i) 1130 continue; 1131 1132 if (!idle_cpu(i)) 1133 return i; 1134 } 1135 } 1136 1137 if (default_cpu == -1) 1138 default_cpu = housekeeping_any_cpu(HK_TYPE_TIMER); 1139 1140 return default_cpu; 1141 } 1142 1143 /* 1144 * When add_timer_on() enqueues a timer into the timer wheel of an 1145 * idle CPU then this timer might expire before the next timer event 1146 * which is scheduled to wake up that CPU. In case of a completely 1147 * idle system the next event might even be infinite time into the 1148 * future. wake_up_idle_cpu() ensures that the CPU is woken up and 1149 * leaves the inner idle loop so the newly added timer is taken into 1150 * account when the CPU goes back to idle and evaluates the timer 1151 * wheel for the next timer event. 1152 */ 1153 static void wake_up_idle_cpu(int cpu) 1154 { 1155 struct rq *rq = cpu_rq(cpu); 1156 1157 if (cpu == smp_processor_id()) 1158 return; 1159 1160 /* 1161 * Set TIF_NEED_RESCHED and send an IPI if in the non-polling 1162 * part of the idle loop. This forces an exit from the idle loop 1163 * and a round trip to schedule(). Now this could be optimized 1164 * because a simple new idle loop iteration is enough to 1165 * re-evaluate the next tick. Provided some re-ordering of tick 1166 * nohz functions that would need to follow TIF_NR_POLLING 1167 * clearing: 1168 * 1169 * - On most architectures, a simple fetch_or on ti::flags with a 1170 * "0" value would be enough to know if an IPI needs to be sent. 1171 * 1172 * - x86 needs to perform a last need_resched() check between 1173 * monitor and mwait which doesn't take timers into account. 1174 * There a dedicated TIF_TIMER flag would be required to 1175 * fetch_or here and be checked along with TIF_NEED_RESCHED 1176 * before mwait(). 1177 * 1178 * However, remote timer enqueue is not such a frequent event 1179 * and testing of the above solutions didn't appear to report 1180 * much benefits. 1181 */ 1182 if (set_nr_and_not_polling(rq->idle)) 1183 smp_send_reschedule(cpu); 1184 else 1185 trace_sched_wake_idle_without_ipi(cpu); 1186 } 1187 1188 static bool wake_up_full_nohz_cpu(int cpu) 1189 { 1190 /* 1191 * We just need the target to call irq_exit() and re-evaluate 1192 * the next tick. The nohz full kick at least implies that. 1193 * If needed we can still optimize that later with an 1194 * empty IRQ. 1195 */ 1196 if (cpu_is_offline(cpu)) 1197 return true; /* Don't try to wake offline CPUs. */ 1198 if (tick_nohz_full_cpu(cpu)) { 1199 if (cpu != smp_processor_id() || 1200 tick_nohz_tick_stopped()) 1201 tick_nohz_full_kick_cpu(cpu); 1202 return true; 1203 } 1204 1205 return false; 1206 } 1207 1208 /* 1209 * Wake up the specified CPU. If the CPU is going offline, it is the 1210 * caller's responsibility to deal with the lost wakeup, for example, 1211 * by hooking into the CPU_DEAD notifier like timers and hrtimers do. 1212 */ 1213 void wake_up_nohz_cpu(int cpu) 1214 { 1215 if (!wake_up_full_nohz_cpu(cpu)) 1216 wake_up_idle_cpu(cpu); 1217 } 1218 1219 static void nohz_csd_func(void *info) 1220 { 1221 struct rq *rq = info; 1222 int cpu = cpu_of(rq); 1223 unsigned int flags; 1224 1225 /* 1226 * Release the rq::nohz_csd. 1227 */ 1228 flags = atomic_fetch_andnot(NOHZ_KICK_MASK | NOHZ_NEWILB_KICK, nohz_flags(cpu)); 1229 WARN_ON(!(flags & NOHZ_KICK_MASK)); 1230 1231 rq->idle_balance = idle_cpu(cpu); 1232 if (rq->idle_balance && !need_resched()) { 1233 rq->nohz_idle_balance = flags; 1234 raise_softirq_irqoff(SCHED_SOFTIRQ); 1235 } 1236 } 1237 1238 #endif /* CONFIG_NO_HZ_COMMON */ 1239 1240 #ifdef CONFIG_NO_HZ_FULL 1241 static inline bool __need_bw_check(struct rq *rq, struct task_struct *p) 1242 { 1243 if (rq->nr_running != 1) 1244 return false; 1245 1246 if (p->sched_class != &fair_sched_class) 1247 return false; 1248 1249 if (!task_on_rq_queued(p)) 1250 return false; 1251 1252 return true; 1253 } 1254 1255 bool sched_can_stop_tick(struct rq *rq) 1256 { 1257 int fifo_nr_running; 1258 1259 /* Deadline tasks, even if single, need the tick */ 1260 if (rq->dl.dl_nr_running) 1261 return false; 1262 1263 /* 1264 * If there are more than one RR tasks, we need the tick to affect the 1265 * actual RR behaviour. 1266 */ 1267 if (rq->rt.rr_nr_running) { 1268 if (rq->rt.rr_nr_running == 1) 1269 return true; 1270 else 1271 return false; 1272 } 1273 1274 /* 1275 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no 1276 * forced preemption between FIFO tasks. 1277 */ 1278 fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; 1279 if (fifo_nr_running) 1280 return true; 1281 1282 /* 1283 * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; 1284 * if there's more than one we need the tick for involuntary 1285 * preemption. 1286 */ 1287 if (rq->nr_running > 1) 1288 return false; 1289 1290 /* 1291 * If there is one task and it has CFS runtime bandwidth constraints 1292 * and it's on the cpu now we don't want to stop the tick. 1293 * This check prevents clearing the bit if a newly enqueued task here is 1294 * dequeued by migrating while the constrained task continues to run. 1295 * E.g. going from 2->1 without going through pick_next_task(). 1296 */ 1297 if (__need_bw_check(rq, rq->curr)) { 1298 if (cfs_task_bw_constrained(rq->curr)) 1299 return false; 1300 } 1301 1302 return true; 1303 } 1304 #endif /* CONFIG_NO_HZ_FULL */ 1305 #endif /* CONFIG_SMP */ 1306 1307 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ 1308 (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) 1309 /* 1310 * Iterate task_group tree rooted at *from, calling @down when first entering a 1311 * node and @up when leaving it for the final time. 1312 * 1313 * Caller must hold rcu_lock or sufficient equivalent. 1314 */ 1315 int walk_tg_tree_from(struct task_group *from, 1316 tg_visitor down, tg_visitor up, void *data) 1317 { 1318 struct task_group *parent, *child; 1319 int ret; 1320 1321 parent = from; 1322 1323 down: 1324 ret = (*down)(parent, data); 1325 if (ret) 1326 goto out; 1327 list_for_each_entry_rcu(child, &parent->children, siblings) { 1328 parent = child; 1329 goto down; 1330 1331 up: 1332 continue; 1333 } 1334 ret = (*up)(parent, data); 1335 if (ret || parent == from) 1336 goto out; 1337 1338 child = parent; 1339 parent = parent->parent; 1340 if (parent) 1341 goto up; 1342 out: 1343 return ret; 1344 } 1345 1346 int tg_nop(struct task_group *tg, void *data) 1347 { 1348 return 0; 1349 } 1350 #endif 1351 1352 void set_load_weight(struct task_struct *p, bool update_load) 1353 { 1354 int prio = p->static_prio - MAX_RT_PRIO; 1355 struct load_weight lw; 1356 1357 if (task_has_idle_policy(p)) { 1358 lw.weight = scale_load(WEIGHT_IDLEPRIO); 1359 lw.inv_weight = WMULT_IDLEPRIO; 1360 } else { 1361 lw.weight = scale_load(sched_prio_to_weight[prio]); 1362 lw.inv_weight = sched_prio_to_wmult[prio]; 1363 } 1364 1365 /* 1366 * SCHED_OTHER tasks have to update their load when changing their 1367 * weight 1368 */ 1369 if (update_load && p->sched_class == &fair_sched_class) 1370 reweight_task(p, &lw); 1371 else 1372 p->se.load = lw; 1373 } 1374 1375 #ifdef CONFIG_UCLAMP_TASK 1376 /* 1377 * Serializes updates of utilization clamp values 1378 * 1379 * The (slow-path) user-space triggers utilization clamp value updates which 1380 * can require updates on (fast-path) scheduler's data structures used to 1381 * support enqueue/dequeue operations. 1382 * While the per-CPU rq lock protects fast-path update operations, user-space 1383 * requests are serialized using a mutex to reduce the risk of conflicting 1384 * updates or API abuses. 1385 */ 1386 static DEFINE_MUTEX(uclamp_mutex); 1387 1388 /* Max allowed minimum utilization */ 1389 static unsigned int __maybe_unused sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; 1390 1391 /* Max allowed maximum utilization */ 1392 static unsigned int __maybe_unused sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; 1393 1394 /* 1395 * By default RT tasks run at the maximum performance point/capacity of the 1396 * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to 1397 * SCHED_CAPACITY_SCALE. 1398 * 1399 * This knob allows admins to change the default behavior when uclamp is being 1400 * used. In battery powered devices, particularly, running at the maximum 1401 * capacity and frequency will increase energy consumption and shorten the 1402 * battery life. 1403 * 1404 * This knob only affects RT tasks that their uclamp_se->user_defined == false. 1405 * 1406 * This knob will not override the system default sched_util_clamp_min defined 1407 * above. 1408 */ 1409 unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE; 1410 1411 /* All clamps are required to be less or equal than these values */ 1412 static struct uclamp_se uclamp_default[UCLAMP_CNT]; 1413 1414 /* 1415 * This static key is used to reduce the uclamp overhead in the fast path. It 1416 * primarily disables the call to uclamp_rq_{inc, dec}() in 1417 * enqueue/dequeue_task(). 1418 * 1419 * This allows users to continue to enable uclamp in their kernel config with 1420 * minimum uclamp overhead in the fast path. 1421 * 1422 * As soon as userspace modifies any of the uclamp knobs, the static key is 1423 * enabled, since we have an actual users that make use of uclamp 1424 * functionality. 1425 * 1426 * The knobs that would enable this static key are: 1427 * 1428 * * A task modifying its uclamp value with sched_setattr(). 1429 * * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs. 1430 * * An admin modifying the cgroup cpu.uclamp.{min, max} 1431 */ 1432 DEFINE_STATIC_KEY_FALSE(sched_uclamp_used); 1433 1434 static inline unsigned int 1435 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id, 1436 unsigned int clamp_value) 1437 { 1438 /* 1439 * Avoid blocked utilization pushing up the frequency when we go 1440 * idle (which drops the max-clamp) by retaining the last known 1441 * max-clamp. 1442 */ 1443 if (clamp_id == UCLAMP_MAX) { 1444 rq->uclamp_flags |= UCLAMP_FLAG_IDLE; 1445 return clamp_value; 1446 } 1447 1448 return uclamp_none(UCLAMP_MIN); 1449 } 1450 1451 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id, 1452 unsigned int clamp_value) 1453 { 1454 /* Reset max-clamp retention only on idle exit */ 1455 if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 1456 return; 1457 1458 uclamp_rq_set(rq, clamp_id, clamp_value); 1459 } 1460 1461 static inline 1462 unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, 1463 unsigned int clamp_value) 1464 { 1465 struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; 1466 int bucket_id = UCLAMP_BUCKETS - 1; 1467 1468 /* 1469 * Since both min and max clamps are max aggregated, find the 1470 * top most bucket with tasks in. 1471 */ 1472 for ( ; bucket_id >= 0; bucket_id--) { 1473 if (!bucket[bucket_id].tasks) 1474 continue; 1475 return bucket[bucket_id].value; 1476 } 1477 1478 /* No tasks -- default clamp values */ 1479 return uclamp_idle_value(rq, clamp_id, clamp_value); 1480 } 1481 1482 static void __uclamp_update_util_min_rt_default(struct task_struct *p) 1483 { 1484 unsigned int default_util_min; 1485 struct uclamp_se *uc_se; 1486 1487 lockdep_assert_held(&p->pi_lock); 1488 1489 uc_se = &p->uclamp_req[UCLAMP_MIN]; 1490 1491 /* Only sync if user didn't override the default */ 1492 if (uc_se->user_defined) 1493 return; 1494 1495 default_util_min = sysctl_sched_uclamp_util_min_rt_default; 1496 uclamp_se_set(uc_se, default_util_min, false); 1497 } 1498 1499 static void uclamp_update_util_min_rt_default(struct task_struct *p) 1500 { 1501 if (!rt_task(p)) 1502 return; 1503 1504 /* Protect updates to p->uclamp_* */ 1505 guard(task_rq_lock)(p); 1506 __uclamp_update_util_min_rt_default(p); 1507 } 1508 1509 static inline struct uclamp_se 1510 uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) 1511 { 1512 /* Copy by value as we could modify it */ 1513 struct uclamp_se uc_req = p->uclamp_req[clamp_id]; 1514 #ifdef CONFIG_UCLAMP_TASK_GROUP 1515 unsigned int tg_min, tg_max, value; 1516 1517 /* 1518 * Tasks in autogroups or root task group will be 1519 * restricted by system defaults. 1520 */ 1521 if (task_group_is_autogroup(task_group(p))) 1522 return uc_req; 1523 if (task_group(p) == &root_task_group) 1524 return uc_req; 1525 1526 tg_min = task_group(p)->uclamp[UCLAMP_MIN].value; 1527 tg_max = task_group(p)->uclamp[UCLAMP_MAX].value; 1528 value = uc_req.value; 1529 value = clamp(value, tg_min, tg_max); 1530 uclamp_se_set(&uc_req, value, false); 1531 #endif 1532 1533 return uc_req; 1534 } 1535 1536 /* 1537 * The effective clamp bucket index of a task depends on, by increasing 1538 * priority: 1539 * - the task specific clamp value, when explicitly requested from userspace 1540 * - the task group effective clamp value, for tasks not either in the root 1541 * group or in an autogroup 1542 * - the system default clamp value, defined by the sysadmin 1543 */ 1544 static inline struct uclamp_se 1545 uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id) 1546 { 1547 struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id); 1548 struct uclamp_se uc_max = uclamp_default[clamp_id]; 1549 1550 /* System default restrictions always apply */ 1551 if (unlikely(uc_req.value > uc_max.value)) 1552 return uc_max; 1553 1554 return uc_req; 1555 } 1556 1557 unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id) 1558 { 1559 struct uclamp_se uc_eff; 1560 1561 /* Task currently refcounted: use back-annotated (effective) value */ 1562 if (p->uclamp[clamp_id].active) 1563 return (unsigned long)p->uclamp[clamp_id].value; 1564 1565 uc_eff = uclamp_eff_get(p, clamp_id); 1566 1567 return (unsigned long)uc_eff.value; 1568 } 1569 1570 /* 1571 * When a task is enqueued on a rq, the clamp bucket currently defined by the 1572 * task's uclamp::bucket_id is refcounted on that rq. This also immediately 1573 * updates the rq's clamp value if required. 1574 * 1575 * Tasks can have a task-specific value requested from user-space, track 1576 * within each bucket the maximum value for tasks refcounted in it. 1577 * This "local max aggregation" allows to track the exact "requested" value 1578 * for each bucket when all its RUNNABLE tasks require the same clamp. 1579 */ 1580 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, 1581 enum uclamp_id clamp_id) 1582 { 1583 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1584 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1585 struct uclamp_bucket *bucket; 1586 1587 lockdep_assert_rq_held(rq); 1588 1589 /* Update task effective clamp */ 1590 p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id); 1591 1592 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1593 bucket->tasks++; 1594 uc_se->active = true; 1595 1596 uclamp_idle_reset(rq, clamp_id, uc_se->value); 1597 1598 /* 1599 * Local max aggregation: rq buckets always track the max 1600 * "requested" clamp value of its RUNNABLE tasks. 1601 */ 1602 if (bucket->tasks == 1 || uc_se->value > bucket->value) 1603 bucket->value = uc_se->value; 1604 1605 if (uc_se->value > uclamp_rq_get(rq, clamp_id)) 1606 uclamp_rq_set(rq, clamp_id, uc_se->value); 1607 } 1608 1609 /* 1610 * When a task is dequeued from a rq, the clamp bucket refcounted by the task 1611 * is released. If this is the last task reference counting the rq's max 1612 * active clamp value, then the rq's clamp value is updated. 1613 * 1614 * Both refcounted tasks and rq's cached clamp values are expected to be 1615 * always valid. If it's detected they are not, as defensive programming, 1616 * enforce the expected state and warn. 1617 */ 1618 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, 1619 enum uclamp_id clamp_id) 1620 { 1621 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1622 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1623 struct uclamp_bucket *bucket; 1624 unsigned int bkt_clamp; 1625 unsigned int rq_clamp; 1626 1627 lockdep_assert_rq_held(rq); 1628 1629 /* 1630 * If sched_uclamp_used was enabled after task @p was enqueued, 1631 * we could end up with unbalanced call to uclamp_rq_dec_id(). 1632 * 1633 * In this case the uc_se->active flag should be false since no uclamp 1634 * accounting was performed at enqueue time and we can just return 1635 * here. 1636 * 1637 * Need to be careful of the following enqueue/dequeue ordering 1638 * problem too 1639 * 1640 * enqueue(taskA) 1641 * // sched_uclamp_used gets enabled 1642 * enqueue(taskB) 1643 * dequeue(taskA) 1644 * // Must not decrement bucket->tasks here 1645 * dequeue(taskB) 1646 * 1647 * where we could end up with stale data in uc_se and 1648 * bucket[uc_se->bucket_id]. 1649 * 1650 * The following check here eliminates the possibility of such race. 1651 */ 1652 if (unlikely(!uc_se->active)) 1653 return; 1654 1655 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1656 1657 SCHED_WARN_ON(!bucket->tasks); 1658 if (likely(bucket->tasks)) 1659 bucket->tasks--; 1660 1661 uc_se->active = false; 1662 1663 /* 1664 * Keep "local max aggregation" simple and accept to (possibly) 1665 * overboost some RUNNABLE tasks in the same bucket. 1666 * The rq clamp bucket value is reset to its base value whenever 1667 * there are no more RUNNABLE tasks refcounting it. 1668 */ 1669 if (likely(bucket->tasks)) 1670 return; 1671 1672 rq_clamp = uclamp_rq_get(rq, clamp_id); 1673 /* 1674 * Defensive programming: this should never happen. If it happens, 1675 * e.g. due to future modification, warn and fix up the expected value. 1676 */ 1677 SCHED_WARN_ON(bucket->value > rq_clamp); 1678 if (bucket->value >= rq_clamp) { 1679 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value); 1680 uclamp_rq_set(rq, clamp_id, bkt_clamp); 1681 } 1682 } 1683 1684 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) 1685 { 1686 enum uclamp_id clamp_id; 1687 1688 /* 1689 * Avoid any overhead until uclamp is actually used by the userspace. 1690 * 1691 * The condition is constructed such that a NOP is generated when 1692 * sched_uclamp_used is disabled. 1693 */ 1694 if (!static_branch_unlikely(&sched_uclamp_used)) 1695 return; 1696 1697 if (unlikely(!p->sched_class->uclamp_enabled)) 1698 return; 1699 1700 if (p->se.sched_delayed) 1701 return; 1702 1703 for_each_clamp_id(clamp_id) 1704 uclamp_rq_inc_id(rq, p, clamp_id); 1705 1706 /* Reset clamp idle holding when there is one RUNNABLE task */ 1707 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE) 1708 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1709 } 1710 1711 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) 1712 { 1713 enum uclamp_id clamp_id; 1714 1715 /* 1716 * Avoid any overhead until uclamp is actually used by the userspace. 1717 * 1718 * The condition is constructed such that a NOP is generated when 1719 * sched_uclamp_used is disabled. 1720 */ 1721 if (!static_branch_unlikely(&sched_uclamp_used)) 1722 return; 1723 1724 if (unlikely(!p->sched_class->uclamp_enabled)) 1725 return; 1726 1727 if (p->se.sched_delayed) 1728 return; 1729 1730 for_each_clamp_id(clamp_id) 1731 uclamp_rq_dec_id(rq, p, clamp_id); 1732 } 1733 1734 static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p, 1735 enum uclamp_id clamp_id) 1736 { 1737 if (!p->uclamp[clamp_id].active) 1738 return; 1739 1740 uclamp_rq_dec_id(rq, p, clamp_id); 1741 uclamp_rq_inc_id(rq, p, clamp_id); 1742 1743 /* 1744 * Make sure to clear the idle flag if we've transiently reached 0 1745 * active tasks on rq. 1746 */ 1747 if (clamp_id == UCLAMP_MAX && (rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 1748 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1749 } 1750 1751 static inline void 1752 uclamp_update_active(struct task_struct *p) 1753 { 1754 enum uclamp_id clamp_id; 1755 struct rq_flags rf; 1756 struct rq *rq; 1757 1758 /* 1759 * Lock the task and the rq where the task is (or was) queued. 1760 * 1761 * We might lock the (previous) rq of a !RUNNABLE task, but that's the 1762 * price to pay to safely serialize util_{min,max} updates with 1763 * enqueues, dequeues and migration operations. 1764 * This is the same locking schema used by __set_cpus_allowed_ptr(). 1765 */ 1766 rq = task_rq_lock(p, &rf); 1767 1768 /* 1769 * Setting the clamp bucket is serialized by task_rq_lock(). 1770 * If the task is not yet RUNNABLE and its task_struct is not 1771 * affecting a valid clamp bucket, the next time it's enqueued, 1772 * it will already see the updated clamp bucket value. 1773 */ 1774 for_each_clamp_id(clamp_id) 1775 uclamp_rq_reinc_id(rq, p, clamp_id); 1776 1777 task_rq_unlock(rq, p, &rf); 1778 } 1779 1780 #ifdef CONFIG_UCLAMP_TASK_GROUP 1781 static inline void 1782 uclamp_update_active_tasks(struct cgroup_subsys_state *css) 1783 { 1784 struct css_task_iter it; 1785 struct task_struct *p; 1786 1787 css_task_iter_start(css, 0, &it); 1788 while ((p = css_task_iter_next(&it))) 1789 uclamp_update_active(p); 1790 css_task_iter_end(&it); 1791 } 1792 1793 static void cpu_util_update_eff(struct cgroup_subsys_state *css); 1794 #endif 1795 1796 #ifdef CONFIG_SYSCTL 1797 #ifdef CONFIG_UCLAMP_TASK_GROUP 1798 static void uclamp_update_root_tg(void) 1799 { 1800 struct task_group *tg = &root_task_group; 1801 1802 uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN], 1803 sysctl_sched_uclamp_util_min, false); 1804 uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX], 1805 sysctl_sched_uclamp_util_max, false); 1806 1807 guard(rcu)(); 1808 cpu_util_update_eff(&root_task_group.css); 1809 } 1810 #else 1811 static void uclamp_update_root_tg(void) { } 1812 #endif 1813 1814 static void uclamp_sync_util_min_rt_default(void) 1815 { 1816 struct task_struct *g, *p; 1817 1818 /* 1819 * copy_process() sysctl_uclamp 1820 * uclamp_min_rt = X; 1821 * write_lock(&tasklist_lock) read_lock(&tasklist_lock) 1822 * // link thread smp_mb__after_spinlock() 1823 * write_unlock(&tasklist_lock) read_unlock(&tasklist_lock); 1824 * sched_post_fork() for_each_process_thread() 1825 * __uclamp_sync_rt() __uclamp_sync_rt() 1826 * 1827 * Ensures that either sched_post_fork() will observe the new 1828 * uclamp_min_rt or for_each_process_thread() will observe the new 1829 * task. 1830 */ 1831 read_lock(&tasklist_lock); 1832 smp_mb__after_spinlock(); 1833 read_unlock(&tasklist_lock); 1834 1835 guard(rcu)(); 1836 for_each_process_thread(g, p) 1837 uclamp_update_util_min_rt_default(p); 1838 } 1839 1840 static int sysctl_sched_uclamp_handler(const struct ctl_table *table, int write, 1841 void *buffer, size_t *lenp, loff_t *ppos) 1842 { 1843 bool update_root_tg = false; 1844 int old_min, old_max, old_min_rt; 1845 int result; 1846 1847 guard(mutex)(&uclamp_mutex); 1848 1849 old_min = sysctl_sched_uclamp_util_min; 1850 old_max = sysctl_sched_uclamp_util_max; 1851 old_min_rt = sysctl_sched_uclamp_util_min_rt_default; 1852 1853 result = proc_dointvec(table, write, buffer, lenp, ppos); 1854 if (result) 1855 goto undo; 1856 if (!write) 1857 return 0; 1858 1859 if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max || 1860 sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE || 1861 sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) { 1862 1863 result = -EINVAL; 1864 goto undo; 1865 } 1866 1867 if (old_min != sysctl_sched_uclamp_util_min) { 1868 uclamp_se_set(&uclamp_default[UCLAMP_MIN], 1869 sysctl_sched_uclamp_util_min, false); 1870 update_root_tg = true; 1871 } 1872 if (old_max != sysctl_sched_uclamp_util_max) { 1873 uclamp_se_set(&uclamp_default[UCLAMP_MAX], 1874 sysctl_sched_uclamp_util_max, false); 1875 update_root_tg = true; 1876 } 1877 1878 if (update_root_tg) { 1879 static_branch_enable(&sched_uclamp_used); 1880 uclamp_update_root_tg(); 1881 } 1882 1883 if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) { 1884 static_branch_enable(&sched_uclamp_used); 1885 uclamp_sync_util_min_rt_default(); 1886 } 1887 1888 /* 1889 * We update all RUNNABLE tasks only when task groups are in use. 1890 * Otherwise, keep it simple and do just a lazy update at each next 1891 * task enqueue time. 1892 */ 1893 return 0; 1894 1895 undo: 1896 sysctl_sched_uclamp_util_min = old_min; 1897 sysctl_sched_uclamp_util_max = old_max; 1898 sysctl_sched_uclamp_util_min_rt_default = old_min_rt; 1899 return result; 1900 } 1901 #endif 1902 1903 static void uclamp_fork(struct task_struct *p) 1904 { 1905 enum uclamp_id clamp_id; 1906 1907 /* 1908 * We don't need to hold task_rq_lock() when updating p->uclamp_* here 1909 * as the task is still at its early fork stages. 1910 */ 1911 for_each_clamp_id(clamp_id) 1912 p->uclamp[clamp_id].active = false; 1913 1914 if (likely(!p->sched_reset_on_fork)) 1915 return; 1916 1917 for_each_clamp_id(clamp_id) { 1918 uclamp_se_set(&p->uclamp_req[clamp_id], 1919 uclamp_none(clamp_id), false); 1920 } 1921 } 1922 1923 static void uclamp_post_fork(struct task_struct *p) 1924 { 1925 uclamp_update_util_min_rt_default(p); 1926 } 1927 1928 static void __init init_uclamp_rq(struct rq *rq) 1929 { 1930 enum uclamp_id clamp_id; 1931 struct uclamp_rq *uc_rq = rq->uclamp; 1932 1933 for_each_clamp_id(clamp_id) { 1934 uc_rq[clamp_id] = (struct uclamp_rq) { 1935 .value = uclamp_none(clamp_id) 1936 }; 1937 } 1938 1939 rq->uclamp_flags = UCLAMP_FLAG_IDLE; 1940 } 1941 1942 static void __init init_uclamp(void) 1943 { 1944 struct uclamp_se uc_max = {}; 1945 enum uclamp_id clamp_id; 1946 int cpu; 1947 1948 for_each_possible_cpu(cpu) 1949 init_uclamp_rq(cpu_rq(cpu)); 1950 1951 for_each_clamp_id(clamp_id) { 1952 uclamp_se_set(&init_task.uclamp_req[clamp_id], 1953 uclamp_none(clamp_id), false); 1954 } 1955 1956 /* System defaults allow max clamp values for both indexes */ 1957 uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false); 1958 for_each_clamp_id(clamp_id) { 1959 uclamp_default[clamp_id] = uc_max; 1960 #ifdef CONFIG_UCLAMP_TASK_GROUP 1961 root_task_group.uclamp_req[clamp_id] = uc_max; 1962 root_task_group.uclamp[clamp_id] = uc_max; 1963 #endif 1964 } 1965 } 1966 1967 #else /* !CONFIG_UCLAMP_TASK */ 1968 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { } 1969 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { } 1970 static inline void uclamp_fork(struct task_struct *p) { } 1971 static inline void uclamp_post_fork(struct task_struct *p) { } 1972 static inline void init_uclamp(void) { } 1973 #endif /* CONFIG_UCLAMP_TASK */ 1974 1975 bool sched_task_on_rq(struct task_struct *p) 1976 { 1977 return task_on_rq_queued(p); 1978 } 1979 1980 unsigned long get_wchan(struct task_struct *p) 1981 { 1982 unsigned long ip = 0; 1983 unsigned int state; 1984 1985 if (!p || p == current) 1986 return 0; 1987 1988 /* Only get wchan if task is blocked and we can keep it that way. */ 1989 raw_spin_lock_irq(&p->pi_lock); 1990 state = READ_ONCE(p->__state); 1991 smp_rmb(); /* see try_to_wake_up() */ 1992 if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq) 1993 ip = __get_wchan(p); 1994 raw_spin_unlock_irq(&p->pi_lock); 1995 1996 return ip; 1997 } 1998 1999 void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 2000 { 2001 if (!(flags & ENQUEUE_NOCLOCK)) 2002 update_rq_clock(rq); 2003 2004 if (!(flags & ENQUEUE_RESTORE)) { 2005 sched_info_enqueue(rq, p); 2006 psi_enqueue(p, (flags & ENQUEUE_WAKEUP) && !(flags & ENQUEUE_MIGRATED)); 2007 } 2008 2009 p->sched_class->enqueue_task(rq, p, flags); 2010 /* 2011 * Must be after ->enqueue_task() because ENQUEUE_DELAYED can clear 2012 * ->sched_delayed. 2013 */ 2014 uclamp_rq_inc(rq, p); 2015 2016 if (sched_core_enabled(rq)) 2017 sched_core_enqueue(rq, p); 2018 } 2019 2020 /* 2021 * Must only return false when DEQUEUE_SLEEP. 2022 */ 2023 inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags) 2024 { 2025 if (sched_core_enabled(rq)) 2026 sched_core_dequeue(rq, p, flags); 2027 2028 if (!(flags & DEQUEUE_NOCLOCK)) 2029 update_rq_clock(rq); 2030 2031 if (!(flags & DEQUEUE_SAVE)) { 2032 sched_info_dequeue(rq, p); 2033 psi_dequeue(p, flags & DEQUEUE_SLEEP); 2034 } 2035 2036 /* 2037 * Must be before ->dequeue_task() because ->dequeue_task() can 'fail' 2038 * and mark the task ->sched_delayed. 2039 */ 2040 uclamp_rq_dec(rq, p); 2041 return p->sched_class->dequeue_task(rq, p, flags); 2042 } 2043 2044 void activate_task(struct rq *rq, struct task_struct *p, int flags) 2045 { 2046 if (task_on_rq_migrating(p)) 2047 flags |= ENQUEUE_MIGRATED; 2048 if (flags & ENQUEUE_MIGRATED) 2049 sched_mm_cid_migrate_to(rq, p); 2050 2051 enqueue_task(rq, p, flags); 2052 2053 WRITE_ONCE(p->on_rq, TASK_ON_RQ_QUEUED); 2054 ASSERT_EXCLUSIVE_WRITER(p->on_rq); 2055 } 2056 2057 void deactivate_task(struct rq *rq, struct task_struct *p, int flags) 2058 { 2059 SCHED_WARN_ON(flags & DEQUEUE_SLEEP); 2060 2061 WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING); 2062 ASSERT_EXCLUSIVE_WRITER(p->on_rq); 2063 2064 /* 2065 * Code explicitly relies on TASK_ON_RQ_MIGRATING begin set *before* 2066 * dequeue_task() and cleared *after* enqueue_task(). 2067 */ 2068 2069 dequeue_task(rq, p, flags); 2070 } 2071 2072 static void block_task(struct rq *rq, struct task_struct *p, int flags) 2073 { 2074 if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags)) 2075 __block_task(rq, p); 2076 } 2077 2078 /** 2079 * task_curr - is this task currently executing on a CPU? 2080 * @p: the task in question. 2081 * 2082 * Return: 1 if the task is currently executing. 0 otherwise. 2083 */ 2084 inline int task_curr(const struct task_struct *p) 2085 { 2086 return cpu_curr(task_cpu(p)) == p; 2087 } 2088 2089 /* 2090 * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock, 2091 * use the balance_callback list if you want balancing. 2092 * 2093 * this means any call to check_class_changed() must be followed by a call to 2094 * balance_callback(). 2095 */ 2096 void check_class_changed(struct rq *rq, struct task_struct *p, 2097 const struct sched_class *prev_class, 2098 int oldprio) 2099 { 2100 if (prev_class != p->sched_class) { 2101 if (prev_class->switched_from) 2102 prev_class->switched_from(rq, p); 2103 2104 p->sched_class->switched_to(rq, p); 2105 } else if (oldprio != p->prio || dl_task(p)) 2106 p->sched_class->prio_changed(rq, p, oldprio); 2107 } 2108 2109 void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags) 2110 { 2111 if (p->sched_class == rq->curr->sched_class) 2112 rq->curr->sched_class->wakeup_preempt(rq, p, flags); 2113 else if (sched_class_above(p->sched_class, rq->curr->sched_class)) 2114 resched_curr(rq); 2115 2116 /* 2117 * A queue event has occurred, and we're going to schedule. In 2118 * this case, we can save a useless back to back clock update. 2119 */ 2120 if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) 2121 rq_clock_skip_update(rq); 2122 } 2123 2124 static __always_inline 2125 int __task_state_match(struct task_struct *p, unsigned int state) 2126 { 2127 if (READ_ONCE(p->__state) & state) 2128 return 1; 2129 2130 if (READ_ONCE(p->saved_state) & state) 2131 return -1; 2132 2133 return 0; 2134 } 2135 2136 static __always_inline 2137 int task_state_match(struct task_struct *p, unsigned int state) 2138 { 2139 /* 2140 * Serialize against current_save_and_set_rtlock_wait_state(), 2141 * current_restore_rtlock_saved_state(), and __refrigerator(). 2142 */ 2143 guard(raw_spinlock_irq)(&p->pi_lock); 2144 return __task_state_match(p, state); 2145 } 2146 2147 /* 2148 * wait_task_inactive - wait for a thread to unschedule. 2149 * 2150 * Wait for the thread to block in any of the states set in @match_state. 2151 * If it changes, i.e. @p might have woken up, then return zero. When we 2152 * succeed in waiting for @p to be off its CPU, we return a positive number 2153 * (its total switch count). If a second call a short while later returns the 2154 * same number, the caller can be sure that @p has remained unscheduled the 2155 * whole time. 2156 * 2157 * The caller must ensure that the task *will* unschedule sometime soon, 2158 * else this function might spin for a *long* time. This function can't 2159 * be called with interrupts off, or it may introduce deadlock with 2160 * smp_call_function() if an IPI is sent by the same process we are 2161 * waiting to become inactive. 2162 */ 2163 unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state) 2164 { 2165 int running, queued, match; 2166 struct rq_flags rf; 2167 unsigned long ncsw; 2168 struct rq *rq; 2169 2170 for (;;) { 2171 /* 2172 * We do the initial early heuristics without holding 2173 * any task-queue locks at all. We'll only try to get 2174 * the runqueue lock when things look like they will 2175 * work out! 2176 */ 2177 rq = task_rq(p); 2178 2179 /* 2180 * If the task is actively running on another CPU 2181 * still, just relax and busy-wait without holding 2182 * any locks. 2183 * 2184 * NOTE! Since we don't hold any locks, it's not 2185 * even sure that "rq" stays as the right runqueue! 2186 * But we don't care, since "task_on_cpu()" will 2187 * return false if the runqueue has changed and p 2188 * is actually now running somewhere else! 2189 */ 2190 while (task_on_cpu(rq, p)) { 2191 if (!task_state_match(p, match_state)) 2192 return 0; 2193 cpu_relax(); 2194 } 2195 2196 /* 2197 * Ok, time to look more closely! We need the rq 2198 * lock now, to be *sure*. If we're wrong, we'll 2199 * just go back and repeat. 2200 */ 2201 rq = task_rq_lock(p, &rf); 2202 trace_sched_wait_task(p); 2203 running = task_on_cpu(rq, p); 2204 queued = task_on_rq_queued(p); 2205 ncsw = 0; 2206 if ((match = __task_state_match(p, match_state))) { 2207 /* 2208 * When matching on p->saved_state, consider this task 2209 * still queued so it will wait. 2210 */ 2211 if (match < 0) 2212 queued = 1; 2213 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ 2214 } 2215 task_rq_unlock(rq, p, &rf); 2216 2217 /* 2218 * If it changed from the expected state, bail out now. 2219 */ 2220 if (unlikely(!ncsw)) 2221 break; 2222 2223 /* 2224 * Was it really running after all now that we 2225 * checked with the proper locks actually held? 2226 * 2227 * Oops. Go back and try again.. 2228 */ 2229 if (unlikely(running)) { 2230 cpu_relax(); 2231 continue; 2232 } 2233 2234 /* 2235 * It's not enough that it's not actively running, 2236 * it must be off the runqueue _entirely_, and not 2237 * preempted! 2238 * 2239 * So if it was still runnable (but just not actively 2240 * running right now), it's preempted, and we should 2241 * yield - it could be a while. 2242 */ 2243 if (unlikely(queued)) { 2244 ktime_t to = NSEC_PER_SEC / HZ; 2245 2246 set_current_state(TASK_UNINTERRUPTIBLE); 2247 schedule_hrtimeout(&to, HRTIMER_MODE_REL_HARD); 2248 continue; 2249 } 2250 2251 /* 2252 * Ahh, all good. It wasn't running, and it wasn't 2253 * runnable, which means that it will never become 2254 * running in the future either. We're all done! 2255 */ 2256 break; 2257 } 2258 2259 return ncsw; 2260 } 2261 2262 #ifdef CONFIG_SMP 2263 2264 static void 2265 __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx); 2266 2267 static void migrate_disable_switch(struct rq *rq, struct task_struct *p) 2268 { 2269 struct affinity_context ac = { 2270 .new_mask = cpumask_of(rq->cpu), 2271 .flags = SCA_MIGRATE_DISABLE, 2272 }; 2273 2274 if (likely(!p->migration_disabled)) 2275 return; 2276 2277 if (p->cpus_ptr != &p->cpus_mask) 2278 return; 2279 2280 /* 2281 * Violates locking rules! See comment in __do_set_cpus_allowed(). 2282 */ 2283 __do_set_cpus_allowed(p, &ac); 2284 } 2285 2286 void migrate_disable(void) 2287 { 2288 struct task_struct *p = current; 2289 2290 if (p->migration_disabled) { 2291 #ifdef CONFIG_DEBUG_PREEMPT 2292 /* 2293 *Warn about overflow half-way through the range. 2294 */ 2295 WARN_ON_ONCE((s16)p->migration_disabled < 0); 2296 #endif 2297 p->migration_disabled++; 2298 return; 2299 } 2300 2301 guard(preempt)(); 2302 this_rq()->nr_pinned++; 2303 p->migration_disabled = 1; 2304 } 2305 EXPORT_SYMBOL_GPL(migrate_disable); 2306 2307 void migrate_enable(void) 2308 { 2309 struct task_struct *p = current; 2310 struct affinity_context ac = { 2311 .new_mask = &p->cpus_mask, 2312 .flags = SCA_MIGRATE_ENABLE, 2313 }; 2314 2315 #ifdef CONFIG_DEBUG_PREEMPT 2316 /* 2317 * Check both overflow from migrate_disable() and superfluous 2318 * migrate_enable(). 2319 */ 2320 if (WARN_ON_ONCE((s16)p->migration_disabled <= 0)) 2321 return; 2322 #endif 2323 2324 if (p->migration_disabled > 1) { 2325 p->migration_disabled--; 2326 return; 2327 } 2328 2329 /* 2330 * Ensure stop_task runs either before or after this, and that 2331 * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule(). 2332 */ 2333 guard(preempt)(); 2334 if (p->cpus_ptr != &p->cpus_mask) 2335 __set_cpus_allowed_ptr(p, &ac); 2336 /* 2337 * Mustn't clear migration_disabled() until cpus_ptr points back at the 2338 * regular cpus_mask, otherwise things that race (eg. 2339 * select_fallback_rq) get confused. 2340 */ 2341 barrier(); 2342 p->migration_disabled = 0; 2343 this_rq()->nr_pinned--; 2344 } 2345 EXPORT_SYMBOL_GPL(migrate_enable); 2346 2347 static inline bool rq_has_pinned_tasks(struct rq *rq) 2348 { 2349 return rq->nr_pinned; 2350 } 2351 2352 /* 2353 * Per-CPU kthreads are allowed to run on !active && online CPUs, see 2354 * __set_cpus_allowed_ptr() and select_fallback_rq(). 2355 */ 2356 static inline bool is_cpu_allowed(struct task_struct *p, int cpu) 2357 { 2358 /* When not in the task's cpumask, no point in looking further. */ 2359 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 2360 return false; 2361 2362 /* migrate_disabled() must be allowed to finish. */ 2363 if (is_migration_disabled(p)) 2364 return cpu_online(cpu); 2365 2366 /* Non kernel threads are not allowed during either online or offline. */ 2367 if (!(p->flags & PF_KTHREAD)) 2368 return cpu_active(cpu) && task_cpu_possible(cpu, p); 2369 2370 /* KTHREAD_IS_PER_CPU is always allowed. */ 2371 if (kthread_is_per_cpu(p)) 2372 return cpu_online(cpu); 2373 2374 /* Regular kernel threads don't get to stay during offline. */ 2375 if (cpu_dying(cpu)) 2376 return false; 2377 2378 /* But are allowed during online. */ 2379 return cpu_online(cpu); 2380 } 2381 2382 /* 2383 * This is how migration works: 2384 * 2385 * 1) we invoke migration_cpu_stop() on the target CPU using 2386 * stop_one_cpu(). 2387 * 2) stopper starts to run (implicitly forcing the migrated thread 2388 * off the CPU) 2389 * 3) it checks whether the migrated task is still in the wrong runqueue. 2390 * 4) if it's in the wrong runqueue then the migration thread removes 2391 * it and puts it into the right queue. 2392 * 5) stopper completes and stop_one_cpu() returns and the migration 2393 * is done. 2394 */ 2395 2396 /* 2397 * move_queued_task - move a queued task to new rq. 2398 * 2399 * Returns (locked) new rq. Old rq's lock is released. 2400 */ 2401 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, 2402 struct task_struct *p, int new_cpu) 2403 { 2404 lockdep_assert_rq_held(rq); 2405 2406 deactivate_task(rq, p, DEQUEUE_NOCLOCK); 2407 set_task_cpu(p, new_cpu); 2408 rq_unlock(rq, rf); 2409 2410 rq = cpu_rq(new_cpu); 2411 2412 rq_lock(rq, rf); 2413 WARN_ON_ONCE(task_cpu(p) != new_cpu); 2414 activate_task(rq, p, 0); 2415 wakeup_preempt(rq, p, 0); 2416 2417 return rq; 2418 } 2419 2420 struct migration_arg { 2421 struct task_struct *task; 2422 int dest_cpu; 2423 struct set_affinity_pending *pending; 2424 }; 2425 2426 /* 2427 * @refs: number of wait_for_completion() 2428 * @stop_pending: is @stop_work in use 2429 */ 2430 struct set_affinity_pending { 2431 refcount_t refs; 2432 unsigned int stop_pending; 2433 struct completion done; 2434 struct cpu_stop_work stop_work; 2435 struct migration_arg arg; 2436 }; 2437 2438 /* 2439 * Move (not current) task off this CPU, onto the destination CPU. We're doing 2440 * this because either it can't run here any more (set_cpus_allowed() 2441 * away from this CPU, or CPU going down), or because we're 2442 * attempting to rebalance this task on exec (sched_exec). 2443 * 2444 * So we race with normal scheduler movements, but that's OK, as long 2445 * as the task is no longer on this CPU. 2446 */ 2447 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, 2448 struct task_struct *p, int dest_cpu) 2449 { 2450 /* Affinity changed (again). */ 2451 if (!is_cpu_allowed(p, dest_cpu)) 2452 return rq; 2453 2454 rq = move_queued_task(rq, rf, p, dest_cpu); 2455 2456 return rq; 2457 } 2458 2459 /* 2460 * migration_cpu_stop - this will be executed by a high-prio stopper thread 2461 * and performs thread migration by bumping thread off CPU then 2462 * 'pushing' onto another runqueue. 2463 */ 2464 static int migration_cpu_stop(void *data) 2465 { 2466 struct migration_arg *arg = data; 2467 struct set_affinity_pending *pending = arg->pending; 2468 struct task_struct *p = arg->task; 2469 struct rq *rq = this_rq(); 2470 bool complete = false; 2471 struct rq_flags rf; 2472 2473 /* 2474 * The original target CPU might have gone down and we might 2475 * be on another CPU but it doesn't matter. 2476 */ 2477 local_irq_save(rf.flags); 2478 /* 2479 * We need to explicitly wake pending tasks before running 2480 * __migrate_task() such that we will not miss enforcing cpus_ptr 2481 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. 2482 */ 2483 flush_smp_call_function_queue(); 2484 2485 raw_spin_lock(&p->pi_lock); 2486 rq_lock(rq, &rf); 2487 2488 /* 2489 * If we were passed a pending, then ->stop_pending was set, thus 2490 * p->migration_pending must have remained stable. 2491 */ 2492 WARN_ON_ONCE(pending && pending != p->migration_pending); 2493 2494 /* 2495 * If task_rq(p) != rq, it cannot be migrated here, because we're 2496 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because 2497 * we're holding p->pi_lock. 2498 */ 2499 if (task_rq(p) == rq) { 2500 if (is_migration_disabled(p)) 2501 goto out; 2502 2503 if (pending) { 2504 p->migration_pending = NULL; 2505 complete = true; 2506 2507 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) 2508 goto out; 2509 } 2510 2511 if (task_on_rq_queued(p)) { 2512 update_rq_clock(rq); 2513 rq = __migrate_task(rq, &rf, p, arg->dest_cpu); 2514 } else { 2515 p->wake_cpu = arg->dest_cpu; 2516 } 2517 2518 /* 2519 * XXX __migrate_task() can fail, at which point we might end 2520 * up running on a dodgy CPU, AFAICT this can only happen 2521 * during CPU hotplug, at which point we'll get pushed out 2522 * anyway, so it's probably not a big deal. 2523 */ 2524 2525 } else if (pending) { 2526 /* 2527 * This happens when we get migrated between migrate_enable()'s 2528 * preempt_enable() and scheduling the stopper task. At that 2529 * point we're a regular task again and not current anymore. 2530 * 2531 * A !PREEMPT kernel has a giant hole here, which makes it far 2532 * more likely. 2533 */ 2534 2535 /* 2536 * The task moved before the stopper got to run. We're holding 2537 * ->pi_lock, so the allowed mask is stable - if it got 2538 * somewhere allowed, we're done. 2539 */ 2540 if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) { 2541 p->migration_pending = NULL; 2542 complete = true; 2543 goto out; 2544 } 2545 2546 /* 2547 * When migrate_enable() hits a rq mis-match we can't reliably 2548 * determine is_migration_disabled() and so have to chase after 2549 * it. 2550 */ 2551 WARN_ON_ONCE(!pending->stop_pending); 2552 preempt_disable(); 2553 task_rq_unlock(rq, p, &rf); 2554 stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop, 2555 &pending->arg, &pending->stop_work); 2556 preempt_enable(); 2557 return 0; 2558 } 2559 out: 2560 if (pending) 2561 pending->stop_pending = false; 2562 task_rq_unlock(rq, p, &rf); 2563 2564 if (complete) 2565 complete_all(&pending->done); 2566 2567 return 0; 2568 } 2569 2570 int push_cpu_stop(void *arg) 2571 { 2572 struct rq *lowest_rq = NULL, *rq = this_rq(); 2573 struct task_struct *p = arg; 2574 2575 raw_spin_lock_irq(&p->pi_lock); 2576 raw_spin_rq_lock(rq); 2577 2578 if (task_rq(p) != rq) 2579 goto out_unlock; 2580 2581 if (is_migration_disabled(p)) { 2582 p->migration_flags |= MDF_PUSH; 2583 goto out_unlock; 2584 } 2585 2586 p->migration_flags &= ~MDF_PUSH; 2587 2588 if (p->sched_class->find_lock_rq) 2589 lowest_rq = p->sched_class->find_lock_rq(p, rq); 2590 2591 if (!lowest_rq) 2592 goto out_unlock; 2593 2594 // XXX validate p is still the highest prio task 2595 if (task_rq(p) == rq) { 2596 deactivate_task(rq, p, 0); 2597 set_task_cpu(p, lowest_rq->cpu); 2598 activate_task(lowest_rq, p, 0); 2599 resched_curr(lowest_rq); 2600 } 2601 2602 double_unlock_balance(rq, lowest_rq); 2603 2604 out_unlock: 2605 rq->push_busy = false; 2606 raw_spin_rq_unlock(rq); 2607 raw_spin_unlock_irq(&p->pi_lock); 2608 2609 put_task_struct(p); 2610 return 0; 2611 } 2612 2613 /* 2614 * sched_class::set_cpus_allowed must do the below, but is not required to 2615 * actually call this function. 2616 */ 2617 void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx) 2618 { 2619 if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) { 2620 p->cpus_ptr = ctx->new_mask; 2621 return; 2622 } 2623 2624 cpumask_copy(&p->cpus_mask, ctx->new_mask); 2625 p->nr_cpus_allowed = cpumask_weight(ctx->new_mask); 2626 2627 /* 2628 * Swap in a new user_cpus_ptr if SCA_USER flag set 2629 */ 2630 if (ctx->flags & SCA_USER) 2631 swap(p->user_cpus_ptr, ctx->user_mask); 2632 } 2633 2634 static void 2635 __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) 2636 { 2637 struct rq *rq = task_rq(p); 2638 bool queued, running; 2639 2640 /* 2641 * This here violates the locking rules for affinity, since we're only 2642 * supposed to change these variables while holding both rq->lock and 2643 * p->pi_lock. 2644 * 2645 * HOWEVER, it magically works, because ttwu() is the only code that 2646 * accesses these variables under p->pi_lock and only does so after 2647 * smp_cond_load_acquire(&p->on_cpu, !VAL), and we're in __schedule() 2648 * before finish_task(). 2649 * 2650 * XXX do further audits, this smells like something putrid. 2651 */ 2652 if (ctx->flags & SCA_MIGRATE_DISABLE) 2653 SCHED_WARN_ON(!p->on_cpu); 2654 else 2655 lockdep_assert_held(&p->pi_lock); 2656 2657 queued = task_on_rq_queued(p); 2658 running = task_current(rq, p); 2659 2660 if (queued) { 2661 /* 2662 * Because __kthread_bind() calls this on blocked tasks without 2663 * holding rq->lock. 2664 */ 2665 lockdep_assert_rq_held(rq); 2666 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 2667 } 2668 if (running) 2669 put_prev_task(rq, p); 2670 2671 p->sched_class->set_cpus_allowed(p, ctx); 2672 2673 if (queued) 2674 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 2675 if (running) 2676 set_next_task(rq, p); 2677 } 2678 2679 /* 2680 * Used for kthread_bind() and select_fallback_rq(), in both cases the user 2681 * affinity (if any) should be destroyed too. 2682 */ 2683 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) 2684 { 2685 struct affinity_context ac = { 2686 .new_mask = new_mask, 2687 .user_mask = NULL, 2688 .flags = SCA_USER, /* clear the user requested mask */ 2689 }; 2690 union cpumask_rcuhead { 2691 cpumask_t cpumask; 2692 struct rcu_head rcu; 2693 }; 2694 2695 __do_set_cpus_allowed(p, &ac); 2696 2697 /* 2698 * Because this is called with p->pi_lock held, it is not possible 2699 * to use kfree() here (when PREEMPT_RT=y), therefore punt to using 2700 * kfree_rcu(). 2701 */ 2702 kfree_rcu((union cpumask_rcuhead *)ac.user_mask, rcu); 2703 } 2704 2705 int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, 2706 int node) 2707 { 2708 cpumask_t *user_mask; 2709 unsigned long flags; 2710 2711 /* 2712 * Always clear dst->user_cpus_ptr first as their user_cpus_ptr's 2713 * may differ by now due to racing. 2714 */ 2715 dst->user_cpus_ptr = NULL; 2716 2717 /* 2718 * This check is racy and losing the race is a valid situation. 2719 * It is not worth the extra overhead of taking the pi_lock on 2720 * every fork/clone. 2721 */ 2722 if (data_race(!src->user_cpus_ptr)) 2723 return 0; 2724 2725 user_mask = alloc_user_cpus_ptr(node); 2726 if (!user_mask) 2727 return -ENOMEM; 2728 2729 /* 2730 * Use pi_lock to protect content of user_cpus_ptr 2731 * 2732 * Though unlikely, user_cpus_ptr can be reset to NULL by a concurrent 2733 * do_set_cpus_allowed(). 2734 */ 2735 raw_spin_lock_irqsave(&src->pi_lock, flags); 2736 if (src->user_cpus_ptr) { 2737 swap(dst->user_cpus_ptr, user_mask); 2738 cpumask_copy(dst->user_cpus_ptr, src->user_cpus_ptr); 2739 } 2740 raw_spin_unlock_irqrestore(&src->pi_lock, flags); 2741 2742 if (unlikely(user_mask)) 2743 kfree(user_mask); 2744 2745 return 0; 2746 } 2747 2748 static inline struct cpumask *clear_user_cpus_ptr(struct task_struct *p) 2749 { 2750 struct cpumask *user_mask = NULL; 2751 2752 swap(p->user_cpus_ptr, user_mask); 2753 2754 return user_mask; 2755 } 2756 2757 void release_user_cpus_ptr(struct task_struct *p) 2758 { 2759 kfree(clear_user_cpus_ptr(p)); 2760 } 2761 2762 /* 2763 * This function is wildly self concurrent; here be dragons. 2764 * 2765 * 2766 * When given a valid mask, __set_cpus_allowed_ptr() must block until the 2767 * designated task is enqueued on an allowed CPU. If that task is currently 2768 * running, we have to kick it out using the CPU stopper. 2769 * 2770 * Migrate-Disable comes along and tramples all over our nice sandcastle. 2771 * Consider: 2772 * 2773 * Initial conditions: P0->cpus_mask = [0, 1] 2774 * 2775 * P0@CPU0 P1 2776 * 2777 * migrate_disable(); 2778 * <preempted> 2779 * set_cpus_allowed_ptr(P0, [1]); 2780 * 2781 * P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes 2782 * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region). 2783 * This means we need the following scheme: 2784 * 2785 * P0@CPU0 P1 2786 * 2787 * migrate_disable(); 2788 * <preempted> 2789 * set_cpus_allowed_ptr(P0, [1]); 2790 * <blocks> 2791 * <resumes> 2792 * migrate_enable(); 2793 * __set_cpus_allowed_ptr(); 2794 * <wakes local stopper> 2795 * `--> <woken on migration completion> 2796 * 2797 * Now the fun stuff: there may be several P1-like tasks, i.e. multiple 2798 * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any 2799 * task p are serialized by p->pi_lock, which we can leverage: the one that 2800 * should come into effect at the end of the Migrate-Disable region is the last 2801 * one. This means we only need to track a single cpumask (i.e. p->cpus_mask), 2802 * but we still need to properly signal those waiting tasks at the appropriate 2803 * moment. 2804 * 2805 * This is implemented using struct set_affinity_pending. The first 2806 * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will 2807 * setup an instance of that struct and install it on the targeted task_struct. 2808 * Any and all further callers will reuse that instance. Those then wait for 2809 * a completion signaled at the tail of the CPU stopper callback (1), triggered 2810 * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()). 2811 * 2812 * 2813 * (1) In the cases covered above. There is one more where the completion is 2814 * signaled within affine_move_task() itself: when a subsequent affinity request 2815 * occurs after the stopper bailed out due to the targeted task still being 2816 * Migrate-Disable. Consider: 2817 * 2818 * Initial conditions: P0->cpus_mask = [0, 1] 2819 * 2820 * CPU0 P1 P2 2821 * <P0> 2822 * migrate_disable(); 2823 * <preempted> 2824 * set_cpus_allowed_ptr(P0, [1]); 2825 * <blocks> 2826 * <migration/0> 2827 * migration_cpu_stop() 2828 * is_migration_disabled() 2829 * <bails> 2830 * set_cpus_allowed_ptr(P0, [0, 1]); 2831 * <signal completion> 2832 * <awakes> 2833 * 2834 * Note that the above is safe vs a concurrent migrate_enable(), as any 2835 * pending affinity completion is preceded by an uninstallation of 2836 * p->migration_pending done with p->pi_lock held. 2837 */ 2838 static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf, 2839 int dest_cpu, unsigned int flags) 2840 __releases(rq->lock) 2841 __releases(p->pi_lock) 2842 { 2843 struct set_affinity_pending my_pending = { }, *pending = NULL; 2844 bool stop_pending, complete = false; 2845 2846 /* Can the task run on the task's current CPU? If so, we're done */ 2847 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) { 2848 struct task_struct *push_task = NULL; 2849 2850 if ((flags & SCA_MIGRATE_ENABLE) && 2851 (p->migration_flags & MDF_PUSH) && !rq->push_busy) { 2852 rq->push_busy = true; 2853 push_task = get_task_struct(p); 2854 } 2855 2856 /* 2857 * If there are pending waiters, but no pending stop_work, 2858 * then complete now. 2859 */ 2860 pending = p->migration_pending; 2861 if (pending && !pending->stop_pending) { 2862 p->migration_pending = NULL; 2863 complete = true; 2864 } 2865 2866 preempt_disable(); 2867 task_rq_unlock(rq, p, rf); 2868 if (push_task) { 2869 stop_one_cpu_nowait(rq->cpu, push_cpu_stop, 2870 p, &rq->push_work); 2871 } 2872 preempt_enable(); 2873 2874 if (complete) 2875 complete_all(&pending->done); 2876 2877 return 0; 2878 } 2879 2880 if (!(flags & SCA_MIGRATE_ENABLE)) { 2881 /* serialized by p->pi_lock */ 2882 if (!p->migration_pending) { 2883 /* Install the request */ 2884 refcount_set(&my_pending.refs, 1); 2885 init_completion(&my_pending.done); 2886 my_pending.arg = (struct migration_arg) { 2887 .task = p, 2888 .dest_cpu = dest_cpu, 2889 .pending = &my_pending, 2890 }; 2891 2892 p->migration_pending = &my_pending; 2893 } else { 2894 pending = p->migration_pending; 2895 refcount_inc(&pending->refs); 2896 /* 2897 * Affinity has changed, but we've already installed a 2898 * pending. migration_cpu_stop() *must* see this, else 2899 * we risk a completion of the pending despite having a 2900 * task on a disallowed CPU. 2901 * 2902 * Serialized by p->pi_lock, so this is safe. 2903 */ 2904 pending->arg.dest_cpu = dest_cpu; 2905 } 2906 } 2907 pending = p->migration_pending; 2908 /* 2909 * - !MIGRATE_ENABLE: 2910 * we'll have installed a pending if there wasn't one already. 2911 * 2912 * - MIGRATE_ENABLE: 2913 * we're here because the current CPU isn't matching anymore, 2914 * the only way that can happen is because of a concurrent 2915 * set_cpus_allowed_ptr() call, which should then still be 2916 * pending completion. 2917 * 2918 * Either way, we really should have a @pending here. 2919 */ 2920 if (WARN_ON_ONCE(!pending)) { 2921 task_rq_unlock(rq, p, rf); 2922 return -EINVAL; 2923 } 2924 2925 if (task_on_cpu(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { 2926 /* 2927 * MIGRATE_ENABLE gets here because 'p == current', but for 2928 * anything else we cannot do is_migration_disabled(), punt 2929 * and have the stopper function handle it all race-free. 2930 */ 2931 stop_pending = pending->stop_pending; 2932 if (!stop_pending) 2933 pending->stop_pending = true; 2934 2935 if (flags & SCA_MIGRATE_ENABLE) 2936 p->migration_flags &= ~MDF_PUSH; 2937 2938 preempt_disable(); 2939 task_rq_unlock(rq, p, rf); 2940 if (!stop_pending) { 2941 stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, 2942 &pending->arg, &pending->stop_work); 2943 } 2944 preempt_enable(); 2945 2946 if (flags & SCA_MIGRATE_ENABLE) 2947 return 0; 2948 } else { 2949 2950 if (!is_migration_disabled(p)) { 2951 if (task_on_rq_queued(p)) 2952 rq = move_queued_task(rq, rf, p, dest_cpu); 2953 2954 if (!pending->stop_pending) { 2955 p->migration_pending = NULL; 2956 complete = true; 2957 } 2958 } 2959 task_rq_unlock(rq, p, rf); 2960 2961 if (complete) 2962 complete_all(&pending->done); 2963 } 2964 2965 wait_for_completion(&pending->done); 2966 2967 if (refcount_dec_and_test(&pending->refs)) 2968 wake_up_var(&pending->refs); /* No UaF, just an address */ 2969 2970 /* 2971 * Block the original owner of &pending until all subsequent callers 2972 * have seen the completion and decremented the refcount 2973 */ 2974 wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs)); 2975 2976 /* ARGH */ 2977 WARN_ON_ONCE(my_pending.stop_pending); 2978 2979 return 0; 2980 } 2981 2982 /* 2983 * Called with both p->pi_lock and rq->lock held; drops both before returning. 2984 */ 2985 static int __set_cpus_allowed_ptr_locked(struct task_struct *p, 2986 struct affinity_context *ctx, 2987 struct rq *rq, 2988 struct rq_flags *rf) 2989 __releases(rq->lock) 2990 __releases(p->pi_lock) 2991 { 2992 const struct cpumask *cpu_allowed_mask = task_cpu_possible_mask(p); 2993 const struct cpumask *cpu_valid_mask = cpu_active_mask; 2994 bool kthread = p->flags & PF_KTHREAD; 2995 unsigned int dest_cpu; 2996 int ret = 0; 2997 2998 update_rq_clock(rq); 2999 3000 if (kthread || is_migration_disabled(p)) { 3001 /* 3002 * Kernel threads are allowed on online && !active CPUs, 3003 * however, during cpu-hot-unplug, even these might get pushed 3004 * away if not KTHREAD_IS_PER_CPU. 3005 * 3006 * Specifically, migration_disabled() tasks must not fail the 3007 * cpumask_any_and_distribute() pick below, esp. so on 3008 * SCA_MIGRATE_ENABLE, otherwise we'll not call 3009 * set_cpus_allowed_common() and actually reset p->cpus_ptr. 3010 */ 3011 cpu_valid_mask = cpu_online_mask; 3012 } 3013 3014 if (!kthread && !cpumask_subset(ctx->new_mask, cpu_allowed_mask)) { 3015 ret = -EINVAL; 3016 goto out; 3017 } 3018 3019 /* 3020 * Must re-check here, to close a race against __kthread_bind(), 3021 * sched_setaffinity() is not guaranteed to observe the flag. 3022 */ 3023 if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) { 3024 ret = -EINVAL; 3025 goto out; 3026 } 3027 3028 if (!(ctx->flags & SCA_MIGRATE_ENABLE)) { 3029 if (cpumask_equal(&p->cpus_mask, ctx->new_mask)) { 3030 if (ctx->flags & SCA_USER) 3031 swap(p->user_cpus_ptr, ctx->user_mask); 3032 goto out; 3033 } 3034 3035 if (WARN_ON_ONCE(p == current && 3036 is_migration_disabled(p) && 3037 !cpumask_test_cpu(task_cpu(p), ctx->new_mask))) { 3038 ret = -EBUSY; 3039 goto out; 3040 } 3041 } 3042 3043 /* 3044 * Picking a ~random cpu helps in cases where we are changing affinity 3045 * for groups of tasks (ie. cpuset), so that load balancing is not 3046 * immediately required to distribute the tasks within their new mask. 3047 */ 3048 dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, ctx->new_mask); 3049 if (dest_cpu >= nr_cpu_ids) { 3050 ret = -EINVAL; 3051 goto out; 3052 } 3053 3054 __do_set_cpus_allowed(p, ctx); 3055 3056 return affine_move_task(rq, p, rf, dest_cpu, ctx->flags); 3057 3058 out: 3059 task_rq_unlock(rq, p, rf); 3060 3061 return ret; 3062 } 3063 3064 /* 3065 * Change a given task's CPU affinity. Migrate the thread to a 3066 * proper CPU and schedule it away if the CPU it's executing on 3067 * is removed from the allowed bitmask. 3068 * 3069 * NOTE: the caller must have a valid reference to the task, the 3070 * task must not exit() & deallocate itself prematurely. The 3071 * call is not atomic; no spinlocks may be held. 3072 */ 3073 int __set_cpus_allowed_ptr(struct task_struct *p, struct affinity_context *ctx) 3074 { 3075 struct rq_flags rf; 3076 struct rq *rq; 3077 3078 rq = task_rq_lock(p, &rf); 3079 /* 3080 * Masking should be skipped if SCA_USER or any of the SCA_MIGRATE_* 3081 * flags are set. 3082 */ 3083 if (p->user_cpus_ptr && 3084 !(ctx->flags & (SCA_USER | SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) && 3085 cpumask_and(rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr)) 3086 ctx->new_mask = rq->scratch_mask; 3087 3088 return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf); 3089 } 3090 3091 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) 3092 { 3093 struct affinity_context ac = { 3094 .new_mask = new_mask, 3095 .flags = 0, 3096 }; 3097 3098 return __set_cpus_allowed_ptr(p, &ac); 3099 } 3100 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); 3101 3102 /* 3103 * Change a given task's CPU affinity to the intersection of its current 3104 * affinity mask and @subset_mask, writing the resulting mask to @new_mask. 3105 * If user_cpus_ptr is defined, use it as the basis for restricting CPU 3106 * affinity or use cpu_online_mask instead. 3107 * 3108 * If the resulting mask is empty, leave the affinity unchanged and return 3109 * -EINVAL. 3110 */ 3111 static int restrict_cpus_allowed_ptr(struct task_struct *p, 3112 struct cpumask *new_mask, 3113 const struct cpumask *subset_mask) 3114 { 3115 struct affinity_context ac = { 3116 .new_mask = new_mask, 3117 .flags = 0, 3118 }; 3119 struct rq_flags rf; 3120 struct rq *rq; 3121 int err; 3122 3123 rq = task_rq_lock(p, &rf); 3124 3125 /* 3126 * Forcefully restricting the affinity of a deadline task is 3127 * likely to cause problems, so fail and noisily override the 3128 * mask entirely. 3129 */ 3130 if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { 3131 err = -EPERM; 3132 goto err_unlock; 3133 } 3134 3135 if (!cpumask_and(new_mask, task_user_cpus(p), subset_mask)) { 3136 err = -EINVAL; 3137 goto err_unlock; 3138 } 3139 3140 return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf); 3141 3142 err_unlock: 3143 task_rq_unlock(rq, p, &rf); 3144 return err; 3145 } 3146 3147 /* 3148 * Restrict the CPU affinity of task @p so that it is a subset of 3149 * task_cpu_possible_mask() and point @p->user_cpus_ptr to a copy of the 3150 * old affinity mask. If the resulting mask is empty, we warn and walk 3151 * up the cpuset hierarchy until we find a suitable mask. 3152 */ 3153 void force_compatible_cpus_allowed_ptr(struct task_struct *p) 3154 { 3155 cpumask_var_t new_mask; 3156 const struct cpumask *override_mask = task_cpu_possible_mask(p); 3157 3158 alloc_cpumask_var(&new_mask, GFP_KERNEL); 3159 3160 /* 3161 * __migrate_task() can fail silently in the face of concurrent 3162 * offlining of the chosen destination CPU, so take the hotplug 3163 * lock to ensure that the migration succeeds. 3164 */ 3165 cpus_read_lock(); 3166 if (!cpumask_available(new_mask)) 3167 goto out_set_mask; 3168 3169 if (!restrict_cpus_allowed_ptr(p, new_mask, override_mask)) 3170 goto out_free_mask; 3171 3172 /* 3173 * We failed to find a valid subset of the affinity mask for the 3174 * task, so override it based on its cpuset hierarchy. 3175 */ 3176 cpuset_cpus_allowed(p, new_mask); 3177 override_mask = new_mask; 3178 3179 out_set_mask: 3180 if (printk_ratelimit()) { 3181 printk_deferred("Overriding affinity for process %d (%s) to CPUs %*pbl\n", 3182 task_pid_nr(p), p->comm, 3183 cpumask_pr_args(override_mask)); 3184 } 3185 3186 WARN_ON(set_cpus_allowed_ptr(p, override_mask)); 3187 out_free_mask: 3188 cpus_read_unlock(); 3189 free_cpumask_var(new_mask); 3190 } 3191 3192 /* 3193 * Restore the affinity of a task @p which was previously restricted by a 3194 * call to force_compatible_cpus_allowed_ptr(). 3195 * 3196 * It is the caller's responsibility to serialise this with any calls to 3197 * force_compatible_cpus_allowed_ptr(@p). 3198 */ 3199 void relax_compatible_cpus_allowed_ptr(struct task_struct *p) 3200 { 3201 struct affinity_context ac = { 3202 .new_mask = task_user_cpus(p), 3203 .flags = 0, 3204 }; 3205 int ret; 3206 3207 /* 3208 * Try to restore the old affinity mask with __sched_setaffinity(). 3209 * Cpuset masking will be done there too. 3210 */ 3211 ret = __sched_setaffinity(p, &ac); 3212 WARN_ON_ONCE(ret); 3213 } 3214 3215 void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 3216 { 3217 #ifdef CONFIG_SCHED_DEBUG 3218 unsigned int state = READ_ONCE(p->__state); 3219 3220 /* 3221 * We should never call set_task_cpu() on a blocked task, 3222 * ttwu() will sort out the placement. 3223 */ 3224 WARN_ON_ONCE(state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq); 3225 3226 /* 3227 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING, 3228 * because schedstat_wait_{start,end} rebase migrating task's wait_start 3229 * time relying on p->on_rq. 3230 */ 3231 WARN_ON_ONCE(state == TASK_RUNNING && 3232 p->sched_class == &fair_sched_class && 3233 (p->on_rq && !task_on_rq_migrating(p))); 3234 3235 #ifdef CONFIG_LOCKDEP 3236 /* 3237 * The caller should hold either p->pi_lock or rq->lock, when changing 3238 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. 3239 * 3240 * sched_move_task() holds both and thus holding either pins the cgroup, 3241 * see task_group(). 3242 * 3243 * Furthermore, all task_rq users should acquire both locks, see 3244 * task_rq_lock(). 3245 */ 3246 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || 3247 lockdep_is_held(__rq_lockp(task_rq(p))))); 3248 #endif 3249 /* 3250 * Clearly, migrating tasks to offline CPUs is a fairly daft thing. 3251 */ 3252 WARN_ON_ONCE(!cpu_online(new_cpu)); 3253 3254 WARN_ON_ONCE(is_migration_disabled(p)); 3255 #endif 3256 3257 trace_sched_migrate_task(p, new_cpu); 3258 3259 if (task_cpu(p) != new_cpu) { 3260 if (p->sched_class->migrate_task_rq) 3261 p->sched_class->migrate_task_rq(p, new_cpu); 3262 p->se.nr_migrations++; 3263 rseq_migrate(p); 3264 sched_mm_cid_migrate_from(p); 3265 perf_event_task_migrate(p); 3266 } 3267 3268 __set_task_cpu(p, new_cpu); 3269 } 3270 3271 #ifdef CONFIG_NUMA_BALANCING 3272 static void __migrate_swap_task(struct task_struct *p, int cpu) 3273 { 3274 if (task_on_rq_queued(p)) { 3275 struct rq *src_rq, *dst_rq; 3276 struct rq_flags srf, drf; 3277 3278 src_rq = task_rq(p); 3279 dst_rq = cpu_rq(cpu); 3280 3281 rq_pin_lock(src_rq, &srf); 3282 rq_pin_lock(dst_rq, &drf); 3283 3284 deactivate_task(src_rq, p, 0); 3285 set_task_cpu(p, cpu); 3286 activate_task(dst_rq, p, 0); 3287 wakeup_preempt(dst_rq, p, 0); 3288 3289 rq_unpin_lock(dst_rq, &drf); 3290 rq_unpin_lock(src_rq, &srf); 3291 3292 } else { 3293 /* 3294 * Task isn't running anymore; make it appear like we migrated 3295 * it before it went to sleep. This means on wakeup we make the 3296 * previous CPU our target instead of where it really is. 3297 */ 3298 p->wake_cpu = cpu; 3299 } 3300 } 3301 3302 struct migration_swap_arg { 3303 struct task_struct *src_task, *dst_task; 3304 int src_cpu, dst_cpu; 3305 }; 3306 3307 static int migrate_swap_stop(void *data) 3308 { 3309 struct migration_swap_arg *arg = data; 3310 struct rq *src_rq, *dst_rq; 3311 3312 if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu)) 3313 return -EAGAIN; 3314 3315 src_rq = cpu_rq(arg->src_cpu); 3316 dst_rq = cpu_rq(arg->dst_cpu); 3317 3318 guard(double_raw_spinlock)(&arg->src_task->pi_lock, &arg->dst_task->pi_lock); 3319 guard(double_rq_lock)(src_rq, dst_rq); 3320 3321 if (task_cpu(arg->dst_task) != arg->dst_cpu) 3322 return -EAGAIN; 3323 3324 if (task_cpu(arg->src_task) != arg->src_cpu) 3325 return -EAGAIN; 3326 3327 if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr)) 3328 return -EAGAIN; 3329 3330 if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr)) 3331 return -EAGAIN; 3332 3333 __migrate_swap_task(arg->src_task, arg->dst_cpu); 3334 __migrate_swap_task(arg->dst_task, arg->src_cpu); 3335 3336 return 0; 3337 } 3338 3339 /* 3340 * Cross migrate two tasks 3341 */ 3342 int migrate_swap(struct task_struct *cur, struct task_struct *p, 3343 int target_cpu, int curr_cpu) 3344 { 3345 struct migration_swap_arg arg; 3346 int ret = -EINVAL; 3347 3348 arg = (struct migration_swap_arg){ 3349 .src_task = cur, 3350 .src_cpu = curr_cpu, 3351 .dst_task = p, 3352 .dst_cpu = target_cpu, 3353 }; 3354 3355 if (arg.src_cpu == arg.dst_cpu) 3356 goto out; 3357 3358 /* 3359 * These three tests are all lockless; this is OK since all of them 3360 * will be re-checked with proper locks held further down the line. 3361 */ 3362 if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) 3363 goto out; 3364 3365 if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr)) 3366 goto out; 3367 3368 if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr)) 3369 goto out; 3370 3371 trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); 3372 ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); 3373 3374 out: 3375 return ret; 3376 } 3377 #endif /* CONFIG_NUMA_BALANCING */ 3378 3379 /*** 3380 * kick_process - kick a running thread to enter/exit the kernel 3381 * @p: the to-be-kicked thread 3382 * 3383 * Cause a process which is running on another CPU to enter 3384 * kernel-mode, without any delay. (to get signals handled.) 3385 * 3386 * NOTE: this function doesn't have to take the runqueue lock, 3387 * because all it wants to ensure is that the remote task enters 3388 * the kernel. If the IPI races and the task has been migrated 3389 * to another CPU then no harm is done and the purpose has been 3390 * achieved as well. 3391 */ 3392 void kick_process(struct task_struct *p) 3393 { 3394 guard(preempt)(); 3395 int cpu = task_cpu(p); 3396 3397 if ((cpu != smp_processor_id()) && task_curr(p)) 3398 smp_send_reschedule(cpu); 3399 } 3400 EXPORT_SYMBOL_GPL(kick_process); 3401 3402 /* 3403 * ->cpus_ptr is protected by both rq->lock and p->pi_lock 3404 * 3405 * A few notes on cpu_active vs cpu_online: 3406 * 3407 * - cpu_active must be a subset of cpu_online 3408 * 3409 * - on CPU-up we allow per-CPU kthreads on the online && !active CPU, 3410 * see __set_cpus_allowed_ptr(). At this point the newly online 3411 * CPU isn't yet part of the sched domains, and balancing will not 3412 * see it. 3413 * 3414 * - on CPU-down we clear cpu_active() to mask the sched domains and 3415 * avoid the load balancer to place new tasks on the to be removed 3416 * CPU. Existing tasks will remain running there and will be taken 3417 * off. 3418 * 3419 * This means that fallback selection must not select !active CPUs. 3420 * And can assume that any active CPU must be online. Conversely 3421 * select_task_rq() below may allow selection of !active CPUs in order 3422 * to satisfy the above rules. 3423 */ 3424 static int select_fallback_rq(int cpu, struct task_struct *p) 3425 { 3426 int nid = cpu_to_node(cpu); 3427 const struct cpumask *nodemask = NULL; 3428 enum { cpuset, possible, fail } state = cpuset; 3429 int dest_cpu; 3430 3431 /* 3432 * If the node that the CPU is on has been offlined, cpu_to_node() 3433 * will return -1. There is no CPU on the node, and we should 3434 * select the CPU on the other node. 3435 */ 3436 if (nid != -1) { 3437 nodemask = cpumask_of_node(nid); 3438 3439 /* Look for allowed, online CPU in same node. */ 3440 for_each_cpu(dest_cpu, nodemask) { 3441 if (is_cpu_allowed(p, dest_cpu)) 3442 return dest_cpu; 3443 } 3444 } 3445 3446 for (;;) { 3447 /* Any allowed, online CPU? */ 3448 for_each_cpu(dest_cpu, p->cpus_ptr) { 3449 if (!is_cpu_allowed(p, dest_cpu)) 3450 continue; 3451 3452 goto out; 3453 } 3454 3455 /* No more Mr. Nice Guy. */ 3456 switch (state) { 3457 case cpuset: 3458 if (cpuset_cpus_allowed_fallback(p)) { 3459 state = possible; 3460 break; 3461 } 3462 fallthrough; 3463 case possible: 3464 /* 3465 * XXX When called from select_task_rq() we only 3466 * hold p->pi_lock and again violate locking order. 3467 * 3468 * More yuck to audit. 3469 */ 3470 do_set_cpus_allowed(p, task_cpu_possible_mask(p)); 3471 state = fail; 3472 break; 3473 case fail: 3474 BUG(); 3475 break; 3476 } 3477 } 3478 3479 out: 3480 if (state != cpuset) { 3481 /* 3482 * Don't tell them about moving exiting tasks or 3483 * kernel threads (both mm NULL), since they never 3484 * leave kernel. 3485 */ 3486 if (p->mm && printk_ratelimit()) { 3487 printk_deferred("process %d (%s) no longer affine to cpu%d\n", 3488 task_pid_nr(p), p->comm, cpu); 3489 } 3490 } 3491 3492 return dest_cpu; 3493 } 3494 3495 /* 3496 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable. 3497 */ 3498 static inline 3499 int select_task_rq(struct task_struct *p, int cpu, int wake_flags) 3500 { 3501 lockdep_assert_held(&p->pi_lock); 3502 3503 if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) 3504 cpu = p->sched_class->select_task_rq(p, cpu, wake_flags); 3505 else 3506 cpu = cpumask_any(p->cpus_ptr); 3507 3508 /* 3509 * In order not to call set_task_cpu() on a blocking task we need 3510 * to rely on ttwu() to place the task on a valid ->cpus_ptr 3511 * CPU. 3512 * 3513 * Since this is common to all placement strategies, this lives here. 3514 * 3515 * [ this allows ->select_task() to simply return task_cpu(p) and 3516 * not worry about this generic constraint ] 3517 */ 3518 if (unlikely(!is_cpu_allowed(p, cpu))) 3519 cpu = select_fallback_rq(task_cpu(p), p); 3520 3521 return cpu; 3522 } 3523 3524 void sched_set_stop_task(int cpu, struct task_struct *stop) 3525 { 3526 static struct lock_class_key stop_pi_lock; 3527 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; 3528 struct task_struct *old_stop = cpu_rq(cpu)->stop; 3529 3530 if (stop) { 3531 /* 3532 * Make it appear like a SCHED_FIFO task, its something 3533 * userspace knows about and won't get confused about. 3534 * 3535 * Also, it will make PI more or less work without too 3536 * much confusion -- but then, stop work should not 3537 * rely on PI working anyway. 3538 */ 3539 sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); 3540 3541 stop->sched_class = &stop_sched_class; 3542 3543 /* 3544 * The PI code calls rt_mutex_setprio() with ->pi_lock held to 3545 * adjust the effective priority of a task. As a result, 3546 * rt_mutex_setprio() can trigger (RT) balancing operations, 3547 * which can then trigger wakeups of the stop thread to push 3548 * around the current task. 3549 * 3550 * The stop task itself will never be part of the PI-chain, it 3551 * never blocks, therefore that ->pi_lock recursion is safe. 3552 * Tell lockdep about this by placing the stop->pi_lock in its 3553 * own class. 3554 */ 3555 lockdep_set_class(&stop->pi_lock, &stop_pi_lock); 3556 } 3557 3558 cpu_rq(cpu)->stop = stop; 3559 3560 if (old_stop) { 3561 /* 3562 * Reset it back to a normal scheduling class so that 3563 * it can die in pieces. 3564 */ 3565 old_stop->sched_class = &rt_sched_class; 3566 } 3567 } 3568 3569 #else /* CONFIG_SMP */ 3570 3571 static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { } 3572 3573 static inline bool rq_has_pinned_tasks(struct rq *rq) 3574 { 3575 return false; 3576 } 3577 3578 #endif /* !CONFIG_SMP */ 3579 3580 static void 3581 ttwu_stat(struct task_struct *p, int cpu, int wake_flags) 3582 { 3583 struct rq *rq; 3584 3585 if (!schedstat_enabled()) 3586 return; 3587 3588 rq = this_rq(); 3589 3590 #ifdef CONFIG_SMP 3591 if (cpu == rq->cpu) { 3592 __schedstat_inc(rq->ttwu_local); 3593 __schedstat_inc(p->stats.nr_wakeups_local); 3594 } else { 3595 struct sched_domain *sd; 3596 3597 __schedstat_inc(p->stats.nr_wakeups_remote); 3598 3599 guard(rcu)(); 3600 for_each_domain(rq->cpu, sd) { 3601 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { 3602 __schedstat_inc(sd->ttwu_wake_remote); 3603 break; 3604 } 3605 } 3606 } 3607 3608 if (wake_flags & WF_MIGRATED) 3609 __schedstat_inc(p->stats.nr_wakeups_migrate); 3610 #endif /* CONFIG_SMP */ 3611 3612 __schedstat_inc(rq->ttwu_count); 3613 __schedstat_inc(p->stats.nr_wakeups); 3614 3615 if (wake_flags & WF_SYNC) 3616 __schedstat_inc(p->stats.nr_wakeups_sync); 3617 } 3618 3619 /* 3620 * Mark the task runnable. 3621 */ 3622 static inline void ttwu_do_wakeup(struct task_struct *p) 3623 { 3624 WRITE_ONCE(p->__state, TASK_RUNNING); 3625 trace_sched_wakeup(p); 3626 } 3627 3628 static void 3629 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 3630 struct rq_flags *rf) 3631 { 3632 int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; 3633 3634 lockdep_assert_rq_held(rq); 3635 3636 if (p->sched_contributes_to_load) 3637 rq->nr_uninterruptible--; 3638 3639 #ifdef CONFIG_SMP 3640 if (wake_flags & WF_MIGRATED) 3641 en_flags |= ENQUEUE_MIGRATED; 3642 else 3643 #endif 3644 if (p->in_iowait) { 3645 delayacct_blkio_end(p); 3646 atomic_dec(&task_rq(p)->nr_iowait); 3647 } 3648 3649 activate_task(rq, p, en_flags); 3650 wakeup_preempt(rq, p, wake_flags); 3651 3652 ttwu_do_wakeup(p); 3653 3654 #ifdef CONFIG_SMP 3655 if (p->sched_class->task_woken) { 3656 /* 3657 * Our task @p is fully woken up and running; so it's safe to 3658 * drop the rq->lock, hereafter rq is only used for statistics. 3659 */ 3660 rq_unpin_lock(rq, rf); 3661 p->sched_class->task_woken(rq, p); 3662 rq_repin_lock(rq, rf); 3663 } 3664 3665 if (rq->idle_stamp) { 3666 u64 delta = rq_clock(rq) - rq->idle_stamp; 3667 u64 max = 2*rq->max_idle_balance_cost; 3668 3669 update_avg(&rq->avg_idle, delta); 3670 3671 if (rq->avg_idle > max) 3672 rq->avg_idle = max; 3673 3674 rq->idle_stamp = 0; 3675 } 3676 #endif 3677 } 3678 3679 /* 3680 * Consider @p being inside a wait loop: 3681 * 3682 * for (;;) { 3683 * set_current_state(TASK_UNINTERRUPTIBLE); 3684 * 3685 * if (CONDITION) 3686 * break; 3687 * 3688 * schedule(); 3689 * } 3690 * __set_current_state(TASK_RUNNING); 3691 * 3692 * between set_current_state() and schedule(). In this case @p is still 3693 * runnable, so all that needs doing is change p->state back to TASK_RUNNING in 3694 * an atomic manner. 3695 * 3696 * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq 3697 * then schedule() must still happen and p->state can be changed to 3698 * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we 3699 * need to do a full wakeup with enqueue. 3700 * 3701 * Returns: %true when the wakeup is done, 3702 * %false otherwise. 3703 */ 3704 static int ttwu_runnable(struct task_struct *p, int wake_flags) 3705 { 3706 struct rq_flags rf; 3707 struct rq *rq; 3708 int ret = 0; 3709 3710 rq = __task_rq_lock(p, &rf); 3711 if (task_on_rq_queued(p)) { 3712 update_rq_clock(rq); 3713 if (p->se.sched_delayed) 3714 enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED); 3715 if (!task_on_cpu(rq, p)) { 3716 /* 3717 * When on_rq && !on_cpu the task is preempted, see if 3718 * it should preempt the task that is current now. 3719 */ 3720 wakeup_preempt(rq, p, wake_flags); 3721 } 3722 ttwu_do_wakeup(p); 3723 ret = 1; 3724 } 3725 __task_rq_unlock(rq, &rf); 3726 3727 return ret; 3728 } 3729 3730 #ifdef CONFIG_SMP 3731 void sched_ttwu_pending(void *arg) 3732 { 3733 struct llist_node *llist = arg; 3734 struct rq *rq = this_rq(); 3735 struct task_struct *p, *t; 3736 struct rq_flags rf; 3737 3738 if (!llist) 3739 return; 3740 3741 rq_lock_irqsave(rq, &rf); 3742 update_rq_clock(rq); 3743 3744 llist_for_each_entry_safe(p, t, llist, wake_entry.llist) { 3745 if (WARN_ON_ONCE(p->on_cpu)) 3746 smp_cond_load_acquire(&p->on_cpu, !VAL); 3747 3748 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) 3749 set_task_cpu(p, cpu_of(rq)); 3750 3751 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 3752 } 3753 3754 /* 3755 * Must be after enqueueing at least once task such that 3756 * idle_cpu() does not observe a false-negative -- if it does, 3757 * it is possible for select_idle_siblings() to stack a number 3758 * of tasks on this CPU during that window. 3759 * 3760 * It is OK to clear ttwu_pending when another task pending. 3761 * We will receive IPI after local IRQ enabled and then enqueue it. 3762 * Since now nr_running > 0, idle_cpu() will always get correct result. 3763 */ 3764 WRITE_ONCE(rq->ttwu_pending, 0); 3765 rq_unlock_irqrestore(rq, &rf); 3766 } 3767 3768 /* 3769 * Prepare the scene for sending an IPI for a remote smp_call 3770 * 3771 * Returns true if the caller can proceed with sending the IPI. 3772 * Returns false otherwise. 3773 */ 3774 bool call_function_single_prep_ipi(int cpu) 3775 { 3776 if (set_nr_if_polling(cpu_rq(cpu)->idle)) { 3777 trace_sched_wake_idle_without_ipi(cpu); 3778 return false; 3779 } 3780 3781 return true; 3782 } 3783 3784 /* 3785 * Queue a task on the target CPUs wake_list and wake the CPU via IPI if 3786 * necessary. The wakee CPU on receipt of the IPI will queue the task 3787 * via sched_ttwu_wakeup() for activation so the wakee incurs the cost 3788 * of the wakeup instead of the waker. 3789 */ 3790 static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3791 { 3792 struct rq *rq = cpu_rq(cpu); 3793 3794 p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); 3795 3796 WRITE_ONCE(rq->ttwu_pending, 1); 3797 __smp_call_single_queue(cpu, &p->wake_entry.llist); 3798 } 3799 3800 void wake_up_if_idle(int cpu) 3801 { 3802 struct rq *rq = cpu_rq(cpu); 3803 3804 guard(rcu)(); 3805 if (is_idle_task(rcu_dereference(rq->curr))) { 3806 guard(rq_lock_irqsave)(rq); 3807 if (is_idle_task(rq->curr)) 3808 resched_curr(rq); 3809 } 3810 } 3811 3812 bool cpus_equal_capacity(int this_cpu, int that_cpu) 3813 { 3814 if (!sched_asym_cpucap_active()) 3815 return true; 3816 3817 if (this_cpu == that_cpu) 3818 return true; 3819 3820 return arch_scale_cpu_capacity(this_cpu) == arch_scale_cpu_capacity(that_cpu); 3821 } 3822 3823 bool cpus_share_cache(int this_cpu, int that_cpu) 3824 { 3825 if (this_cpu == that_cpu) 3826 return true; 3827 3828 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 3829 } 3830 3831 /* 3832 * Whether CPUs are share cache resources, which means LLC on non-cluster 3833 * machines and LLC tag or L2 on machines with clusters. 3834 */ 3835 bool cpus_share_resources(int this_cpu, int that_cpu) 3836 { 3837 if (this_cpu == that_cpu) 3838 return true; 3839 3840 return per_cpu(sd_share_id, this_cpu) == per_cpu(sd_share_id, that_cpu); 3841 } 3842 3843 static inline bool ttwu_queue_cond(struct task_struct *p, int cpu) 3844 { 3845 /* 3846 * Do not complicate things with the async wake_list while the CPU is 3847 * in hotplug state. 3848 */ 3849 if (!cpu_active(cpu)) 3850 return false; 3851 3852 /* Ensure the task will still be allowed to run on the CPU. */ 3853 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 3854 return false; 3855 3856 /* 3857 * If the CPU does not share cache, then queue the task on the 3858 * remote rqs wakelist to avoid accessing remote data. 3859 */ 3860 if (!cpus_share_cache(smp_processor_id(), cpu)) 3861 return true; 3862 3863 if (cpu == smp_processor_id()) 3864 return false; 3865 3866 /* 3867 * If the wakee cpu is idle, or the task is descheduling and the 3868 * only running task on the CPU, then use the wakelist to offload 3869 * the task activation to the idle (or soon-to-be-idle) CPU as 3870 * the current CPU is likely busy. nr_running is checked to 3871 * avoid unnecessary task stacking. 3872 * 3873 * Note that we can only get here with (wakee) p->on_rq=0, 3874 * p->on_cpu can be whatever, we've done the dequeue, so 3875 * the wakee has been accounted out of ->nr_running. 3876 */ 3877 if (!cpu_rq(cpu)->nr_running) 3878 return true; 3879 3880 return false; 3881 } 3882 3883 static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3884 { 3885 if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) { 3886 sched_clock_cpu(cpu); /* Sync clocks across CPUs */ 3887 __ttwu_queue_wakelist(p, cpu, wake_flags); 3888 return true; 3889 } 3890 3891 return false; 3892 } 3893 3894 #else /* !CONFIG_SMP */ 3895 3896 static inline bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3897 { 3898 return false; 3899 } 3900 3901 #endif /* CONFIG_SMP */ 3902 3903 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) 3904 { 3905 struct rq *rq = cpu_rq(cpu); 3906 struct rq_flags rf; 3907 3908 if (ttwu_queue_wakelist(p, cpu, wake_flags)) 3909 return; 3910 3911 rq_lock(rq, &rf); 3912 update_rq_clock(rq); 3913 ttwu_do_activate(rq, p, wake_flags, &rf); 3914 rq_unlock(rq, &rf); 3915 } 3916 3917 /* 3918 * Invoked from try_to_wake_up() to check whether the task can be woken up. 3919 * 3920 * The caller holds p::pi_lock if p != current or has preemption 3921 * disabled when p == current. 3922 * 3923 * The rules of saved_state: 3924 * 3925 * The related locking code always holds p::pi_lock when updating 3926 * p::saved_state, which means the code is fully serialized in both cases. 3927 * 3928 * For PREEMPT_RT, the lock wait and lock wakeups happen via TASK_RTLOCK_WAIT. 3929 * No other bits set. This allows to distinguish all wakeup scenarios. 3930 * 3931 * For FREEZER, the wakeup happens via TASK_FROZEN. No other bits set. This 3932 * allows us to prevent early wakeup of tasks before they can be run on 3933 * asymmetric ISA architectures (eg ARMv9). 3934 */ 3935 static __always_inline 3936 bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success) 3937 { 3938 int match; 3939 3940 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) { 3941 WARN_ON_ONCE((state & TASK_RTLOCK_WAIT) && 3942 state != TASK_RTLOCK_WAIT); 3943 } 3944 3945 *success = !!(match = __task_state_match(p, state)); 3946 3947 /* 3948 * Saved state preserves the task state across blocking on 3949 * an RT lock or TASK_FREEZABLE tasks. If the state matches, 3950 * set p::saved_state to TASK_RUNNING, but do not wake the task 3951 * because it waits for a lock wakeup or __thaw_task(). Also 3952 * indicate success because from the regular waker's point of 3953 * view this has succeeded. 3954 * 3955 * After acquiring the lock the task will restore p::__state 3956 * from p::saved_state which ensures that the regular 3957 * wakeup is not lost. The restore will also set 3958 * p::saved_state to TASK_RUNNING so any further tests will 3959 * not result in false positives vs. @success 3960 */ 3961 if (match < 0) 3962 p->saved_state = TASK_RUNNING; 3963 3964 return match > 0; 3965 } 3966 3967 /* 3968 * Notes on Program-Order guarantees on SMP systems. 3969 * 3970 * MIGRATION 3971 * 3972 * The basic program-order guarantee on SMP systems is that when a task [t] 3973 * migrates, all its activity on its old CPU [c0] happens-before any subsequent 3974 * execution on its new CPU [c1]. 3975 * 3976 * For migration (of runnable tasks) this is provided by the following means: 3977 * 3978 * A) UNLOCK of the rq(c0)->lock scheduling out task t 3979 * B) migration for t is required to synchronize *both* rq(c0)->lock and 3980 * rq(c1)->lock (if not at the same time, then in that order). 3981 * C) LOCK of the rq(c1)->lock scheduling in task 3982 * 3983 * Release/acquire chaining guarantees that B happens after A and C after B. 3984 * Note: the CPU doing B need not be c0 or c1 3985 * 3986 * Example: 3987 * 3988 * CPU0 CPU1 CPU2 3989 * 3990 * LOCK rq(0)->lock 3991 * sched-out X 3992 * sched-in Y 3993 * UNLOCK rq(0)->lock 3994 * 3995 * LOCK rq(0)->lock // orders against CPU0 3996 * dequeue X 3997 * UNLOCK rq(0)->lock 3998 * 3999 * LOCK rq(1)->lock 4000 * enqueue X 4001 * UNLOCK rq(1)->lock 4002 * 4003 * LOCK rq(1)->lock // orders against CPU2 4004 * sched-out Z 4005 * sched-in X 4006 * UNLOCK rq(1)->lock 4007 * 4008 * 4009 * BLOCKING -- aka. SLEEP + WAKEUP 4010 * 4011 * For blocking we (obviously) need to provide the same guarantee as for 4012 * migration. However the means are completely different as there is no lock 4013 * chain to provide order. Instead we do: 4014 * 4015 * 1) smp_store_release(X->on_cpu, 0) -- finish_task() 4016 * 2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up() 4017 * 4018 * Example: 4019 * 4020 * CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule) 4021 * 4022 * LOCK rq(0)->lock LOCK X->pi_lock 4023 * dequeue X 4024 * sched-out X 4025 * smp_store_release(X->on_cpu, 0); 4026 * 4027 * smp_cond_load_acquire(&X->on_cpu, !VAL); 4028 * X->state = WAKING 4029 * set_task_cpu(X,2) 4030 * 4031 * LOCK rq(2)->lock 4032 * enqueue X 4033 * X->state = RUNNING 4034 * UNLOCK rq(2)->lock 4035 * 4036 * LOCK rq(2)->lock // orders against CPU1 4037 * sched-out Z 4038 * sched-in X 4039 * UNLOCK rq(2)->lock 4040 * 4041 * UNLOCK X->pi_lock 4042 * UNLOCK rq(0)->lock 4043 * 4044 * 4045 * However, for wakeups there is a second guarantee we must provide, namely we 4046 * must ensure that CONDITION=1 done by the caller can not be reordered with 4047 * accesses to the task state; see try_to_wake_up() and set_current_state(). 4048 */ 4049 4050 /** 4051 * try_to_wake_up - wake up a thread 4052 * @p: the thread to be awakened 4053 * @state: the mask of task states that can be woken 4054 * @wake_flags: wake modifier flags (WF_*) 4055 * 4056 * Conceptually does: 4057 * 4058 * If (@state & @p->state) @p->state = TASK_RUNNING. 4059 * 4060 * If the task was not queued/runnable, also place it back on a runqueue. 4061 * 4062 * This function is atomic against schedule() which would dequeue the task. 4063 * 4064 * It issues a full memory barrier before accessing @p->state, see the comment 4065 * with set_current_state(). 4066 * 4067 * Uses p->pi_lock to serialize against concurrent wake-ups. 4068 * 4069 * Relies on p->pi_lock stabilizing: 4070 * - p->sched_class 4071 * - p->cpus_ptr 4072 * - p->sched_task_group 4073 * in order to do migration, see its use of select_task_rq()/set_task_cpu(). 4074 * 4075 * Tries really hard to only take one task_rq(p)->lock for performance. 4076 * Takes rq->lock in: 4077 * - ttwu_runnable() -- old rq, unavoidable, see comment there; 4078 * - ttwu_queue() -- new rq, for enqueue of the task; 4079 * - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us. 4080 * 4081 * As a consequence we race really badly with just about everything. See the 4082 * many memory barriers and their comments for details. 4083 * 4084 * Return: %true if @p->state changes (an actual wakeup was done), 4085 * %false otherwise. 4086 */ 4087 int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 4088 { 4089 guard(preempt)(); 4090 int cpu, success = 0; 4091 4092 if (p == current) { 4093 /* 4094 * We're waking current, this means 'p->on_rq' and 'task_cpu(p) 4095 * == smp_processor_id()'. Together this means we can special 4096 * case the whole 'p->on_rq && ttwu_runnable()' case below 4097 * without taking any locks. 4098 * 4099 * Specifically, given current runs ttwu() we must be before 4100 * schedule()'s block_task(), as such this must not observe 4101 * sched_delayed. 4102 * 4103 * In particular: 4104 * - we rely on Program-Order guarantees for all the ordering, 4105 * - we're serialized against set_special_state() by virtue of 4106 * it disabling IRQs (this allows not taking ->pi_lock). 4107 */ 4108 SCHED_WARN_ON(p->se.sched_delayed); 4109 if (!ttwu_state_match(p, state, &success)) 4110 goto out; 4111 4112 trace_sched_waking(p); 4113 ttwu_do_wakeup(p); 4114 goto out; 4115 } 4116 4117 /* 4118 * If we are going to wake up a thread waiting for CONDITION we 4119 * need to ensure that CONDITION=1 done by the caller can not be 4120 * reordered with p->state check below. This pairs with smp_store_mb() 4121 * in set_current_state() that the waiting thread does. 4122 */ 4123 scoped_guard (raw_spinlock_irqsave, &p->pi_lock) { 4124 smp_mb__after_spinlock(); 4125 if (!ttwu_state_match(p, state, &success)) 4126 break; 4127 4128 trace_sched_waking(p); 4129 4130 /* 4131 * Ensure we load p->on_rq _after_ p->state, otherwise it would 4132 * be possible to, falsely, observe p->on_rq == 0 and get stuck 4133 * in smp_cond_load_acquire() below. 4134 * 4135 * sched_ttwu_pending() try_to_wake_up() 4136 * STORE p->on_rq = 1 LOAD p->state 4137 * UNLOCK rq->lock 4138 * 4139 * __schedule() (switch to task 'p') 4140 * LOCK rq->lock smp_rmb(); 4141 * smp_mb__after_spinlock(); 4142 * UNLOCK rq->lock 4143 * 4144 * [task p] 4145 * STORE p->state = UNINTERRUPTIBLE LOAD p->on_rq 4146 * 4147 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 4148 * __schedule(). See the comment for smp_mb__after_spinlock(). 4149 * 4150 * A similar smp_rmb() lives in __task_needs_rq_lock(). 4151 */ 4152 smp_rmb(); 4153 if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags)) 4154 break; 4155 4156 #ifdef CONFIG_SMP 4157 /* 4158 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be 4159 * possible to, falsely, observe p->on_cpu == 0. 4160 * 4161 * One must be running (->on_cpu == 1) in order to remove oneself 4162 * from the runqueue. 4163 * 4164 * __schedule() (switch to task 'p') try_to_wake_up() 4165 * STORE p->on_cpu = 1 LOAD p->on_rq 4166 * UNLOCK rq->lock 4167 * 4168 * __schedule() (put 'p' to sleep) 4169 * LOCK rq->lock smp_rmb(); 4170 * smp_mb__after_spinlock(); 4171 * STORE p->on_rq = 0 LOAD p->on_cpu 4172 * 4173 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 4174 * __schedule(). See the comment for smp_mb__after_spinlock(). 4175 * 4176 * Form a control-dep-acquire with p->on_rq == 0 above, to ensure 4177 * schedule()'s deactivate_task() has 'happened' and p will no longer 4178 * care about it's own p->state. See the comment in __schedule(). 4179 */ 4180 smp_acquire__after_ctrl_dep(); 4181 4182 /* 4183 * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq 4184 * == 0), which means we need to do an enqueue, change p->state to 4185 * TASK_WAKING such that we can unlock p->pi_lock before doing the 4186 * enqueue, such as ttwu_queue_wakelist(). 4187 */ 4188 WRITE_ONCE(p->__state, TASK_WAKING); 4189 4190 /* 4191 * If the owning (remote) CPU is still in the middle of schedule() with 4192 * this task as prev, considering queueing p on the remote CPUs wake_list 4193 * which potentially sends an IPI instead of spinning on p->on_cpu to 4194 * let the waker make forward progress. This is safe because IRQs are 4195 * disabled and the IPI will deliver after on_cpu is cleared. 4196 * 4197 * Ensure we load task_cpu(p) after p->on_cpu: 4198 * 4199 * set_task_cpu(p, cpu); 4200 * STORE p->cpu = @cpu 4201 * __schedule() (switch to task 'p') 4202 * LOCK rq->lock 4203 * smp_mb__after_spin_lock() smp_cond_load_acquire(&p->on_cpu) 4204 * STORE p->on_cpu = 1 LOAD p->cpu 4205 * 4206 * to ensure we observe the correct CPU on which the task is currently 4207 * scheduling. 4208 */ 4209 if (smp_load_acquire(&p->on_cpu) && 4210 ttwu_queue_wakelist(p, task_cpu(p), wake_flags)) 4211 break; 4212 4213 /* 4214 * If the owning (remote) CPU is still in the middle of schedule() with 4215 * this task as prev, wait until it's done referencing the task. 4216 * 4217 * Pairs with the smp_store_release() in finish_task(). 4218 * 4219 * This ensures that tasks getting woken will be fully ordered against 4220 * their previous state and preserve Program Order. 4221 */ 4222 smp_cond_load_acquire(&p->on_cpu, !VAL); 4223 4224 cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU); 4225 if (task_cpu(p) != cpu) { 4226 if (p->in_iowait) { 4227 delayacct_blkio_end(p); 4228 atomic_dec(&task_rq(p)->nr_iowait); 4229 } 4230 4231 wake_flags |= WF_MIGRATED; 4232 psi_ttwu_dequeue(p); 4233 set_task_cpu(p, cpu); 4234 } 4235 #else 4236 cpu = task_cpu(p); 4237 #endif /* CONFIG_SMP */ 4238 4239 ttwu_queue(p, cpu, wake_flags); 4240 } 4241 out: 4242 if (success) 4243 ttwu_stat(p, task_cpu(p), wake_flags); 4244 4245 return success; 4246 } 4247 4248 static bool __task_needs_rq_lock(struct task_struct *p) 4249 { 4250 unsigned int state = READ_ONCE(p->__state); 4251 4252 /* 4253 * Since pi->lock blocks try_to_wake_up(), we don't need rq->lock when 4254 * the task is blocked. Make sure to check @state since ttwu() can drop 4255 * locks at the end, see ttwu_queue_wakelist(). 4256 */ 4257 if (state == TASK_RUNNING || state == TASK_WAKING) 4258 return true; 4259 4260 /* 4261 * Ensure we load p->on_rq after p->__state, otherwise it would be 4262 * possible to, falsely, observe p->on_rq == 0. 4263 * 4264 * See try_to_wake_up() for a longer comment. 4265 */ 4266 smp_rmb(); 4267 if (p->on_rq) 4268 return true; 4269 4270 #ifdef CONFIG_SMP 4271 /* 4272 * Ensure the task has finished __schedule() and will not be referenced 4273 * anymore. Again, see try_to_wake_up() for a longer comment. 4274 */ 4275 smp_rmb(); 4276 smp_cond_load_acquire(&p->on_cpu, !VAL); 4277 #endif 4278 4279 return false; 4280 } 4281 4282 /** 4283 * task_call_func - Invoke a function on task in fixed state 4284 * @p: Process for which the function is to be invoked, can be @current. 4285 * @func: Function to invoke. 4286 * @arg: Argument to function. 4287 * 4288 * Fix the task in it's current state by avoiding wakeups and or rq operations 4289 * and call @func(@arg) on it. This function can use ->on_rq and task_curr() 4290 * to work out what the state is, if required. Given that @func can be invoked 4291 * with a runqueue lock held, it had better be quite lightweight. 4292 * 4293 * Returns: 4294 * Whatever @func returns 4295 */ 4296 int task_call_func(struct task_struct *p, task_call_f func, void *arg) 4297 { 4298 struct rq *rq = NULL; 4299 struct rq_flags rf; 4300 int ret; 4301 4302 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 4303 4304 if (__task_needs_rq_lock(p)) 4305 rq = __task_rq_lock(p, &rf); 4306 4307 /* 4308 * At this point the task is pinned; either: 4309 * - blocked and we're holding off wakeups (pi->lock) 4310 * - woken, and we're holding off enqueue (rq->lock) 4311 * - queued, and we're holding off schedule (rq->lock) 4312 * - running, and we're holding off de-schedule (rq->lock) 4313 * 4314 * The called function (@func) can use: task_curr(), p->on_rq and 4315 * p->__state to differentiate between these states. 4316 */ 4317 ret = func(p, arg); 4318 4319 if (rq) 4320 rq_unlock(rq, &rf); 4321 4322 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); 4323 return ret; 4324 } 4325 4326 /** 4327 * cpu_curr_snapshot - Return a snapshot of the currently running task 4328 * @cpu: The CPU on which to snapshot the task. 4329 * 4330 * Returns the task_struct pointer of the task "currently" running on 4331 * the specified CPU. 4332 * 4333 * If the specified CPU was offline, the return value is whatever it 4334 * is, perhaps a pointer to the task_struct structure of that CPU's idle 4335 * task, but there is no guarantee. Callers wishing a useful return 4336 * value must take some action to ensure that the specified CPU remains 4337 * online throughout. 4338 * 4339 * This function executes full memory barriers before and after fetching 4340 * the pointer, which permits the caller to confine this function's fetch 4341 * with respect to the caller's accesses to other shared variables. 4342 */ 4343 struct task_struct *cpu_curr_snapshot(int cpu) 4344 { 4345 struct rq *rq = cpu_rq(cpu); 4346 struct task_struct *t; 4347 struct rq_flags rf; 4348 4349 rq_lock_irqsave(rq, &rf); 4350 smp_mb__after_spinlock(); /* Pairing determined by caller's synchronization design. */ 4351 t = rcu_dereference(cpu_curr(cpu)); 4352 rq_unlock_irqrestore(rq, &rf); 4353 smp_mb(); /* Pairing determined by caller's synchronization design. */ 4354 4355 return t; 4356 } 4357 4358 /** 4359 * wake_up_process - Wake up a specific process 4360 * @p: The process to be woken up. 4361 * 4362 * Attempt to wake up the nominated process and move it to the set of runnable 4363 * processes. 4364 * 4365 * Return: 1 if the process was woken up, 0 if it was already running. 4366 * 4367 * This function executes a full memory barrier before accessing the task state. 4368 */ 4369 int wake_up_process(struct task_struct *p) 4370 { 4371 return try_to_wake_up(p, TASK_NORMAL, 0); 4372 } 4373 EXPORT_SYMBOL(wake_up_process); 4374 4375 int wake_up_state(struct task_struct *p, unsigned int state) 4376 { 4377 return try_to_wake_up(p, state, 0); 4378 } 4379 4380 /* 4381 * Perform scheduler related setup for a newly forked process p. 4382 * p is forked by current. 4383 * 4384 * __sched_fork() is basic setup used by init_idle() too: 4385 */ 4386 static void __sched_fork(unsigned long clone_flags, struct task_struct *p) 4387 { 4388 p->on_rq = 0; 4389 4390 p->se.on_rq = 0; 4391 p->se.exec_start = 0; 4392 p->se.sum_exec_runtime = 0; 4393 p->se.prev_sum_exec_runtime = 0; 4394 p->se.nr_migrations = 0; 4395 p->se.vruntime = 0; 4396 p->se.vlag = 0; 4397 INIT_LIST_HEAD(&p->se.group_node); 4398 4399 /* A delayed task cannot be in clone(). */ 4400 SCHED_WARN_ON(p->se.sched_delayed); 4401 4402 #ifdef CONFIG_FAIR_GROUP_SCHED 4403 p->se.cfs_rq = NULL; 4404 #endif 4405 4406 #ifdef CONFIG_SCHEDSTATS 4407 /* Even if schedstat is disabled, there should not be garbage */ 4408 memset(&p->stats, 0, sizeof(p->stats)); 4409 #endif 4410 4411 init_dl_entity(&p->dl); 4412 4413 INIT_LIST_HEAD(&p->rt.run_list); 4414 p->rt.timeout = 0; 4415 p->rt.time_slice = sched_rr_timeslice; 4416 p->rt.on_rq = 0; 4417 p->rt.on_list = 0; 4418 4419 #ifdef CONFIG_PREEMPT_NOTIFIERS 4420 INIT_HLIST_HEAD(&p->preempt_notifiers); 4421 #endif 4422 4423 #ifdef CONFIG_COMPACTION 4424 p->capture_control = NULL; 4425 #endif 4426 init_numa_balancing(clone_flags, p); 4427 #ifdef CONFIG_SMP 4428 p->wake_entry.u_flags = CSD_TYPE_TTWU; 4429 p->migration_pending = NULL; 4430 #endif 4431 init_sched_mm_cid(p); 4432 } 4433 4434 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); 4435 4436 #ifdef CONFIG_NUMA_BALANCING 4437 4438 int sysctl_numa_balancing_mode; 4439 4440 static void __set_numabalancing_state(bool enabled) 4441 { 4442 if (enabled) 4443 static_branch_enable(&sched_numa_balancing); 4444 else 4445 static_branch_disable(&sched_numa_balancing); 4446 } 4447 4448 void set_numabalancing_state(bool enabled) 4449 { 4450 if (enabled) 4451 sysctl_numa_balancing_mode = NUMA_BALANCING_NORMAL; 4452 else 4453 sysctl_numa_balancing_mode = NUMA_BALANCING_DISABLED; 4454 __set_numabalancing_state(enabled); 4455 } 4456 4457 #ifdef CONFIG_PROC_SYSCTL 4458 static void reset_memory_tiering(void) 4459 { 4460 struct pglist_data *pgdat; 4461 4462 for_each_online_pgdat(pgdat) { 4463 pgdat->nbp_threshold = 0; 4464 pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 4465 pgdat->nbp_th_start = jiffies_to_msecs(jiffies); 4466 } 4467 } 4468 4469 static int sysctl_numa_balancing(const struct ctl_table *table, int write, 4470 void *buffer, size_t *lenp, loff_t *ppos) 4471 { 4472 struct ctl_table t; 4473 int err; 4474 int state = sysctl_numa_balancing_mode; 4475 4476 if (write && !capable(CAP_SYS_ADMIN)) 4477 return -EPERM; 4478 4479 t = *table; 4480 t.data = &state; 4481 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 4482 if (err < 0) 4483 return err; 4484 if (write) { 4485 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 4486 (state & NUMA_BALANCING_MEMORY_TIERING)) 4487 reset_memory_tiering(); 4488 sysctl_numa_balancing_mode = state; 4489 __set_numabalancing_state(state); 4490 } 4491 return err; 4492 } 4493 #endif 4494 #endif 4495 4496 #ifdef CONFIG_SCHEDSTATS 4497 4498 DEFINE_STATIC_KEY_FALSE(sched_schedstats); 4499 4500 static void set_schedstats(bool enabled) 4501 { 4502 if (enabled) 4503 static_branch_enable(&sched_schedstats); 4504 else 4505 static_branch_disable(&sched_schedstats); 4506 } 4507 4508 void force_schedstat_enabled(void) 4509 { 4510 if (!schedstat_enabled()) { 4511 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n"); 4512 static_branch_enable(&sched_schedstats); 4513 } 4514 } 4515 4516 static int __init setup_schedstats(char *str) 4517 { 4518 int ret = 0; 4519 if (!str) 4520 goto out; 4521 4522 if (!strcmp(str, "enable")) { 4523 set_schedstats(true); 4524 ret = 1; 4525 } else if (!strcmp(str, "disable")) { 4526 set_schedstats(false); 4527 ret = 1; 4528 } 4529 out: 4530 if (!ret) 4531 pr_warn("Unable to parse schedstats=\n"); 4532 4533 return ret; 4534 } 4535 __setup("schedstats=", setup_schedstats); 4536 4537 #ifdef CONFIG_PROC_SYSCTL 4538 static int sysctl_schedstats(const struct ctl_table *table, int write, void *buffer, 4539 size_t *lenp, loff_t *ppos) 4540 { 4541 struct ctl_table t; 4542 int err; 4543 int state = static_branch_likely(&sched_schedstats); 4544 4545 if (write && !capable(CAP_SYS_ADMIN)) 4546 return -EPERM; 4547 4548 t = *table; 4549 t.data = &state; 4550 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 4551 if (err < 0) 4552 return err; 4553 if (write) 4554 set_schedstats(state); 4555 return err; 4556 } 4557 #endif /* CONFIG_PROC_SYSCTL */ 4558 #endif /* CONFIG_SCHEDSTATS */ 4559 4560 #ifdef CONFIG_SYSCTL 4561 static struct ctl_table sched_core_sysctls[] = { 4562 #ifdef CONFIG_SCHEDSTATS 4563 { 4564 .procname = "sched_schedstats", 4565 .data = NULL, 4566 .maxlen = sizeof(unsigned int), 4567 .mode = 0644, 4568 .proc_handler = sysctl_schedstats, 4569 .extra1 = SYSCTL_ZERO, 4570 .extra2 = SYSCTL_ONE, 4571 }, 4572 #endif /* CONFIG_SCHEDSTATS */ 4573 #ifdef CONFIG_UCLAMP_TASK 4574 { 4575 .procname = "sched_util_clamp_min", 4576 .data = &sysctl_sched_uclamp_util_min, 4577 .maxlen = sizeof(unsigned int), 4578 .mode = 0644, 4579 .proc_handler = sysctl_sched_uclamp_handler, 4580 }, 4581 { 4582 .procname = "sched_util_clamp_max", 4583 .data = &sysctl_sched_uclamp_util_max, 4584 .maxlen = sizeof(unsigned int), 4585 .mode = 0644, 4586 .proc_handler = sysctl_sched_uclamp_handler, 4587 }, 4588 { 4589 .procname = "sched_util_clamp_min_rt_default", 4590 .data = &sysctl_sched_uclamp_util_min_rt_default, 4591 .maxlen = sizeof(unsigned int), 4592 .mode = 0644, 4593 .proc_handler = sysctl_sched_uclamp_handler, 4594 }, 4595 #endif /* CONFIG_UCLAMP_TASK */ 4596 #ifdef CONFIG_NUMA_BALANCING 4597 { 4598 .procname = "numa_balancing", 4599 .data = NULL, /* filled in by handler */ 4600 .maxlen = sizeof(unsigned int), 4601 .mode = 0644, 4602 .proc_handler = sysctl_numa_balancing, 4603 .extra1 = SYSCTL_ZERO, 4604 .extra2 = SYSCTL_FOUR, 4605 }, 4606 #endif /* CONFIG_NUMA_BALANCING */ 4607 }; 4608 static int __init sched_core_sysctl_init(void) 4609 { 4610 register_sysctl_init("kernel", sched_core_sysctls); 4611 return 0; 4612 } 4613 late_initcall(sched_core_sysctl_init); 4614 #endif /* CONFIG_SYSCTL */ 4615 4616 /* 4617 * fork()/clone()-time setup: 4618 */ 4619 int sched_fork(unsigned long clone_flags, struct task_struct *p) 4620 { 4621 __sched_fork(clone_flags, p); 4622 /* 4623 * We mark the process as NEW here. This guarantees that 4624 * nobody will actually run it, and a signal or other external 4625 * event cannot wake it up and insert it on the runqueue either. 4626 */ 4627 p->__state = TASK_NEW; 4628 4629 /* 4630 * Make sure we do not leak PI boosting priority to the child. 4631 */ 4632 p->prio = current->normal_prio; 4633 4634 uclamp_fork(p); 4635 4636 /* 4637 * Revert to default priority/policy on fork if requested. 4638 */ 4639 if (unlikely(p->sched_reset_on_fork)) { 4640 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 4641 p->policy = SCHED_NORMAL; 4642 p->static_prio = NICE_TO_PRIO(0); 4643 p->rt_priority = 0; 4644 } else if (PRIO_TO_NICE(p->static_prio) < 0) 4645 p->static_prio = NICE_TO_PRIO(0); 4646 4647 p->prio = p->normal_prio = p->static_prio; 4648 set_load_weight(p, false); 4649 p->se.custom_slice = 0; 4650 p->se.slice = sysctl_sched_base_slice; 4651 4652 /* 4653 * We don't need the reset flag anymore after the fork. It has 4654 * fulfilled its duty: 4655 */ 4656 p->sched_reset_on_fork = 0; 4657 } 4658 4659 if (dl_prio(p->prio)) 4660 return -EAGAIN; 4661 else if (rt_prio(p->prio)) 4662 p->sched_class = &rt_sched_class; 4663 else 4664 p->sched_class = &fair_sched_class; 4665 4666 init_entity_runnable_average(&p->se); 4667 4668 4669 #ifdef CONFIG_SCHED_INFO 4670 if (likely(sched_info_on())) 4671 memset(&p->sched_info, 0, sizeof(p->sched_info)); 4672 #endif 4673 #if defined(CONFIG_SMP) 4674 p->on_cpu = 0; 4675 #endif 4676 init_task_preempt_count(p); 4677 #ifdef CONFIG_SMP 4678 plist_node_init(&p->pushable_tasks, MAX_PRIO); 4679 RB_CLEAR_NODE(&p->pushable_dl_tasks); 4680 #endif 4681 return 0; 4682 } 4683 4684 void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) 4685 { 4686 unsigned long flags; 4687 4688 /* 4689 * Because we're not yet on the pid-hash, p->pi_lock isn't strictly 4690 * required yet, but lockdep gets upset if rules are violated. 4691 */ 4692 raw_spin_lock_irqsave(&p->pi_lock, flags); 4693 #ifdef CONFIG_CGROUP_SCHED 4694 if (1) { 4695 struct task_group *tg; 4696 tg = container_of(kargs->cset->subsys[cpu_cgrp_id], 4697 struct task_group, css); 4698 tg = autogroup_task_group(p, tg); 4699 p->sched_task_group = tg; 4700 } 4701 #endif 4702 rseq_migrate(p); 4703 /* 4704 * We're setting the CPU for the first time, we don't migrate, 4705 * so use __set_task_cpu(). 4706 */ 4707 __set_task_cpu(p, smp_processor_id()); 4708 if (p->sched_class->task_fork) 4709 p->sched_class->task_fork(p); 4710 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 4711 } 4712 4713 void sched_post_fork(struct task_struct *p) 4714 { 4715 uclamp_post_fork(p); 4716 } 4717 4718 unsigned long to_ratio(u64 period, u64 runtime) 4719 { 4720 if (runtime == RUNTIME_INF) 4721 return BW_UNIT; 4722 4723 /* 4724 * Doing this here saves a lot of checks in all 4725 * the calling paths, and returning zero seems 4726 * safe for them anyway. 4727 */ 4728 if (period == 0) 4729 return 0; 4730 4731 return div64_u64(runtime << BW_SHIFT, period); 4732 } 4733 4734 /* 4735 * wake_up_new_task - wake up a newly created task for the first time. 4736 * 4737 * This function will do some initial scheduler statistics housekeeping 4738 * that must be done for every newly created context, then puts the task 4739 * on the runqueue and wakes it. 4740 */ 4741 void wake_up_new_task(struct task_struct *p) 4742 { 4743 struct rq_flags rf; 4744 struct rq *rq; 4745 4746 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 4747 WRITE_ONCE(p->__state, TASK_RUNNING); 4748 #ifdef CONFIG_SMP 4749 /* 4750 * Fork balancing, do it here and not earlier because: 4751 * - cpus_ptr can change in the fork path 4752 * - any previously selected CPU might disappear through hotplug 4753 * 4754 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq, 4755 * as we're not fully set-up yet. 4756 */ 4757 p->recent_used_cpu = task_cpu(p); 4758 rseq_migrate(p); 4759 __set_task_cpu(p, select_task_rq(p, task_cpu(p), WF_FORK)); 4760 #endif 4761 rq = __task_rq_lock(p, &rf); 4762 update_rq_clock(rq); 4763 post_init_entity_util_avg(p); 4764 4765 activate_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_INITIAL); 4766 trace_sched_wakeup_new(p); 4767 wakeup_preempt(rq, p, WF_FORK); 4768 #ifdef CONFIG_SMP 4769 if (p->sched_class->task_woken) { 4770 /* 4771 * Nothing relies on rq->lock after this, so it's fine to 4772 * drop it. 4773 */ 4774 rq_unpin_lock(rq, &rf); 4775 p->sched_class->task_woken(rq, p); 4776 rq_repin_lock(rq, &rf); 4777 } 4778 #endif 4779 task_rq_unlock(rq, p, &rf); 4780 } 4781 4782 #ifdef CONFIG_PREEMPT_NOTIFIERS 4783 4784 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key); 4785 4786 void preempt_notifier_inc(void) 4787 { 4788 static_branch_inc(&preempt_notifier_key); 4789 } 4790 EXPORT_SYMBOL_GPL(preempt_notifier_inc); 4791 4792 void preempt_notifier_dec(void) 4793 { 4794 static_branch_dec(&preempt_notifier_key); 4795 } 4796 EXPORT_SYMBOL_GPL(preempt_notifier_dec); 4797 4798 /** 4799 * preempt_notifier_register - tell me when current is being preempted & rescheduled 4800 * @notifier: notifier struct to register 4801 */ 4802 void preempt_notifier_register(struct preempt_notifier *notifier) 4803 { 4804 if (!static_branch_unlikely(&preempt_notifier_key)) 4805 WARN(1, "registering preempt_notifier while notifiers disabled\n"); 4806 4807 hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); 4808 } 4809 EXPORT_SYMBOL_GPL(preempt_notifier_register); 4810 4811 /** 4812 * preempt_notifier_unregister - no longer interested in preemption notifications 4813 * @notifier: notifier struct to unregister 4814 * 4815 * This is *not* safe to call from within a preemption notifier. 4816 */ 4817 void preempt_notifier_unregister(struct preempt_notifier *notifier) 4818 { 4819 hlist_del(¬ifier->link); 4820 } 4821 EXPORT_SYMBOL_GPL(preempt_notifier_unregister); 4822 4823 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) 4824 { 4825 struct preempt_notifier *notifier; 4826 4827 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4828 notifier->ops->sched_in(notifier, raw_smp_processor_id()); 4829 } 4830 4831 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4832 { 4833 if (static_branch_unlikely(&preempt_notifier_key)) 4834 __fire_sched_in_preempt_notifiers(curr); 4835 } 4836 4837 static void 4838 __fire_sched_out_preempt_notifiers(struct task_struct *curr, 4839 struct task_struct *next) 4840 { 4841 struct preempt_notifier *notifier; 4842 4843 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4844 notifier->ops->sched_out(notifier, next); 4845 } 4846 4847 static __always_inline void 4848 fire_sched_out_preempt_notifiers(struct task_struct *curr, 4849 struct task_struct *next) 4850 { 4851 if (static_branch_unlikely(&preempt_notifier_key)) 4852 __fire_sched_out_preempt_notifiers(curr, next); 4853 } 4854 4855 #else /* !CONFIG_PREEMPT_NOTIFIERS */ 4856 4857 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4858 { 4859 } 4860 4861 static inline void 4862 fire_sched_out_preempt_notifiers(struct task_struct *curr, 4863 struct task_struct *next) 4864 { 4865 } 4866 4867 #endif /* CONFIG_PREEMPT_NOTIFIERS */ 4868 4869 static inline void prepare_task(struct task_struct *next) 4870 { 4871 #ifdef CONFIG_SMP 4872 /* 4873 * Claim the task as running, we do this before switching to it 4874 * such that any running task will have this set. 4875 * 4876 * See the smp_load_acquire(&p->on_cpu) case in ttwu() and 4877 * its ordering comment. 4878 */ 4879 WRITE_ONCE(next->on_cpu, 1); 4880 #endif 4881 } 4882 4883 static inline void finish_task(struct task_struct *prev) 4884 { 4885 #ifdef CONFIG_SMP 4886 /* 4887 * This must be the very last reference to @prev from this CPU. After 4888 * p->on_cpu is cleared, the task can be moved to a different CPU. We 4889 * must ensure this doesn't happen until the switch is completely 4890 * finished. 4891 * 4892 * In particular, the load of prev->state in finish_task_switch() must 4893 * happen before this. 4894 * 4895 * Pairs with the smp_cond_load_acquire() in try_to_wake_up(). 4896 */ 4897 smp_store_release(&prev->on_cpu, 0); 4898 #endif 4899 } 4900 4901 #ifdef CONFIG_SMP 4902 4903 static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) 4904 { 4905 void (*func)(struct rq *rq); 4906 struct balance_callback *next; 4907 4908 lockdep_assert_rq_held(rq); 4909 4910 while (head) { 4911 func = (void (*)(struct rq *))head->func; 4912 next = head->next; 4913 head->next = NULL; 4914 head = next; 4915 4916 func(rq); 4917 } 4918 } 4919 4920 static void balance_push(struct rq *rq); 4921 4922 /* 4923 * balance_push_callback is a right abuse of the callback interface and plays 4924 * by significantly different rules. 4925 * 4926 * Where the normal balance_callback's purpose is to be ran in the same context 4927 * that queued it (only later, when it's safe to drop rq->lock again), 4928 * balance_push_callback is specifically targeted at __schedule(). 4929 * 4930 * This abuse is tolerated because it places all the unlikely/odd cases behind 4931 * a single test, namely: rq->balance_callback == NULL. 4932 */ 4933 struct balance_callback balance_push_callback = { 4934 .next = NULL, 4935 .func = balance_push, 4936 }; 4937 4938 static inline struct balance_callback * 4939 __splice_balance_callbacks(struct rq *rq, bool split) 4940 { 4941 struct balance_callback *head = rq->balance_callback; 4942 4943 if (likely(!head)) 4944 return NULL; 4945 4946 lockdep_assert_rq_held(rq); 4947 /* 4948 * Must not take balance_push_callback off the list when 4949 * splice_balance_callbacks() and balance_callbacks() are not 4950 * in the same rq->lock section. 4951 * 4952 * In that case it would be possible for __schedule() to interleave 4953 * and observe the list empty. 4954 */ 4955 if (split && head == &balance_push_callback) 4956 head = NULL; 4957 else 4958 rq->balance_callback = NULL; 4959 4960 return head; 4961 } 4962 4963 struct balance_callback *splice_balance_callbacks(struct rq *rq) 4964 { 4965 return __splice_balance_callbacks(rq, true); 4966 } 4967 4968 static void __balance_callbacks(struct rq *rq) 4969 { 4970 do_balance_callbacks(rq, __splice_balance_callbacks(rq, false)); 4971 } 4972 4973 void balance_callbacks(struct rq *rq, struct balance_callback *head) 4974 { 4975 unsigned long flags; 4976 4977 if (unlikely(head)) { 4978 raw_spin_rq_lock_irqsave(rq, flags); 4979 do_balance_callbacks(rq, head); 4980 raw_spin_rq_unlock_irqrestore(rq, flags); 4981 } 4982 } 4983 4984 #else 4985 4986 static inline void __balance_callbacks(struct rq *rq) 4987 { 4988 } 4989 4990 #endif 4991 4992 static inline void 4993 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) 4994 { 4995 /* 4996 * Since the runqueue lock will be released by the next 4997 * task (which is an invalid locking op but in the case 4998 * of the scheduler it's an obvious special-case), so we 4999 * do an early lockdep release here: 5000 */ 5001 rq_unpin_lock(rq, rf); 5002 spin_release(&__rq_lockp(rq)->dep_map, _THIS_IP_); 5003 #ifdef CONFIG_DEBUG_SPINLOCK 5004 /* this is a valid case when another task releases the spinlock */ 5005 rq_lockp(rq)->owner = next; 5006 #endif 5007 } 5008 5009 static inline void finish_lock_switch(struct rq *rq) 5010 { 5011 /* 5012 * If we are tracking spinlock dependencies then we have to 5013 * fix up the runqueue lock - which gets 'carried over' from 5014 * prev into current: 5015 */ 5016 spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_); 5017 __balance_callbacks(rq); 5018 raw_spin_rq_unlock_irq(rq); 5019 } 5020 5021 /* 5022 * NOP if the arch has not defined these: 5023 */ 5024 5025 #ifndef prepare_arch_switch 5026 # define prepare_arch_switch(next) do { } while (0) 5027 #endif 5028 5029 #ifndef finish_arch_post_lock_switch 5030 # define finish_arch_post_lock_switch() do { } while (0) 5031 #endif 5032 5033 static inline void kmap_local_sched_out(void) 5034 { 5035 #ifdef CONFIG_KMAP_LOCAL 5036 if (unlikely(current->kmap_ctrl.idx)) 5037 __kmap_local_sched_out(); 5038 #endif 5039 } 5040 5041 static inline void kmap_local_sched_in(void) 5042 { 5043 #ifdef CONFIG_KMAP_LOCAL 5044 if (unlikely(current->kmap_ctrl.idx)) 5045 __kmap_local_sched_in(); 5046 #endif 5047 } 5048 5049 /** 5050 * prepare_task_switch - prepare to switch tasks 5051 * @rq: the runqueue preparing to switch 5052 * @prev: the current task that is being switched out 5053 * @next: the task we are going to switch to. 5054 * 5055 * This is called with the rq lock held and interrupts off. It must 5056 * be paired with a subsequent finish_task_switch after the context 5057 * switch. 5058 * 5059 * prepare_task_switch sets up locking and calls architecture specific 5060 * hooks. 5061 */ 5062 static inline void 5063 prepare_task_switch(struct rq *rq, struct task_struct *prev, 5064 struct task_struct *next) 5065 { 5066 kcov_prepare_switch(prev); 5067 sched_info_switch(rq, prev, next); 5068 perf_event_task_sched_out(prev, next); 5069 rseq_preempt(prev); 5070 fire_sched_out_preempt_notifiers(prev, next); 5071 kmap_local_sched_out(); 5072 prepare_task(next); 5073 prepare_arch_switch(next); 5074 } 5075 5076 /** 5077 * finish_task_switch - clean up after a task-switch 5078 * @prev: the thread we just switched away from. 5079 * 5080 * finish_task_switch must be called after the context switch, paired 5081 * with a prepare_task_switch call before the context switch. 5082 * finish_task_switch will reconcile locking set up by prepare_task_switch, 5083 * and do any other architecture-specific cleanup actions. 5084 * 5085 * Note that we may have delayed dropping an mm in context_switch(). If 5086 * so, we finish that here outside of the runqueue lock. (Doing it 5087 * with the lock held can cause deadlocks; see schedule() for 5088 * details.) 5089 * 5090 * The context switch have flipped the stack from under us and restored the 5091 * local variables which were saved when this task called schedule() in the 5092 * past. 'prev == current' is still correct but we need to recalculate this_rq 5093 * because prev may have moved to another CPU. 5094 */ 5095 static struct rq *finish_task_switch(struct task_struct *prev) 5096 __releases(rq->lock) 5097 { 5098 struct rq *rq = this_rq(); 5099 struct mm_struct *mm = rq->prev_mm; 5100 unsigned int prev_state; 5101 5102 /* 5103 * The previous task will have left us with a preempt_count of 2 5104 * because it left us after: 5105 * 5106 * schedule() 5107 * preempt_disable(); // 1 5108 * __schedule() 5109 * raw_spin_lock_irq(&rq->lock) // 2 5110 * 5111 * Also, see FORK_PREEMPT_COUNT. 5112 */ 5113 if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 5114 "corrupted preempt_count: %s/%d/0x%x\n", 5115 current->comm, current->pid, preempt_count())) 5116 preempt_count_set(FORK_PREEMPT_COUNT); 5117 5118 rq->prev_mm = NULL; 5119 5120 /* 5121 * A task struct has one reference for the use as "current". 5122 * If a task dies, then it sets TASK_DEAD in tsk->state and calls 5123 * schedule one last time. The schedule call will never return, and 5124 * the scheduled task must drop that reference. 5125 * 5126 * We must observe prev->state before clearing prev->on_cpu (in 5127 * finish_task), otherwise a concurrent wakeup can get prev 5128 * running on another CPU and we could rave with its RUNNING -> DEAD 5129 * transition, resulting in a double drop. 5130 */ 5131 prev_state = READ_ONCE(prev->__state); 5132 vtime_task_switch(prev); 5133 perf_event_task_sched_in(prev, current); 5134 finish_task(prev); 5135 tick_nohz_task_switch(); 5136 finish_lock_switch(rq); 5137 finish_arch_post_lock_switch(); 5138 kcov_finish_switch(current); 5139 /* 5140 * kmap_local_sched_out() is invoked with rq::lock held and 5141 * interrupts disabled. There is no requirement for that, but the 5142 * sched out code does not have an interrupt enabled section. 5143 * Restoring the maps on sched in does not require interrupts being 5144 * disabled either. 5145 */ 5146 kmap_local_sched_in(); 5147 5148 fire_sched_in_preempt_notifiers(current); 5149 /* 5150 * When switching through a kernel thread, the loop in 5151 * membarrier_{private,global}_expedited() may have observed that 5152 * kernel thread and not issued an IPI. It is therefore possible to 5153 * schedule between user->kernel->user threads without passing though 5154 * switch_mm(). Membarrier requires a barrier after storing to 5155 * rq->curr, before returning to userspace, so provide them here: 5156 * 5157 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly 5158 * provided by mmdrop_lazy_tlb(), 5159 * - a sync_core for SYNC_CORE. 5160 */ 5161 if (mm) { 5162 membarrier_mm_sync_core_before_usermode(mm); 5163 mmdrop_lazy_tlb_sched(mm); 5164 } 5165 5166 if (unlikely(prev_state == TASK_DEAD)) { 5167 if (prev->sched_class->task_dead) 5168 prev->sched_class->task_dead(prev); 5169 5170 /* Task is done with its stack. */ 5171 put_task_stack(prev); 5172 5173 put_task_struct_rcu_user(prev); 5174 } 5175 5176 return rq; 5177 } 5178 5179 /** 5180 * schedule_tail - first thing a freshly forked thread must call. 5181 * @prev: the thread we just switched away from. 5182 */ 5183 asmlinkage __visible void schedule_tail(struct task_struct *prev) 5184 __releases(rq->lock) 5185 { 5186 /* 5187 * New tasks start with FORK_PREEMPT_COUNT, see there and 5188 * finish_task_switch() for details. 5189 * 5190 * finish_task_switch() will drop rq->lock() and lower preempt_count 5191 * and the preempt_enable() will end up enabling preemption (on 5192 * PREEMPT_COUNT kernels). 5193 */ 5194 5195 finish_task_switch(prev); 5196 preempt_enable(); 5197 5198 if (current->set_child_tid) 5199 put_user(task_pid_vnr(current), current->set_child_tid); 5200 5201 calculate_sigpending(); 5202 } 5203 5204 /* 5205 * context_switch - switch to the new MM and the new thread's register state. 5206 */ 5207 static __always_inline struct rq * 5208 context_switch(struct rq *rq, struct task_struct *prev, 5209 struct task_struct *next, struct rq_flags *rf) 5210 { 5211 prepare_task_switch(rq, prev, next); 5212 5213 /* 5214 * For paravirt, this is coupled with an exit in switch_to to 5215 * combine the page table reload and the switch backend into 5216 * one hypercall. 5217 */ 5218 arch_start_context_switch(prev); 5219 5220 /* 5221 * kernel -> kernel lazy + transfer active 5222 * user -> kernel lazy + mmgrab_lazy_tlb() active 5223 * 5224 * kernel -> user switch + mmdrop_lazy_tlb() active 5225 * user -> user switch 5226 * 5227 * switch_mm_cid() needs to be updated if the barriers provided 5228 * by context_switch() are modified. 5229 */ 5230 if (!next->mm) { // to kernel 5231 enter_lazy_tlb(prev->active_mm, next); 5232 5233 next->active_mm = prev->active_mm; 5234 if (prev->mm) // from user 5235 mmgrab_lazy_tlb(prev->active_mm); 5236 else 5237 prev->active_mm = NULL; 5238 } else { // to user 5239 membarrier_switch_mm(rq, prev->active_mm, next->mm); 5240 /* 5241 * sys_membarrier() requires an smp_mb() between setting 5242 * rq->curr / membarrier_switch_mm() and returning to userspace. 5243 * 5244 * The below provides this either through switch_mm(), or in 5245 * case 'prev->active_mm == next->mm' through 5246 * finish_task_switch()'s mmdrop(). 5247 */ 5248 switch_mm_irqs_off(prev->active_mm, next->mm, next); 5249 lru_gen_use_mm(next->mm); 5250 5251 if (!prev->mm) { // from kernel 5252 /* will mmdrop_lazy_tlb() in finish_task_switch(). */ 5253 rq->prev_mm = prev->active_mm; 5254 prev->active_mm = NULL; 5255 } 5256 } 5257 5258 /* switch_mm_cid() requires the memory barriers above. */ 5259 switch_mm_cid(rq, prev, next); 5260 5261 prepare_lock_switch(rq, next, rf); 5262 5263 /* Here we just switch the register state and the stack. */ 5264 switch_to(prev, next, prev); 5265 barrier(); 5266 5267 return finish_task_switch(prev); 5268 } 5269 5270 /* 5271 * nr_running and nr_context_switches: 5272 * 5273 * externally visible scheduler statistics: current number of runnable 5274 * threads, total number of context switches performed since bootup. 5275 */ 5276 unsigned int nr_running(void) 5277 { 5278 unsigned int i, sum = 0; 5279 5280 for_each_online_cpu(i) 5281 sum += cpu_rq(i)->nr_running; 5282 5283 return sum; 5284 } 5285 5286 /* 5287 * Check if only the current task is running on the CPU. 5288 * 5289 * Caution: this function does not check that the caller has disabled 5290 * preemption, thus the result might have a time-of-check-to-time-of-use 5291 * race. The caller is responsible to use it correctly, for example: 5292 * 5293 * - from a non-preemptible section (of course) 5294 * 5295 * - from a thread that is bound to a single CPU 5296 * 5297 * - in a loop with very short iterations (e.g. a polling loop) 5298 */ 5299 bool single_task_running(void) 5300 { 5301 return raw_rq()->nr_running == 1; 5302 } 5303 EXPORT_SYMBOL(single_task_running); 5304 5305 unsigned long long nr_context_switches_cpu(int cpu) 5306 { 5307 return cpu_rq(cpu)->nr_switches; 5308 } 5309 5310 unsigned long long nr_context_switches(void) 5311 { 5312 int i; 5313 unsigned long long sum = 0; 5314 5315 for_each_possible_cpu(i) 5316 sum += cpu_rq(i)->nr_switches; 5317 5318 return sum; 5319 } 5320 5321 /* 5322 * Consumers of these two interfaces, like for example the cpuidle menu 5323 * governor, are using nonsensical data. Preferring shallow idle state selection 5324 * for a CPU that has IO-wait which might not even end up running the task when 5325 * it does become runnable. 5326 */ 5327 5328 unsigned int nr_iowait_cpu(int cpu) 5329 { 5330 return atomic_read(&cpu_rq(cpu)->nr_iowait); 5331 } 5332 5333 /* 5334 * IO-wait accounting, and how it's mostly bollocks (on SMP). 5335 * 5336 * The idea behind IO-wait account is to account the idle time that we could 5337 * have spend running if it were not for IO. That is, if we were to improve the 5338 * storage performance, we'd have a proportional reduction in IO-wait time. 5339 * 5340 * This all works nicely on UP, where, when a task blocks on IO, we account 5341 * idle time as IO-wait, because if the storage were faster, it could've been 5342 * running and we'd not be idle. 5343 * 5344 * This has been extended to SMP, by doing the same for each CPU. This however 5345 * is broken. 5346 * 5347 * Imagine for instance the case where two tasks block on one CPU, only the one 5348 * CPU will have IO-wait accounted, while the other has regular idle. Even 5349 * though, if the storage were faster, both could've ran at the same time, 5350 * utilising both CPUs. 5351 * 5352 * This means, that when looking globally, the current IO-wait accounting on 5353 * SMP is a lower bound, by reason of under accounting. 5354 * 5355 * Worse, since the numbers are provided per CPU, they are sometimes 5356 * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly 5357 * associated with any one particular CPU, it can wake to another CPU than it 5358 * blocked on. This means the per CPU IO-wait number is meaningless. 5359 * 5360 * Task CPU affinities can make all that even more 'interesting'. 5361 */ 5362 5363 unsigned int nr_iowait(void) 5364 { 5365 unsigned int i, sum = 0; 5366 5367 for_each_possible_cpu(i) 5368 sum += nr_iowait_cpu(i); 5369 5370 return sum; 5371 } 5372 5373 #ifdef CONFIG_SMP 5374 5375 /* 5376 * sched_exec - execve() is a valuable balancing opportunity, because at 5377 * this point the task has the smallest effective memory and cache footprint. 5378 */ 5379 void sched_exec(void) 5380 { 5381 struct task_struct *p = current; 5382 struct migration_arg arg; 5383 int dest_cpu; 5384 5385 scoped_guard (raw_spinlock_irqsave, &p->pi_lock) { 5386 dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC); 5387 if (dest_cpu == smp_processor_id()) 5388 return; 5389 5390 if (unlikely(!cpu_active(dest_cpu))) 5391 return; 5392 5393 arg = (struct migration_arg){ p, dest_cpu }; 5394 } 5395 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); 5396 } 5397 5398 #endif 5399 5400 DEFINE_PER_CPU(struct kernel_stat, kstat); 5401 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); 5402 5403 EXPORT_PER_CPU_SYMBOL(kstat); 5404 EXPORT_PER_CPU_SYMBOL(kernel_cpustat); 5405 5406 /* 5407 * The function fair_sched_class.update_curr accesses the struct curr 5408 * and its field curr->exec_start; when called from task_sched_runtime(), 5409 * we observe a high rate of cache misses in practice. 5410 * Prefetching this data results in improved performance. 5411 */ 5412 static inline void prefetch_curr_exec_start(struct task_struct *p) 5413 { 5414 #ifdef CONFIG_FAIR_GROUP_SCHED 5415 struct sched_entity *curr = p->se.cfs_rq->curr; 5416 #else 5417 struct sched_entity *curr = task_rq(p)->cfs.curr; 5418 #endif 5419 prefetch(curr); 5420 prefetch(&curr->exec_start); 5421 } 5422 5423 /* 5424 * Return accounted runtime for the task. 5425 * In case the task is currently running, return the runtime plus current's 5426 * pending runtime that have not been accounted yet. 5427 */ 5428 unsigned long long task_sched_runtime(struct task_struct *p) 5429 { 5430 struct rq_flags rf; 5431 struct rq *rq; 5432 u64 ns; 5433 5434 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP) 5435 /* 5436 * 64-bit doesn't need locks to atomically read a 64-bit value. 5437 * So we have a optimization chance when the task's delta_exec is 0. 5438 * Reading ->on_cpu is racy, but this is OK. 5439 * 5440 * If we race with it leaving CPU, we'll take a lock. So we're correct. 5441 * If we race with it entering CPU, unaccounted time is 0. This is 5442 * indistinguishable from the read occurring a few cycles earlier. 5443 * If we see ->on_cpu without ->on_rq, the task is leaving, and has 5444 * been accounted, so we're correct here as well. 5445 */ 5446 if (!p->on_cpu || !task_on_rq_queued(p)) 5447 return p->se.sum_exec_runtime; 5448 #endif 5449 5450 rq = task_rq_lock(p, &rf); 5451 /* 5452 * Must be ->curr _and_ ->on_rq. If dequeued, we would 5453 * project cycles that may never be accounted to this 5454 * thread, breaking clock_gettime(). 5455 */ 5456 if (task_current(rq, p) && task_on_rq_queued(p)) { 5457 prefetch_curr_exec_start(p); 5458 update_rq_clock(rq); 5459 p->sched_class->update_curr(rq); 5460 } 5461 ns = p->se.sum_exec_runtime; 5462 task_rq_unlock(rq, p, &rf); 5463 5464 return ns; 5465 } 5466 5467 #ifdef CONFIG_SCHED_DEBUG 5468 static u64 cpu_resched_latency(struct rq *rq) 5469 { 5470 int latency_warn_ms = READ_ONCE(sysctl_resched_latency_warn_ms); 5471 u64 resched_latency, now = rq_clock(rq); 5472 static bool warned_once; 5473 5474 if (sysctl_resched_latency_warn_once && warned_once) 5475 return 0; 5476 5477 if (!need_resched() || !latency_warn_ms) 5478 return 0; 5479 5480 if (system_state == SYSTEM_BOOTING) 5481 return 0; 5482 5483 if (!rq->last_seen_need_resched_ns) { 5484 rq->last_seen_need_resched_ns = now; 5485 rq->ticks_without_resched = 0; 5486 return 0; 5487 } 5488 5489 rq->ticks_without_resched++; 5490 resched_latency = now - rq->last_seen_need_resched_ns; 5491 if (resched_latency <= latency_warn_ms * NSEC_PER_MSEC) 5492 return 0; 5493 5494 warned_once = true; 5495 5496 return resched_latency; 5497 } 5498 5499 static int __init setup_resched_latency_warn_ms(char *str) 5500 { 5501 long val; 5502 5503 if ((kstrtol(str, 0, &val))) { 5504 pr_warn("Unable to set resched_latency_warn_ms\n"); 5505 return 1; 5506 } 5507 5508 sysctl_resched_latency_warn_ms = val; 5509 return 1; 5510 } 5511 __setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms); 5512 #else 5513 static inline u64 cpu_resched_latency(struct rq *rq) { return 0; } 5514 #endif /* CONFIG_SCHED_DEBUG */ 5515 5516 /* 5517 * This function gets called by the timer code, with HZ frequency. 5518 * We call it with interrupts disabled. 5519 */ 5520 void sched_tick(void) 5521 { 5522 int cpu = smp_processor_id(); 5523 struct rq *rq = cpu_rq(cpu); 5524 struct task_struct *curr; 5525 struct rq_flags rf; 5526 unsigned long hw_pressure; 5527 u64 resched_latency; 5528 5529 if (housekeeping_cpu(cpu, HK_TYPE_TICK)) 5530 arch_scale_freq_tick(); 5531 5532 sched_clock_tick(); 5533 5534 rq_lock(rq, &rf); 5535 5536 curr = rq->curr; 5537 psi_account_irqtime(rq, curr, NULL); 5538 5539 update_rq_clock(rq); 5540 hw_pressure = arch_scale_hw_pressure(cpu_of(rq)); 5541 update_hw_load_avg(rq_clock_task(rq), rq, hw_pressure); 5542 curr->sched_class->task_tick(rq, curr, 0); 5543 if (sched_feat(LATENCY_WARN)) 5544 resched_latency = cpu_resched_latency(rq); 5545 calc_global_load_tick(rq); 5546 sched_core_tick(rq); 5547 task_tick_mm_cid(rq, curr); 5548 5549 rq_unlock(rq, &rf); 5550 5551 if (sched_feat(LATENCY_WARN) && resched_latency) 5552 resched_latency_warn(cpu, resched_latency); 5553 5554 perf_event_task_tick(); 5555 5556 if (curr->flags & PF_WQ_WORKER) 5557 wq_worker_tick(curr); 5558 5559 #ifdef CONFIG_SMP 5560 rq->idle_balance = idle_cpu(cpu); 5561 sched_balance_trigger(rq); 5562 #endif 5563 } 5564 5565 #ifdef CONFIG_NO_HZ_FULL 5566 5567 struct tick_work { 5568 int cpu; 5569 atomic_t state; 5570 struct delayed_work work; 5571 }; 5572 /* Values for ->state, see diagram below. */ 5573 #define TICK_SCHED_REMOTE_OFFLINE 0 5574 #define TICK_SCHED_REMOTE_OFFLINING 1 5575 #define TICK_SCHED_REMOTE_RUNNING 2 5576 5577 /* 5578 * State diagram for ->state: 5579 * 5580 * 5581 * TICK_SCHED_REMOTE_OFFLINE 5582 * | ^ 5583 * | | 5584 * | | sched_tick_remote() 5585 * | | 5586 * | | 5587 * +--TICK_SCHED_REMOTE_OFFLINING 5588 * | ^ 5589 * | | 5590 * sched_tick_start() | | sched_tick_stop() 5591 * | | 5592 * V | 5593 * TICK_SCHED_REMOTE_RUNNING 5594 * 5595 * 5596 * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote() 5597 * and sched_tick_start() are happy to leave the state in RUNNING. 5598 */ 5599 5600 static struct tick_work __percpu *tick_work_cpu; 5601 5602 static void sched_tick_remote(struct work_struct *work) 5603 { 5604 struct delayed_work *dwork = to_delayed_work(work); 5605 struct tick_work *twork = container_of(dwork, struct tick_work, work); 5606 int cpu = twork->cpu; 5607 struct rq *rq = cpu_rq(cpu); 5608 int os; 5609 5610 /* 5611 * Handle the tick only if it appears the remote CPU is running in full 5612 * dynticks mode. The check is racy by nature, but missing a tick or 5613 * having one too much is no big deal because the scheduler tick updates 5614 * statistics and checks timeslices in a time-independent way, regardless 5615 * of when exactly it is running. 5616 */ 5617 if (tick_nohz_tick_stopped_cpu(cpu)) { 5618 guard(rq_lock_irq)(rq); 5619 struct task_struct *curr = rq->curr; 5620 5621 if (cpu_online(cpu)) { 5622 update_rq_clock(rq); 5623 5624 if (!is_idle_task(curr)) { 5625 /* 5626 * Make sure the next tick runs within a 5627 * reasonable amount of time. 5628 */ 5629 u64 delta = rq_clock_task(rq) - curr->se.exec_start; 5630 WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3); 5631 } 5632 curr->sched_class->task_tick(rq, curr, 0); 5633 5634 calc_load_nohz_remote(rq); 5635 } 5636 } 5637 5638 /* 5639 * Run the remote tick once per second (1Hz). This arbitrary 5640 * frequency is large enough to avoid overload but short enough 5641 * to keep scheduler internal stats reasonably up to date. But 5642 * first update state to reflect hotplug activity if required. 5643 */ 5644 os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING); 5645 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE); 5646 if (os == TICK_SCHED_REMOTE_RUNNING) 5647 queue_delayed_work(system_unbound_wq, dwork, HZ); 5648 } 5649 5650 static void sched_tick_start(int cpu) 5651 { 5652 int os; 5653 struct tick_work *twork; 5654 5655 if (housekeeping_cpu(cpu, HK_TYPE_TICK)) 5656 return; 5657 5658 WARN_ON_ONCE(!tick_work_cpu); 5659 5660 twork = per_cpu_ptr(tick_work_cpu, cpu); 5661 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING); 5662 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING); 5663 if (os == TICK_SCHED_REMOTE_OFFLINE) { 5664 twork->cpu = cpu; 5665 INIT_DELAYED_WORK(&twork->work, sched_tick_remote); 5666 queue_delayed_work(system_unbound_wq, &twork->work, HZ); 5667 } 5668 } 5669 5670 #ifdef CONFIG_HOTPLUG_CPU 5671 static void sched_tick_stop(int cpu) 5672 { 5673 struct tick_work *twork; 5674 int os; 5675 5676 if (housekeeping_cpu(cpu, HK_TYPE_TICK)) 5677 return; 5678 5679 WARN_ON_ONCE(!tick_work_cpu); 5680 5681 twork = per_cpu_ptr(tick_work_cpu, cpu); 5682 /* There cannot be competing actions, but don't rely on stop-machine. */ 5683 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING); 5684 WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING); 5685 /* Don't cancel, as this would mess up the state machine. */ 5686 } 5687 #endif /* CONFIG_HOTPLUG_CPU */ 5688 5689 int __init sched_tick_offload_init(void) 5690 { 5691 tick_work_cpu = alloc_percpu(struct tick_work); 5692 BUG_ON(!tick_work_cpu); 5693 return 0; 5694 } 5695 5696 #else /* !CONFIG_NO_HZ_FULL */ 5697 static inline void sched_tick_start(int cpu) { } 5698 static inline void sched_tick_stop(int cpu) { } 5699 #endif 5700 5701 #if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \ 5702 defined(CONFIG_TRACE_PREEMPT_TOGGLE)) 5703 /* 5704 * If the value passed in is equal to the current preempt count 5705 * then we just disabled preemption. Start timing the latency. 5706 */ 5707 static inline void preempt_latency_start(int val) 5708 { 5709 if (preempt_count() == val) { 5710 unsigned long ip = get_lock_parent_ip(); 5711 #ifdef CONFIG_DEBUG_PREEMPT 5712 current->preempt_disable_ip = ip; 5713 #endif 5714 trace_preempt_off(CALLER_ADDR0, ip); 5715 } 5716 } 5717 5718 void preempt_count_add(int val) 5719 { 5720 #ifdef CONFIG_DEBUG_PREEMPT 5721 /* 5722 * Underflow? 5723 */ 5724 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) 5725 return; 5726 #endif 5727 __preempt_count_add(val); 5728 #ifdef CONFIG_DEBUG_PREEMPT 5729 /* 5730 * Spinlock count overflowing soon? 5731 */ 5732 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= 5733 PREEMPT_MASK - 10); 5734 #endif 5735 preempt_latency_start(val); 5736 } 5737 EXPORT_SYMBOL(preempt_count_add); 5738 NOKPROBE_SYMBOL(preempt_count_add); 5739 5740 /* 5741 * If the value passed in equals to the current preempt count 5742 * then we just enabled preemption. Stop timing the latency. 5743 */ 5744 static inline void preempt_latency_stop(int val) 5745 { 5746 if (preempt_count() == val) 5747 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); 5748 } 5749 5750 void preempt_count_sub(int val) 5751 { 5752 #ifdef CONFIG_DEBUG_PREEMPT 5753 /* 5754 * Underflow? 5755 */ 5756 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 5757 return; 5758 /* 5759 * Is the spinlock portion underflowing? 5760 */ 5761 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && 5762 !(preempt_count() & PREEMPT_MASK))) 5763 return; 5764 #endif 5765 5766 preempt_latency_stop(val); 5767 __preempt_count_sub(val); 5768 } 5769 EXPORT_SYMBOL(preempt_count_sub); 5770 NOKPROBE_SYMBOL(preempt_count_sub); 5771 5772 #else 5773 static inline void preempt_latency_start(int val) { } 5774 static inline void preempt_latency_stop(int val) { } 5775 #endif 5776 5777 static inline unsigned long get_preempt_disable_ip(struct task_struct *p) 5778 { 5779 #ifdef CONFIG_DEBUG_PREEMPT 5780 return p->preempt_disable_ip; 5781 #else 5782 return 0; 5783 #endif 5784 } 5785 5786 /* 5787 * Print scheduling while atomic bug: 5788 */ 5789 static noinline void __schedule_bug(struct task_struct *prev) 5790 { 5791 /* Save this before calling printk(), since that will clobber it */ 5792 unsigned long preempt_disable_ip = get_preempt_disable_ip(current); 5793 5794 if (oops_in_progress) 5795 return; 5796 5797 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", 5798 prev->comm, prev->pid, preempt_count()); 5799 5800 debug_show_held_locks(prev); 5801 print_modules(); 5802 if (irqs_disabled()) 5803 print_irqtrace_events(prev); 5804 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) { 5805 pr_err("Preemption disabled at:"); 5806 print_ip_sym(KERN_ERR, preempt_disable_ip); 5807 } 5808 check_panic_on_warn("scheduling while atomic"); 5809 5810 dump_stack(); 5811 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5812 } 5813 5814 /* 5815 * Various schedule()-time debugging checks and statistics: 5816 */ 5817 static inline void schedule_debug(struct task_struct *prev, bool preempt) 5818 { 5819 #ifdef CONFIG_SCHED_STACK_END_CHECK 5820 if (task_stack_end_corrupted(prev)) 5821 panic("corrupted stack end detected inside scheduler\n"); 5822 5823 if (task_scs_end_corrupted(prev)) 5824 panic("corrupted shadow stack detected inside scheduler\n"); 5825 #endif 5826 5827 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 5828 if (!preempt && READ_ONCE(prev->__state) && prev->non_block_count) { 5829 printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n", 5830 prev->comm, prev->pid, prev->non_block_count); 5831 dump_stack(); 5832 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5833 } 5834 #endif 5835 5836 if (unlikely(in_atomic_preempt_off())) { 5837 __schedule_bug(prev); 5838 preempt_count_set(PREEMPT_DISABLED); 5839 } 5840 rcu_sleep_check(); 5841 SCHED_WARN_ON(ct_state() == CT_STATE_USER); 5842 5843 profile_hit(SCHED_PROFILING, __builtin_return_address(0)); 5844 5845 schedstat_inc(this_rq()->sched_count); 5846 } 5847 5848 static void prev_balance(struct rq *rq, struct task_struct *prev, 5849 struct rq_flags *rf) 5850 { 5851 #ifdef CONFIG_SMP 5852 const struct sched_class *class; 5853 /* 5854 * We must do the balancing pass before put_prev_task(), such 5855 * that when we release the rq->lock the task is in the same 5856 * state as before we took rq->lock. 5857 * 5858 * We can terminate the balance pass as soon as we know there is 5859 * a runnable task of @class priority or higher. 5860 */ 5861 for_class_range(class, prev->sched_class, &idle_sched_class) { 5862 if (class->balance(rq, prev, rf)) 5863 break; 5864 } 5865 #endif 5866 } 5867 5868 /* 5869 * Pick up the highest-prio task: 5870 */ 5871 static inline struct task_struct * 5872 __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 5873 { 5874 const struct sched_class *class; 5875 struct task_struct *p; 5876 5877 rq->dl_server = NULL; 5878 5879 /* 5880 * Optimization: we know that if all tasks are in the fair class we can 5881 * call that function directly, but only if the @prev task wasn't of a 5882 * higher scheduling class, because otherwise those lose the 5883 * opportunity to pull in more work from other CPUs. 5884 */ 5885 if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) && 5886 rq->nr_running == rq->cfs.h_nr_running)) { 5887 5888 p = pick_next_task_fair(rq, prev, rf); 5889 if (unlikely(p == RETRY_TASK)) 5890 goto restart; 5891 5892 /* Assume the next prioritized class is idle_sched_class */ 5893 if (!p) { 5894 p = pick_task_idle(rq); 5895 put_prev_set_next_task(rq, prev, p); 5896 } 5897 5898 return p; 5899 } 5900 5901 restart: 5902 prev_balance(rq, prev, rf); 5903 5904 for_each_class(class) { 5905 if (class->pick_next_task) { 5906 p = class->pick_next_task(rq, prev); 5907 if (p) 5908 return p; 5909 } else { 5910 p = class->pick_task(rq); 5911 if (p) { 5912 put_prev_set_next_task(rq, prev, p); 5913 return p; 5914 } 5915 } 5916 } 5917 5918 BUG(); /* The idle class should always have a runnable task. */ 5919 } 5920 5921 #ifdef CONFIG_SCHED_CORE 5922 static inline bool is_task_rq_idle(struct task_struct *t) 5923 { 5924 return (task_rq(t)->idle == t); 5925 } 5926 5927 static inline bool cookie_equals(struct task_struct *a, unsigned long cookie) 5928 { 5929 return is_task_rq_idle(a) || (a->core_cookie == cookie); 5930 } 5931 5932 static inline bool cookie_match(struct task_struct *a, struct task_struct *b) 5933 { 5934 if (is_task_rq_idle(a) || is_task_rq_idle(b)) 5935 return true; 5936 5937 return a->core_cookie == b->core_cookie; 5938 } 5939 5940 static inline struct task_struct *pick_task(struct rq *rq) 5941 { 5942 const struct sched_class *class; 5943 struct task_struct *p; 5944 5945 rq->dl_server = NULL; 5946 5947 for_each_class(class) { 5948 p = class->pick_task(rq); 5949 if (p) 5950 return p; 5951 } 5952 5953 BUG(); /* The idle class should always have a runnable task. */ 5954 } 5955 5956 extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi); 5957 5958 static void queue_core_balance(struct rq *rq); 5959 5960 static struct task_struct * 5961 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 5962 { 5963 struct task_struct *next, *p, *max = NULL; 5964 const struct cpumask *smt_mask; 5965 bool fi_before = false; 5966 bool core_clock_updated = (rq == rq->core); 5967 unsigned long cookie; 5968 int i, cpu, occ = 0; 5969 struct rq *rq_i; 5970 bool need_sync; 5971 5972 if (!sched_core_enabled(rq)) 5973 return __pick_next_task(rq, prev, rf); 5974 5975 cpu = cpu_of(rq); 5976 5977 /* Stopper task is switching into idle, no need core-wide selection. */ 5978 if (cpu_is_offline(cpu)) { 5979 /* 5980 * Reset core_pick so that we don't enter the fastpath when 5981 * coming online. core_pick would already be migrated to 5982 * another cpu during offline. 5983 */ 5984 rq->core_pick = NULL; 5985 rq->core_dl_server = NULL; 5986 return __pick_next_task(rq, prev, rf); 5987 } 5988 5989 /* 5990 * If there were no {en,de}queues since we picked (IOW, the task 5991 * pointers are all still valid), and we haven't scheduled the last 5992 * pick yet, do so now. 5993 * 5994 * rq->core_pick can be NULL if no selection was made for a CPU because 5995 * it was either offline or went offline during a sibling's core-wide 5996 * selection. In this case, do a core-wide selection. 5997 */ 5998 if (rq->core->core_pick_seq == rq->core->core_task_seq && 5999 rq->core->core_pick_seq != rq->core_sched_seq && 6000 rq->core_pick) { 6001 WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); 6002 6003 next = rq->core_pick; 6004 rq->dl_server = rq->core_dl_server; 6005 rq->core_pick = NULL; 6006 rq->core_dl_server = NULL; 6007 goto out_set_next; 6008 } 6009 6010 prev_balance(rq, prev, rf); 6011 6012 smt_mask = cpu_smt_mask(cpu); 6013 need_sync = !!rq->core->core_cookie; 6014 6015 /* reset state */ 6016 rq->core->core_cookie = 0UL; 6017 if (rq->core->core_forceidle_count) { 6018 if (!core_clock_updated) { 6019 update_rq_clock(rq->core); 6020 core_clock_updated = true; 6021 } 6022 sched_core_account_forceidle(rq); 6023 /* reset after accounting force idle */ 6024 rq->core->core_forceidle_start = 0; 6025 rq->core->core_forceidle_count = 0; 6026 rq->core->core_forceidle_occupation = 0; 6027 need_sync = true; 6028 fi_before = true; 6029 } 6030 6031 /* 6032 * core->core_task_seq, core->core_pick_seq, rq->core_sched_seq 6033 * 6034 * @task_seq guards the task state ({en,de}queues) 6035 * @pick_seq is the @task_seq we did a selection on 6036 * @sched_seq is the @pick_seq we scheduled 6037 * 6038 * However, preemptions can cause multiple picks on the same task set. 6039 * 'Fix' this by also increasing @task_seq for every pick. 6040 */ 6041 rq->core->core_task_seq++; 6042 6043 /* 6044 * Optimize for common case where this CPU has no cookies 6045 * and there are no cookied tasks running on siblings. 6046 */ 6047 if (!need_sync) { 6048 next = pick_task(rq); 6049 if (!next->core_cookie) { 6050 rq->core_pick = NULL; 6051 rq->core_dl_server = NULL; 6052 /* 6053 * For robustness, update the min_vruntime_fi for 6054 * unconstrained picks as well. 6055 */ 6056 WARN_ON_ONCE(fi_before); 6057 task_vruntime_update(rq, next, false); 6058 goto out_set_next; 6059 } 6060 } 6061 6062 /* 6063 * For each thread: do the regular task pick and find the max prio task 6064 * amongst them. 6065 * 6066 * Tie-break prio towards the current CPU 6067 */ 6068 for_each_cpu_wrap(i, smt_mask, cpu) { 6069 rq_i = cpu_rq(i); 6070 6071 /* 6072 * Current cpu always has its clock updated on entrance to 6073 * pick_next_task(). If the current cpu is not the core, 6074 * the core may also have been updated above. 6075 */ 6076 if (i != cpu && (rq_i != rq->core || !core_clock_updated)) 6077 update_rq_clock(rq_i); 6078 6079 rq_i->core_pick = p = pick_task(rq_i); 6080 rq_i->core_dl_server = rq_i->dl_server; 6081 6082 if (!max || prio_less(max, p, fi_before)) 6083 max = p; 6084 } 6085 6086 cookie = rq->core->core_cookie = max->core_cookie; 6087 6088 /* 6089 * For each thread: try and find a runnable task that matches @max or 6090 * force idle. 6091 */ 6092 for_each_cpu(i, smt_mask) { 6093 rq_i = cpu_rq(i); 6094 p = rq_i->core_pick; 6095 6096 if (!cookie_equals(p, cookie)) { 6097 p = NULL; 6098 if (cookie) 6099 p = sched_core_find(rq_i, cookie); 6100 if (!p) 6101 p = idle_sched_class.pick_task(rq_i); 6102 } 6103 6104 rq_i->core_pick = p; 6105 rq_i->core_dl_server = NULL; 6106 6107 if (p == rq_i->idle) { 6108 if (rq_i->nr_running) { 6109 rq->core->core_forceidle_count++; 6110 if (!fi_before) 6111 rq->core->core_forceidle_seq++; 6112 } 6113 } else { 6114 occ++; 6115 } 6116 } 6117 6118 if (schedstat_enabled() && rq->core->core_forceidle_count) { 6119 rq->core->core_forceidle_start = rq_clock(rq->core); 6120 rq->core->core_forceidle_occupation = occ; 6121 } 6122 6123 rq->core->core_pick_seq = rq->core->core_task_seq; 6124 next = rq->core_pick; 6125 rq->core_sched_seq = rq->core->core_pick_seq; 6126 6127 /* Something should have been selected for current CPU */ 6128 WARN_ON_ONCE(!next); 6129 6130 /* 6131 * Reschedule siblings 6132 * 6133 * NOTE: L1TF -- at this point we're no longer running the old task and 6134 * sending an IPI (below) ensures the sibling will no longer be running 6135 * their task. This ensures there is no inter-sibling overlap between 6136 * non-matching user state. 6137 */ 6138 for_each_cpu(i, smt_mask) { 6139 rq_i = cpu_rq(i); 6140 6141 /* 6142 * An online sibling might have gone offline before a task 6143 * could be picked for it, or it might be offline but later 6144 * happen to come online, but its too late and nothing was 6145 * picked for it. That's Ok - it will pick tasks for itself, 6146 * so ignore it. 6147 */ 6148 if (!rq_i->core_pick) 6149 continue; 6150 6151 /* 6152 * Update for new !FI->FI transitions, or if continuing to be in !FI: 6153 * fi_before fi update? 6154 * 0 0 1 6155 * 0 1 1 6156 * 1 0 1 6157 * 1 1 0 6158 */ 6159 if (!(fi_before && rq->core->core_forceidle_count)) 6160 task_vruntime_update(rq_i, rq_i->core_pick, !!rq->core->core_forceidle_count); 6161 6162 rq_i->core_pick->core_occupation = occ; 6163 6164 if (i == cpu) { 6165 rq_i->core_pick = NULL; 6166 rq_i->core_dl_server = NULL; 6167 continue; 6168 } 6169 6170 /* Did we break L1TF mitigation requirements? */ 6171 WARN_ON_ONCE(!cookie_match(next, rq_i->core_pick)); 6172 6173 if (rq_i->curr == rq_i->core_pick) { 6174 rq_i->core_pick = NULL; 6175 rq_i->core_dl_server = NULL; 6176 continue; 6177 } 6178 6179 resched_curr(rq_i); 6180 } 6181 6182 out_set_next: 6183 put_prev_set_next_task(rq, prev, next); 6184 if (rq->core->core_forceidle_count && next == rq->idle) 6185 queue_core_balance(rq); 6186 6187 return next; 6188 } 6189 6190 static bool try_steal_cookie(int this, int that) 6191 { 6192 struct rq *dst = cpu_rq(this), *src = cpu_rq(that); 6193 struct task_struct *p; 6194 unsigned long cookie; 6195 bool success = false; 6196 6197 guard(irq)(); 6198 guard(double_rq_lock)(dst, src); 6199 6200 cookie = dst->core->core_cookie; 6201 if (!cookie) 6202 return false; 6203 6204 if (dst->curr != dst->idle) 6205 return false; 6206 6207 p = sched_core_find(src, cookie); 6208 if (!p) 6209 return false; 6210 6211 do { 6212 if (p == src->core_pick || p == src->curr) 6213 goto next; 6214 6215 if (!is_cpu_allowed(p, this)) 6216 goto next; 6217 6218 if (p->core_occupation > dst->idle->core_occupation) 6219 goto next; 6220 /* 6221 * sched_core_find() and sched_core_next() will ensure 6222 * that task @p is not throttled now, we also need to 6223 * check whether the runqueue of the destination CPU is 6224 * being throttled. 6225 */ 6226 if (sched_task_is_throttled(p, this)) 6227 goto next; 6228 6229 deactivate_task(src, p, 0); 6230 set_task_cpu(p, this); 6231 activate_task(dst, p, 0); 6232 6233 resched_curr(dst); 6234 6235 success = true; 6236 break; 6237 6238 next: 6239 p = sched_core_next(p, cookie); 6240 } while (p); 6241 6242 return success; 6243 } 6244 6245 static bool steal_cookie_task(int cpu, struct sched_domain *sd) 6246 { 6247 int i; 6248 6249 for_each_cpu_wrap(i, sched_domain_span(sd), cpu + 1) { 6250 if (i == cpu) 6251 continue; 6252 6253 if (need_resched()) 6254 break; 6255 6256 if (try_steal_cookie(cpu, i)) 6257 return true; 6258 } 6259 6260 return false; 6261 } 6262 6263 static void sched_core_balance(struct rq *rq) 6264 { 6265 struct sched_domain *sd; 6266 int cpu = cpu_of(rq); 6267 6268 guard(preempt)(); 6269 guard(rcu)(); 6270 6271 raw_spin_rq_unlock_irq(rq); 6272 for_each_domain(cpu, sd) { 6273 if (need_resched()) 6274 break; 6275 6276 if (steal_cookie_task(cpu, sd)) 6277 break; 6278 } 6279 raw_spin_rq_lock_irq(rq); 6280 } 6281 6282 static DEFINE_PER_CPU(struct balance_callback, core_balance_head); 6283 6284 static void queue_core_balance(struct rq *rq) 6285 { 6286 if (!sched_core_enabled(rq)) 6287 return; 6288 6289 if (!rq->core->core_cookie) 6290 return; 6291 6292 if (!rq->nr_running) /* not forced idle */ 6293 return; 6294 6295 queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance); 6296 } 6297 6298 DEFINE_LOCK_GUARD_1(core_lock, int, 6299 sched_core_lock(*_T->lock, &_T->flags), 6300 sched_core_unlock(*_T->lock, &_T->flags), 6301 unsigned long flags) 6302 6303 static void sched_core_cpu_starting(unsigned int cpu) 6304 { 6305 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 6306 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; 6307 int t; 6308 6309 guard(core_lock)(&cpu); 6310 6311 WARN_ON_ONCE(rq->core != rq); 6312 6313 /* if we're the first, we'll be our own leader */ 6314 if (cpumask_weight(smt_mask) == 1) 6315 return; 6316 6317 /* find the leader */ 6318 for_each_cpu(t, smt_mask) { 6319 if (t == cpu) 6320 continue; 6321 rq = cpu_rq(t); 6322 if (rq->core == rq) { 6323 core_rq = rq; 6324 break; 6325 } 6326 } 6327 6328 if (WARN_ON_ONCE(!core_rq)) /* whoopsie */ 6329 return; 6330 6331 /* install and validate core_rq */ 6332 for_each_cpu(t, smt_mask) { 6333 rq = cpu_rq(t); 6334 6335 if (t == cpu) 6336 rq->core = core_rq; 6337 6338 WARN_ON_ONCE(rq->core != core_rq); 6339 } 6340 } 6341 6342 static void sched_core_cpu_deactivate(unsigned int cpu) 6343 { 6344 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 6345 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; 6346 int t; 6347 6348 guard(core_lock)(&cpu); 6349 6350 /* if we're the last man standing, nothing to do */ 6351 if (cpumask_weight(smt_mask) == 1) { 6352 WARN_ON_ONCE(rq->core != rq); 6353 return; 6354 } 6355 6356 /* if we're not the leader, nothing to do */ 6357 if (rq->core != rq) 6358 return; 6359 6360 /* find a new leader */ 6361 for_each_cpu(t, smt_mask) { 6362 if (t == cpu) 6363 continue; 6364 core_rq = cpu_rq(t); 6365 break; 6366 } 6367 6368 if (WARN_ON_ONCE(!core_rq)) /* impossible */ 6369 return; 6370 6371 /* copy the shared state to the new leader */ 6372 core_rq->core_task_seq = rq->core_task_seq; 6373 core_rq->core_pick_seq = rq->core_pick_seq; 6374 core_rq->core_cookie = rq->core_cookie; 6375 core_rq->core_forceidle_count = rq->core_forceidle_count; 6376 core_rq->core_forceidle_seq = rq->core_forceidle_seq; 6377 core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; 6378 6379 /* 6380 * Accounting edge for forced idle is handled in pick_next_task(). 6381 * Don't need another one here, since the hotplug thread shouldn't 6382 * have a cookie. 6383 */ 6384 core_rq->core_forceidle_start = 0; 6385 6386 /* install new leader */ 6387 for_each_cpu(t, smt_mask) { 6388 rq = cpu_rq(t); 6389 rq->core = core_rq; 6390 } 6391 } 6392 6393 static inline void sched_core_cpu_dying(unsigned int cpu) 6394 { 6395 struct rq *rq = cpu_rq(cpu); 6396 6397 if (rq->core != rq) 6398 rq->core = rq; 6399 } 6400 6401 #else /* !CONFIG_SCHED_CORE */ 6402 6403 static inline void sched_core_cpu_starting(unsigned int cpu) {} 6404 static inline void sched_core_cpu_deactivate(unsigned int cpu) {} 6405 static inline void sched_core_cpu_dying(unsigned int cpu) {} 6406 6407 static struct task_struct * 6408 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6409 { 6410 return __pick_next_task(rq, prev, rf); 6411 } 6412 6413 #endif /* CONFIG_SCHED_CORE */ 6414 6415 /* 6416 * Constants for the sched_mode argument of __schedule(). 6417 * 6418 * The mode argument allows RT enabled kernels to differentiate a 6419 * preemption from blocking on an 'sleeping' spin/rwlock. 6420 */ 6421 #define SM_IDLE (-1) 6422 #define SM_NONE 0 6423 #define SM_PREEMPT 1 6424 #define SM_RTLOCK_WAIT 2 6425 6426 /* 6427 * __schedule() is the main scheduler function. 6428 * 6429 * The main means of driving the scheduler and thus entering this function are: 6430 * 6431 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. 6432 * 6433 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return 6434 * paths. For example, see arch/x86/entry_64.S. 6435 * 6436 * To drive preemption between tasks, the scheduler sets the flag in timer 6437 * interrupt handler sched_tick(). 6438 * 6439 * 3. Wakeups don't really cause entry into schedule(). They add a 6440 * task to the run-queue and that's it. 6441 * 6442 * Now, if the new task added to the run-queue preempts the current 6443 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets 6444 * called on the nearest possible occasion: 6445 * 6446 * - If the kernel is preemptible (CONFIG_PREEMPTION=y): 6447 * 6448 * - in syscall or exception context, at the next outmost 6449 * preempt_enable(). (this might be as soon as the wake_up()'s 6450 * spin_unlock()!) 6451 * 6452 * - in IRQ context, return from interrupt-handler to 6453 * preemptible context 6454 * 6455 * - If the kernel is not preemptible (CONFIG_PREEMPTION is not set) 6456 * then at the next: 6457 * 6458 * - cond_resched() call 6459 * - explicit schedule() call 6460 * - return from syscall or exception to user-space 6461 * - return from interrupt-handler to user-space 6462 * 6463 * WARNING: must be called with preemption disabled! 6464 */ 6465 static void __sched notrace __schedule(int sched_mode) 6466 { 6467 struct task_struct *prev, *next; 6468 /* 6469 * On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted 6470 * as a preemption by schedule_debug() and RCU. 6471 */ 6472 bool preempt = sched_mode > SM_NONE; 6473 unsigned long *switch_count; 6474 unsigned long prev_state; 6475 struct rq_flags rf; 6476 struct rq *rq; 6477 int cpu; 6478 6479 cpu = smp_processor_id(); 6480 rq = cpu_rq(cpu); 6481 prev = rq->curr; 6482 6483 schedule_debug(prev, preempt); 6484 6485 if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) 6486 hrtick_clear(rq); 6487 6488 local_irq_disable(); 6489 rcu_note_context_switch(preempt); 6490 6491 /* 6492 * Make sure that signal_pending_state()->signal_pending() below 6493 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) 6494 * done by the caller to avoid the race with signal_wake_up(): 6495 * 6496 * __set_current_state(@state) signal_wake_up() 6497 * schedule() set_tsk_thread_flag(p, TIF_SIGPENDING) 6498 * wake_up_state(p, state) 6499 * LOCK rq->lock LOCK p->pi_state 6500 * smp_mb__after_spinlock() smp_mb__after_spinlock() 6501 * if (signal_pending_state()) if (p->state & @state) 6502 * 6503 * Also, the membarrier system call requires a full memory barrier 6504 * after coming from user-space, before storing to rq->curr; this 6505 * barrier matches a full barrier in the proximity of the membarrier 6506 * system call exit. 6507 */ 6508 rq_lock(rq, &rf); 6509 smp_mb__after_spinlock(); 6510 6511 /* Promote REQ to ACT */ 6512 rq->clock_update_flags <<= 1; 6513 update_rq_clock(rq); 6514 rq->clock_update_flags = RQCF_UPDATED; 6515 6516 switch_count = &prev->nivcsw; 6517 6518 /* Task state changes only considers SM_PREEMPT as preemption */ 6519 preempt = sched_mode == SM_PREEMPT; 6520 6521 /* 6522 * We must load prev->state once (task_struct::state is volatile), such 6523 * that we form a control dependency vs deactivate_task() below. 6524 */ 6525 prev_state = READ_ONCE(prev->__state); 6526 if (sched_mode == SM_IDLE) { 6527 if (!rq->nr_running) { 6528 next = prev; 6529 goto picked; 6530 } 6531 } else if (!preempt && prev_state) { 6532 if (signal_pending_state(prev_state, prev)) { 6533 WRITE_ONCE(prev->__state, TASK_RUNNING); 6534 } else { 6535 int flags = DEQUEUE_NOCLOCK; 6536 6537 prev->sched_contributes_to_load = 6538 (prev_state & TASK_UNINTERRUPTIBLE) && 6539 !(prev_state & TASK_NOLOAD) && 6540 !(prev_state & TASK_FROZEN); 6541 6542 if (unlikely(is_special_task_state(prev_state))) 6543 flags |= DEQUEUE_SPECIAL; 6544 6545 /* 6546 * __schedule() ttwu() 6547 * prev_state = prev->state; if (p->on_rq && ...) 6548 * if (prev_state) goto out; 6549 * p->on_rq = 0; smp_acquire__after_ctrl_dep(); 6550 * p->state = TASK_WAKING 6551 * 6552 * Where __schedule() and ttwu() have matching control dependencies. 6553 * 6554 * After this, schedule() must not care about p->state any more. 6555 */ 6556 block_task(rq, prev, flags); 6557 } 6558 switch_count = &prev->nvcsw; 6559 } 6560 6561 next = pick_next_task(rq, prev, &rf); 6562 picked: 6563 clear_tsk_need_resched(prev); 6564 clear_preempt_need_resched(); 6565 #ifdef CONFIG_SCHED_DEBUG 6566 rq->last_seen_need_resched_ns = 0; 6567 #endif 6568 6569 if (likely(prev != next)) { 6570 rq->nr_switches++; 6571 /* 6572 * RCU users of rcu_dereference(rq->curr) may not see 6573 * changes to task_struct made by pick_next_task(). 6574 */ 6575 RCU_INIT_POINTER(rq->curr, next); 6576 /* 6577 * The membarrier system call requires each architecture 6578 * to have a full memory barrier after updating 6579 * rq->curr, before returning to user-space. 6580 * 6581 * Here are the schemes providing that barrier on the 6582 * various architectures: 6583 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC, 6584 * RISC-V. switch_mm() relies on membarrier_arch_switch_mm() 6585 * on PowerPC and on RISC-V. 6586 * - finish_lock_switch() for weakly-ordered 6587 * architectures where spin_unlock is a full barrier, 6588 * - switch_to() for arm64 (weakly-ordered, spin_unlock 6589 * is a RELEASE barrier), 6590 * 6591 * The barrier matches a full barrier in the proximity of 6592 * the membarrier system call entry. 6593 * 6594 * On RISC-V, this barrier pairing is also needed for the 6595 * SYNC_CORE command when switching between processes, cf. 6596 * the inline comments in membarrier_arch_switch_mm(). 6597 */ 6598 ++*switch_count; 6599 6600 migrate_disable_switch(rq, prev); 6601 psi_account_irqtime(rq, prev, next); 6602 psi_sched_switch(prev, next, !task_on_rq_queued(prev)); 6603 6604 trace_sched_switch(preempt, prev, next, prev_state); 6605 6606 /* Also unlocks the rq: */ 6607 rq = context_switch(rq, prev, next, &rf); 6608 } else { 6609 rq_unpin_lock(rq, &rf); 6610 __balance_callbacks(rq); 6611 raw_spin_rq_unlock_irq(rq); 6612 } 6613 } 6614 6615 void __noreturn do_task_dead(void) 6616 { 6617 /* Causes final put_task_struct in finish_task_switch(): */ 6618 set_special_state(TASK_DEAD); 6619 6620 /* Tell freezer to ignore us: */ 6621 current->flags |= PF_NOFREEZE; 6622 6623 __schedule(SM_NONE); 6624 BUG(); 6625 6626 /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */ 6627 for (;;) 6628 cpu_relax(); 6629 } 6630 6631 static inline void sched_submit_work(struct task_struct *tsk) 6632 { 6633 static DEFINE_WAIT_OVERRIDE_MAP(sched_map, LD_WAIT_CONFIG); 6634 unsigned int task_flags; 6635 6636 /* 6637 * Establish LD_WAIT_CONFIG context to ensure none of the code called 6638 * will use a blocking primitive -- which would lead to recursion. 6639 */ 6640 lock_map_acquire_try(&sched_map); 6641 6642 task_flags = tsk->flags; 6643 /* 6644 * If a worker goes to sleep, notify and ask workqueue whether it 6645 * wants to wake up a task to maintain concurrency. 6646 */ 6647 if (task_flags & PF_WQ_WORKER) 6648 wq_worker_sleeping(tsk); 6649 else if (task_flags & PF_IO_WORKER) 6650 io_wq_worker_sleeping(tsk); 6651 6652 /* 6653 * spinlock and rwlock must not flush block requests. This will 6654 * deadlock if the callback attempts to acquire a lock which is 6655 * already acquired. 6656 */ 6657 SCHED_WARN_ON(current->__state & TASK_RTLOCK_WAIT); 6658 6659 /* 6660 * If we are going to sleep and we have plugged IO queued, 6661 * make sure to submit it to avoid deadlocks. 6662 */ 6663 blk_flush_plug(tsk->plug, true); 6664 6665 lock_map_release(&sched_map); 6666 } 6667 6668 static void sched_update_worker(struct task_struct *tsk) 6669 { 6670 if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_BLOCK_TS)) { 6671 if (tsk->flags & PF_BLOCK_TS) 6672 blk_plug_invalidate_ts(tsk); 6673 if (tsk->flags & PF_WQ_WORKER) 6674 wq_worker_running(tsk); 6675 else if (tsk->flags & PF_IO_WORKER) 6676 io_wq_worker_running(tsk); 6677 } 6678 } 6679 6680 static __always_inline void __schedule_loop(int sched_mode) 6681 { 6682 do { 6683 preempt_disable(); 6684 __schedule(sched_mode); 6685 sched_preempt_enable_no_resched(); 6686 } while (need_resched()); 6687 } 6688 6689 asmlinkage __visible void __sched schedule(void) 6690 { 6691 struct task_struct *tsk = current; 6692 6693 #ifdef CONFIG_RT_MUTEXES 6694 lockdep_assert(!tsk->sched_rt_mutex); 6695 #endif 6696 6697 if (!task_is_running(tsk)) 6698 sched_submit_work(tsk); 6699 __schedule_loop(SM_NONE); 6700 sched_update_worker(tsk); 6701 } 6702 EXPORT_SYMBOL(schedule); 6703 6704 /* 6705 * synchronize_rcu_tasks() makes sure that no task is stuck in preempted 6706 * state (have scheduled out non-voluntarily) by making sure that all 6707 * tasks have either left the run queue or have gone into user space. 6708 * As idle tasks do not do either, they must not ever be preempted 6709 * (schedule out non-voluntarily). 6710 * 6711 * schedule_idle() is similar to schedule_preempt_disable() except that it 6712 * never enables preemption because it does not call sched_submit_work(). 6713 */ 6714 void __sched schedule_idle(void) 6715 { 6716 /* 6717 * As this skips calling sched_submit_work(), which the idle task does 6718 * regardless because that function is a NOP when the task is in a 6719 * TASK_RUNNING state, make sure this isn't used someplace that the 6720 * current task can be in any other state. Note, idle is always in the 6721 * TASK_RUNNING state. 6722 */ 6723 WARN_ON_ONCE(current->__state); 6724 do { 6725 __schedule(SM_IDLE); 6726 } while (need_resched()); 6727 } 6728 6729 #if defined(CONFIG_CONTEXT_TRACKING_USER) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) 6730 asmlinkage __visible void __sched schedule_user(void) 6731 { 6732 /* 6733 * If we come here after a random call to set_need_resched(), 6734 * or we have been woken up remotely but the IPI has not yet arrived, 6735 * we haven't yet exited the RCU idle mode. Do it here manually until 6736 * we find a better solution. 6737 * 6738 * NB: There are buggy callers of this function. Ideally we 6739 * should warn if prev_state != CT_STATE_USER, but that will trigger 6740 * too frequently to make sense yet. 6741 */ 6742 enum ctx_state prev_state = exception_enter(); 6743 schedule(); 6744 exception_exit(prev_state); 6745 } 6746 #endif 6747 6748 /** 6749 * schedule_preempt_disabled - called with preemption disabled 6750 * 6751 * Returns with preemption disabled. Note: preempt_count must be 1 6752 */ 6753 void __sched schedule_preempt_disabled(void) 6754 { 6755 sched_preempt_enable_no_resched(); 6756 schedule(); 6757 preempt_disable(); 6758 } 6759 6760 #ifdef CONFIG_PREEMPT_RT 6761 void __sched notrace schedule_rtlock(void) 6762 { 6763 __schedule_loop(SM_RTLOCK_WAIT); 6764 } 6765 NOKPROBE_SYMBOL(schedule_rtlock); 6766 #endif 6767 6768 static void __sched notrace preempt_schedule_common(void) 6769 { 6770 do { 6771 /* 6772 * Because the function tracer can trace preempt_count_sub() 6773 * and it also uses preempt_enable/disable_notrace(), if 6774 * NEED_RESCHED is set, the preempt_enable_notrace() called 6775 * by the function tracer will call this function again and 6776 * cause infinite recursion. 6777 * 6778 * Preemption must be disabled here before the function 6779 * tracer can trace. Break up preempt_disable() into two 6780 * calls. One to disable preemption without fear of being 6781 * traced. The other to still record the preemption latency, 6782 * which can also be traced by the function tracer. 6783 */ 6784 preempt_disable_notrace(); 6785 preempt_latency_start(1); 6786 __schedule(SM_PREEMPT); 6787 preempt_latency_stop(1); 6788 preempt_enable_no_resched_notrace(); 6789 6790 /* 6791 * Check again in case we missed a preemption opportunity 6792 * between schedule and now. 6793 */ 6794 } while (need_resched()); 6795 } 6796 6797 #ifdef CONFIG_PREEMPTION 6798 /* 6799 * This is the entry point to schedule() from in-kernel preemption 6800 * off of preempt_enable. 6801 */ 6802 asmlinkage __visible void __sched notrace preempt_schedule(void) 6803 { 6804 /* 6805 * If there is a non-zero preempt_count or interrupts are disabled, 6806 * we do not want to preempt the current task. Just return.. 6807 */ 6808 if (likely(!preemptible())) 6809 return; 6810 preempt_schedule_common(); 6811 } 6812 NOKPROBE_SYMBOL(preempt_schedule); 6813 EXPORT_SYMBOL(preempt_schedule); 6814 6815 #ifdef CONFIG_PREEMPT_DYNAMIC 6816 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 6817 #ifndef preempt_schedule_dynamic_enabled 6818 #define preempt_schedule_dynamic_enabled preempt_schedule 6819 #define preempt_schedule_dynamic_disabled NULL 6820 #endif 6821 DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); 6822 EXPORT_STATIC_CALL_TRAMP(preempt_schedule); 6823 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 6824 static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule); 6825 void __sched notrace dynamic_preempt_schedule(void) 6826 { 6827 if (!static_branch_unlikely(&sk_dynamic_preempt_schedule)) 6828 return; 6829 preempt_schedule(); 6830 } 6831 NOKPROBE_SYMBOL(dynamic_preempt_schedule); 6832 EXPORT_SYMBOL(dynamic_preempt_schedule); 6833 #endif 6834 #endif 6835 6836 /** 6837 * preempt_schedule_notrace - preempt_schedule called by tracing 6838 * 6839 * The tracing infrastructure uses preempt_enable_notrace to prevent 6840 * recursion and tracing preempt enabling caused by the tracing 6841 * infrastructure itself. But as tracing can happen in areas coming 6842 * from userspace or just about to enter userspace, a preempt enable 6843 * can occur before user_exit() is called. This will cause the scheduler 6844 * to be called when the system is still in usermode. 6845 * 6846 * To prevent this, the preempt_enable_notrace will use this function 6847 * instead of preempt_schedule() to exit user context if needed before 6848 * calling the scheduler. 6849 */ 6850 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) 6851 { 6852 enum ctx_state prev_ctx; 6853 6854 if (likely(!preemptible())) 6855 return; 6856 6857 do { 6858 /* 6859 * Because the function tracer can trace preempt_count_sub() 6860 * and it also uses preempt_enable/disable_notrace(), if 6861 * NEED_RESCHED is set, the preempt_enable_notrace() called 6862 * by the function tracer will call this function again and 6863 * cause infinite recursion. 6864 * 6865 * Preemption must be disabled here before the function 6866 * tracer can trace. Break up preempt_disable() into two 6867 * calls. One to disable preemption without fear of being 6868 * traced. The other to still record the preemption latency, 6869 * which can also be traced by the function tracer. 6870 */ 6871 preempt_disable_notrace(); 6872 preempt_latency_start(1); 6873 /* 6874 * Needs preempt disabled in case user_exit() is traced 6875 * and the tracer calls preempt_enable_notrace() causing 6876 * an infinite recursion. 6877 */ 6878 prev_ctx = exception_enter(); 6879 __schedule(SM_PREEMPT); 6880 exception_exit(prev_ctx); 6881 6882 preempt_latency_stop(1); 6883 preempt_enable_no_resched_notrace(); 6884 } while (need_resched()); 6885 } 6886 EXPORT_SYMBOL_GPL(preempt_schedule_notrace); 6887 6888 #ifdef CONFIG_PREEMPT_DYNAMIC 6889 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 6890 #ifndef preempt_schedule_notrace_dynamic_enabled 6891 #define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace 6892 #define preempt_schedule_notrace_dynamic_disabled NULL 6893 #endif 6894 DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); 6895 EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); 6896 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 6897 static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace); 6898 void __sched notrace dynamic_preempt_schedule_notrace(void) 6899 { 6900 if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace)) 6901 return; 6902 preempt_schedule_notrace(); 6903 } 6904 NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace); 6905 EXPORT_SYMBOL(dynamic_preempt_schedule_notrace); 6906 #endif 6907 #endif 6908 6909 #endif /* CONFIG_PREEMPTION */ 6910 6911 /* 6912 * This is the entry point to schedule() from kernel preemption 6913 * off of IRQ context. 6914 * Note, that this is called and return with IRQs disabled. This will 6915 * protect us against recursive calling from IRQ contexts. 6916 */ 6917 asmlinkage __visible void __sched preempt_schedule_irq(void) 6918 { 6919 enum ctx_state prev_state; 6920 6921 /* Catch callers which need to be fixed */ 6922 BUG_ON(preempt_count() || !irqs_disabled()); 6923 6924 prev_state = exception_enter(); 6925 6926 do { 6927 preempt_disable(); 6928 local_irq_enable(); 6929 __schedule(SM_PREEMPT); 6930 local_irq_disable(); 6931 sched_preempt_enable_no_resched(); 6932 } while (need_resched()); 6933 6934 exception_exit(prev_state); 6935 } 6936 6937 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, 6938 void *key) 6939 { 6940 WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~(WF_SYNC|WF_CURRENT_CPU)); 6941 return try_to_wake_up(curr->private, mode, wake_flags); 6942 } 6943 EXPORT_SYMBOL(default_wake_function); 6944 6945 void __setscheduler_prio(struct task_struct *p, int prio) 6946 { 6947 if (dl_prio(prio)) 6948 p->sched_class = &dl_sched_class; 6949 else if (rt_prio(prio)) 6950 p->sched_class = &rt_sched_class; 6951 else 6952 p->sched_class = &fair_sched_class; 6953 6954 p->prio = prio; 6955 } 6956 6957 #ifdef CONFIG_RT_MUTEXES 6958 6959 /* 6960 * Would be more useful with typeof()/auto_type but they don't mix with 6961 * bit-fields. Since it's a local thing, use int. Keep the generic sounding 6962 * name such that if someone were to implement this function we get to compare 6963 * notes. 6964 */ 6965 #define fetch_and_set(x, v) ({ int _x = (x); (x) = (v); _x; }) 6966 6967 void rt_mutex_pre_schedule(void) 6968 { 6969 lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1)); 6970 sched_submit_work(current); 6971 } 6972 6973 void rt_mutex_schedule(void) 6974 { 6975 lockdep_assert(current->sched_rt_mutex); 6976 __schedule_loop(SM_NONE); 6977 } 6978 6979 void rt_mutex_post_schedule(void) 6980 { 6981 sched_update_worker(current); 6982 lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); 6983 } 6984 6985 /* 6986 * rt_mutex_setprio - set the current priority of a task 6987 * @p: task to boost 6988 * @pi_task: donor task 6989 * 6990 * This function changes the 'effective' priority of a task. It does 6991 * not touch ->normal_prio like __setscheduler(). 6992 * 6993 * Used by the rt_mutex code to implement priority inheritance 6994 * logic. Call site only calls if the priority of the task changed. 6995 */ 6996 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) 6997 { 6998 int prio, oldprio, queued, running, queue_flag = 6999 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 7000 const struct sched_class *prev_class; 7001 struct rq_flags rf; 7002 struct rq *rq; 7003 7004 /* XXX used to be waiter->prio, not waiter->task->prio */ 7005 prio = __rt_effective_prio(pi_task, p->normal_prio); 7006 7007 /* 7008 * If nothing changed; bail early. 7009 */ 7010 if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) 7011 return; 7012 7013 rq = __task_rq_lock(p, &rf); 7014 update_rq_clock(rq); 7015 /* 7016 * Set under pi_lock && rq->lock, such that the value can be used under 7017 * either lock. 7018 * 7019 * Note that there is loads of tricky to make this pointer cache work 7020 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to 7021 * ensure a task is de-boosted (pi_task is set to NULL) before the 7022 * task is allowed to run again (and can exit). This ensures the pointer 7023 * points to a blocked task -- which guarantees the task is present. 7024 */ 7025 p->pi_top_task = pi_task; 7026 7027 /* 7028 * For FIFO/RR we only need to set prio, if that matches we're done. 7029 */ 7030 if (prio == p->prio && !dl_prio(prio)) 7031 goto out_unlock; 7032 7033 /* 7034 * Idle task boosting is a no-no in general. There is one 7035 * exception, when PREEMPT_RT and NOHZ is active: 7036 * 7037 * The idle task calls get_next_timer_interrupt() and holds 7038 * the timer wheel base->lock on the CPU and another CPU wants 7039 * to access the timer (probably to cancel it). We can safely 7040 * ignore the boosting request, as the idle CPU runs this code 7041 * with interrupts disabled and will complete the lock 7042 * protected section without being interrupted. So there is no 7043 * real need to boost. 7044 */ 7045 if (unlikely(p == rq->idle)) { 7046 WARN_ON(p != rq->curr); 7047 WARN_ON(p->pi_blocked_on); 7048 goto out_unlock; 7049 } 7050 7051 trace_sched_pi_setprio(p, pi_task); 7052 oldprio = p->prio; 7053 7054 if (oldprio == prio) 7055 queue_flag &= ~DEQUEUE_MOVE; 7056 7057 prev_class = p->sched_class; 7058 queued = task_on_rq_queued(p); 7059 running = task_current(rq, p); 7060 if (queued) 7061 dequeue_task(rq, p, queue_flag); 7062 if (running) 7063 put_prev_task(rq, p); 7064 7065 /* 7066 * Boosting condition are: 7067 * 1. -rt task is running and holds mutex A 7068 * --> -dl task blocks on mutex A 7069 * 7070 * 2. -dl task is running and holds mutex A 7071 * --> -dl task blocks on mutex A and could preempt the 7072 * running task 7073 */ 7074 if (dl_prio(prio)) { 7075 if (!dl_prio(p->normal_prio) || 7076 (pi_task && dl_prio(pi_task->prio) && 7077 dl_entity_preempt(&pi_task->dl, &p->dl))) { 7078 p->dl.pi_se = pi_task->dl.pi_se; 7079 queue_flag |= ENQUEUE_REPLENISH; 7080 } else { 7081 p->dl.pi_se = &p->dl; 7082 } 7083 } else if (rt_prio(prio)) { 7084 if (dl_prio(oldprio)) 7085 p->dl.pi_se = &p->dl; 7086 if (oldprio < prio) 7087 queue_flag |= ENQUEUE_HEAD; 7088 } else { 7089 if (dl_prio(oldprio)) 7090 p->dl.pi_se = &p->dl; 7091 if (rt_prio(oldprio)) 7092 p->rt.timeout = 0; 7093 } 7094 7095 __setscheduler_prio(p, prio); 7096 7097 if (queued) 7098 enqueue_task(rq, p, queue_flag); 7099 if (running) 7100 set_next_task(rq, p); 7101 7102 check_class_changed(rq, p, prev_class, oldprio); 7103 out_unlock: 7104 /* Avoid rq from going away on us: */ 7105 preempt_disable(); 7106 7107 rq_unpin_lock(rq, &rf); 7108 __balance_callbacks(rq); 7109 raw_spin_rq_unlock(rq); 7110 7111 preempt_enable(); 7112 } 7113 #endif 7114 7115 #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) 7116 int __sched __cond_resched(void) 7117 { 7118 if (should_resched(0)) { 7119 preempt_schedule_common(); 7120 return 1; 7121 } 7122 /* 7123 * In preemptible kernels, ->rcu_read_lock_nesting tells the tick 7124 * whether the current CPU is in an RCU read-side critical section, 7125 * so the tick can report quiescent states even for CPUs looping 7126 * in kernel context. In contrast, in non-preemptible kernels, 7127 * RCU readers leave no in-memory hints, which means that CPU-bound 7128 * processes executing in kernel context might never report an 7129 * RCU quiescent state. Therefore, the following code causes 7130 * cond_resched() to report a quiescent state, but only when RCU 7131 * is in urgent need of one. 7132 */ 7133 #ifndef CONFIG_PREEMPT_RCU 7134 rcu_all_qs(); 7135 #endif 7136 return 0; 7137 } 7138 EXPORT_SYMBOL(__cond_resched); 7139 #endif 7140 7141 #ifdef CONFIG_PREEMPT_DYNAMIC 7142 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 7143 #define cond_resched_dynamic_enabled __cond_resched 7144 #define cond_resched_dynamic_disabled ((void *)&__static_call_return0) 7145 DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); 7146 EXPORT_STATIC_CALL_TRAMP(cond_resched); 7147 7148 #define might_resched_dynamic_enabled __cond_resched 7149 #define might_resched_dynamic_disabled ((void *)&__static_call_return0) 7150 DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); 7151 EXPORT_STATIC_CALL_TRAMP(might_resched); 7152 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 7153 static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); 7154 int __sched dynamic_cond_resched(void) 7155 { 7156 klp_sched_try_switch(); 7157 if (!static_branch_unlikely(&sk_dynamic_cond_resched)) 7158 return 0; 7159 return __cond_resched(); 7160 } 7161 EXPORT_SYMBOL(dynamic_cond_resched); 7162 7163 static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched); 7164 int __sched dynamic_might_resched(void) 7165 { 7166 if (!static_branch_unlikely(&sk_dynamic_might_resched)) 7167 return 0; 7168 return __cond_resched(); 7169 } 7170 EXPORT_SYMBOL(dynamic_might_resched); 7171 #endif 7172 #endif 7173 7174 /* 7175 * __cond_resched_lock() - if a reschedule is pending, drop the given lock, 7176 * call schedule, and on return reacquire the lock. 7177 * 7178 * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level 7179 * operations here to prevent schedule() from being called twice (once via 7180 * spin_unlock(), once by hand). 7181 */ 7182 int __cond_resched_lock(spinlock_t *lock) 7183 { 7184 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7185 int ret = 0; 7186 7187 lockdep_assert_held(lock); 7188 7189 if (spin_needbreak(lock) || resched) { 7190 spin_unlock(lock); 7191 if (!_cond_resched()) 7192 cpu_relax(); 7193 ret = 1; 7194 spin_lock(lock); 7195 } 7196 return ret; 7197 } 7198 EXPORT_SYMBOL(__cond_resched_lock); 7199 7200 int __cond_resched_rwlock_read(rwlock_t *lock) 7201 { 7202 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7203 int ret = 0; 7204 7205 lockdep_assert_held_read(lock); 7206 7207 if (rwlock_needbreak(lock) || resched) { 7208 read_unlock(lock); 7209 if (!_cond_resched()) 7210 cpu_relax(); 7211 ret = 1; 7212 read_lock(lock); 7213 } 7214 return ret; 7215 } 7216 EXPORT_SYMBOL(__cond_resched_rwlock_read); 7217 7218 int __cond_resched_rwlock_write(rwlock_t *lock) 7219 { 7220 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7221 int ret = 0; 7222 7223 lockdep_assert_held_write(lock); 7224 7225 if (rwlock_needbreak(lock) || resched) { 7226 write_unlock(lock); 7227 if (!_cond_resched()) 7228 cpu_relax(); 7229 ret = 1; 7230 write_lock(lock); 7231 } 7232 return ret; 7233 } 7234 EXPORT_SYMBOL(__cond_resched_rwlock_write); 7235 7236 #ifdef CONFIG_PREEMPT_DYNAMIC 7237 7238 #ifdef CONFIG_GENERIC_ENTRY 7239 #include <linux/entry-common.h> 7240 #endif 7241 7242 /* 7243 * SC:cond_resched 7244 * SC:might_resched 7245 * SC:preempt_schedule 7246 * SC:preempt_schedule_notrace 7247 * SC:irqentry_exit_cond_resched 7248 * 7249 * 7250 * NONE: 7251 * cond_resched <- __cond_resched 7252 * might_resched <- RET0 7253 * preempt_schedule <- NOP 7254 * preempt_schedule_notrace <- NOP 7255 * irqentry_exit_cond_resched <- NOP 7256 * 7257 * VOLUNTARY: 7258 * cond_resched <- __cond_resched 7259 * might_resched <- __cond_resched 7260 * preempt_schedule <- NOP 7261 * preempt_schedule_notrace <- NOP 7262 * irqentry_exit_cond_resched <- NOP 7263 * 7264 * FULL: 7265 * cond_resched <- RET0 7266 * might_resched <- RET0 7267 * preempt_schedule <- preempt_schedule 7268 * preempt_schedule_notrace <- preempt_schedule_notrace 7269 * irqentry_exit_cond_resched <- irqentry_exit_cond_resched 7270 */ 7271 7272 enum { 7273 preempt_dynamic_undefined = -1, 7274 preempt_dynamic_none, 7275 preempt_dynamic_voluntary, 7276 preempt_dynamic_full, 7277 }; 7278 7279 int preempt_dynamic_mode = preempt_dynamic_undefined; 7280 7281 int sched_dynamic_mode(const char *str) 7282 { 7283 if (!strcmp(str, "none")) 7284 return preempt_dynamic_none; 7285 7286 if (!strcmp(str, "voluntary")) 7287 return preempt_dynamic_voluntary; 7288 7289 if (!strcmp(str, "full")) 7290 return preempt_dynamic_full; 7291 7292 return -EINVAL; 7293 } 7294 7295 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 7296 #define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled) 7297 #define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled) 7298 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 7299 #define preempt_dynamic_enable(f) static_key_enable(&sk_dynamic_##f.key) 7300 #define preempt_dynamic_disable(f) static_key_disable(&sk_dynamic_##f.key) 7301 #else 7302 #error "Unsupported PREEMPT_DYNAMIC mechanism" 7303 #endif 7304 7305 static DEFINE_MUTEX(sched_dynamic_mutex); 7306 static bool klp_override; 7307 7308 static void __sched_dynamic_update(int mode) 7309 { 7310 /* 7311 * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in 7312 * the ZERO state, which is invalid. 7313 */ 7314 if (!klp_override) 7315 preempt_dynamic_enable(cond_resched); 7316 preempt_dynamic_enable(might_resched); 7317 preempt_dynamic_enable(preempt_schedule); 7318 preempt_dynamic_enable(preempt_schedule_notrace); 7319 preempt_dynamic_enable(irqentry_exit_cond_resched); 7320 7321 switch (mode) { 7322 case preempt_dynamic_none: 7323 if (!klp_override) 7324 preempt_dynamic_enable(cond_resched); 7325 preempt_dynamic_disable(might_resched); 7326 preempt_dynamic_disable(preempt_schedule); 7327 preempt_dynamic_disable(preempt_schedule_notrace); 7328 preempt_dynamic_disable(irqentry_exit_cond_resched); 7329 if (mode != preempt_dynamic_mode) 7330 pr_info("Dynamic Preempt: none\n"); 7331 break; 7332 7333 case preempt_dynamic_voluntary: 7334 if (!klp_override) 7335 preempt_dynamic_enable(cond_resched); 7336 preempt_dynamic_enable(might_resched); 7337 preempt_dynamic_disable(preempt_schedule); 7338 preempt_dynamic_disable(preempt_schedule_notrace); 7339 preempt_dynamic_disable(irqentry_exit_cond_resched); 7340 if (mode != preempt_dynamic_mode) 7341 pr_info("Dynamic Preempt: voluntary\n"); 7342 break; 7343 7344 case preempt_dynamic_full: 7345 if (!klp_override) 7346 preempt_dynamic_disable(cond_resched); 7347 preempt_dynamic_disable(might_resched); 7348 preempt_dynamic_enable(preempt_schedule); 7349 preempt_dynamic_enable(preempt_schedule_notrace); 7350 preempt_dynamic_enable(irqentry_exit_cond_resched); 7351 if (mode != preempt_dynamic_mode) 7352 pr_info("Dynamic Preempt: full\n"); 7353 break; 7354 } 7355 7356 preempt_dynamic_mode = mode; 7357 } 7358 7359 void sched_dynamic_update(int mode) 7360 { 7361 mutex_lock(&sched_dynamic_mutex); 7362 __sched_dynamic_update(mode); 7363 mutex_unlock(&sched_dynamic_mutex); 7364 } 7365 7366 #ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL 7367 7368 static int klp_cond_resched(void) 7369 { 7370 __klp_sched_try_switch(); 7371 return __cond_resched(); 7372 } 7373 7374 void sched_dynamic_klp_enable(void) 7375 { 7376 mutex_lock(&sched_dynamic_mutex); 7377 7378 klp_override = true; 7379 static_call_update(cond_resched, klp_cond_resched); 7380 7381 mutex_unlock(&sched_dynamic_mutex); 7382 } 7383 7384 void sched_dynamic_klp_disable(void) 7385 { 7386 mutex_lock(&sched_dynamic_mutex); 7387 7388 klp_override = false; 7389 __sched_dynamic_update(preempt_dynamic_mode); 7390 7391 mutex_unlock(&sched_dynamic_mutex); 7392 } 7393 7394 #endif /* CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ 7395 7396 static int __init setup_preempt_mode(char *str) 7397 { 7398 int mode = sched_dynamic_mode(str); 7399 if (mode < 0) { 7400 pr_warn("Dynamic Preempt: unsupported mode: %s\n", str); 7401 return 0; 7402 } 7403 7404 sched_dynamic_update(mode); 7405 return 1; 7406 } 7407 __setup("preempt=", setup_preempt_mode); 7408 7409 static void __init preempt_dynamic_init(void) 7410 { 7411 if (preempt_dynamic_mode == preempt_dynamic_undefined) { 7412 if (IS_ENABLED(CONFIG_PREEMPT_NONE)) { 7413 sched_dynamic_update(preempt_dynamic_none); 7414 } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) { 7415 sched_dynamic_update(preempt_dynamic_voluntary); 7416 } else { 7417 /* Default static call setting, nothing to do */ 7418 WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT)); 7419 preempt_dynamic_mode = preempt_dynamic_full; 7420 pr_info("Dynamic Preempt: full\n"); 7421 } 7422 } 7423 } 7424 7425 #define PREEMPT_MODEL_ACCESSOR(mode) \ 7426 bool preempt_model_##mode(void) \ 7427 { \ 7428 WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \ 7429 return preempt_dynamic_mode == preempt_dynamic_##mode; \ 7430 } \ 7431 EXPORT_SYMBOL_GPL(preempt_model_##mode) 7432 7433 PREEMPT_MODEL_ACCESSOR(none); 7434 PREEMPT_MODEL_ACCESSOR(voluntary); 7435 PREEMPT_MODEL_ACCESSOR(full); 7436 7437 #else /* !CONFIG_PREEMPT_DYNAMIC: */ 7438 7439 static inline void preempt_dynamic_init(void) { } 7440 7441 #endif /* CONFIG_PREEMPT_DYNAMIC */ 7442 7443 int io_schedule_prepare(void) 7444 { 7445 int old_iowait = current->in_iowait; 7446 7447 current->in_iowait = 1; 7448 blk_flush_plug(current->plug, true); 7449 return old_iowait; 7450 } 7451 7452 void io_schedule_finish(int token) 7453 { 7454 current->in_iowait = token; 7455 } 7456 7457 /* 7458 * This task is about to go to sleep on IO. Increment rq->nr_iowait so 7459 * that process accounting knows that this is a task in IO wait state. 7460 */ 7461 long __sched io_schedule_timeout(long timeout) 7462 { 7463 int token; 7464 long ret; 7465 7466 token = io_schedule_prepare(); 7467 ret = schedule_timeout(timeout); 7468 io_schedule_finish(token); 7469 7470 return ret; 7471 } 7472 EXPORT_SYMBOL(io_schedule_timeout); 7473 7474 void __sched io_schedule(void) 7475 { 7476 int token; 7477 7478 token = io_schedule_prepare(); 7479 schedule(); 7480 io_schedule_finish(token); 7481 } 7482 EXPORT_SYMBOL(io_schedule); 7483 7484 void sched_show_task(struct task_struct *p) 7485 { 7486 unsigned long free = 0; 7487 int ppid; 7488 7489 if (!try_get_task_stack(p)) 7490 return; 7491 7492 pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p)); 7493 7494 if (task_is_running(p)) 7495 pr_cont(" running task "); 7496 #ifdef CONFIG_DEBUG_STACK_USAGE 7497 free = stack_not_used(p); 7498 #endif 7499 ppid = 0; 7500 rcu_read_lock(); 7501 if (pid_alive(p)) 7502 ppid = task_pid_nr(rcu_dereference(p->real_parent)); 7503 rcu_read_unlock(); 7504 pr_cont(" stack:%-5lu pid:%-5d tgid:%-5d ppid:%-6d flags:0x%08lx\n", 7505 free, task_pid_nr(p), task_tgid_nr(p), 7506 ppid, read_task_thread_flags(p)); 7507 7508 print_worker_info(KERN_INFO, p); 7509 print_stop_info(KERN_INFO, p); 7510 show_stack(p, NULL, KERN_INFO); 7511 put_task_stack(p); 7512 } 7513 EXPORT_SYMBOL_GPL(sched_show_task); 7514 7515 static inline bool 7516 state_filter_match(unsigned long state_filter, struct task_struct *p) 7517 { 7518 unsigned int state = READ_ONCE(p->__state); 7519 7520 /* no filter, everything matches */ 7521 if (!state_filter) 7522 return true; 7523 7524 /* filter, but doesn't match */ 7525 if (!(state & state_filter)) 7526 return false; 7527 7528 /* 7529 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows 7530 * TASK_KILLABLE). 7531 */ 7532 if (state_filter == TASK_UNINTERRUPTIBLE && (state & TASK_NOLOAD)) 7533 return false; 7534 7535 return true; 7536 } 7537 7538 7539 void show_state_filter(unsigned int state_filter) 7540 { 7541 struct task_struct *g, *p; 7542 7543 rcu_read_lock(); 7544 for_each_process_thread(g, p) { 7545 /* 7546 * reset the NMI-timeout, listing all files on a slow 7547 * console might take a lot of time: 7548 * Also, reset softlockup watchdogs on all CPUs, because 7549 * another CPU might be blocked waiting for us to process 7550 * an IPI. 7551 */ 7552 touch_nmi_watchdog(); 7553 touch_all_softlockup_watchdogs(); 7554 if (state_filter_match(state_filter, p)) 7555 sched_show_task(p); 7556 } 7557 7558 #ifdef CONFIG_SCHED_DEBUG 7559 if (!state_filter) 7560 sysrq_sched_debug_show(); 7561 #endif 7562 rcu_read_unlock(); 7563 /* 7564 * Only show locks if all tasks are dumped: 7565 */ 7566 if (!state_filter) 7567 debug_show_all_locks(); 7568 } 7569 7570 /** 7571 * init_idle - set up an idle thread for a given CPU 7572 * @idle: task in question 7573 * @cpu: CPU the idle task belongs to 7574 * 7575 * NOTE: this function does not set the idle thread's NEED_RESCHED 7576 * flag, to make booting more robust. 7577 */ 7578 void __init init_idle(struct task_struct *idle, int cpu) 7579 { 7580 #ifdef CONFIG_SMP 7581 struct affinity_context ac = (struct affinity_context) { 7582 .new_mask = cpumask_of(cpu), 7583 .flags = 0, 7584 }; 7585 #endif 7586 struct rq *rq = cpu_rq(cpu); 7587 unsigned long flags; 7588 7589 __sched_fork(0, idle); 7590 7591 raw_spin_lock_irqsave(&idle->pi_lock, flags); 7592 raw_spin_rq_lock(rq); 7593 7594 idle->__state = TASK_RUNNING; 7595 idle->se.exec_start = sched_clock(); 7596 /* 7597 * PF_KTHREAD should already be set at this point; regardless, make it 7598 * look like a proper per-CPU kthread. 7599 */ 7600 idle->flags |= PF_KTHREAD | PF_NO_SETAFFINITY; 7601 kthread_set_per_cpu(idle, cpu); 7602 7603 #ifdef CONFIG_SMP 7604 /* 7605 * It's possible that init_idle() gets called multiple times on a task, 7606 * in that case do_set_cpus_allowed() will not do the right thing. 7607 * 7608 * And since this is boot we can forgo the serialization. 7609 */ 7610 set_cpus_allowed_common(idle, &ac); 7611 #endif 7612 /* 7613 * We're having a chicken and egg problem, even though we are 7614 * holding rq->lock, the CPU isn't yet set to this CPU so the 7615 * lockdep check in task_group() will fail. 7616 * 7617 * Similar case to sched_fork(). / Alternatively we could 7618 * use task_rq_lock() here and obtain the other rq->lock. 7619 * 7620 * Silence PROVE_RCU 7621 */ 7622 rcu_read_lock(); 7623 __set_task_cpu(idle, cpu); 7624 rcu_read_unlock(); 7625 7626 rq->idle = idle; 7627 rcu_assign_pointer(rq->curr, idle); 7628 idle->on_rq = TASK_ON_RQ_QUEUED; 7629 #ifdef CONFIG_SMP 7630 idle->on_cpu = 1; 7631 #endif 7632 raw_spin_rq_unlock(rq); 7633 raw_spin_unlock_irqrestore(&idle->pi_lock, flags); 7634 7635 /* Set the preempt count _outside_ the spinlocks! */ 7636 init_idle_preempt_count(idle, cpu); 7637 7638 /* 7639 * The idle tasks have their own, simple scheduling class: 7640 */ 7641 idle->sched_class = &idle_sched_class; 7642 ftrace_graph_init_idle_task(idle, cpu); 7643 vtime_init_idle(idle, cpu); 7644 #ifdef CONFIG_SMP 7645 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); 7646 #endif 7647 } 7648 7649 #ifdef CONFIG_SMP 7650 7651 int cpuset_cpumask_can_shrink(const struct cpumask *cur, 7652 const struct cpumask *trial) 7653 { 7654 int ret = 1; 7655 7656 if (cpumask_empty(cur)) 7657 return ret; 7658 7659 ret = dl_cpuset_cpumask_can_shrink(cur, trial); 7660 7661 return ret; 7662 } 7663 7664 int task_can_attach(struct task_struct *p) 7665 { 7666 int ret = 0; 7667 7668 /* 7669 * Kthreads which disallow setaffinity shouldn't be moved 7670 * to a new cpuset; we don't want to change their CPU 7671 * affinity and isolating such threads by their set of 7672 * allowed nodes is unnecessary. Thus, cpusets are not 7673 * applicable for such threads. This prevents checking for 7674 * success of set_cpus_allowed_ptr() on all attached tasks 7675 * before cpus_mask may be changed. 7676 */ 7677 if (p->flags & PF_NO_SETAFFINITY) 7678 ret = -EINVAL; 7679 7680 return ret; 7681 } 7682 7683 bool sched_smp_initialized __read_mostly; 7684 7685 #ifdef CONFIG_NUMA_BALANCING 7686 /* Migrate current task p to target_cpu */ 7687 int migrate_task_to(struct task_struct *p, int target_cpu) 7688 { 7689 struct migration_arg arg = { p, target_cpu }; 7690 int curr_cpu = task_cpu(p); 7691 7692 if (curr_cpu == target_cpu) 7693 return 0; 7694 7695 if (!cpumask_test_cpu(target_cpu, p->cpus_ptr)) 7696 return -EINVAL; 7697 7698 /* TODO: This is not properly updating schedstats */ 7699 7700 trace_sched_move_numa(p, curr_cpu, target_cpu); 7701 return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); 7702 } 7703 7704 /* 7705 * Requeue a task on a given node and accurately track the number of NUMA 7706 * tasks on the runqueues 7707 */ 7708 void sched_setnuma(struct task_struct *p, int nid) 7709 { 7710 bool queued, running; 7711 struct rq_flags rf; 7712 struct rq *rq; 7713 7714 rq = task_rq_lock(p, &rf); 7715 queued = task_on_rq_queued(p); 7716 running = task_current(rq, p); 7717 7718 if (queued) 7719 dequeue_task(rq, p, DEQUEUE_SAVE); 7720 if (running) 7721 put_prev_task(rq, p); 7722 7723 p->numa_preferred_nid = nid; 7724 7725 if (queued) 7726 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 7727 if (running) 7728 set_next_task(rq, p); 7729 task_rq_unlock(rq, p, &rf); 7730 } 7731 #endif /* CONFIG_NUMA_BALANCING */ 7732 7733 #ifdef CONFIG_HOTPLUG_CPU 7734 /* 7735 * Ensure that the idle task is using init_mm right before its CPU goes 7736 * offline. 7737 */ 7738 void idle_task_exit(void) 7739 { 7740 struct mm_struct *mm = current->active_mm; 7741 7742 BUG_ON(cpu_online(smp_processor_id())); 7743 BUG_ON(current != this_rq()->idle); 7744 7745 if (mm != &init_mm) { 7746 switch_mm(mm, &init_mm, current); 7747 finish_arch_post_lock_switch(); 7748 } 7749 7750 /* finish_cpu(), as ran on the BP, will clean up the active_mm state */ 7751 } 7752 7753 static int __balance_push_cpu_stop(void *arg) 7754 { 7755 struct task_struct *p = arg; 7756 struct rq *rq = this_rq(); 7757 struct rq_flags rf; 7758 int cpu; 7759 7760 raw_spin_lock_irq(&p->pi_lock); 7761 rq_lock(rq, &rf); 7762 7763 update_rq_clock(rq); 7764 7765 if (task_rq(p) == rq && task_on_rq_queued(p)) { 7766 cpu = select_fallback_rq(rq->cpu, p); 7767 rq = __migrate_task(rq, &rf, p, cpu); 7768 } 7769 7770 rq_unlock(rq, &rf); 7771 raw_spin_unlock_irq(&p->pi_lock); 7772 7773 put_task_struct(p); 7774 7775 return 0; 7776 } 7777 7778 static DEFINE_PER_CPU(struct cpu_stop_work, push_work); 7779 7780 /* 7781 * Ensure we only run per-cpu kthreads once the CPU goes !active. 7782 * 7783 * This is enabled below SCHED_AP_ACTIVE; when !cpu_active(), but only 7784 * effective when the hotplug motion is down. 7785 */ 7786 static void balance_push(struct rq *rq) 7787 { 7788 struct task_struct *push_task = rq->curr; 7789 7790 lockdep_assert_rq_held(rq); 7791 7792 /* 7793 * Ensure the thing is persistent until balance_push_set(.on = false); 7794 */ 7795 rq->balance_callback = &balance_push_callback; 7796 7797 /* 7798 * Only active while going offline and when invoked on the outgoing 7799 * CPU. 7800 */ 7801 if (!cpu_dying(rq->cpu) || rq != this_rq()) 7802 return; 7803 7804 /* 7805 * Both the cpu-hotplug and stop task are in this case and are 7806 * required to complete the hotplug process. 7807 */ 7808 if (kthread_is_per_cpu(push_task) || 7809 is_migration_disabled(push_task)) { 7810 7811 /* 7812 * If this is the idle task on the outgoing CPU try to wake 7813 * up the hotplug control thread which might wait for the 7814 * last task to vanish. The rcuwait_active() check is 7815 * accurate here because the waiter is pinned on this CPU 7816 * and can't obviously be running in parallel. 7817 * 7818 * On RT kernels this also has to check whether there are 7819 * pinned and scheduled out tasks on the runqueue. They 7820 * need to leave the migrate disabled section first. 7821 */ 7822 if (!rq->nr_running && !rq_has_pinned_tasks(rq) && 7823 rcuwait_active(&rq->hotplug_wait)) { 7824 raw_spin_rq_unlock(rq); 7825 rcuwait_wake_up(&rq->hotplug_wait); 7826 raw_spin_rq_lock(rq); 7827 } 7828 return; 7829 } 7830 7831 get_task_struct(push_task); 7832 /* 7833 * Temporarily drop rq->lock such that we can wake-up the stop task. 7834 * Both preemption and IRQs are still disabled. 7835 */ 7836 preempt_disable(); 7837 raw_spin_rq_unlock(rq); 7838 stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task, 7839 this_cpu_ptr(&push_work)); 7840 preempt_enable(); 7841 /* 7842 * At this point need_resched() is true and we'll take the loop in 7843 * schedule(). The next pick is obviously going to be the stop task 7844 * which kthread_is_per_cpu() and will push this task away. 7845 */ 7846 raw_spin_rq_lock(rq); 7847 } 7848 7849 static void balance_push_set(int cpu, bool on) 7850 { 7851 struct rq *rq = cpu_rq(cpu); 7852 struct rq_flags rf; 7853 7854 rq_lock_irqsave(rq, &rf); 7855 if (on) { 7856 WARN_ON_ONCE(rq->balance_callback); 7857 rq->balance_callback = &balance_push_callback; 7858 } else if (rq->balance_callback == &balance_push_callback) { 7859 rq->balance_callback = NULL; 7860 } 7861 rq_unlock_irqrestore(rq, &rf); 7862 } 7863 7864 /* 7865 * Invoked from a CPUs hotplug control thread after the CPU has been marked 7866 * inactive. All tasks which are not per CPU kernel threads are either 7867 * pushed off this CPU now via balance_push() or placed on a different CPU 7868 * during wakeup. Wait until the CPU is quiescent. 7869 */ 7870 static void balance_hotplug_wait(void) 7871 { 7872 struct rq *rq = this_rq(); 7873 7874 rcuwait_wait_event(&rq->hotplug_wait, 7875 rq->nr_running == 1 && !rq_has_pinned_tasks(rq), 7876 TASK_UNINTERRUPTIBLE); 7877 } 7878 7879 #else 7880 7881 static inline void balance_push(struct rq *rq) 7882 { 7883 } 7884 7885 static inline void balance_push_set(int cpu, bool on) 7886 { 7887 } 7888 7889 static inline void balance_hotplug_wait(void) 7890 { 7891 } 7892 7893 #endif /* CONFIG_HOTPLUG_CPU */ 7894 7895 void set_rq_online(struct rq *rq) 7896 { 7897 if (!rq->online) { 7898 const struct sched_class *class; 7899 7900 cpumask_set_cpu(rq->cpu, rq->rd->online); 7901 rq->online = 1; 7902 7903 for_each_class(class) { 7904 if (class->rq_online) 7905 class->rq_online(rq); 7906 } 7907 } 7908 } 7909 7910 void set_rq_offline(struct rq *rq) 7911 { 7912 if (rq->online) { 7913 const struct sched_class *class; 7914 7915 update_rq_clock(rq); 7916 for_each_class(class) { 7917 if (class->rq_offline) 7918 class->rq_offline(rq); 7919 } 7920 7921 cpumask_clear_cpu(rq->cpu, rq->rd->online); 7922 rq->online = 0; 7923 } 7924 } 7925 7926 static inline void sched_set_rq_online(struct rq *rq, int cpu) 7927 { 7928 struct rq_flags rf; 7929 7930 rq_lock_irqsave(rq, &rf); 7931 if (rq->rd) { 7932 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 7933 set_rq_online(rq); 7934 } 7935 rq_unlock_irqrestore(rq, &rf); 7936 } 7937 7938 static inline void sched_set_rq_offline(struct rq *rq, int cpu) 7939 { 7940 struct rq_flags rf; 7941 7942 rq_lock_irqsave(rq, &rf); 7943 if (rq->rd) { 7944 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 7945 set_rq_offline(rq); 7946 } 7947 rq_unlock_irqrestore(rq, &rf); 7948 } 7949 7950 /* 7951 * used to mark begin/end of suspend/resume: 7952 */ 7953 static int num_cpus_frozen; 7954 7955 /* 7956 * Update cpusets according to cpu_active mask. If cpusets are 7957 * disabled, cpuset_update_active_cpus() becomes a simple wrapper 7958 * around partition_sched_domains(). 7959 * 7960 * If we come here as part of a suspend/resume, don't touch cpusets because we 7961 * want to restore it back to its original state upon resume anyway. 7962 */ 7963 static void cpuset_cpu_active(void) 7964 { 7965 if (cpuhp_tasks_frozen) { 7966 /* 7967 * num_cpus_frozen tracks how many CPUs are involved in suspend 7968 * resume sequence. As long as this is not the last online 7969 * operation in the resume sequence, just build a single sched 7970 * domain, ignoring cpusets. 7971 */ 7972 partition_sched_domains(1, NULL, NULL); 7973 if (--num_cpus_frozen) 7974 return; 7975 /* 7976 * This is the last CPU online operation. So fall through and 7977 * restore the original sched domains by considering the 7978 * cpuset configurations. 7979 */ 7980 cpuset_force_rebuild(); 7981 } 7982 cpuset_update_active_cpus(); 7983 } 7984 7985 static int cpuset_cpu_inactive(unsigned int cpu) 7986 { 7987 if (!cpuhp_tasks_frozen) { 7988 int ret = dl_bw_check_overflow(cpu); 7989 7990 if (ret) 7991 return ret; 7992 cpuset_update_active_cpus(); 7993 } else { 7994 num_cpus_frozen++; 7995 partition_sched_domains(1, NULL, NULL); 7996 } 7997 return 0; 7998 } 7999 8000 static inline void sched_smt_present_inc(int cpu) 8001 { 8002 #ifdef CONFIG_SCHED_SMT 8003 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 8004 static_branch_inc_cpuslocked(&sched_smt_present); 8005 #endif 8006 } 8007 8008 static inline void sched_smt_present_dec(int cpu) 8009 { 8010 #ifdef CONFIG_SCHED_SMT 8011 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 8012 static_branch_dec_cpuslocked(&sched_smt_present); 8013 #endif 8014 } 8015 8016 int sched_cpu_activate(unsigned int cpu) 8017 { 8018 struct rq *rq = cpu_rq(cpu); 8019 8020 /* 8021 * Clear the balance_push callback and prepare to schedule 8022 * regular tasks. 8023 */ 8024 balance_push_set(cpu, false); 8025 8026 /* 8027 * When going up, increment the number of cores with SMT present. 8028 */ 8029 sched_smt_present_inc(cpu); 8030 set_cpu_active(cpu, true); 8031 8032 if (sched_smp_initialized) { 8033 sched_update_numa(cpu, true); 8034 sched_domains_numa_masks_set(cpu); 8035 cpuset_cpu_active(); 8036 } 8037 8038 /* 8039 * Put the rq online, if not already. This happens: 8040 * 8041 * 1) In the early boot process, because we build the real domains 8042 * after all CPUs have been brought up. 8043 * 8044 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the 8045 * domains. 8046 */ 8047 sched_set_rq_online(rq, cpu); 8048 8049 return 0; 8050 } 8051 8052 int sched_cpu_deactivate(unsigned int cpu) 8053 { 8054 struct rq *rq = cpu_rq(cpu); 8055 int ret; 8056 8057 /* 8058 * Remove CPU from nohz.idle_cpus_mask to prevent participating in 8059 * load balancing when not active 8060 */ 8061 nohz_balance_exit_idle(rq); 8062 8063 set_cpu_active(cpu, false); 8064 8065 /* 8066 * From this point forward, this CPU will refuse to run any task that 8067 * is not: migrate_disable() or KTHREAD_IS_PER_CPU, and will actively 8068 * push those tasks away until this gets cleared, see 8069 * sched_cpu_dying(). 8070 */ 8071 balance_push_set(cpu, true); 8072 8073 /* 8074 * We've cleared cpu_active_mask / set balance_push, wait for all 8075 * preempt-disabled and RCU users of this state to go away such that 8076 * all new such users will observe it. 8077 * 8078 * Specifically, we rely on ttwu to no longer target this CPU, see 8079 * ttwu_queue_cond() and is_cpu_allowed(). 8080 * 8081 * Do sync before park smpboot threads to take care the RCU boost case. 8082 */ 8083 synchronize_rcu(); 8084 8085 sched_set_rq_offline(rq, cpu); 8086 8087 /* 8088 * When going down, decrement the number of cores with SMT present. 8089 */ 8090 sched_smt_present_dec(cpu); 8091 8092 #ifdef CONFIG_SCHED_SMT 8093 sched_core_cpu_deactivate(cpu); 8094 #endif 8095 8096 if (!sched_smp_initialized) 8097 return 0; 8098 8099 sched_update_numa(cpu, false); 8100 ret = cpuset_cpu_inactive(cpu); 8101 if (ret) { 8102 sched_smt_present_inc(cpu); 8103 sched_set_rq_online(rq, cpu); 8104 balance_push_set(cpu, false); 8105 set_cpu_active(cpu, true); 8106 sched_update_numa(cpu, true); 8107 return ret; 8108 } 8109 sched_domains_numa_masks_clear(cpu); 8110 return 0; 8111 } 8112 8113 static void sched_rq_cpu_starting(unsigned int cpu) 8114 { 8115 struct rq *rq = cpu_rq(cpu); 8116 8117 rq->calc_load_update = calc_load_update; 8118 update_max_interval(); 8119 } 8120 8121 int sched_cpu_starting(unsigned int cpu) 8122 { 8123 sched_core_cpu_starting(cpu); 8124 sched_rq_cpu_starting(cpu); 8125 sched_tick_start(cpu); 8126 return 0; 8127 } 8128 8129 #ifdef CONFIG_HOTPLUG_CPU 8130 8131 /* 8132 * Invoked immediately before the stopper thread is invoked to bring the 8133 * CPU down completely. At this point all per CPU kthreads except the 8134 * hotplug thread (current) and the stopper thread (inactive) have been 8135 * either parked or have been unbound from the outgoing CPU. Ensure that 8136 * any of those which might be on the way out are gone. 8137 * 8138 * If after this point a bound task is being woken on this CPU then the 8139 * responsible hotplug callback has failed to do it's job. 8140 * sched_cpu_dying() will catch it with the appropriate fireworks. 8141 */ 8142 int sched_cpu_wait_empty(unsigned int cpu) 8143 { 8144 balance_hotplug_wait(); 8145 return 0; 8146 } 8147 8148 /* 8149 * Since this CPU is going 'away' for a while, fold any nr_active delta we 8150 * might have. Called from the CPU stopper task after ensuring that the 8151 * stopper is the last running task on the CPU, so nr_active count is 8152 * stable. We need to take the tear-down thread which is calling this into 8153 * account, so we hand in adjust = 1 to the load calculation. 8154 * 8155 * Also see the comment "Global load-average calculations". 8156 */ 8157 static void calc_load_migrate(struct rq *rq) 8158 { 8159 long delta = calc_load_fold_active(rq, 1); 8160 8161 if (delta) 8162 atomic_long_add(delta, &calc_load_tasks); 8163 } 8164 8165 static void dump_rq_tasks(struct rq *rq, const char *loglvl) 8166 { 8167 struct task_struct *g, *p; 8168 int cpu = cpu_of(rq); 8169 8170 lockdep_assert_rq_held(rq); 8171 8172 printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running); 8173 for_each_process_thread(g, p) { 8174 if (task_cpu(p) != cpu) 8175 continue; 8176 8177 if (!task_on_rq_queued(p)) 8178 continue; 8179 8180 printk("%s\tpid: %d, name: %s\n", loglvl, p->pid, p->comm); 8181 } 8182 } 8183 8184 int sched_cpu_dying(unsigned int cpu) 8185 { 8186 struct rq *rq = cpu_rq(cpu); 8187 struct rq_flags rf; 8188 8189 /* Handle pending wakeups and then migrate everything off */ 8190 sched_tick_stop(cpu); 8191 8192 rq_lock_irqsave(rq, &rf); 8193 if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) { 8194 WARN(true, "Dying CPU not properly vacated!"); 8195 dump_rq_tasks(rq, KERN_WARNING); 8196 } 8197 rq_unlock_irqrestore(rq, &rf); 8198 8199 calc_load_migrate(rq); 8200 update_max_interval(); 8201 hrtick_clear(rq); 8202 sched_core_cpu_dying(cpu); 8203 return 0; 8204 } 8205 #endif 8206 8207 void __init sched_init_smp(void) 8208 { 8209 sched_init_numa(NUMA_NO_NODE); 8210 8211 /* 8212 * There's no userspace yet to cause hotplug operations; hence all the 8213 * CPU masks are stable and all blatant races in the below code cannot 8214 * happen. 8215 */ 8216 mutex_lock(&sched_domains_mutex); 8217 sched_init_domains(cpu_active_mask); 8218 mutex_unlock(&sched_domains_mutex); 8219 8220 /* Move init over to a non-isolated CPU */ 8221 if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0) 8222 BUG(); 8223 current->flags &= ~PF_NO_SETAFFINITY; 8224 sched_init_granularity(); 8225 8226 init_sched_rt_class(); 8227 init_sched_dl_class(); 8228 8229 sched_smp_initialized = true; 8230 } 8231 8232 static int __init migration_init(void) 8233 { 8234 sched_cpu_starting(smp_processor_id()); 8235 return 0; 8236 } 8237 early_initcall(migration_init); 8238 8239 #else 8240 void __init sched_init_smp(void) 8241 { 8242 sched_init_granularity(); 8243 } 8244 #endif /* CONFIG_SMP */ 8245 8246 int in_sched_functions(unsigned long addr) 8247 { 8248 return in_lock_functions(addr) || 8249 (addr >= (unsigned long)__sched_text_start 8250 && addr < (unsigned long)__sched_text_end); 8251 } 8252 8253 #ifdef CONFIG_CGROUP_SCHED 8254 /* 8255 * Default task group. 8256 * Every task in system belongs to this group at bootup. 8257 */ 8258 struct task_group root_task_group; 8259 LIST_HEAD(task_groups); 8260 8261 /* Cacheline aligned slab cache for task_group */ 8262 static struct kmem_cache *task_group_cache __ro_after_init; 8263 #endif 8264 8265 void __init sched_init(void) 8266 { 8267 unsigned long ptr = 0; 8268 int i; 8269 8270 /* Make sure the linker didn't screw up */ 8271 BUG_ON(&idle_sched_class != &fair_sched_class + 1 || 8272 &fair_sched_class != &rt_sched_class + 1 || 8273 &rt_sched_class != &dl_sched_class + 1); 8274 #ifdef CONFIG_SMP 8275 BUG_ON(&dl_sched_class != &stop_sched_class + 1); 8276 #endif 8277 8278 wait_bit_init(); 8279 8280 #ifdef CONFIG_FAIR_GROUP_SCHED 8281 ptr += 2 * nr_cpu_ids * sizeof(void **); 8282 #endif 8283 #ifdef CONFIG_RT_GROUP_SCHED 8284 ptr += 2 * nr_cpu_ids * sizeof(void **); 8285 #endif 8286 if (ptr) { 8287 ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); 8288 8289 #ifdef CONFIG_FAIR_GROUP_SCHED 8290 root_task_group.se = (struct sched_entity **)ptr; 8291 ptr += nr_cpu_ids * sizeof(void **); 8292 8293 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 8294 ptr += nr_cpu_ids * sizeof(void **); 8295 8296 root_task_group.shares = ROOT_TASK_GROUP_LOAD; 8297 init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL); 8298 #endif /* CONFIG_FAIR_GROUP_SCHED */ 8299 #ifdef CONFIG_RT_GROUP_SCHED 8300 root_task_group.rt_se = (struct sched_rt_entity **)ptr; 8301 ptr += nr_cpu_ids * sizeof(void **); 8302 8303 root_task_group.rt_rq = (struct rt_rq **)ptr; 8304 ptr += nr_cpu_ids * sizeof(void **); 8305 8306 #endif /* CONFIG_RT_GROUP_SCHED */ 8307 } 8308 8309 #ifdef CONFIG_SMP 8310 init_defrootdomain(); 8311 #endif 8312 8313 #ifdef CONFIG_RT_GROUP_SCHED 8314 init_rt_bandwidth(&root_task_group.rt_bandwidth, 8315 global_rt_period(), global_rt_runtime()); 8316 #endif /* CONFIG_RT_GROUP_SCHED */ 8317 8318 #ifdef CONFIG_CGROUP_SCHED 8319 task_group_cache = KMEM_CACHE(task_group, 0); 8320 8321 list_add(&root_task_group.list, &task_groups); 8322 INIT_LIST_HEAD(&root_task_group.children); 8323 INIT_LIST_HEAD(&root_task_group.siblings); 8324 autogroup_init(&init_task); 8325 #endif /* CONFIG_CGROUP_SCHED */ 8326 8327 for_each_possible_cpu(i) { 8328 struct rq *rq; 8329 8330 rq = cpu_rq(i); 8331 raw_spin_lock_init(&rq->__lock); 8332 rq->nr_running = 0; 8333 rq->calc_load_active = 0; 8334 rq->calc_load_update = jiffies + LOAD_FREQ; 8335 init_cfs_rq(&rq->cfs); 8336 init_rt_rq(&rq->rt); 8337 init_dl_rq(&rq->dl); 8338 #ifdef CONFIG_FAIR_GROUP_SCHED 8339 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); 8340 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 8341 /* 8342 * How much CPU bandwidth does root_task_group get? 8343 * 8344 * In case of task-groups formed through the cgroup filesystem, it 8345 * gets 100% of the CPU resources in the system. This overall 8346 * system CPU resource is divided among the tasks of 8347 * root_task_group and its child task-groups in a fair manner, 8348 * based on each entity's (task or task-group's) weight 8349 * (se->load.weight). 8350 * 8351 * In other words, if root_task_group has 10 tasks of weight 8352 * 1024) and two child groups A0 and A1 (of weight 1024 each), 8353 * then A0's share of the CPU resource is: 8354 * 8355 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% 8356 * 8357 * We achieve this by letting root_task_group's tasks sit 8358 * directly in rq->cfs (i.e root_task_group->se[] = NULL). 8359 */ 8360 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); 8361 #endif /* CONFIG_FAIR_GROUP_SCHED */ 8362 8363 #ifdef CONFIG_RT_GROUP_SCHED 8364 /* 8365 * This is required for init cpu because rt.c:__enable_runtime() 8366 * starts working after scheduler_running, which is not the case 8367 * yet. 8368 */ 8369 rq->rt.rt_runtime = global_rt_runtime(); 8370 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); 8371 #endif 8372 #ifdef CONFIG_SMP 8373 rq->sd = NULL; 8374 rq->rd = NULL; 8375 rq->cpu_capacity = SCHED_CAPACITY_SCALE; 8376 rq->balance_callback = &balance_push_callback; 8377 rq->active_balance = 0; 8378 rq->next_balance = jiffies; 8379 rq->push_cpu = 0; 8380 rq->cpu = i; 8381 rq->online = 0; 8382 rq->idle_stamp = 0; 8383 rq->avg_idle = 2*sysctl_sched_migration_cost; 8384 rq->max_idle_balance_cost = sysctl_sched_migration_cost; 8385 8386 INIT_LIST_HEAD(&rq->cfs_tasks); 8387 8388 rq_attach_root(rq, &def_root_domain); 8389 #ifdef CONFIG_NO_HZ_COMMON 8390 rq->last_blocked_load_update_tick = jiffies; 8391 atomic_set(&rq->nohz_flags, 0); 8392 8393 INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq); 8394 #endif 8395 #ifdef CONFIG_HOTPLUG_CPU 8396 rcuwait_init(&rq->hotplug_wait); 8397 #endif 8398 #endif /* CONFIG_SMP */ 8399 hrtick_rq_init(rq); 8400 atomic_set(&rq->nr_iowait, 0); 8401 fair_server_init(rq); 8402 8403 #ifdef CONFIG_SCHED_CORE 8404 rq->core = rq; 8405 rq->core_pick = NULL; 8406 rq->core_dl_server = NULL; 8407 rq->core_enabled = 0; 8408 rq->core_tree = RB_ROOT; 8409 rq->core_forceidle_count = 0; 8410 rq->core_forceidle_occupation = 0; 8411 rq->core_forceidle_start = 0; 8412 8413 rq->core_cookie = 0UL; 8414 #endif 8415 zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i)); 8416 } 8417 8418 set_load_weight(&init_task, false); 8419 init_task.se.slice = sysctl_sched_base_slice, 8420 8421 /* 8422 * The boot idle thread does lazy MMU switching as well: 8423 */ 8424 mmgrab_lazy_tlb(&init_mm); 8425 enter_lazy_tlb(&init_mm, current); 8426 8427 /* 8428 * The idle task doesn't need the kthread struct to function, but it 8429 * is dressed up as a per-CPU kthread and thus needs to play the part 8430 * if we want to avoid special-casing it in code that deals with per-CPU 8431 * kthreads. 8432 */ 8433 WARN_ON(!set_kthread_struct(current)); 8434 8435 /* 8436 * Make us the idle thread. Technically, schedule() should not be 8437 * called from this thread, however somewhere below it might be, 8438 * but because we are the idle thread, we just pick up running again 8439 * when this runqueue becomes "idle". 8440 */ 8441 init_idle(current, smp_processor_id()); 8442 8443 calc_load_update = jiffies + LOAD_FREQ; 8444 8445 #ifdef CONFIG_SMP 8446 idle_thread_set_boot_cpu(); 8447 balance_push_set(smp_processor_id(), false); 8448 #endif 8449 init_sched_fair_class(); 8450 8451 psi_init(); 8452 8453 init_uclamp(); 8454 8455 preempt_dynamic_init(); 8456 8457 scheduler_running = 1; 8458 } 8459 8460 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 8461 8462 void __might_sleep(const char *file, int line) 8463 { 8464 unsigned int state = get_current_state(); 8465 /* 8466 * Blocking primitives will set (and therefore destroy) current->state, 8467 * since we will exit with TASK_RUNNING make sure we enter with it, 8468 * otherwise we will destroy state. 8469 */ 8470 WARN_ONCE(state != TASK_RUNNING && current->task_state_change, 8471 "do not call blocking ops when !TASK_RUNNING; " 8472 "state=%x set at [<%p>] %pS\n", state, 8473 (void *)current->task_state_change, 8474 (void *)current->task_state_change); 8475 8476 __might_resched(file, line, 0); 8477 } 8478 EXPORT_SYMBOL(__might_sleep); 8479 8480 static void print_preempt_disable_ip(int preempt_offset, unsigned long ip) 8481 { 8482 if (!IS_ENABLED(CONFIG_DEBUG_PREEMPT)) 8483 return; 8484 8485 if (preempt_count() == preempt_offset) 8486 return; 8487 8488 pr_err("Preemption disabled at:"); 8489 print_ip_sym(KERN_ERR, ip); 8490 } 8491 8492 static inline bool resched_offsets_ok(unsigned int offsets) 8493 { 8494 unsigned int nested = preempt_count(); 8495 8496 nested += rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT; 8497 8498 return nested == offsets; 8499 } 8500 8501 void __might_resched(const char *file, int line, unsigned int offsets) 8502 { 8503 /* Ratelimiting timestamp: */ 8504 static unsigned long prev_jiffy; 8505 8506 unsigned long preempt_disable_ip; 8507 8508 /* WARN_ON_ONCE() by default, no rate limit required: */ 8509 rcu_sleep_check(); 8510 8511 if ((resched_offsets_ok(offsets) && !irqs_disabled() && 8512 !is_idle_task(current) && !current->non_block_count) || 8513 system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || 8514 oops_in_progress) 8515 return; 8516 8517 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 8518 return; 8519 prev_jiffy = jiffies; 8520 8521 /* Save this before calling printk(), since that will clobber it: */ 8522 preempt_disable_ip = get_preempt_disable_ip(current); 8523 8524 pr_err("BUG: sleeping function called from invalid context at %s:%d\n", 8525 file, line); 8526 pr_err("in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n", 8527 in_atomic(), irqs_disabled(), current->non_block_count, 8528 current->pid, current->comm); 8529 pr_err("preempt_count: %x, expected: %x\n", preempt_count(), 8530 offsets & MIGHT_RESCHED_PREEMPT_MASK); 8531 8532 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) { 8533 pr_err("RCU nest depth: %d, expected: %u\n", 8534 rcu_preempt_depth(), offsets >> MIGHT_RESCHED_RCU_SHIFT); 8535 } 8536 8537 if (task_stack_end_corrupted(current)) 8538 pr_emerg("Thread overran stack, or stack corrupted\n"); 8539 8540 debug_show_held_locks(current); 8541 if (irqs_disabled()) 8542 print_irqtrace_events(current); 8543 8544 print_preempt_disable_ip(offsets & MIGHT_RESCHED_PREEMPT_MASK, 8545 preempt_disable_ip); 8546 8547 dump_stack(); 8548 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 8549 } 8550 EXPORT_SYMBOL(__might_resched); 8551 8552 void __cant_sleep(const char *file, int line, int preempt_offset) 8553 { 8554 static unsigned long prev_jiffy; 8555 8556 if (irqs_disabled()) 8557 return; 8558 8559 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 8560 return; 8561 8562 if (preempt_count() > preempt_offset) 8563 return; 8564 8565 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 8566 return; 8567 prev_jiffy = jiffies; 8568 8569 printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line); 8570 printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", 8571 in_atomic(), irqs_disabled(), 8572 current->pid, current->comm); 8573 8574 debug_show_held_locks(current); 8575 dump_stack(); 8576 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 8577 } 8578 EXPORT_SYMBOL_GPL(__cant_sleep); 8579 8580 #ifdef CONFIG_SMP 8581 void __cant_migrate(const char *file, int line) 8582 { 8583 static unsigned long prev_jiffy; 8584 8585 if (irqs_disabled()) 8586 return; 8587 8588 if (is_migration_disabled(current)) 8589 return; 8590 8591 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 8592 return; 8593 8594 if (preempt_count() > 0) 8595 return; 8596 8597 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 8598 return; 8599 prev_jiffy = jiffies; 8600 8601 pr_err("BUG: assuming non migratable context at %s:%d\n", file, line); 8602 pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n", 8603 in_atomic(), irqs_disabled(), is_migration_disabled(current), 8604 current->pid, current->comm); 8605 8606 debug_show_held_locks(current); 8607 dump_stack(); 8608 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 8609 } 8610 EXPORT_SYMBOL_GPL(__cant_migrate); 8611 #endif 8612 #endif 8613 8614 #ifdef CONFIG_MAGIC_SYSRQ 8615 void normalize_rt_tasks(void) 8616 { 8617 struct task_struct *g, *p; 8618 struct sched_attr attr = { 8619 .sched_policy = SCHED_NORMAL, 8620 }; 8621 8622 read_lock(&tasklist_lock); 8623 for_each_process_thread(g, p) { 8624 /* 8625 * Only normalize user tasks: 8626 */ 8627 if (p->flags & PF_KTHREAD) 8628 continue; 8629 8630 p->se.exec_start = 0; 8631 schedstat_set(p->stats.wait_start, 0); 8632 schedstat_set(p->stats.sleep_start, 0); 8633 schedstat_set(p->stats.block_start, 0); 8634 8635 if (!rt_or_dl_task(p)) { 8636 /* 8637 * Renice negative nice level userspace 8638 * tasks back to 0: 8639 */ 8640 if (task_nice(p) < 0) 8641 set_user_nice(p, 0); 8642 continue; 8643 } 8644 8645 __sched_setscheduler(p, &attr, false, false); 8646 } 8647 read_unlock(&tasklist_lock); 8648 } 8649 8650 #endif /* CONFIG_MAGIC_SYSRQ */ 8651 8652 #if defined(CONFIG_KGDB_KDB) 8653 /* 8654 * These functions are only useful for KDB. 8655 * 8656 * They can only be called when the whole system has been 8657 * stopped - every CPU needs to be quiescent, and no scheduling 8658 * activity can take place. Using them for anything else would 8659 * be a serious bug, and as a result, they aren't even visible 8660 * under any other configuration. 8661 */ 8662 8663 /** 8664 * curr_task - return the current task for a given CPU. 8665 * @cpu: the processor in question. 8666 * 8667 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 8668 * 8669 * Return: The current task for @cpu. 8670 */ 8671 struct task_struct *curr_task(int cpu) 8672 { 8673 return cpu_curr(cpu); 8674 } 8675 8676 #endif /* defined(CONFIG_KGDB_KDB) */ 8677 8678 #ifdef CONFIG_CGROUP_SCHED 8679 /* task_group_lock serializes the addition/removal of task groups */ 8680 static DEFINE_SPINLOCK(task_group_lock); 8681 8682 static inline void alloc_uclamp_sched_group(struct task_group *tg, 8683 struct task_group *parent) 8684 { 8685 #ifdef CONFIG_UCLAMP_TASK_GROUP 8686 enum uclamp_id clamp_id; 8687 8688 for_each_clamp_id(clamp_id) { 8689 uclamp_se_set(&tg->uclamp_req[clamp_id], 8690 uclamp_none(clamp_id), false); 8691 tg->uclamp[clamp_id] = parent->uclamp[clamp_id]; 8692 } 8693 #endif 8694 } 8695 8696 static void sched_free_group(struct task_group *tg) 8697 { 8698 free_fair_sched_group(tg); 8699 free_rt_sched_group(tg); 8700 autogroup_free(tg); 8701 kmem_cache_free(task_group_cache, tg); 8702 } 8703 8704 static void sched_free_group_rcu(struct rcu_head *rcu) 8705 { 8706 sched_free_group(container_of(rcu, struct task_group, rcu)); 8707 } 8708 8709 static void sched_unregister_group(struct task_group *tg) 8710 { 8711 unregister_fair_sched_group(tg); 8712 unregister_rt_sched_group(tg); 8713 /* 8714 * We have to wait for yet another RCU grace period to expire, as 8715 * print_cfs_stats() might run concurrently. 8716 */ 8717 call_rcu(&tg->rcu, sched_free_group_rcu); 8718 } 8719 8720 /* allocate runqueue etc for a new task group */ 8721 struct task_group *sched_create_group(struct task_group *parent) 8722 { 8723 struct task_group *tg; 8724 8725 tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO); 8726 if (!tg) 8727 return ERR_PTR(-ENOMEM); 8728 8729 if (!alloc_fair_sched_group(tg, parent)) 8730 goto err; 8731 8732 if (!alloc_rt_sched_group(tg, parent)) 8733 goto err; 8734 8735 alloc_uclamp_sched_group(tg, parent); 8736 8737 return tg; 8738 8739 err: 8740 sched_free_group(tg); 8741 return ERR_PTR(-ENOMEM); 8742 } 8743 8744 void sched_online_group(struct task_group *tg, struct task_group *parent) 8745 { 8746 unsigned long flags; 8747 8748 spin_lock_irqsave(&task_group_lock, flags); 8749 list_add_rcu(&tg->list, &task_groups); 8750 8751 /* Root should already exist: */ 8752 WARN_ON(!parent); 8753 8754 tg->parent = parent; 8755 INIT_LIST_HEAD(&tg->children); 8756 list_add_rcu(&tg->siblings, &parent->children); 8757 spin_unlock_irqrestore(&task_group_lock, flags); 8758 8759 online_fair_sched_group(tg); 8760 } 8761 8762 /* RCU callback to free various structures associated with a task group */ 8763 static void sched_unregister_group_rcu(struct rcu_head *rhp) 8764 { 8765 /* Now it should be safe to free those cfs_rqs: */ 8766 sched_unregister_group(container_of(rhp, struct task_group, rcu)); 8767 } 8768 8769 void sched_destroy_group(struct task_group *tg) 8770 { 8771 /* Wait for possible concurrent references to cfs_rqs complete: */ 8772 call_rcu(&tg->rcu, sched_unregister_group_rcu); 8773 } 8774 8775 void sched_release_group(struct task_group *tg) 8776 { 8777 unsigned long flags; 8778 8779 /* 8780 * Unlink first, to avoid walk_tg_tree_from() from finding us (via 8781 * sched_cfs_period_timer()). 8782 * 8783 * For this to be effective, we have to wait for all pending users of 8784 * this task group to leave their RCU critical section to ensure no new 8785 * user will see our dying task group any more. Specifically ensure 8786 * that tg_unthrottle_up() won't add decayed cfs_rq's to it. 8787 * 8788 * We therefore defer calling unregister_fair_sched_group() to 8789 * sched_unregister_group() which is guarantied to get called only after the 8790 * current RCU grace period has expired. 8791 */ 8792 spin_lock_irqsave(&task_group_lock, flags); 8793 list_del_rcu(&tg->list); 8794 list_del_rcu(&tg->siblings); 8795 spin_unlock_irqrestore(&task_group_lock, flags); 8796 } 8797 8798 static struct task_group *sched_get_task_group(struct task_struct *tsk) 8799 { 8800 struct task_group *tg; 8801 8802 /* 8803 * All callers are synchronized by task_rq_lock(); we do not use RCU 8804 * which is pointless here. Thus, we pass "true" to task_css_check() 8805 * to prevent lockdep warnings. 8806 */ 8807 tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), 8808 struct task_group, css); 8809 tg = autogroup_task_group(tsk, tg); 8810 8811 return tg; 8812 } 8813 8814 static void sched_change_group(struct task_struct *tsk, struct task_group *group) 8815 { 8816 tsk->sched_task_group = group; 8817 8818 #ifdef CONFIG_FAIR_GROUP_SCHED 8819 if (tsk->sched_class->task_change_group) 8820 tsk->sched_class->task_change_group(tsk); 8821 else 8822 #endif 8823 set_task_rq(tsk, task_cpu(tsk)); 8824 } 8825 8826 /* 8827 * Change task's runqueue when it moves between groups. 8828 * 8829 * The caller of this function should have put the task in its new group by 8830 * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect 8831 * its new group. 8832 */ 8833 void sched_move_task(struct task_struct *tsk) 8834 { 8835 int queued, running, queue_flags = 8836 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 8837 struct task_group *group; 8838 struct rq *rq; 8839 8840 CLASS(task_rq_lock, rq_guard)(tsk); 8841 rq = rq_guard.rq; 8842 8843 /* 8844 * Esp. with SCHED_AUTOGROUP enabled it is possible to get superfluous 8845 * group changes. 8846 */ 8847 group = sched_get_task_group(tsk); 8848 if (group == tsk->sched_task_group) 8849 return; 8850 8851 update_rq_clock(rq); 8852 8853 running = task_current(rq, tsk); 8854 queued = task_on_rq_queued(tsk); 8855 8856 if (queued) 8857 dequeue_task(rq, tsk, queue_flags); 8858 if (running) 8859 put_prev_task(rq, tsk); 8860 8861 sched_change_group(tsk, group); 8862 8863 if (queued) 8864 enqueue_task(rq, tsk, queue_flags); 8865 if (running) { 8866 set_next_task(rq, tsk); 8867 /* 8868 * After changing group, the running task may have joined a 8869 * throttled one but it's still the running task. Trigger a 8870 * resched to make sure that task can still run. 8871 */ 8872 resched_curr(rq); 8873 } 8874 } 8875 8876 static inline struct task_group *css_tg(struct cgroup_subsys_state *css) 8877 { 8878 return css ? container_of(css, struct task_group, css) : NULL; 8879 } 8880 8881 static struct cgroup_subsys_state * 8882 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 8883 { 8884 struct task_group *parent = css_tg(parent_css); 8885 struct task_group *tg; 8886 8887 if (!parent) { 8888 /* This is early initialization for the top cgroup */ 8889 return &root_task_group.css; 8890 } 8891 8892 tg = sched_create_group(parent); 8893 if (IS_ERR(tg)) 8894 return ERR_PTR(-ENOMEM); 8895 8896 return &tg->css; 8897 } 8898 8899 /* Expose task group only after completing cgroup initialization */ 8900 static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) 8901 { 8902 struct task_group *tg = css_tg(css); 8903 struct task_group *parent = css_tg(css->parent); 8904 8905 if (parent) 8906 sched_online_group(tg, parent); 8907 8908 #ifdef CONFIG_UCLAMP_TASK_GROUP 8909 /* Propagate the effective uclamp value for the new group */ 8910 guard(mutex)(&uclamp_mutex); 8911 guard(rcu)(); 8912 cpu_util_update_eff(css); 8913 #endif 8914 8915 return 0; 8916 } 8917 8918 static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) 8919 { 8920 struct task_group *tg = css_tg(css); 8921 8922 sched_release_group(tg); 8923 } 8924 8925 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) 8926 { 8927 struct task_group *tg = css_tg(css); 8928 8929 /* 8930 * Relies on the RCU grace period between css_released() and this. 8931 */ 8932 sched_unregister_group(tg); 8933 } 8934 8935 #ifdef CONFIG_RT_GROUP_SCHED 8936 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) 8937 { 8938 struct task_struct *task; 8939 struct cgroup_subsys_state *css; 8940 8941 cgroup_taskset_for_each(task, css, tset) { 8942 if (!sched_rt_can_attach(css_tg(css), task)) 8943 return -EINVAL; 8944 } 8945 return 0; 8946 } 8947 #endif 8948 8949 static void cpu_cgroup_attach(struct cgroup_taskset *tset) 8950 { 8951 struct task_struct *task; 8952 struct cgroup_subsys_state *css; 8953 8954 cgroup_taskset_for_each(task, css, tset) 8955 sched_move_task(task); 8956 } 8957 8958 #ifdef CONFIG_UCLAMP_TASK_GROUP 8959 static void cpu_util_update_eff(struct cgroup_subsys_state *css) 8960 { 8961 struct cgroup_subsys_state *top_css = css; 8962 struct uclamp_se *uc_parent = NULL; 8963 struct uclamp_se *uc_se = NULL; 8964 unsigned int eff[UCLAMP_CNT]; 8965 enum uclamp_id clamp_id; 8966 unsigned int clamps; 8967 8968 lockdep_assert_held(&uclamp_mutex); 8969 SCHED_WARN_ON(!rcu_read_lock_held()); 8970 8971 css_for_each_descendant_pre(css, top_css) { 8972 uc_parent = css_tg(css)->parent 8973 ? css_tg(css)->parent->uclamp : NULL; 8974 8975 for_each_clamp_id(clamp_id) { 8976 /* Assume effective clamps matches requested clamps */ 8977 eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value; 8978 /* Cap effective clamps with parent's effective clamps */ 8979 if (uc_parent && 8980 eff[clamp_id] > uc_parent[clamp_id].value) { 8981 eff[clamp_id] = uc_parent[clamp_id].value; 8982 } 8983 } 8984 /* Ensure protection is always capped by limit */ 8985 eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]); 8986 8987 /* Propagate most restrictive effective clamps */ 8988 clamps = 0x0; 8989 uc_se = css_tg(css)->uclamp; 8990 for_each_clamp_id(clamp_id) { 8991 if (eff[clamp_id] == uc_se[clamp_id].value) 8992 continue; 8993 uc_se[clamp_id].value = eff[clamp_id]; 8994 uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]); 8995 clamps |= (0x1 << clamp_id); 8996 } 8997 if (!clamps) { 8998 css = css_rightmost_descendant(css); 8999 continue; 9000 } 9001 9002 /* Immediately update descendants RUNNABLE tasks */ 9003 uclamp_update_active_tasks(css); 9004 } 9005 } 9006 9007 /* 9008 * Integer 10^N with a given N exponent by casting to integer the literal "1eN" 9009 * C expression. Since there is no way to convert a macro argument (N) into a 9010 * character constant, use two levels of macros. 9011 */ 9012 #define _POW10(exp) ((unsigned int)1e##exp) 9013 #define POW10(exp) _POW10(exp) 9014 9015 struct uclamp_request { 9016 #define UCLAMP_PERCENT_SHIFT 2 9017 #define UCLAMP_PERCENT_SCALE (100 * POW10(UCLAMP_PERCENT_SHIFT)) 9018 s64 percent; 9019 u64 util; 9020 int ret; 9021 }; 9022 9023 static inline struct uclamp_request 9024 capacity_from_percent(char *buf) 9025 { 9026 struct uclamp_request req = { 9027 .percent = UCLAMP_PERCENT_SCALE, 9028 .util = SCHED_CAPACITY_SCALE, 9029 .ret = 0, 9030 }; 9031 9032 buf = strim(buf); 9033 if (strcmp(buf, "max")) { 9034 req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT, 9035 &req.percent); 9036 if (req.ret) 9037 return req; 9038 if ((u64)req.percent > UCLAMP_PERCENT_SCALE) { 9039 req.ret = -ERANGE; 9040 return req; 9041 } 9042 9043 req.util = req.percent << SCHED_CAPACITY_SHIFT; 9044 req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE); 9045 } 9046 9047 return req; 9048 } 9049 9050 static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf, 9051 size_t nbytes, loff_t off, 9052 enum uclamp_id clamp_id) 9053 { 9054 struct uclamp_request req; 9055 struct task_group *tg; 9056 9057 req = capacity_from_percent(buf); 9058 if (req.ret) 9059 return req.ret; 9060 9061 static_branch_enable(&sched_uclamp_used); 9062 9063 guard(mutex)(&uclamp_mutex); 9064 guard(rcu)(); 9065 9066 tg = css_tg(of_css(of)); 9067 if (tg->uclamp_req[clamp_id].value != req.util) 9068 uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false); 9069 9070 /* 9071 * Because of not recoverable conversion rounding we keep track of the 9072 * exact requested value 9073 */ 9074 tg->uclamp_pct[clamp_id] = req.percent; 9075 9076 /* Update effective clamps to track the most restrictive value */ 9077 cpu_util_update_eff(of_css(of)); 9078 9079 return nbytes; 9080 } 9081 9082 static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of, 9083 char *buf, size_t nbytes, 9084 loff_t off) 9085 { 9086 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN); 9087 } 9088 9089 static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of, 9090 char *buf, size_t nbytes, 9091 loff_t off) 9092 { 9093 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX); 9094 } 9095 9096 static inline void cpu_uclamp_print(struct seq_file *sf, 9097 enum uclamp_id clamp_id) 9098 { 9099 struct task_group *tg; 9100 u64 util_clamp; 9101 u64 percent; 9102 u32 rem; 9103 9104 scoped_guard (rcu) { 9105 tg = css_tg(seq_css(sf)); 9106 util_clamp = tg->uclamp_req[clamp_id].value; 9107 } 9108 9109 if (util_clamp == SCHED_CAPACITY_SCALE) { 9110 seq_puts(sf, "max\n"); 9111 return; 9112 } 9113 9114 percent = tg->uclamp_pct[clamp_id]; 9115 percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem); 9116 seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem); 9117 } 9118 9119 static int cpu_uclamp_min_show(struct seq_file *sf, void *v) 9120 { 9121 cpu_uclamp_print(sf, UCLAMP_MIN); 9122 return 0; 9123 } 9124 9125 static int cpu_uclamp_max_show(struct seq_file *sf, void *v) 9126 { 9127 cpu_uclamp_print(sf, UCLAMP_MAX); 9128 return 0; 9129 } 9130 #endif /* CONFIG_UCLAMP_TASK_GROUP */ 9131 9132 #ifdef CONFIG_FAIR_GROUP_SCHED 9133 static int cpu_shares_write_u64(struct cgroup_subsys_state *css, 9134 struct cftype *cftype, u64 shareval) 9135 { 9136 if (shareval > scale_load_down(ULONG_MAX)) 9137 shareval = MAX_SHARES; 9138 return sched_group_set_shares(css_tg(css), scale_load(shareval)); 9139 } 9140 9141 static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, 9142 struct cftype *cft) 9143 { 9144 struct task_group *tg = css_tg(css); 9145 9146 return (u64) scale_load_down(tg->shares); 9147 } 9148 9149 #ifdef CONFIG_CFS_BANDWIDTH 9150 static DEFINE_MUTEX(cfs_constraints_mutex); 9151 9152 const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ 9153 static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ 9154 /* More than 203 days if BW_SHIFT equals 20. */ 9155 static const u64 max_cfs_runtime = MAX_BW * NSEC_PER_USEC; 9156 9157 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); 9158 9159 static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota, 9160 u64 burst) 9161 { 9162 int i, ret = 0, runtime_enabled, runtime_was_enabled; 9163 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9164 9165 if (tg == &root_task_group) 9166 return -EINVAL; 9167 9168 /* 9169 * Ensure we have at some amount of bandwidth every period. This is 9170 * to prevent reaching a state of large arrears when throttled via 9171 * entity_tick() resulting in prolonged exit starvation. 9172 */ 9173 if (quota < min_cfs_quota_period || period < min_cfs_quota_period) 9174 return -EINVAL; 9175 9176 /* 9177 * Likewise, bound things on the other side by preventing insane quota 9178 * periods. This also allows us to normalize in computing quota 9179 * feasibility. 9180 */ 9181 if (period > max_cfs_quota_period) 9182 return -EINVAL; 9183 9184 /* 9185 * Bound quota to defend quota against overflow during bandwidth shift. 9186 */ 9187 if (quota != RUNTIME_INF && quota > max_cfs_runtime) 9188 return -EINVAL; 9189 9190 if (quota != RUNTIME_INF && (burst > quota || 9191 burst + quota > max_cfs_runtime)) 9192 return -EINVAL; 9193 9194 /* 9195 * Prevent race between setting of cfs_rq->runtime_enabled and 9196 * unthrottle_offline_cfs_rqs(). 9197 */ 9198 guard(cpus_read_lock)(); 9199 guard(mutex)(&cfs_constraints_mutex); 9200 9201 ret = __cfs_schedulable(tg, period, quota); 9202 if (ret) 9203 return ret; 9204 9205 runtime_enabled = quota != RUNTIME_INF; 9206 runtime_was_enabled = cfs_b->quota != RUNTIME_INF; 9207 /* 9208 * If we need to toggle cfs_bandwidth_used, off->on must occur 9209 * before making related changes, and on->off must occur afterwards 9210 */ 9211 if (runtime_enabled && !runtime_was_enabled) 9212 cfs_bandwidth_usage_inc(); 9213 9214 scoped_guard (raw_spinlock_irq, &cfs_b->lock) { 9215 cfs_b->period = ns_to_ktime(period); 9216 cfs_b->quota = quota; 9217 cfs_b->burst = burst; 9218 9219 __refill_cfs_bandwidth_runtime(cfs_b); 9220 9221 /* 9222 * Restart the period timer (if active) to handle new 9223 * period expiry: 9224 */ 9225 if (runtime_enabled) 9226 start_cfs_bandwidth(cfs_b); 9227 } 9228 9229 for_each_online_cpu(i) { 9230 struct cfs_rq *cfs_rq = tg->cfs_rq[i]; 9231 struct rq *rq = cfs_rq->rq; 9232 9233 guard(rq_lock_irq)(rq); 9234 cfs_rq->runtime_enabled = runtime_enabled; 9235 cfs_rq->runtime_remaining = 0; 9236 9237 if (cfs_rq->throttled) 9238 unthrottle_cfs_rq(cfs_rq); 9239 } 9240 9241 if (runtime_was_enabled && !runtime_enabled) 9242 cfs_bandwidth_usage_dec(); 9243 9244 return 0; 9245 } 9246 9247 static int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) 9248 { 9249 u64 quota, period, burst; 9250 9251 period = ktime_to_ns(tg->cfs_bandwidth.period); 9252 burst = tg->cfs_bandwidth.burst; 9253 if (cfs_quota_us < 0) 9254 quota = RUNTIME_INF; 9255 else if ((u64)cfs_quota_us <= U64_MAX / NSEC_PER_USEC) 9256 quota = (u64)cfs_quota_us * NSEC_PER_USEC; 9257 else 9258 return -EINVAL; 9259 9260 return tg_set_cfs_bandwidth(tg, period, quota, burst); 9261 } 9262 9263 static long tg_get_cfs_quota(struct task_group *tg) 9264 { 9265 u64 quota_us; 9266 9267 if (tg->cfs_bandwidth.quota == RUNTIME_INF) 9268 return -1; 9269 9270 quota_us = tg->cfs_bandwidth.quota; 9271 do_div(quota_us, NSEC_PER_USEC); 9272 9273 return quota_us; 9274 } 9275 9276 static int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) 9277 { 9278 u64 quota, period, burst; 9279 9280 if ((u64)cfs_period_us > U64_MAX / NSEC_PER_USEC) 9281 return -EINVAL; 9282 9283 period = (u64)cfs_period_us * NSEC_PER_USEC; 9284 quota = tg->cfs_bandwidth.quota; 9285 burst = tg->cfs_bandwidth.burst; 9286 9287 return tg_set_cfs_bandwidth(tg, period, quota, burst); 9288 } 9289 9290 static long tg_get_cfs_period(struct task_group *tg) 9291 { 9292 u64 cfs_period_us; 9293 9294 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); 9295 do_div(cfs_period_us, NSEC_PER_USEC); 9296 9297 return cfs_period_us; 9298 } 9299 9300 static int tg_set_cfs_burst(struct task_group *tg, long cfs_burst_us) 9301 { 9302 u64 quota, period, burst; 9303 9304 if ((u64)cfs_burst_us > U64_MAX / NSEC_PER_USEC) 9305 return -EINVAL; 9306 9307 burst = (u64)cfs_burst_us * NSEC_PER_USEC; 9308 period = ktime_to_ns(tg->cfs_bandwidth.period); 9309 quota = tg->cfs_bandwidth.quota; 9310 9311 return tg_set_cfs_bandwidth(tg, period, quota, burst); 9312 } 9313 9314 static long tg_get_cfs_burst(struct task_group *tg) 9315 { 9316 u64 burst_us; 9317 9318 burst_us = tg->cfs_bandwidth.burst; 9319 do_div(burst_us, NSEC_PER_USEC); 9320 9321 return burst_us; 9322 } 9323 9324 static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, 9325 struct cftype *cft) 9326 { 9327 return tg_get_cfs_quota(css_tg(css)); 9328 } 9329 9330 static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css, 9331 struct cftype *cftype, s64 cfs_quota_us) 9332 { 9333 return tg_set_cfs_quota(css_tg(css), cfs_quota_us); 9334 } 9335 9336 static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css, 9337 struct cftype *cft) 9338 { 9339 return tg_get_cfs_period(css_tg(css)); 9340 } 9341 9342 static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css, 9343 struct cftype *cftype, u64 cfs_period_us) 9344 { 9345 return tg_set_cfs_period(css_tg(css), cfs_period_us); 9346 } 9347 9348 static u64 cpu_cfs_burst_read_u64(struct cgroup_subsys_state *css, 9349 struct cftype *cft) 9350 { 9351 return tg_get_cfs_burst(css_tg(css)); 9352 } 9353 9354 static int cpu_cfs_burst_write_u64(struct cgroup_subsys_state *css, 9355 struct cftype *cftype, u64 cfs_burst_us) 9356 { 9357 return tg_set_cfs_burst(css_tg(css), cfs_burst_us); 9358 } 9359 9360 struct cfs_schedulable_data { 9361 struct task_group *tg; 9362 u64 period, quota; 9363 }; 9364 9365 /* 9366 * normalize group quota/period to be quota/max_period 9367 * note: units are usecs 9368 */ 9369 static u64 normalize_cfs_quota(struct task_group *tg, 9370 struct cfs_schedulable_data *d) 9371 { 9372 u64 quota, period; 9373 9374 if (tg == d->tg) { 9375 period = d->period; 9376 quota = d->quota; 9377 } else { 9378 period = tg_get_cfs_period(tg); 9379 quota = tg_get_cfs_quota(tg); 9380 } 9381 9382 /* note: these should typically be equivalent */ 9383 if (quota == RUNTIME_INF || quota == -1) 9384 return RUNTIME_INF; 9385 9386 return to_ratio(period, quota); 9387 } 9388 9389 static int tg_cfs_schedulable_down(struct task_group *tg, void *data) 9390 { 9391 struct cfs_schedulable_data *d = data; 9392 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9393 s64 quota = 0, parent_quota = -1; 9394 9395 if (!tg->parent) { 9396 quota = RUNTIME_INF; 9397 } else { 9398 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; 9399 9400 quota = normalize_cfs_quota(tg, d); 9401 parent_quota = parent_b->hierarchical_quota; 9402 9403 /* 9404 * Ensure max(child_quota) <= parent_quota. On cgroup2, 9405 * always take the non-RUNTIME_INF min. On cgroup1, only 9406 * inherit when no limit is set. In both cases this is used 9407 * by the scheduler to determine if a given CFS task has a 9408 * bandwidth constraint at some higher level. 9409 */ 9410 if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) { 9411 if (quota == RUNTIME_INF) 9412 quota = parent_quota; 9413 else if (parent_quota != RUNTIME_INF) 9414 quota = min(quota, parent_quota); 9415 } else { 9416 if (quota == RUNTIME_INF) 9417 quota = parent_quota; 9418 else if (parent_quota != RUNTIME_INF && quota > parent_quota) 9419 return -EINVAL; 9420 } 9421 } 9422 cfs_b->hierarchical_quota = quota; 9423 9424 return 0; 9425 } 9426 9427 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) 9428 { 9429 struct cfs_schedulable_data data = { 9430 .tg = tg, 9431 .period = period, 9432 .quota = quota, 9433 }; 9434 9435 if (quota != RUNTIME_INF) { 9436 do_div(data.period, NSEC_PER_USEC); 9437 do_div(data.quota, NSEC_PER_USEC); 9438 } 9439 9440 guard(rcu)(); 9441 return walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); 9442 } 9443 9444 static int cpu_cfs_stat_show(struct seq_file *sf, void *v) 9445 { 9446 struct task_group *tg = css_tg(seq_css(sf)); 9447 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9448 9449 seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); 9450 seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); 9451 seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); 9452 9453 if (schedstat_enabled() && tg != &root_task_group) { 9454 struct sched_statistics *stats; 9455 u64 ws = 0; 9456 int i; 9457 9458 for_each_possible_cpu(i) { 9459 stats = __schedstats_from_se(tg->se[i]); 9460 ws += schedstat_val(stats->wait_sum); 9461 } 9462 9463 seq_printf(sf, "wait_sum %llu\n", ws); 9464 } 9465 9466 seq_printf(sf, "nr_bursts %d\n", cfs_b->nr_burst); 9467 seq_printf(sf, "burst_time %llu\n", cfs_b->burst_time); 9468 9469 return 0; 9470 } 9471 9472 static u64 throttled_time_self(struct task_group *tg) 9473 { 9474 int i; 9475 u64 total = 0; 9476 9477 for_each_possible_cpu(i) { 9478 total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time); 9479 } 9480 9481 return total; 9482 } 9483 9484 static int cpu_cfs_local_stat_show(struct seq_file *sf, void *v) 9485 { 9486 struct task_group *tg = css_tg(seq_css(sf)); 9487 9488 seq_printf(sf, "throttled_time %llu\n", throttled_time_self(tg)); 9489 9490 return 0; 9491 } 9492 #endif /* CONFIG_CFS_BANDWIDTH */ 9493 #endif /* CONFIG_FAIR_GROUP_SCHED */ 9494 9495 #ifdef CONFIG_RT_GROUP_SCHED 9496 static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, 9497 struct cftype *cft, s64 val) 9498 { 9499 return sched_group_set_rt_runtime(css_tg(css), val); 9500 } 9501 9502 static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, 9503 struct cftype *cft) 9504 { 9505 return sched_group_rt_runtime(css_tg(css)); 9506 } 9507 9508 static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, 9509 struct cftype *cftype, u64 rt_period_us) 9510 { 9511 return sched_group_set_rt_period(css_tg(css), rt_period_us); 9512 } 9513 9514 static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, 9515 struct cftype *cft) 9516 { 9517 return sched_group_rt_period(css_tg(css)); 9518 } 9519 #endif /* CONFIG_RT_GROUP_SCHED */ 9520 9521 #ifdef CONFIG_FAIR_GROUP_SCHED 9522 static s64 cpu_idle_read_s64(struct cgroup_subsys_state *css, 9523 struct cftype *cft) 9524 { 9525 return css_tg(css)->idle; 9526 } 9527 9528 static int cpu_idle_write_s64(struct cgroup_subsys_state *css, 9529 struct cftype *cft, s64 idle) 9530 { 9531 return sched_group_set_idle(css_tg(css), idle); 9532 } 9533 #endif 9534 9535 static struct cftype cpu_legacy_files[] = { 9536 #ifdef CONFIG_FAIR_GROUP_SCHED 9537 { 9538 .name = "shares", 9539 .read_u64 = cpu_shares_read_u64, 9540 .write_u64 = cpu_shares_write_u64, 9541 }, 9542 { 9543 .name = "idle", 9544 .read_s64 = cpu_idle_read_s64, 9545 .write_s64 = cpu_idle_write_s64, 9546 }, 9547 #endif 9548 #ifdef CONFIG_CFS_BANDWIDTH 9549 { 9550 .name = "cfs_quota_us", 9551 .read_s64 = cpu_cfs_quota_read_s64, 9552 .write_s64 = cpu_cfs_quota_write_s64, 9553 }, 9554 { 9555 .name = "cfs_period_us", 9556 .read_u64 = cpu_cfs_period_read_u64, 9557 .write_u64 = cpu_cfs_period_write_u64, 9558 }, 9559 { 9560 .name = "cfs_burst_us", 9561 .read_u64 = cpu_cfs_burst_read_u64, 9562 .write_u64 = cpu_cfs_burst_write_u64, 9563 }, 9564 { 9565 .name = "stat", 9566 .seq_show = cpu_cfs_stat_show, 9567 }, 9568 { 9569 .name = "stat.local", 9570 .seq_show = cpu_cfs_local_stat_show, 9571 }, 9572 #endif 9573 #ifdef CONFIG_RT_GROUP_SCHED 9574 { 9575 .name = "rt_runtime_us", 9576 .read_s64 = cpu_rt_runtime_read, 9577 .write_s64 = cpu_rt_runtime_write, 9578 }, 9579 { 9580 .name = "rt_period_us", 9581 .read_u64 = cpu_rt_period_read_uint, 9582 .write_u64 = cpu_rt_period_write_uint, 9583 }, 9584 #endif 9585 #ifdef CONFIG_UCLAMP_TASK_GROUP 9586 { 9587 .name = "uclamp.min", 9588 .flags = CFTYPE_NOT_ON_ROOT, 9589 .seq_show = cpu_uclamp_min_show, 9590 .write = cpu_uclamp_min_write, 9591 }, 9592 { 9593 .name = "uclamp.max", 9594 .flags = CFTYPE_NOT_ON_ROOT, 9595 .seq_show = cpu_uclamp_max_show, 9596 .write = cpu_uclamp_max_write, 9597 }, 9598 #endif 9599 { } /* Terminate */ 9600 }; 9601 9602 static int cpu_extra_stat_show(struct seq_file *sf, 9603 struct cgroup_subsys_state *css) 9604 { 9605 #ifdef CONFIG_CFS_BANDWIDTH 9606 { 9607 struct task_group *tg = css_tg(css); 9608 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9609 u64 throttled_usec, burst_usec; 9610 9611 throttled_usec = cfs_b->throttled_time; 9612 do_div(throttled_usec, NSEC_PER_USEC); 9613 burst_usec = cfs_b->burst_time; 9614 do_div(burst_usec, NSEC_PER_USEC); 9615 9616 seq_printf(sf, "nr_periods %d\n" 9617 "nr_throttled %d\n" 9618 "throttled_usec %llu\n" 9619 "nr_bursts %d\n" 9620 "burst_usec %llu\n", 9621 cfs_b->nr_periods, cfs_b->nr_throttled, 9622 throttled_usec, cfs_b->nr_burst, burst_usec); 9623 } 9624 #endif 9625 return 0; 9626 } 9627 9628 static int cpu_local_stat_show(struct seq_file *sf, 9629 struct cgroup_subsys_state *css) 9630 { 9631 #ifdef CONFIG_CFS_BANDWIDTH 9632 { 9633 struct task_group *tg = css_tg(css); 9634 u64 throttled_self_usec; 9635 9636 throttled_self_usec = throttled_time_self(tg); 9637 do_div(throttled_self_usec, NSEC_PER_USEC); 9638 9639 seq_printf(sf, "throttled_usec %llu\n", 9640 throttled_self_usec); 9641 } 9642 #endif 9643 return 0; 9644 } 9645 9646 #ifdef CONFIG_FAIR_GROUP_SCHED 9647 static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, 9648 struct cftype *cft) 9649 { 9650 struct task_group *tg = css_tg(css); 9651 u64 weight = scale_load_down(tg->shares); 9652 9653 return DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024); 9654 } 9655 9656 static int cpu_weight_write_u64(struct cgroup_subsys_state *css, 9657 struct cftype *cft, u64 weight) 9658 { 9659 /* 9660 * cgroup weight knobs should use the common MIN, DFL and MAX 9661 * values which are 1, 100 and 10000 respectively. While it loses 9662 * a bit of range on both ends, it maps pretty well onto the shares 9663 * value used by scheduler and the round-trip conversions preserve 9664 * the original value over the entire range. 9665 */ 9666 if (weight < CGROUP_WEIGHT_MIN || weight > CGROUP_WEIGHT_MAX) 9667 return -ERANGE; 9668 9669 weight = DIV_ROUND_CLOSEST_ULL(weight * 1024, CGROUP_WEIGHT_DFL); 9670 9671 return sched_group_set_shares(css_tg(css), scale_load(weight)); 9672 } 9673 9674 static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css, 9675 struct cftype *cft) 9676 { 9677 unsigned long weight = scale_load_down(css_tg(css)->shares); 9678 int last_delta = INT_MAX; 9679 int prio, delta; 9680 9681 /* find the closest nice value to the current weight */ 9682 for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) { 9683 delta = abs(sched_prio_to_weight[prio] - weight); 9684 if (delta >= last_delta) 9685 break; 9686 last_delta = delta; 9687 } 9688 9689 return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO); 9690 } 9691 9692 static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css, 9693 struct cftype *cft, s64 nice) 9694 { 9695 unsigned long weight; 9696 int idx; 9697 9698 if (nice < MIN_NICE || nice > MAX_NICE) 9699 return -ERANGE; 9700 9701 idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO; 9702 idx = array_index_nospec(idx, 40); 9703 weight = sched_prio_to_weight[idx]; 9704 9705 return sched_group_set_shares(css_tg(css), scale_load(weight)); 9706 } 9707 #endif 9708 9709 static void __maybe_unused cpu_period_quota_print(struct seq_file *sf, 9710 long period, long quota) 9711 { 9712 if (quota < 0) 9713 seq_puts(sf, "max"); 9714 else 9715 seq_printf(sf, "%ld", quota); 9716 9717 seq_printf(sf, " %ld\n", period); 9718 } 9719 9720 /* caller should put the current value in *@periodp before calling */ 9721 static int __maybe_unused cpu_period_quota_parse(char *buf, 9722 u64 *periodp, u64 *quotap) 9723 { 9724 char tok[21]; /* U64_MAX */ 9725 9726 if (sscanf(buf, "%20s %llu", tok, periodp) < 1) 9727 return -EINVAL; 9728 9729 *periodp *= NSEC_PER_USEC; 9730 9731 if (sscanf(tok, "%llu", quotap)) 9732 *quotap *= NSEC_PER_USEC; 9733 else if (!strcmp(tok, "max")) 9734 *quotap = RUNTIME_INF; 9735 else 9736 return -EINVAL; 9737 9738 return 0; 9739 } 9740 9741 #ifdef CONFIG_CFS_BANDWIDTH 9742 static int cpu_max_show(struct seq_file *sf, void *v) 9743 { 9744 struct task_group *tg = css_tg(seq_css(sf)); 9745 9746 cpu_period_quota_print(sf, tg_get_cfs_period(tg), tg_get_cfs_quota(tg)); 9747 return 0; 9748 } 9749 9750 static ssize_t cpu_max_write(struct kernfs_open_file *of, 9751 char *buf, size_t nbytes, loff_t off) 9752 { 9753 struct task_group *tg = css_tg(of_css(of)); 9754 u64 period = tg_get_cfs_period(tg); 9755 u64 burst = tg->cfs_bandwidth.burst; 9756 u64 quota; 9757 int ret; 9758 9759 ret = cpu_period_quota_parse(buf, &period, "a); 9760 if (!ret) 9761 ret = tg_set_cfs_bandwidth(tg, period, quota, burst); 9762 return ret ?: nbytes; 9763 } 9764 #endif 9765 9766 static struct cftype cpu_files[] = { 9767 #ifdef CONFIG_FAIR_GROUP_SCHED 9768 { 9769 .name = "weight", 9770 .flags = CFTYPE_NOT_ON_ROOT, 9771 .read_u64 = cpu_weight_read_u64, 9772 .write_u64 = cpu_weight_write_u64, 9773 }, 9774 { 9775 .name = "weight.nice", 9776 .flags = CFTYPE_NOT_ON_ROOT, 9777 .read_s64 = cpu_weight_nice_read_s64, 9778 .write_s64 = cpu_weight_nice_write_s64, 9779 }, 9780 { 9781 .name = "idle", 9782 .flags = CFTYPE_NOT_ON_ROOT, 9783 .read_s64 = cpu_idle_read_s64, 9784 .write_s64 = cpu_idle_write_s64, 9785 }, 9786 #endif 9787 #ifdef CONFIG_CFS_BANDWIDTH 9788 { 9789 .name = "max", 9790 .flags = CFTYPE_NOT_ON_ROOT, 9791 .seq_show = cpu_max_show, 9792 .write = cpu_max_write, 9793 }, 9794 { 9795 .name = "max.burst", 9796 .flags = CFTYPE_NOT_ON_ROOT, 9797 .read_u64 = cpu_cfs_burst_read_u64, 9798 .write_u64 = cpu_cfs_burst_write_u64, 9799 }, 9800 #endif 9801 #ifdef CONFIG_UCLAMP_TASK_GROUP 9802 { 9803 .name = "uclamp.min", 9804 .flags = CFTYPE_NOT_ON_ROOT, 9805 .seq_show = cpu_uclamp_min_show, 9806 .write = cpu_uclamp_min_write, 9807 }, 9808 { 9809 .name = "uclamp.max", 9810 .flags = CFTYPE_NOT_ON_ROOT, 9811 .seq_show = cpu_uclamp_max_show, 9812 .write = cpu_uclamp_max_write, 9813 }, 9814 #endif 9815 { } /* terminate */ 9816 }; 9817 9818 struct cgroup_subsys cpu_cgrp_subsys = { 9819 .css_alloc = cpu_cgroup_css_alloc, 9820 .css_online = cpu_cgroup_css_online, 9821 .css_released = cpu_cgroup_css_released, 9822 .css_free = cpu_cgroup_css_free, 9823 .css_extra_stat_show = cpu_extra_stat_show, 9824 .css_local_stat_show = cpu_local_stat_show, 9825 #ifdef CONFIG_RT_GROUP_SCHED 9826 .can_attach = cpu_cgroup_can_attach, 9827 #endif 9828 .attach = cpu_cgroup_attach, 9829 .legacy_cftypes = cpu_legacy_files, 9830 .dfl_cftypes = cpu_files, 9831 .early_init = true, 9832 .threaded = true, 9833 }; 9834 9835 #endif /* CONFIG_CGROUP_SCHED */ 9836 9837 void dump_cpu_task(int cpu) 9838 { 9839 if (in_hardirq() && cpu == smp_processor_id()) { 9840 struct pt_regs *regs; 9841 9842 regs = get_irq_regs(); 9843 if (regs) { 9844 show_regs(regs); 9845 return; 9846 } 9847 } 9848 9849 if (trigger_single_cpu_backtrace(cpu)) 9850 return; 9851 9852 pr_info("Task dump for CPU %d:\n", cpu); 9853 sched_show_task(cpu_curr(cpu)); 9854 } 9855 9856 /* 9857 * Nice levels are multiplicative, with a gentle 10% change for every 9858 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to 9859 * nice 1, it will get ~10% less CPU time than another CPU-bound task 9860 * that remained on nice 0. 9861 * 9862 * The "10% effect" is relative and cumulative: from _any_ nice level, 9863 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level 9864 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. 9865 * If a task goes up by ~10% and another task goes down by ~10% then 9866 * the relative distance between them is ~25%.) 9867 */ 9868 const int sched_prio_to_weight[40] = { 9869 /* -20 */ 88761, 71755, 56483, 46273, 36291, 9870 /* -15 */ 29154, 23254, 18705, 14949, 11916, 9871 /* -10 */ 9548, 7620, 6100, 4904, 3906, 9872 /* -5 */ 3121, 2501, 1991, 1586, 1277, 9873 /* 0 */ 1024, 820, 655, 526, 423, 9874 /* 5 */ 335, 272, 215, 172, 137, 9875 /* 10 */ 110, 87, 70, 56, 45, 9876 /* 15 */ 36, 29, 23, 18, 15, 9877 }; 9878 9879 /* 9880 * Inverse (2^32/x) values of the sched_prio_to_weight[] array, pre-calculated. 9881 * 9882 * In cases where the weight does not change often, we can use the 9883 * pre-calculated inverse to speed up arithmetics by turning divisions 9884 * into multiplications: 9885 */ 9886 const u32 sched_prio_to_wmult[40] = { 9887 /* -20 */ 48388, 59856, 76040, 92818, 118348, 9888 /* -15 */ 147320, 184698, 229616, 287308, 360437, 9889 /* -10 */ 449829, 563644, 704093, 875809, 1099582, 9890 /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, 9891 /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, 9892 /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, 9893 /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, 9894 /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, 9895 }; 9896 9897 void call_trace_sched_update_nr_running(struct rq *rq, int count) 9898 { 9899 trace_sched_update_nr_running_tp(rq, count); 9900 } 9901 9902 #ifdef CONFIG_SCHED_MM_CID 9903 9904 /* 9905 * @cid_lock: Guarantee forward-progress of cid allocation. 9906 * 9907 * Concurrency ID allocation within a bitmap is mostly lock-free. The cid_lock 9908 * is only used when contention is detected by the lock-free allocation so 9909 * forward progress can be guaranteed. 9910 */ 9911 DEFINE_RAW_SPINLOCK(cid_lock); 9912 9913 /* 9914 * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock. 9915 * 9916 * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is 9917 * detected, it is set to 1 to ensure that all newly coming allocations are 9918 * serialized by @cid_lock until the allocation which detected contention 9919 * completes and sets @use_cid_lock back to 0. This guarantees forward progress 9920 * of a cid allocation. 9921 */ 9922 int use_cid_lock; 9923 9924 /* 9925 * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid 9926 * concurrently with respect to the execution of the source runqueue context 9927 * switch. 9928 * 9929 * There is one basic properties we want to guarantee here: 9930 * 9931 * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively 9932 * used by a task. That would lead to concurrent allocation of the cid and 9933 * userspace corruption. 9934 * 9935 * Provide this guarantee by introducing a Dekker memory ordering to guarantee 9936 * that a pair of loads observe at least one of a pair of stores, which can be 9937 * shown as: 9938 * 9939 * X = Y = 0 9940 * 9941 * w[X]=1 w[Y]=1 9942 * MB MB 9943 * r[Y]=y r[X]=x 9944 * 9945 * Which guarantees that x==0 && y==0 is impossible. But rather than using 9946 * values 0 and 1, this algorithm cares about specific state transitions of the 9947 * runqueue current task (as updated by the scheduler context switch), and the 9948 * per-mm/cpu cid value. 9949 * 9950 * Let's introduce task (Y) which has task->mm == mm and task (N) which has 9951 * task->mm != mm for the rest of the discussion. There are two scheduler state 9952 * transitions on context switch we care about: 9953 * 9954 * (TSA) Store to rq->curr with transition from (N) to (Y) 9955 * 9956 * (TSB) Store to rq->curr with transition from (Y) to (N) 9957 * 9958 * On the remote-clear side, there is one transition we care about: 9959 * 9960 * (TMA) cmpxchg to *pcpu_cid to set the LAZY flag 9961 * 9962 * There is also a transition to UNSET state which can be performed from all 9963 * sides (scheduler, remote-clear). It is always performed with a cmpxchg which 9964 * guarantees that only a single thread will succeed: 9965 * 9966 * (TMB) cmpxchg to *pcpu_cid to mark UNSET 9967 * 9968 * Just to be clear, what we do _not_ want to happen is a transition to UNSET 9969 * when a thread is actively using the cid (property (1)). 9970 * 9971 * Let's looks at the relevant combinations of TSA/TSB, and TMA transitions. 9972 * 9973 * Scenario A) (TSA)+(TMA) (from next task perspective) 9974 * 9975 * CPU0 CPU1 9976 * 9977 * Context switch CS-1 Remote-clear 9978 * - store to rq->curr: (N)->(Y) (TSA) - cmpxchg to *pcpu_id to LAZY (TMA) 9979 * (implied barrier after cmpxchg) 9980 * - switch_mm_cid() 9981 * - memory barrier (see switch_mm_cid() 9982 * comment explaining how this barrier 9983 * is combined with other scheduler 9984 * barriers) 9985 * - mm_cid_get (next) 9986 * - READ_ONCE(*pcpu_cid) - rcu_dereference(src_rq->curr) 9987 * 9988 * This Dekker ensures that either task (Y) is observed by the 9989 * rcu_dereference() or the LAZY flag is observed by READ_ONCE(), or both are 9990 * observed. 9991 * 9992 * If task (Y) store is observed by rcu_dereference(), it means that there is 9993 * still an active task on the cpu. Remote-clear will therefore not transition 9994 * to UNSET, which fulfills property (1). 9995 * 9996 * If task (Y) is not observed, but the lazy flag is observed by READ_ONCE(), 9997 * it will move its state to UNSET, which clears the percpu cid perhaps 9998 * uselessly (which is not an issue for correctness). Because task (Y) is not 9999 * observed, CPU1 can move ahead to set the state to UNSET. Because moving 10000 * state to UNSET is done with a cmpxchg expecting that the old state has the 10001 * LAZY flag set, only one thread will successfully UNSET. 10002 * 10003 * If both states (LAZY flag and task (Y)) are observed, the thread on CPU0 10004 * will observe the LAZY flag and transition to UNSET (perhaps uselessly), and 10005 * CPU1 will observe task (Y) and do nothing more, which is fine. 10006 * 10007 * What we are effectively preventing with this Dekker is a scenario where 10008 * neither LAZY flag nor store (Y) are observed, which would fail property (1) 10009 * because this would UNSET a cid which is actively used. 10010 */ 10011 10012 void sched_mm_cid_migrate_from(struct task_struct *t) 10013 { 10014 t->migrate_from_cpu = task_cpu(t); 10015 } 10016 10017 static 10018 int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq, 10019 struct task_struct *t, 10020 struct mm_cid *src_pcpu_cid) 10021 { 10022 struct mm_struct *mm = t->mm; 10023 struct task_struct *src_task; 10024 int src_cid, last_mm_cid; 10025 10026 if (!mm) 10027 return -1; 10028 10029 last_mm_cid = t->last_mm_cid; 10030 /* 10031 * If the migrated task has no last cid, or if the current 10032 * task on src rq uses the cid, it means the source cid does not need 10033 * to be moved to the destination cpu. 10034 */ 10035 if (last_mm_cid == -1) 10036 return -1; 10037 src_cid = READ_ONCE(src_pcpu_cid->cid); 10038 if (!mm_cid_is_valid(src_cid) || last_mm_cid != src_cid) 10039 return -1; 10040 10041 /* 10042 * If we observe an active task using the mm on this rq, it means we 10043 * are not the last task to be migrated from this cpu for this mm, so 10044 * there is no need to move src_cid to the destination cpu. 10045 */ 10046 guard(rcu)(); 10047 src_task = rcu_dereference(src_rq->curr); 10048 if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) { 10049 t->last_mm_cid = -1; 10050 return -1; 10051 } 10052 10053 return src_cid; 10054 } 10055 10056 static 10057 int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq, 10058 struct task_struct *t, 10059 struct mm_cid *src_pcpu_cid, 10060 int src_cid) 10061 { 10062 struct task_struct *src_task; 10063 struct mm_struct *mm = t->mm; 10064 int lazy_cid; 10065 10066 if (src_cid == -1) 10067 return -1; 10068 10069 /* 10070 * Attempt to clear the source cpu cid to move it to the destination 10071 * cpu. 10072 */ 10073 lazy_cid = mm_cid_set_lazy_put(src_cid); 10074 if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid)) 10075 return -1; 10076 10077 /* 10078 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 10079 * rq->curr->mm matches the scheduler barrier in context_switch() 10080 * between store to rq->curr and load of prev and next task's 10081 * per-mm/cpu cid. 10082 * 10083 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 10084 * rq->curr->mm_cid_active matches the barrier in 10085 * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and 10086 * sched_mm_cid_after_execve() between store to t->mm_cid_active and 10087 * load of per-mm/cpu cid. 10088 */ 10089 10090 /* 10091 * If we observe an active task using the mm on this rq after setting 10092 * the lazy-put flag, this task will be responsible for transitioning 10093 * from lazy-put flag set to MM_CID_UNSET. 10094 */ 10095 scoped_guard (rcu) { 10096 src_task = rcu_dereference(src_rq->curr); 10097 if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) { 10098 /* 10099 * We observed an active task for this mm, there is therefore 10100 * no point in moving this cid to the destination cpu. 10101 */ 10102 t->last_mm_cid = -1; 10103 return -1; 10104 } 10105 } 10106 10107 /* 10108 * The src_cid is unused, so it can be unset. 10109 */ 10110 if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET)) 10111 return -1; 10112 return src_cid; 10113 } 10114 10115 /* 10116 * Migration to dst cpu. Called with dst_rq lock held. 10117 * Interrupts are disabled, which keeps the window of cid ownership without the 10118 * source rq lock held small. 10119 */ 10120 void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) 10121 { 10122 struct mm_cid *src_pcpu_cid, *dst_pcpu_cid; 10123 struct mm_struct *mm = t->mm; 10124 int src_cid, dst_cid, src_cpu; 10125 struct rq *src_rq; 10126 10127 lockdep_assert_rq_held(dst_rq); 10128 10129 if (!mm) 10130 return; 10131 src_cpu = t->migrate_from_cpu; 10132 if (src_cpu == -1) { 10133 t->last_mm_cid = -1; 10134 return; 10135 } 10136 /* 10137 * Move the src cid if the dst cid is unset. This keeps id 10138 * allocation closest to 0 in cases where few threads migrate around 10139 * many CPUs. 10140 * 10141 * If destination cid is already set, we may have to just clear 10142 * the src cid to ensure compactness in frequent migrations 10143 * scenarios. 10144 * 10145 * It is not useful to clear the src cid when the number of threads is 10146 * greater or equal to the number of allowed CPUs, because user-space 10147 * can expect that the number of allowed cids can reach the number of 10148 * allowed CPUs. 10149 */ 10150 dst_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(dst_rq)); 10151 dst_cid = READ_ONCE(dst_pcpu_cid->cid); 10152 if (!mm_cid_is_unset(dst_cid) && 10153 atomic_read(&mm->mm_users) >= t->nr_cpus_allowed) 10154 return; 10155 src_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, src_cpu); 10156 src_rq = cpu_rq(src_cpu); 10157 src_cid = __sched_mm_cid_migrate_from_fetch_cid(src_rq, t, src_pcpu_cid); 10158 if (src_cid == -1) 10159 return; 10160 src_cid = __sched_mm_cid_migrate_from_try_steal_cid(src_rq, t, src_pcpu_cid, 10161 src_cid); 10162 if (src_cid == -1) 10163 return; 10164 if (!mm_cid_is_unset(dst_cid)) { 10165 __mm_cid_put(mm, src_cid); 10166 return; 10167 } 10168 /* Move src_cid to dst cpu. */ 10169 mm_cid_snapshot_time(dst_rq, mm); 10170 WRITE_ONCE(dst_pcpu_cid->cid, src_cid); 10171 } 10172 10173 static void sched_mm_cid_remote_clear(struct mm_struct *mm, struct mm_cid *pcpu_cid, 10174 int cpu) 10175 { 10176 struct rq *rq = cpu_rq(cpu); 10177 struct task_struct *t; 10178 int cid, lazy_cid; 10179 10180 cid = READ_ONCE(pcpu_cid->cid); 10181 if (!mm_cid_is_valid(cid)) 10182 return; 10183 10184 /* 10185 * Clear the cpu cid if it is set to keep cid allocation compact. If 10186 * there happens to be other tasks left on the source cpu using this 10187 * mm, the next task using this mm will reallocate its cid on context 10188 * switch. 10189 */ 10190 lazy_cid = mm_cid_set_lazy_put(cid); 10191 if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid)) 10192 return; 10193 10194 /* 10195 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 10196 * rq->curr->mm matches the scheduler barrier in context_switch() 10197 * between store to rq->curr and load of prev and next task's 10198 * per-mm/cpu cid. 10199 * 10200 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 10201 * rq->curr->mm_cid_active matches the barrier in 10202 * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and 10203 * sched_mm_cid_after_execve() between store to t->mm_cid_active and 10204 * load of per-mm/cpu cid. 10205 */ 10206 10207 /* 10208 * If we observe an active task using the mm on this rq after setting 10209 * the lazy-put flag, that task will be responsible for transitioning 10210 * from lazy-put flag set to MM_CID_UNSET. 10211 */ 10212 scoped_guard (rcu) { 10213 t = rcu_dereference(rq->curr); 10214 if (READ_ONCE(t->mm_cid_active) && t->mm == mm) 10215 return; 10216 } 10217 10218 /* 10219 * The cid is unused, so it can be unset. 10220 * Disable interrupts to keep the window of cid ownership without rq 10221 * lock small. 10222 */ 10223 scoped_guard (irqsave) { 10224 if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET)) 10225 __mm_cid_put(mm, cid); 10226 } 10227 } 10228 10229 static void sched_mm_cid_remote_clear_old(struct mm_struct *mm, int cpu) 10230 { 10231 struct rq *rq = cpu_rq(cpu); 10232 struct mm_cid *pcpu_cid; 10233 struct task_struct *curr; 10234 u64 rq_clock; 10235 10236 /* 10237 * rq->clock load is racy on 32-bit but one spurious clear once in a 10238 * while is irrelevant. 10239 */ 10240 rq_clock = READ_ONCE(rq->clock); 10241 pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu); 10242 10243 /* 10244 * In order to take care of infrequently scheduled tasks, bump the time 10245 * snapshot associated with this cid if an active task using the mm is 10246 * observed on this rq. 10247 */ 10248 scoped_guard (rcu) { 10249 curr = rcu_dereference(rq->curr); 10250 if (READ_ONCE(curr->mm_cid_active) && curr->mm == mm) { 10251 WRITE_ONCE(pcpu_cid->time, rq_clock); 10252 return; 10253 } 10254 } 10255 10256 if (rq_clock < pcpu_cid->time + SCHED_MM_CID_PERIOD_NS) 10257 return; 10258 sched_mm_cid_remote_clear(mm, pcpu_cid, cpu); 10259 } 10260 10261 static void sched_mm_cid_remote_clear_weight(struct mm_struct *mm, int cpu, 10262 int weight) 10263 { 10264 struct mm_cid *pcpu_cid; 10265 int cid; 10266 10267 pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu); 10268 cid = READ_ONCE(pcpu_cid->cid); 10269 if (!mm_cid_is_valid(cid) || cid < weight) 10270 return; 10271 sched_mm_cid_remote_clear(mm, pcpu_cid, cpu); 10272 } 10273 10274 static void task_mm_cid_work(struct callback_head *work) 10275 { 10276 unsigned long now = jiffies, old_scan, next_scan; 10277 struct task_struct *t = current; 10278 struct cpumask *cidmask; 10279 struct mm_struct *mm; 10280 int weight, cpu; 10281 10282 SCHED_WARN_ON(t != container_of(work, struct task_struct, cid_work)); 10283 10284 work->next = work; /* Prevent double-add */ 10285 if (t->flags & PF_EXITING) 10286 return; 10287 mm = t->mm; 10288 if (!mm) 10289 return; 10290 old_scan = READ_ONCE(mm->mm_cid_next_scan); 10291 next_scan = now + msecs_to_jiffies(MM_CID_SCAN_DELAY); 10292 if (!old_scan) { 10293 unsigned long res; 10294 10295 res = cmpxchg(&mm->mm_cid_next_scan, old_scan, next_scan); 10296 if (res != old_scan) 10297 old_scan = res; 10298 else 10299 old_scan = next_scan; 10300 } 10301 if (time_before(now, old_scan)) 10302 return; 10303 if (!try_cmpxchg(&mm->mm_cid_next_scan, &old_scan, next_scan)) 10304 return; 10305 cidmask = mm_cidmask(mm); 10306 /* Clear cids that were not recently used. */ 10307 for_each_possible_cpu(cpu) 10308 sched_mm_cid_remote_clear_old(mm, cpu); 10309 weight = cpumask_weight(cidmask); 10310 /* 10311 * Clear cids that are greater or equal to the cidmask weight to 10312 * recompact it. 10313 */ 10314 for_each_possible_cpu(cpu) 10315 sched_mm_cid_remote_clear_weight(mm, cpu, weight); 10316 } 10317 10318 void init_sched_mm_cid(struct task_struct *t) 10319 { 10320 struct mm_struct *mm = t->mm; 10321 int mm_users = 0; 10322 10323 if (mm) { 10324 mm_users = atomic_read(&mm->mm_users); 10325 if (mm_users == 1) 10326 mm->mm_cid_next_scan = jiffies + msecs_to_jiffies(MM_CID_SCAN_DELAY); 10327 } 10328 t->cid_work.next = &t->cid_work; /* Protect against double add */ 10329 init_task_work(&t->cid_work, task_mm_cid_work); 10330 } 10331 10332 void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) 10333 { 10334 struct callback_head *work = &curr->cid_work; 10335 unsigned long now = jiffies; 10336 10337 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || 10338 work->next != work) 10339 return; 10340 if (time_before(now, READ_ONCE(curr->mm->mm_cid_next_scan))) 10341 return; 10342 task_work_add(curr, work, TWA_RESUME); 10343 } 10344 10345 void sched_mm_cid_exit_signals(struct task_struct *t) 10346 { 10347 struct mm_struct *mm = t->mm; 10348 struct rq *rq; 10349 10350 if (!mm) 10351 return; 10352 10353 preempt_disable(); 10354 rq = this_rq(); 10355 guard(rq_lock_irqsave)(rq); 10356 preempt_enable_no_resched(); /* holding spinlock */ 10357 WRITE_ONCE(t->mm_cid_active, 0); 10358 /* 10359 * Store t->mm_cid_active before loading per-mm/cpu cid. 10360 * Matches barrier in sched_mm_cid_remote_clear_old(). 10361 */ 10362 smp_mb(); 10363 mm_cid_put(mm); 10364 t->last_mm_cid = t->mm_cid = -1; 10365 } 10366 10367 void sched_mm_cid_before_execve(struct task_struct *t) 10368 { 10369 struct mm_struct *mm = t->mm; 10370 struct rq *rq; 10371 10372 if (!mm) 10373 return; 10374 10375 preempt_disable(); 10376 rq = this_rq(); 10377 guard(rq_lock_irqsave)(rq); 10378 preempt_enable_no_resched(); /* holding spinlock */ 10379 WRITE_ONCE(t->mm_cid_active, 0); 10380 /* 10381 * Store t->mm_cid_active before loading per-mm/cpu cid. 10382 * Matches barrier in sched_mm_cid_remote_clear_old(). 10383 */ 10384 smp_mb(); 10385 mm_cid_put(mm); 10386 t->last_mm_cid = t->mm_cid = -1; 10387 } 10388 10389 void sched_mm_cid_after_execve(struct task_struct *t) 10390 { 10391 struct mm_struct *mm = t->mm; 10392 struct rq *rq; 10393 10394 if (!mm) 10395 return; 10396 10397 preempt_disable(); 10398 rq = this_rq(); 10399 scoped_guard (rq_lock_irqsave, rq) { 10400 preempt_enable_no_resched(); /* holding spinlock */ 10401 WRITE_ONCE(t->mm_cid_active, 1); 10402 /* 10403 * Store t->mm_cid_active before loading per-mm/cpu cid. 10404 * Matches barrier in sched_mm_cid_remote_clear_old(). 10405 */ 10406 smp_mb(); 10407 t->last_mm_cid = t->mm_cid = mm_cid_get(rq, mm); 10408 } 10409 rseq_set_notify_resume(t); 10410 } 10411 10412 void sched_mm_cid_fork(struct task_struct *t) 10413 { 10414 WARN_ON_ONCE(!t->mm || t->mm_cid != -1); 10415 t->mm_cid_active = 1; 10416 } 10417 #endif 10418