1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/sched/core.c 4 * 5 * Core kernel scheduler code and related syscalls 6 * 7 * Copyright (C) 1991-2002 Linus Torvalds 8 */ 9 #define CREATE_TRACE_POINTS 10 #include <trace/events/sched.h> 11 #undef CREATE_TRACE_POINTS 12 13 #include "sched.h" 14 15 #include <linux/nospec.h> 16 17 #include <linux/kcov.h> 18 #include <linux/scs.h> 19 20 #include <asm/switch_to.h> 21 #include <asm/tlb.h> 22 23 #include "../workqueue_internal.h" 24 #include "../../fs/io-wq.h" 25 #include "../smpboot.h" 26 27 #include "pelt.h" 28 #include "smp.h" 29 30 /* 31 * Export tracepoints that act as a bare tracehook (ie: have no trace event 32 * associated with them) to allow external modules to probe them. 33 */ 34 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp); 35 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp); 36 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp); 37 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp); 38 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp); 39 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp); 40 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp); 41 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); 42 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); 43 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); 44 45 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 46 47 #ifdef CONFIG_SCHED_DEBUG 48 /* 49 * Debugging: various feature bits 50 * 51 * If SCHED_DEBUG is disabled, each compilation unit has its own copy of 52 * sysctl_sched_features, defined in sched.h, to allow constants propagation 53 * at compile time and compiler optimization based on features default. 54 */ 55 #define SCHED_FEAT(name, enabled) \ 56 (1UL << __SCHED_FEAT_##name) * enabled | 57 const_debug unsigned int sysctl_sched_features = 58 #include "features.h" 59 0; 60 #undef SCHED_FEAT 61 62 /* 63 * Print a warning if need_resched is set for the given duration (if 64 * LATENCY_WARN is enabled). 65 * 66 * If sysctl_resched_latency_warn_once is set, only one warning will be shown 67 * per boot. 68 */ 69 __read_mostly int sysctl_resched_latency_warn_ms = 100; 70 __read_mostly int sysctl_resched_latency_warn_once = 1; 71 #endif /* CONFIG_SCHED_DEBUG */ 72 73 /* 74 * Number of tasks to iterate in a single balance run. 75 * Limited because this is done with IRQs disabled. 76 */ 77 const_debug unsigned int sysctl_sched_nr_migrate = 32; 78 79 /* 80 * period over which we measure -rt task CPU usage in us. 81 * default: 1s 82 */ 83 unsigned int sysctl_sched_rt_period = 1000000; 84 85 __read_mostly int scheduler_running; 86 87 #ifdef CONFIG_SCHED_CORE 88 89 DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); 90 91 /* kernel prio, less is more */ 92 static inline int __task_prio(struct task_struct *p) 93 { 94 if (p->sched_class == &stop_sched_class) /* trumps deadline */ 95 return -2; 96 97 if (rt_prio(p->prio)) /* includes deadline */ 98 return p->prio; /* [-1, 99] */ 99 100 if (p->sched_class == &idle_sched_class) 101 return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ 102 103 return MAX_RT_PRIO + MAX_NICE; /* 120, squash fair */ 104 } 105 106 /* 107 * l(a,b) 108 * le(a,b) := !l(b,a) 109 * g(a,b) := l(b,a) 110 * ge(a,b) := !l(a,b) 111 */ 112 113 /* real prio, less is less */ 114 static inline bool prio_less(struct task_struct *a, struct task_struct *b, bool in_fi) 115 { 116 117 int pa = __task_prio(a), pb = __task_prio(b); 118 119 if (-pa < -pb) 120 return true; 121 122 if (-pb < -pa) 123 return false; 124 125 if (pa == -1) /* dl_prio() doesn't work because of stop_class above */ 126 return !dl_time_before(a->dl.deadline, b->dl.deadline); 127 128 if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ 129 return cfs_prio_less(a, b, in_fi); 130 131 return false; 132 } 133 134 static inline bool __sched_core_less(struct task_struct *a, struct task_struct *b) 135 { 136 if (a->core_cookie < b->core_cookie) 137 return true; 138 139 if (a->core_cookie > b->core_cookie) 140 return false; 141 142 /* flip prio, so high prio is leftmost */ 143 if (prio_less(b, a, task_rq(a)->core->core_forceidle)) 144 return true; 145 146 return false; 147 } 148 149 #define __node_2_sc(node) rb_entry((node), struct task_struct, core_node) 150 151 static inline bool rb_sched_core_less(struct rb_node *a, const struct rb_node *b) 152 { 153 return __sched_core_less(__node_2_sc(a), __node_2_sc(b)); 154 } 155 156 static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node) 157 { 158 const struct task_struct *p = __node_2_sc(node); 159 unsigned long cookie = (unsigned long)key; 160 161 if (cookie < p->core_cookie) 162 return -1; 163 164 if (cookie > p->core_cookie) 165 return 1; 166 167 return 0; 168 } 169 170 void sched_core_enqueue(struct rq *rq, struct task_struct *p) 171 { 172 rq->core->core_task_seq++; 173 174 if (!p->core_cookie) 175 return; 176 177 rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less); 178 } 179 180 void sched_core_dequeue(struct rq *rq, struct task_struct *p) 181 { 182 rq->core->core_task_seq++; 183 184 if (!sched_core_enqueued(p)) 185 return; 186 187 rb_erase(&p->core_node, &rq->core_tree); 188 RB_CLEAR_NODE(&p->core_node); 189 } 190 191 /* 192 * Find left-most (aka, highest priority) task matching @cookie. 193 */ 194 static struct task_struct *sched_core_find(struct rq *rq, unsigned long cookie) 195 { 196 struct rb_node *node; 197 198 node = rb_find_first((void *)cookie, &rq->core_tree, rb_sched_core_cmp); 199 /* 200 * The idle task always matches any cookie! 201 */ 202 if (!node) 203 return idle_sched_class.pick_task(rq); 204 205 return __node_2_sc(node); 206 } 207 208 static struct task_struct *sched_core_next(struct task_struct *p, unsigned long cookie) 209 { 210 struct rb_node *node = &p->core_node; 211 212 node = rb_next(node); 213 if (!node) 214 return NULL; 215 216 p = container_of(node, struct task_struct, core_node); 217 if (p->core_cookie != cookie) 218 return NULL; 219 220 return p; 221 } 222 223 /* 224 * Magic required such that: 225 * 226 * raw_spin_rq_lock(rq); 227 * ... 228 * raw_spin_rq_unlock(rq); 229 * 230 * ends up locking and unlocking the _same_ lock, and all CPUs 231 * always agree on what rq has what lock. 232 * 233 * XXX entirely possible to selectively enable cores, don't bother for now. 234 */ 235 236 static DEFINE_MUTEX(sched_core_mutex); 237 static atomic_t sched_core_count; 238 static struct cpumask sched_core_mask; 239 240 static void __sched_core_flip(bool enabled) 241 { 242 int cpu, t, i; 243 244 cpus_read_lock(); 245 246 /* 247 * Toggle the online cores, one by one. 248 */ 249 cpumask_copy(&sched_core_mask, cpu_online_mask); 250 for_each_cpu(cpu, &sched_core_mask) { 251 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 252 253 i = 0; 254 local_irq_disable(); 255 for_each_cpu(t, smt_mask) { 256 /* supports up to SMT8 */ 257 raw_spin_lock_nested(&cpu_rq(t)->__lock, i++); 258 } 259 260 for_each_cpu(t, smt_mask) 261 cpu_rq(t)->core_enabled = enabled; 262 263 for_each_cpu(t, smt_mask) 264 raw_spin_unlock(&cpu_rq(t)->__lock); 265 local_irq_enable(); 266 267 cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask); 268 } 269 270 /* 271 * Toggle the offline CPUs. 272 */ 273 cpumask_copy(&sched_core_mask, cpu_possible_mask); 274 cpumask_andnot(&sched_core_mask, &sched_core_mask, cpu_online_mask); 275 276 for_each_cpu(cpu, &sched_core_mask) 277 cpu_rq(cpu)->core_enabled = enabled; 278 279 cpus_read_unlock(); 280 } 281 282 static void sched_core_assert_empty(void) 283 { 284 int cpu; 285 286 for_each_possible_cpu(cpu) 287 WARN_ON_ONCE(!RB_EMPTY_ROOT(&cpu_rq(cpu)->core_tree)); 288 } 289 290 static void __sched_core_enable(void) 291 { 292 static_branch_enable(&__sched_core_enabled); 293 /* 294 * Ensure all previous instances of raw_spin_rq_*lock() have finished 295 * and future ones will observe !sched_core_disabled(). 296 */ 297 synchronize_rcu(); 298 __sched_core_flip(true); 299 sched_core_assert_empty(); 300 } 301 302 static void __sched_core_disable(void) 303 { 304 sched_core_assert_empty(); 305 __sched_core_flip(false); 306 static_branch_disable(&__sched_core_enabled); 307 } 308 309 void sched_core_get(void) 310 { 311 if (atomic_inc_not_zero(&sched_core_count)) 312 return; 313 314 mutex_lock(&sched_core_mutex); 315 if (!atomic_read(&sched_core_count)) 316 __sched_core_enable(); 317 318 smp_mb__before_atomic(); 319 atomic_inc(&sched_core_count); 320 mutex_unlock(&sched_core_mutex); 321 } 322 323 static void __sched_core_put(struct work_struct *work) 324 { 325 if (atomic_dec_and_mutex_lock(&sched_core_count, &sched_core_mutex)) { 326 __sched_core_disable(); 327 mutex_unlock(&sched_core_mutex); 328 } 329 } 330 331 void sched_core_put(void) 332 { 333 static DECLARE_WORK(_work, __sched_core_put); 334 335 /* 336 * "There can be only one" 337 * 338 * Either this is the last one, or we don't actually need to do any 339 * 'work'. If it is the last *again*, we rely on 340 * WORK_STRUCT_PENDING_BIT. 341 */ 342 if (!atomic_add_unless(&sched_core_count, -1, 1)) 343 schedule_work(&_work); 344 } 345 346 #else /* !CONFIG_SCHED_CORE */ 347 348 static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { } 349 static inline void sched_core_dequeue(struct rq *rq, struct task_struct *p) { } 350 351 #endif /* CONFIG_SCHED_CORE */ 352 353 /* 354 * part of the period that we allow rt tasks to run in us. 355 * default: 0.95s 356 */ 357 int sysctl_sched_rt_runtime = 950000; 358 359 360 /* 361 * Serialization rules: 362 * 363 * Lock order: 364 * 365 * p->pi_lock 366 * rq->lock 367 * hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls) 368 * 369 * rq1->lock 370 * rq2->lock where: rq1 < rq2 371 * 372 * Regular state: 373 * 374 * Normal scheduling state is serialized by rq->lock. __schedule() takes the 375 * local CPU's rq->lock, it optionally removes the task from the runqueue and 376 * always looks at the local rq data structures to find the most eligible task 377 * to run next. 378 * 379 * Task enqueue is also under rq->lock, possibly taken from another CPU. 380 * Wakeups from another LLC domain might use an IPI to transfer the enqueue to 381 * the local CPU to avoid bouncing the runqueue state around [ see 382 * ttwu_queue_wakelist() ] 383 * 384 * Task wakeup, specifically wakeups that involve migration, are horribly 385 * complicated to avoid having to take two rq->locks. 386 * 387 * Special state: 388 * 389 * System-calls and anything external will use task_rq_lock() which acquires 390 * both p->pi_lock and rq->lock. As a consequence the state they change is 391 * stable while holding either lock: 392 * 393 * - sched_setaffinity()/ 394 * set_cpus_allowed_ptr(): p->cpus_ptr, p->nr_cpus_allowed 395 * - set_user_nice(): p->se.load, p->*prio 396 * - __sched_setscheduler(): p->sched_class, p->policy, p->*prio, 397 * p->se.load, p->rt_priority, 398 * p->dl.dl_{runtime, deadline, period, flags, bw, density} 399 * - sched_setnuma(): p->numa_preferred_nid 400 * - sched_move_task()/ 401 * cpu_cgroup_fork(): p->sched_task_group 402 * - uclamp_update_active() p->uclamp* 403 * 404 * p->state <- TASK_*: 405 * 406 * is changed locklessly using set_current_state(), __set_current_state() or 407 * set_special_state(), see their respective comments, or by 408 * try_to_wake_up(). This latter uses p->pi_lock to serialize against 409 * concurrent self. 410 * 411 * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }: 412 * 413 * is set by activate_task() and cleared by deactivate_task(), under 414 * rq->lock. Non-zero indicates the task is runnable, the special 415 * ON_RQ_MIGRATING state is used for migration without holding both 416 * rq->locks. It indicates task_cpu() is not stable, see task_rq_lock(). 417 * 418 * p->on_cpu <- { 0, 1 }: 419 * 420 * is set by prepare_task() and cleared by finish_task() such that it will be 421 * set before p is scheduled-in and cleared after p is scheduled-out, both 422 * under rq->lock. Non-zero indicates the task is running on its CPU. 423 * 424 * [ The astute reader will observe that it is possible for two tasks on one 425 * CPU to have ->on_cpu = 1 at the same time. ] 426 * 427 * task_cpu(p): is changed by set_task_cpu(), the rules are: 428 * 429 * - Don't call set_task_cpu() on a blocked task: 430 * 431 * We don't care what CPU we're not running on, this simplifies hotplug, 432 * the CPU assignment of blocked tasks isn't required to be valid. 433 * 434 * - for try_to_wake_up(), called under p->pi_lock: 435 * 436 * This allows try_to_wake_up() to only take one rq->lock, see its comment. 437 * 438 * - for migration called under rq->lock: 439 * [ see task_on_rq_migrating() in task_rq_lock() ] 440 * 441 * o move_queued_task() 442 * o detach_task() 443 * 444 * - for migration called under double_rq_lock(): 445 * 446 * o __migrate_swap_task() 447 * o push_rt_task() / pull_rt_task() 448 * o push_dl_task() / pull_dl_task() 449 * o dl_task_offline_migration() 450 * 451 */ 452 453 void raw_spin_rq_lock_nested(struct rq *rq, int subclass) 454 { 455 raw_spinlock_t *lock; 456 457 /* Matches synchronize_rcu() in __sched_core_enable() */ 458 preempt_disable(); 459 if (sched_core_disabled()) { 460 raw_spin_lock_nested(&rq->__lock, subclass); 461 /* preempt_count *MUST* be > 1 */ 462 preempt_enable_no_resched(); 463 return; 464 } 465 466 for (;;) { 467 lock = __rq_lockp(rq); 468 raw_spin_lock_nested(lock, subclass); 469 if (likely(lock == __rq_lockp(rq))) { 470 /* preempt_count *MUST* be > 1 */ 471 preempt_enable_no_resched(); 472 return; 473 } 474 raw_spin_unlock(lock); 475 } 476 } 477 478 bool raw_spin_rq_trylock(struct rq *rq) 479 { 480 raw_spinlock_t *lock; 481 bool ret; 482 483 /* Matches synchronize_rcu() in __sched_core_enable() */ 484 preempt_disable(); 485 if (sched_core_disabled()) { 486 ret = raw_spin_trylock(&rq->__lock); 487 preempt_enable(); 488 return ret; 489 } 490 491 for (;;) { 492 lock = __rq_lockp(rq); 493 ret = raw_spin_trylock(lock); 494 if (!ret || (likely(lock == __rq_lockp(rq)))) { 495 preempt_enable(); 496 return ret; 497 } 498 raw_spin_unlock(lock); 499 } 500 } 501 502 void raw_spin_rq_unlock(struct rq *rq) 503 { 504 raw_spin_unlock(rq_lockp(rq)); 505 } 506 507 #ifdef CONFIG_SMP 508 /* 509 * double_rq_lock - safely lock two runqueues 510 */ 511 void double_rq_lock(struct rq *rq1, struct rq *rq2) 512 { 513 lockdep_assert_irqs_disabled(); 514 515 if (rq_order_less(rq2, rq1)) 516 swap(rq1, rq2); 517 518 raw_spin_rq_lock(rq1); 519 if (__rq_lockp(rq1) == __rq_lockp(rq2)) 520 return; 521 522 raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING); 523 } 524 #endif 525 526 /* 527 * __task_rq_lock - lock the rq @p resides on. 528 */ 529 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) 530 __acquires(rq->lock) 531 { 532 struct rq *rq; 533 534 lockdep_assert_held(&p->pi_lock); 535 536 for (;;) { 537 rq = task_rq(p); 538 raw_spin_rq_lock(rq); 539 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 540 rq_pin_lock(rq, rf); 541 return rq; 542 } 543 raw_spin_rq_unlock(rq); 544 545 while (unlikely(task_on_rq_migrating(p))) 546 cpu_relax(); 547 } 548 } 549 550 /* 551 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. 552 */ 553 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) 554 __acquires(p->pi_lock) 555 __acquires(rq->lock) 556 { 557 struct rq *rq; 558 559 for (;;) { 560 raw_spin_lock_irqsave(&p->pi_lock, rf->flags); 561 rq = task_rq(p); 562 raw_spin_rq_lock(rq); 563 /* 564 * move_queued_task() task_rq_lock() 565 * 566 * ACQUIRE (rq->lock) 567 * [S] ->on_rq = MIGRATING [L] rq = task_rq() 568 * WMB (__set_task_cpu()) ACQUIRE (rq->lock); 569 * [S] ->cpu = new_cpu [L] task_rq() 570 * [L] ->on_rq 571 * RELEASE (rq->lock) 572 * 573 * If we observe the old CPU in task_rq_lock(), the acquire of 574 * the old rq->lock will fully serialize against the stores. 575 * 576 * If we observe the new CPU in task_rq_lock(), the address 577 * dependency headed by '[L] rq = task_rq()' and the acquire 578 * will pair with the WMB to ensure we then also see migrating. 579 */ 580 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 581 rq_pin_lock(rq, rf); 582 return rq; 583 } 584 raw_spin_rq_unlock(rq); 585 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 586 587 while (unlikely(task_on_rq_migrating(p))) 588 cpu_relax(); 589 } 590 } 591 592 /* 593 * RQ-clock updating methods: 594 */ 595 596 static void update_rq_clock_task(struct rq *rq, s64 delta) 597 { 598 /* 599 * In theory, the compile should just see 0 here, and optimize out the call 600 * to sched_rt_avg_update. But I don't trust it... 601 */ 602 s64 __maybe_unused steal = 0, irq_delta = 0; 603 604 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 605 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 606 607 /* 608 * Since irq_time is only updated on {soft,}irq_exit, we might run into 609 * this case when a previous update_rq_clock() happened inside a 610 * {soft,}irq region. 611 * 612 * When this happens, we stop ->clock_task and only update the 613 * prev_irq_time stamp to account for the part that fit, so that a next 614 * update will consume the rest. This ensures ->clock_task is 615 * monotonic. 616 * 617 * It does however cause some slight miss-attribution of {soft,}irq 618 * time, a more accurate solution would be to update the irq_time using 619 * the current rq->clock timestamp, except that would require using 620 * atomic ops. 621 */ 622 if (irq_delta > delta) 623 irq_delta = delta; 624 625 rq->prev_irq_time += irq_delta; 626 delta -= irq_delta; 627 #endif 628 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 629 if (static_key_false((¶virt_steal_rq_enabled))) { 630 steal = paravirt_steal_clock(cpu_of(rq)); 631 steal -= rq->prev_steal_time_rq; 632 633 if (unlikely(steal > delta)) 634 steal = delta; 635 636 rq->prev_steal_time_rq += steal; 637 delta -= steal; 638 } 639 #endif 640 641 rq->clock_task += delta; 642 643 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 644 if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) 645 update_irq_load_avg(rq, irq_delta + steal); 646 #endif 647 update_rq_clock_pelt(rq, delta); 648 } 649 650 void update_rq_clock(struct rq *rq) 651 { 652 s64 delta; 653 654 lockdep_assert_rq_held(rq); 655 656 if (rq->clock_update_flags & RQCF_ACT_SKIP) 657 return; 658 659 #ifdef CONFIG_SCHED_DEBUG 660 if (sched_feat(WARN_DOUBLE_CLOCK)) 661 SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED); 662 rq->clock_update_flags |= RQCF_UPDATED; 663 #endif 664 665 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; 666 if (delta < 0) 667 return; 668 rq->clock += delta; 669 update_rq_clock_task(rq, delta); 670 } 671 672 #ifdef CONFIG_SCHED_HRTICK 673 /* 674 * Use HR-timers to deliver accurate preemption points. 675 */ 676 677 static void hrtick_clear(struct rq *rq) 678 { 679 if (hrtimer_active(&rq->hrtick_timer)) 680 hrtimer_cancel(&rq->hrtick_timer); 681 } 682 683 /* 684 * High-resolution timer tick. 685 * Runs from hardirq context with interrupts disabled. 686 */ 687 static enum hrtimer_restart hrtick(struct hrtimer *timer) 688 { 689 struct rq *rq = container_of(timer, struct rq, hrtick_timer); 690 struct rq_flags rf; 691 692 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 693 694 rq_lock(rq, &rf); 695 update_rq_clock(rq); 696 rq->curr->sched_class->task_tick(rq, rq->curr, 1); 697 rq_unlock(rq, &rf); 698 699 return HRTIMER_NORESTART; 700 } 701 702 #ifdef CONFIG_SMP 703 704 static void __hrtick_restart(struct rq *rq) 705 { 706 struct hrtimer *timer = &rq->hrtick_timer; 707 ktime_t time = rq->hrtick_time; 708 709 hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD); 710 } 711 712 /* 713 * called from hardirq (IPI) context 714 */ 715 static void __hrtick_start(void *arg) 716 { 717 struct rq *rq = arg; 718 struct rq_flags rf; 719 720 rq_lock(rq, &rf); 721 __hrtick_restart(rq); 722 rq_unlock(rq, &rf); 723 } 724 725 /* 726 * Called to set the hrtick timer state. 727 * 728 * called with rq->lock held and irqs disabled 729 */ 730 void hrtick_start(struct rq *rq, u64 delay) 731 { 732 struct hrtimer *timer = &rq->hrtick_timer; 733 s64 delta; 734 735 /* 736 * Don't schedule slices shorter than 10000ns, that just 737 * doesn't make sense and can cause timer DoS. 738 */ 739 delta = max_t(s64, delay, 10000LL); 740 rq->hrtick_time = ktime_add_ns(timer->base->get_time(), delta); 741 742 if (rq == this_rq()) 743 __hrtick_restart(rq); 744 else 745 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); 746 } 747 748 #else 749 /* 750 * Called to set the hrtick timer state. 751 * 752 * called with rq->lock held and irqs disabled 753 */ 754 void hrtick_start(struct rq *rq, u64 delay) 755 { 756 /* 757 * Don't schedule slices shorter than 10000ns, that just 758 * doesn't make sense. Rely on vruntime for fairness. 759 */ 760 delay = max_t(u64, delay, 10000LL); 761 hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), 762 HRTIMER_MODE_REL_PINNED_HARD); 763 } 764 765 #endif /* CONFIG_SMP */ 766 767 static void hrtick_rq_init(struct rq *rq) 768 { 769 #ifdef CONFIG_SMP 770 INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq); 771 #endif 772 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 773 rq->hrtick_timer.function = hrtick; 774 } 775 #else /* CONFIG_SCHED_HRTICK */ 776 static inline void hrtick_clear(struct rq *rq) 777 { 778 } 779 780 static inline void hrtick_rq_init(struct rq *rq) 781 { 782 } 783 #endif /* CONFIG_SCHED_HRTICK */ 784 785 /* 786 * cmpxchg based fetch_or, macro so it works for different integer types 787 */ 788 #define fetch_or(ptr, mask) \ 789 ({ \ 790 typeof(ptr) _ptr = (ptr); \ 791 typeof(mask) _mask = (mask); \ 792 typeof(*_ptr) _old, _val = *_ptr; \ 793 \ 794 for (;;) { \ 795 _old = cmpxchg(_ptr, _val, _val | _mask); \ 796 if (_old == _val) \ 797 break; \ 798 _val = _old; \ 799 } \ 800 _old; \ 801 }) 802 803 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) 804 /* 805 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, 806 * this avoids any races wrt polling state changes and thereby avoids 807 * spurious IPIs. 808 */ 809 static bool set_nr_and_not_polling(struct task_struct *p) 810 { 811 struct thread_info *ti = task_thread_info(p); 812 return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); 813 } 814 815 /* 816 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. 817 * 818 * If this returns true, then the idle task promises to call 819 * sched_ttwu_pending() and reschedule soon. 820 */ 821 static bool set_nr_if_polling(struct task_struct *p) 822 { 823 struct thread_info *ti = task_thread_info(p); 824 typeof(ti->flags) old, val = READ_ONCE(ti->flags); 825 826 for (;;) { 827 if (!(val & _TIF_POLLING_NRFLAG)) 828 return false; 829 if (val & _TIF_NEED_RESCHED) 830 return true; 831 old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED); 832 if (old == val) 833 break; 834 val = old; 835 } 836 return true; 837 } 838 839 #else 840 static bool set_nr_and_not_polling(struct task_struct *p) 841 { 842 set_tsk_need_resched(p); 843 return true; 844 } 845 846 #ifdef CONFIG_SMP 847 static bool set_nr_if_polling(struct task_struct *p) 848 { 849 return false; 850 } 851 #endif 852 #endif 853 854 static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task) 855 { 856 struct wake_q_node *node = &task->wake_q; 857 858 /* 859 * Atomically grab the task, if ->wake_q is !nil already it means 860 * it's already queued (either by us or someone else) and will get the 861 * wakeup due to that. 862 * 863 * In order to ensure that a pending wakeup will observe our pending 864 * state, even in the failed case, an explicit smp_mb() must be used. 865 */ 866 smp_mb__before_atomic(); 867 if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL))) 868 return false; 869 870 /* 871 * The head is context local, there can be no concurrency. 872 */ 873 *head->lastp = node; 874 head->lastp = &node->next; 875 return true; 876 } 877 878 /** 879 * wake_q_add() - queue a wakeup for 'later' waking. 880 * @head: the wake_q_head to add @task to 881 * @task: the task to queue for 'later' wakeup 882 * 883 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 884 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 885 * instantly. 886 * 887 * This function must be used as-if it were wake_up_process(); IOW the task 888 * must be ready to be woken at this location. 889 */ 890 void wake_q_add(struct wake_q_head *head, struct task_struct *task) 891 { 892 if (__wake_q_add(head, task)) 893 get_task_struct(task); 894 } 895 896 /** 897 * wake_q_add_safe() - safely queue a wakeup for 'later' waking. 898 * @head: the wake_q_head to add @task to 899 * @task: the task to queue for 'later' wakeup 900 * 901 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 902 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 903 * instantly. 904 * 905 * This function must be used as-if it were wake_up_process(); IOW the task 906 * must be ready to be woken at this location. 907 * 908 * This function is essentially a task-safe equivalent to wake_q_add(). Callers 909 * that already hold reference to @task can call the 'safe' version and trust 910 * wake_q to do the right thing depending whether or not the @task is already 911 * queued for wakeup. 912 */ 913 void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task) 914 { 915 if (!__wake_q_add(head, task)) 916 put_task_struct(task); 917 } 918 919 void wake_up_q(struct wake_q_head *head) 920 { 921 struct wake_q_node *node = head->first; 922 923 while (node != WAKE_Q_TAIL) { 924 struct task_struct *task; 925 926 task = container_of(node, struct task_struct, wake_q); 927 /* Task can safely be re-inserted now: */ 928 node = node->next; 929 task->wake_q.next = NULL; 930 931 /* 932 * wake_up_process() executes a full barrier, which pairs with 933 * the queueing in wake_q_add() so as not to miss wakeups. 934 */ 935 wake_up_process(task); 936 put_task_struct(task); 937 } 938 } 939 940 /* 941 * resched_curr - mark rq's current task 'to be rescheduled now'. 942 * 943 * On UP this means the setting of the need_resched flag, on SMP it 944 * might also involve a cross-CPU call to trigger the scheduler on 945 * the target CPU. 946 */ 947 void resched_curr(struct rq *rq) 948 { 949 struct task_struct *curr = rq->curr; 950 int cpu; 951 952 lockdep_assert_rq_held(rq); 953 954 if (test_tsk_need_resched(curr)) 955 return; 956 957 cpu = cpu_of(rq); 958 959 if (cpu == smp_processor_id()) { 960 set_tsk_need_resched(curr); 961 set_preempt_need_resched(); 962 return; 963 } 964 965 if (set_nr_and_not_polling(curr)) 966 smp_send_reschedule(cpu); 967 else 968 trace_sched_wake_idle_without_ipi(cpu); 969 } 970 971 void resched_cpu(int cpu) 972 { 973 struct rq *rq = cpu_rq(cpu); 974 unsigned long flags; 975 976 raw_spin_rq_lock_irqsave(rq, flags); 977 if (cpu_online(cpu) || cpu == smp_processor_id()) 978 resched_curr(rq); 979 raw_spin_rq_unlock_irqrestore(rq, flags); 980 } 981 982 #ifdef CONFIG_SMP 983 #ifdef CONFIG_NO_HZ_COMMON 984 /* 985 * In the semi idle case, use the nearest busy CPU for migrating timers 986 * from an idle CPU. This is good for power-savings. 987 * 988 * We don't do similar optimization for completely idle system, as 989 * selecting an idle CPU will add more delays to the timers than intended 990 * (as that CPU's timer base may not be uptodate wrt jiffies etc). 991 */ 992 int get_nohz_timer_target(void) 993 { 994 int i, cpu = smp_processor_id(), default_cpu = -1; 995 struct sched_domain *sd; 996 997 if (housekeeping_cpu(cpu, HK_FLAG_TIMER)) { 998 if (!idle_cpu(cpu)) 999 return cpu; 1000 default_cpu = cpu; 1001 } 1002 1003 rcu_read_lock(); 1004 for_each_domain(cpu, sd) { 1005 for_each_cpu_and(i, sched_domain_span(sd), 1006 housekeeping_cpumask(HK_FLAG_TIMER)) { 1007 if (cpu == i) 1008 continue; 1009 1010 if (!idle_cpu(i)) { 1011 cpu = i; 1012 goto unlock; 1013 } 1014 } 1015 } 1016 1017 if (default_cpu == -1) 1018 default_cpu = housekeeping_any_cpu(HK_FLAG_TIMER); 1019 cpu = default_cpu; 1020 unlock: 1021 rcu_read_unlock(); 1022 return cpu; 1023 } 1024 1025 /* 1026 * When add_timer_on() enqueues a timer into the timer wheel of an 1027 * idle CPU then this timer might expire before the next timer event 1028 * which is scheduled to wake up that CPU. In case of a completely 1029 * idle system the next event might even be infinite time into the 1030 * future. wake_up_idle_cpu() ensures that the CPU is woken up and 1031 * leaves the inner idle loop so the newly added timer is taken into 1032 * account when the CPU goes back to idle and evaluates the timer 1033 * wheel for the next timer event. 1034 */ 1035 static void wake_up_idle_cpu(int cpu) 1036 { 1037 struct rq *rq = cpu_rq(cpu); 1038 1039 if (cpu == smp_processor_id()) 1040 return; 1041 1042 if (set_nr_and_not_polling(rq->idle)) 1043 smp_send_reschedule(cpu); 1044 else 1045 trace_sched_wake_idle_without_ipi(cpu); 1046 } 1047 1048 static bool wake_up_full_nohz_cpu(int cpu) 1049 { 1050 /* 1051 * We just need the target to call irq_exit() and re-evaluate 1052 * the next tick. The nohz full kick at least implies that. 1053 * If needed we can still optimize that later with an 1054 * empty IRQ. 1055 */ 1056 if (cpu_is_offline(cpu)) 1057 return true; /* Don't try to wake offline CPUs. */ 1058 if (tick_nohz_full_cpu(cpu)) { 1059 if (cpu != smp_processor_id() || 1060 tick_nohz_tick_stopped()) 1061 tick_nohz_full_kick_cpu(cpu); 1062 return true; 1063 } 1064 1065 return false; 1066 } 1067 1068 /* 1069 * Wake up the specified CPU. If the CPU is going offline, it is the 1070 * caller's responsibility to deal with the lost wakeup, for example, 1071 * by hooking into the CPU_DEAD notifier like timers and hrtimers do. 1072 */ 1073 void wake_up_nohz_cpu(int cpu) 1074 { 1075 if (!wake_up_full_nohz_cpu(cpu)) 1076 wake_up_idle_cpu(cpu); 1077 } 1078 1079 static void nohz_csd_func(void *info) 1080 { 1081 struct rq *rq = info; 1082 int cpu = cpu_of(rq); 1083 unsigned int flags; 1084 1085 /* 1086 * Release the rq::nohz_csd. 1087 */ 1088 flags = atomic_fetch_andnot(NOHZ_KICK_MASK | NOHZ_NEWILB_KICK, nohz_flags(cpu)); 1089 WARN_ON(!(flags & NOHZ_KICK_MASK)); 1090 1091 rq->idle_balance = idle_cpu(cpu); 1092 if (rq->idle_balance && !need_resched()) { 1093 rq->nohz_idle_balance = flags; 1094 raise_softirq_irqoff(SCHED_SOFTIRQ); 1095 } 1096 } 1097 1098 #endif /* CONFIG_NO_HZ_COMMON */ 1099 1100 #ifdef CONFIG_NO_HZ_FULL 1101 bool sched_can_stop_tick(struct rq *rq) 1102 { 1103 int fifo_nr_running; 1104 1105 /* Deadline tasks, even if single, need the tick */ 1106 if (rq->dl.dl_nr_running) 1107 return false; 1108 1109 /* 1110 * If there are more than one RR tasks, we need the tick to affect the 1111 * actual RR behaviour. 1112 */ 1113 if (rq->rt.rr_nr_running) { 1114 if (rq->rt.rr_nr_running == 1) 1115 return true; 1116 else 1117 return false; 1118 } 1119 1120 /* 1121 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no 1122 * forced preemption between FIFO tasks. 1123 */ 1124 fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; 1125 if (fifo_nr_running) 1126 return true; 1127 1128 /* 1129 * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; 1130 * if there's more than one we need the tick for involuntary 1131 * preemption. 1132 */ 1133 if (rq->nr_running > 1) 1134 return false; 1135 1136 return true; 1137 } 1138 #endif /* CONFIG_NO_HZ_FULL */ 1139 #endif /* CONFIG_SMP */ 1140 1141 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ 1142 (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) 1143 /* 1144 * Iterate task_group tree rooted at *from, calling @down when first entering a 1145 * node and @up when leaving it for the final time. 1146 * 1147 * Caller must hold rcu_lock or sufficient equivalent. 1148 */ 1149 int walk_tg_tree_from(struct task_group *from, 1150 tg_visitor down, tg_visitor up, void *data) 1151 { 1152 struct task_group *parent, *child; 1153 int ret; 1154 1155 parent = from; 1156 1157 down: 1158 ret = (*down)(parent, data); 1159 if (ret) 1160 goto out; 1161 list_for_each_entry_rcu(child, &parent->children, siblings) { 1162 parent = child; 1163 goto down; 1164 1165 up: 1166 continue; 1167 } 1168 ret = (*up)(parent, data); 1169 if (ret || parent == from) 1170 goto out; 1171 1172 child = parent; 1173 parent = parent->parent; 1174 if (parent) 1175 goto up; 1176 out: 1177 return ret; 1178 } 1179 1180 int tg_nop(struct task_group *tg, void *data) 1181 { 1182 return 0; 1183 } 1184 #endif 1185 1186 static void set_load_weight(struct task_struct *p, bool update_load) 1187 { 1188 int prio = p->static_prio - MAX_RT_PRIO; 1189 struct load_weight *load = &p->se.load; 1190 1191 /* 1192 * SCHED_IDLE tasks get minimal weight: 1193 */ 1194 if (task_has_idle_policy(p)) { 1195 load->weight = scale_load(WEIGHT_IDLEPRIO); 1196 load->inv_weight = WMULT_IDLEPRIO; 1197 return; 1198 } 1199 1200 /* 1201 * SCHED_OTHER tasks have to update their load when changing their 1202 * weight 1203 */ 1204 if (update_load && p->sched_class == &fair_sched_class) { 1205 reweight_task(p, prio); 1206 } else { 1207 load->weight = scale_load(sched_prio_to_weight[prio]); 1208 load->inv_weight = sched_prio_to_wmult[prio]; 1209 } 1210 } 1211 1212 #ifdef CONFIG_UCLAMP_TASK 1213 /* 1214 * Serializes updates of utilization clamp values 1215 * 1216 * The (slow-path) user-space triggers utilization clamp value updates which 1217 * can require updates on (fast-path) scheduler's data structures used to 1218 * support enqueue/dequeue operations. 1219 * While the per-CPU rq lock protects fast-path update operations, user-space 1220 * requests are serialized using a mutex to reduce the risk of conflicting 1221 * updates or API abuses. 1222 */ 1223 static DEFINE_MUTEX(uclamp_mutex); 1224 1225 /* Max allowed minimum utilization */ 1226 unsigned int sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; 1227 1228 /* Max allowed maximum utilization */ 1229 unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; 1230 1231 /* 1232 * By default RT tasks run at the maximum performance point/capacity of the 1233 * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to 1234 * SCHED_CAPACITY_SCALE. 1235 * 1236 * This knob allows admins to change the default behavior when uclamp is being 1237 * used. In battery powered devices, particularly, running at the maximum 1238 * capacity and frequency will increase energy consumption and shorten the 1239 * battery life. 1240 * 1241 * This knob only affects RT tasks that their uclamp_se->user_defined == false. 1242 * 1243 * This knob will not override the system default sched_util_clamp_min defined 1244 * above. 1245 */ 1246 unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE; 1247 1248 /* All clamps are required to be less or equal than these values */ 1249 static struct uclamp_se uclamp_default[UCLAMP_CNT]; 1250 1251 /* 1252 * This static key is used to reduce the uclamp overhead in the fast path. It 1253 * primarily disables the call to uclamp_rq_{inc, dec}() in 1254 * enqueue/dequeue_task(). 1255 * 1256 * This allows users to continue to enable uclamp in their kernel config with 1257 * minimum uclamp overhead in the fast path. 1258 * 1259 * As soon as userspace modifies any of the uclamp knobs, the static key is 1260 * enabled, since we have an actual users that make use of uclamp 1261 * functionality. 1262 * 1263 * The knobs that would enable this static key are: 1264 * 1265 * * A task modifying its uclamp value with sched_setattr(). 1266 * * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs. 1267 * * An admin modifying the cgroup cpu.uclamp.{min, max} 1268 */ 1269 DEFINE_STATIC_KEY_FALSE(sched_uclamp_used); 1270 1271 /* Integer rounded range for each bucket */ 1272 #define UCLAMP_BUCKET_DELTA DIV_ROUND_CLOSEST(SCHED_CAPACITY_SCALE, UCLAMP_BUCKETS) 1273 1274 #define for_each_clamp_id(clamp_id) \ 1275 for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++) 1276 1277 static inline unsigned int uclamp_bucket_id(unsigned int clamp_value) 1278 { 1279 return min_t(unsigned int, clamp_value / UCLAMP_BUCKET_DELTA, UCLAMP_BUCKETS - 1); 1280 } 1281 1282 static inline unsigned int uclamp_none(enum uclamp_id clamp_id) 1283 { 1284 if (clamp_id == UCLAMP_MIN) 1285 return 0; 1286 return SCHED_CAPACITY_SCALE; 1287 } 1288 1289 static inline void uclamp_se_set(struct uclamp_se *uc_se, 1290 unsigned int value, bool user_defined) 1291 { 1292 uc_se->value = value; 1293 uc_se->bucket_id = uclamp_bucket_id(value); 1294 uc_se->user_defined = user_defined; 1295 } 1296 1297 static inline unsigned int 1298 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id, 1299 unsigned int clamp_value) 1300 { 1301 /* 1302 * Avoid blocked utilization pushing up the frequency when we go 1303 * idle (which drops the max-clamp) by retaining the last known 1304 * max-clamp. 1305 */ 1306 if (clamp_id == UCLAMP_MAX) { 1307 rq->uclamp_flags |= UCLAMP_FLAG_IDLE; 1308 return clamp_value; 1309 } 1310 1311 return uclamp_none(UCLAMP_MIN); 1312 } 1313 1314 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id, 1315 unsigned int clamp_value) 1316 { 1317 /* Reset max-clamp retention only on idle exit */ 1318 if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 1319 return; 1320 1321 WRITE_ONCE(rq->uclamp[clamp_id].value, clamp_value); 1322 } 1323 1324 static inline 1325 unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, 1326 unsigned int clamp_value) 1327 { 1328 struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; 1329 int bucket_id = UCLAMP_BUCKETS - 1; 1330 1331 /* 1332 * Since both min and max clamps are max aggregated, find the 1333 * top most bucket with tasks in. 1334 */ 1335 for ( ; bucket_id >= 0; bucket_id--) { 1336 if (!bucket[bucket_id].tasks) 1337 continue; 1338 return bucket[bucket_id].value; 1339 } 1340 1341 /* No tasks -- default clamp values */ 1342 return uclamp_idle_value(rq, clamp_id, clamp_value); 1343 } 1344 1345 static void __uclamp_update_util_min_rt_default(struct task_struct *p) 1346 { 1347 unsigned int default_util_min; 1348 struct uclamp_se *uc_se; 1349 1350 lockdep_assert_held(&p->pi_lock); 1351 1352 uc_se = &p->uclamp_req[UCLAMP_MIN]; 1353 1354 /* Only sync if user didn't override the default */ 1355 if (uc_se->user_defined) 1356 return; 1357 1358 default_util_min = sysctl_sched_uclamp_util_min_rt_default; 1359 uclamp_se_set(uc_se, default_util_min, false); 1360 } 1361 1362 static void uclamp_update_util_min_rt_default(struct task_struct *p) 1363 { 1364 struct rq_flags rf; 1365 struct rq *rq; 1366 1367 if (!rt_task(p)) 1368 return; 1369 1370 /* Protect updates to p->uclamp_* */ 1371 rq = task_rq_lock(p, &rf); 1372 __uclamp_update_util_min_rt_default(p); 1373 task_rq_unlock(rq, p, &rf); 1374 } 1375 1376 static void uclamp_sync_util_min_rt_default(void) 1377 { 1378 struct task_struct *g, *p; 1379 1380 /* 1381 * copy_process() sysctl_uclamp 1382 * uclamp_min_rt = X; 1383 * write_lock(&tasklist_lock) read_lock(&tasklist_lock) 1384 * // link thread smp_mb__after_spinlock() 1385 * write_unlock(&tasklist_lock) read_unlock(&tasklist_lock); 1386 * sched_post_fork() for_each_process_thread() 1387 * __uclamp_sync_rt() __uclamp_sync_rt() 1388 * 1389 * Ensures that either sched_post_fork() will observe the new 1390 * uclamp_min_rt or for_each_process_thread() will observe the new 1391 * task. 1392 */ 1393 read_lock(&tasklist_lock); 1394 smp_mb__after_spinlock(); 1395 read_unlock(&tasklist_lock); 1396 1397 rcu_read_lock(); 1398 for_each_process_thread(g, p) 1399 uclamp_update_util_min_rt_default(p); 1400 rcu_read_unlock(); 1401 } 1402 1403 static inline struct uclamp_se 1404 uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) 1405 { 1406 /* Copy by value as we could modify it */ 1407 struct uclamp_se uc_req = p->uclamp_req[clamp_id]; 1408 #ifdef CONFIG_UCLAMP_TASK_GROUP 1409 unsigned int tg_min, tg_max, value; 1410 1411 /* 1412 * Tasks in autogroups or root task group will be 1413 * restricted by system defaults. 1414 */ 1415 if (task_group_is_autogroup(task_group(p))) 1416 return uc_req; 1417 if (task_group(p) == &root_task_group) 1418 return uc_req; 1419 1420 tg_min = task_group(p)->uclamp[UCLAMP_MIN].value; 1421 tg_max = task_group(p)->uclamp[UCLAMP_MAX].value; 1422 value = uc_req.value; 1423 value = clamp(value, tg_min, tg_max); 1424 uclamp_se_set(&uc_req, value, false); 1425 #endif 1426 1427 return uc_req; 1428 } 1429 1430 /* 1431 * The effective clamp bucket index of a task depends on, by increasing 1432 * priority: 1433 * - the task specific clamp value, when explicitly requested from userspace 1434 * - the task group effective clamp value, for tasks not either in the root 1435 * group or in an autogroup 1436 * - the system default clamp value, defined by the sysadmin 1437 */ 1438 static inline struct uclamp_se 1439 uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id) 1440 { 1441 struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id); 1442 struct uclamp_se uc_max = uclamp_default[clamp_id]; 1443 1444 /* System default restrictions always apply */ 1445 if (unlikely(uc_req.value > uc_max.value)) 1446 return uc_max; 1447 1448 return uc_req; 1449 } 1450 1451 unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id) 1452 { 1453 struct uclamp_se uc_eff; 1454 1455 /* Task currently refcounted: use back-annotated (effective) value */ 1456 if (p->uclamp[clamp_id].active) 1457 return (unsigned long)p->uclamp[clamp_id].value; 1458 1459 uc_eff = uclamp_eff_get(p, clamp_id); 1460 1461 return (unsigned long)uc_eff.value; 1462 } 1463 1464 /* 1465 * When a task is enqueued on a rq, the clamp bucket currently defined by the 1466 * task's uclamp::bucket_id is refcounted on that rq. This also immediately 1467 * updates the rq's clamp value if required. 1468 * 1469 * Tasks can have a task-specific value requested from user-space, track 1470 * within each bucket the maximum value for tasks refcounted in it. 1471 * This "local max aggregation" allows to track the exact "requested" value 1472 * for each bucket when all its RUNNABLE tasks require the same clamp. 1473 */ 1474 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, 1475 enum uclamp_id clamp_id) 1476 { 1477 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1478 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1479 struct uclamp_bucket *bucket; 1480 1481 lockdep_assert_rq_held(rq); 1482 1483 /* Update task effective clamp */ 1484 p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id); 1485 1486 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1487 bucket->tasks++; 1488 uc_se->active = true; 1489 1490 uclamp_idle_reset(rq, clamp_id, uc_se->value); 1491 1492 /* 1493 * Local max aggregation: rq buckets always track the max 1494 * "requested" clamp value of its RUNNABLE tasks. 1495 */ 1496 if (bucket->tasks == 1 || uc_se->value > bucket->value) 1497 bucket->value = uc_se->value; 1498 1499 if (uc_se->value > READ_ONCE(uc_rq->value)) 1500 WRITE_ONCE(uc_rq->value, uc_se->value); 1501 } 1502 1503 /* 1504 * When a task is dequeued from a rq, the clamp bucket refcounted by the task 1505 * is released. If this is the last task reference counting the rq's max 1506 * active clamp value, then the rq's clamp value is updated. 1507 * 1508 * Both refcounted tasks and rq's cached clamp values are expected to be 1509 * always valid. If it's detected they are not, as defensive programming, 1510 * enforce the expected state and warn. 1511 */ 1512 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, 1513 enum uclamp_id clamp_id) 1514 { 1515 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1516 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1517 struct uclamp_bucket *bucket; 1518 unsigned int bkt_clamp; 1519 unsigned int rq_clamp; 1520 1521 lockdep_assert_rq_held(rq); 1522 1523 /* 1524 * If sched_uclamp_used was enabled after task @p was enqueued, 1525 * we could end up with unbalanced call to uclamp_rq_dec_id(). 1526 * 1527 * In this case the uc_se->active flag should be false since no uclamp 1528 * accounting was performed at enqueue time and we can just return 1529 * here. 1530 * 1531 * Need to be careful of the following enqueue/dequeue ordering 1532 * problem too 1533 * 1534 * enqueue(taskA) 1535 * // sched_uclamp_used gets enabled 1536 * enqueue(taskB) 1537 * dequeue(taskA) 1538 * // Must not decrement bucket->tasks here 1539 * dequeue(taskB) 1540 * 1541 * where we could end up with stale data in uc_se and 1542 * bucket[uc_se->bucket_id]. 1543 * 1544 * The following check here eliminates the possibility of such race. 1545 */ 1546 if (unlikely(!uc_se->active)) 1547 return; 1548 1549 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1550 1551 SCHED_WARN_ON(!bucket->tasks); 1552 if (likely(bucket->tasks)) 1553 bucket->tasks--; 1554 1555 uc_se->active = false; 1556 1557 /* 1558 * Keep "local max aggregation" simple and accept to (possibly) 1559 * overboost some RUNNABLE tasks in the same bucket. 1560 * The rq clamp bucket value is reset to its base value whenever 1561 * there are no more RUNNABLE tasks refcounting it. 1562 */ 1563 if (likely(bucket->tasks)) 1564 return; 1565 1566 rq_clamp = READ_ONCE(uc_rq->value); 1567 /* 1568 * Defensive programming: this should never happen. If it happens, 1569 * e.g. due to future modification, warn and fixup the expected value. 1570 */ 1571 SCHED_WARN_ON(bucket->value > rq_clamp); 1572 if (bucket->value >= rq_clamp) { 1573 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value); 1574 WRITE_ONCE(uc_rq->value, bkt_clamp); 1575 } 1576 } 1577 1578 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) 1579 { 1580 enum uclamp_id clamp_id; 1581 1582 /* 1583 * Avoid any overhead until uclamp is actually used by the userspace. 1584 * 1585 * The condition is constructed such that a NOP is generated when 1586 * sched_uclamp_used is disabled. 1587 */ 1588 if (!static_branch_unlikely(&sched_uclamp_used)) 1589 return; 1590 1591 if (unlikely(!p->sched_class->uclamp_enabled)) 1592 return; 1593 1594 for_each_clamp_id(clamp_id) 1595 uclamp_rq_inc_id(rq, p, clamp_id); 1596 1597 /* Reset clamp idle holding when there is one RUNNABLE task */ 1598 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE) 1599 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1600 } 1601 1602 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) 1603 { 1604 enum uclamp_id clamp_id; 1605 1606 /* 1607 * Avoid any overhead until uclamp is actually used by the userspace. 1608 * 1609 * The condition is constructed such that a NOP is generated when 1610 * sched_uclamp_used is disabled. 1611 */ 1612 if (!static_branch_unlikely(&sched_uclamp_used)) 1613 return; 1614 1615 if (unlikely(!p->sched_class->uclamp_enabled)) 1616 return; 1617 1618 for_each_clamp_id(clamp_id) 1619 uclamp_rq_dec_id(rq, p, clamp_id); 1620 } 1621 1622 static inline void 1623 uclamp_update_active(struct task_struct *p) 1624 { 1625 enum uclamp_id clamp_id; 1626 struct rq_flags rf; 1627 struct rq *rq; 1628 1629 /* 1630 * Lock the task and the rq where the task is (or was) queued. 1631 * 1632 * We might lock the (previous) rq of a !RUNNABLE task, but that's the 1633 * price to pay to safely serialize util_{min,max} updates with 1634 * enqueues, dequeues and migration operations. 1635 * This is the same locking schema used by __set_cpus_allowed_ptr(). 1636 */ 1637 rq = task_rq_lock(p, &rf); 1638 1639 /* 1640 * Setting the clamp bucket is serialized by task_rq_lock(). 1641 * If the task is not yet RUNNABLE and its task_struct is not 1642 * affecting a valid clamp bucket, the next time it's enqueued, 1643 * it will already see the updated clamp bucket value. 1644 */ 1645 for_each_clamp_id(clamp_id) { 1646 if (p->uclamp[clamp_id].active) { 1647 uclamp_rq_dec_id(rq, p, clamp_id); 1648 uclamp_rq_inc_id(rq, p, clamp_id); 1649 } 1650 } 1651 1652 task_rq_unlock(rq, p, &rf); 1653 } 1654 1655 #ifdef CONFIG_UCLAMP_TASK_GROUP 1656 static inline void 1657 uclamp_update_active_tasks(struct cgroup_subsys_state *css) 1658 { 1659 struct css_task_iter it; 1660 struct task_struct *p; 1661 1662 css_task_iter_start(css, 0, &it); 1663 while ((p = css_task_iter_next(&it))) 1664 uclamp_update_active(p); 1665 css_task_iter_end(&it); 1666 } 1667 1668 static void cpu_util_update_eff(struct cgroup_subsys_state *css); 1669 static void uclamp_update_root_tg(void) 1670 { 1671 struct task_group *tg = &root_task_group; 1672 1673 uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN], 1674 sysctl_sched_uclamp_util_min, false); 1675 uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX], 1676 sysctl_sched_uclamp_util_max, false); 1677 1678 rcu_read_lock(); 1679 cpu_util_update_eff(&root_task_group.css); 1680 rcu_read_unlock(); 1681 } 1682 #else 1683 static void uclamp_update_root_tg(void) { } 1684 #endif 1685 1686 int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, 1687 void *buffer, size_t *lenp, loff_t *ppos) 1688 { 1689 bool update_root_tg = false; 1690 int old_min, old_max, old_min_rt; 1691 int result; 1692 1693 mutex_lock(&uclamp_mutex); 1694 old_min = sysctl_sched_uclamp_util_min; 1695 old_max = sysctl_sched_uclamp_util_max; 1696 old_min_rt = sysctl_sched_uclamp_util_min_rt_default; 1697 1698 result = proc_dointvec(table, write, buffer, lenp, ppos); 1699 if (result) 1700 goto undo; 1701 if (!write) 1702 goto done; 1703 1704 if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max || 1705 sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE || 1706 sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) { 1707 1708 result = -EINVAL; 1709 goto undo; 1710 } 1711 1712 if (old_min != sysctl_sched_uclamp_util_min) { 1713 uclamp_se_set(&uclamp_default[UCLAMP_MIN], 1714 sysctl_sched_uclamp_util_min, false); 1715 update_root_tg = true; 1716 } 1717 if (old_max != sysctl_sched_uclamp_util_max) { 1718 uclamp_se_set(&uclamp_default[UCLAMP_MAX], 1719 sysctl_sched_uclamp_util_max, false); 1720 update_root_tg = true; 1721 } 1722 1723 if (update_root_tg) { 1724 static_branch_enable(&sched_uclamp_used); 1725 uclamp_update_root_tg(); 1726 } 1727 1728 if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) { 1729 static_branch_enable(&sched_uclamp_used); 1730 uclamp_sync_util_min_rt_default(); 1731 } 1732 1733 /* 1734 * We update all RUNNABLE tasks only when task groups are in use. 1735 * Otherwise, keep it simple and do just a lazy update at each next 1736 * task enqueue time. 1737 */ 1738 1739 goto done; 1740 1741 undo: 1742 sysctl_sched_uclamp_util_min = old_min; 1743 sysctl_sched_uclamp_util_max = old_max; 1744 sysctl_sched_uclamp_util_min_rt_default = old_min_rt; 1745 done: 1746 mutex_unlock(&uclamp_mutex); 1747 1748 return result; 1749 } 1750 1751 static int uclamp_validate(struct task_struct *p, 1752 const struct sched_attr *attr) 1753 { 1754 int util_min = p->uclamp_req[UCLAMP_MIN].value; 1755 int util_max = p->uclamp_req[UCLAMP_MAX].value; 1756 1757 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) { 1758 util_min = attr->sched_util_min; 1759 1760 if (util_min + 1 > SCHED_CAPACITY_SCALE + 1) 1761 return -EINVAL; 1762 } 1763 1764 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) { 1765 util_max = attr->sched_util_max; 1766 1767 if (util_max + 1 > SCHED_CAPACITY_SCALE + 1) 1768 return -EINVAL; 1769 } 1770 1771 if (util_min != -1 && util_max != -1 && util_min > util_max) 1772 return -EINVAL; 1773 1774 /* 1775 * We have valid uclamp attributes; make sure uclamp is enabled. 1776 * 1777 * We need to do that here, because enabling static branches is a 1778 * blocking operation which obviously cannot be done while holding 1779 * scheduler locks. 1780 */ 1781 static_branch_enable(&sched_uclamp_used); 1782 1783 return 0; 1784 } 1785 1786 static bool uclamp_reset(const struct sched_attr *attr, 1787 enum uclamp_id clamp_id, 1788 struct uclamp_se *uc_se) 1789 { 1790 /* Reset on sched class change for a non user-defined clamp value. */ 1791 if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)) && 1792 !uc_se->user_defined) 1793 return true; 1794 1795 /* Reset on sched_util_{min,max} == -1. */ 1796 if (clamp_id == UCLAMP_MIN && 1797 attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN && 1798 attr->sched_util_min == -1) { 1799 return true; 1800 } 1801 1802 if (clamp_id == UCLAMP_MAX && 1803 attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX && 1804 attr->sched_util_max == -1) { 1805 return true; 1806 } 1807 1808 return false; 1809 } 1810 1811 static void __setscheduler_uclamp(struct task_struct *p, 1812 const struct sched_attr *attr) 1813 { 1814 enum uclamp_id clamp_id; 1815 1816 for_each_clamp_id(clamp_id) { 1817 struct uclamp_se *uc_se = &p->uclamp_req[clamp_id]; 1818 unsigned int value; 1819 1820 if (!uclamp_reset(attr, clamp_id, uc_se)) 1821 continue; 1822 1823 /* 1824 * RT by default have a 100% boost value that could be modified 1825 * at runtime. 1826 */ 1827 if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN)) 1828 value = sysctl_sched_uclamp_util_min_rt_default; 1829 else 1830 value = uclamp_none(clamp_id); 1831 1832 uclamp_se_set(uc_se, value, false); 1833 1834 } 1835 1836 if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP))) 1837 return; 1838 1839 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN && 1840 attr->sched_util_min != -1) { 1841 uclamp_se_set(&p->uclamp_req[UCLAMP_MIN], 1842 attr->sched_util_min, true); 1843 } 1844 1845 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX && 1846 attr->sched_util_max != -1) { 1847 uclamp_se_set(&p->uclamp_req[UCLAMP_MAX], 1848 attr->sched_util_max, true); 1849 } 1850 } 1851 1852 static void uclamp_fork(struct task_struct *p) 1853 { 1854 enum uclamp_id clamp_id; 1855 1856 /* 1857 * We don't need to hold task_rq_lock() when updating p->uclamp_* here 1858 * as the task is still at its early fork stages. 1859 */ 1860 for_each_clamp_id(clamp_id) 1861 p->uclamp[clamp_id].active = false; 1862 1863 if (likely(!p->sched_reset_on_fork)) 1864 return; 1865 1866 for_each_clamp_id(clamp_id) { 1867 uclamp_se_set(&p->uclamp_req[clamp_id], 1868 uclamp_none(clamp_id), false); 1869 } 1870 } 1871 1872 static void uclamp_post_fork(struct task_struct *p) 1873 { 1874 uclamp_update_util_min_rt_default(p); 1875 } 1876 1877 static void __init init_uclamp_rq(struct rq *rq) 1878 { 1879 enum uclamp_id clamp_id; 1880 struct uclamp_rq *uc_rq = rq->uclamp; 1881 1882 for_each_clamp_id(clamp_id) { 1883 uc_rq[clamp_id] = (struct uclamp_rq) { 1884 .value = uclamp_none(clamp_id) 1885 }; 1886 } 1887 1888 rq->uclamp_flags = 0; 1889 } 1890 1891 static void __init init_uclamp(void) 1892 { 1893 struct uclamp_se uc_max = {}; 1894 enum uclamp_id clamp_id; 1895 int cpu; 1896 1897 for_each_possible_cpu(cpu) 1898 init_uclamp_rq(cpu_rq(cpu)); 1899 1900 for_each_clamp_id(clamp_id) { 1901 uclamp_se_set(&init_task.uclamp_req[clamp_id], 1902 uclamp_none(clamp_id), false); 1903 } 1904 1905 /* System defaults allow max clamp values for both indexes */ 1906 uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false); 1907 for_each_clamp_id(clamp_id) { 1908 uclamp_default[clamp_id] = uc_max; 1909 #ifdef CONFIG_UCLAMP_TASK_GROUP 1910 root_task_group.uclamp_req[clamp_id] = uc_max; 1911 root_task_group.uclamp[clamp_id] = uc_max; 1912 #endif 1913 } 1914 } 1915 1916 #else /* CONFIG_UCLAMP_TASK */ 1917 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { } 1918 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { } 1919 static inline int uclamp_validate(struct task_struct *p, 1920 const struct sched_attr *attr) 1921 { 1922 return -EOPNOTSUPP; 1923 } 1924 static void __setscheduler_uclamp(struct task_struct *p, 1925 const struct sched_attr *attr) { } 1926 static inline void uclamp_fork(struct task_struct *p) { } 1927 static inline void uclamp_post_fork(struct task_struct *p) { } 1928 static inline void init_uclamp(void) { } 1929 #endif /* CONFIG_UCLAMP_TASK */ 1930 1931 bool sched_task_on_rq(struct task_struct *p) 1932 { 1933 return task_on_rq_queued(p); 1934 } 1935 1936 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 1937 { 1938 if (!(flags & ENQUEUE_NOCLOCK)) 1939 update_rq_clock(rq); 1940 1941 if (!(flags & ENQUEUE_RESTORE)) { 1942 sched_info_enqueue(rq, p); 1943 psi_enqueue(p, flags & ENQUEUE_WAKEUP); 1944 } 1945 1946 uclamp_rq_inc(rq, p); 1947 p->sched_class->enqueue_task(rq, p, flags); 1948 1949 if (sched_core_enabled(rq)) 1950 sched_core_enqueue(rq, p); 1951 } 1952 1953 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) 1954 { 1955 if (sched_core_enabled(rq)) 1956 sched_core_dequeue(rq, p); 1957 1958 if (!(flags & DEQUEUE_NOCLOCK)) 1959 update_rq_clock(rq); 1960 1961 if (!(flags & DEQUEUE_SAVE)) { 1962 sched_info_dequeue(rq, p); 1963 psi_dequeue(p, flags & DEQUEUE_SLEEP); 1964 } 1965 1966 uclamp_rq_dec(rq, p); 1967 p->sched_class->dequeue_task(rq, p, flags); 1968 } 1969 1970 void activate_task(struct rq *rq, struct task_struct *p, int flags) 1971 { 1972 enqueue_task(rq, p, flags); 1973 1974 p->on_rq = TASK_ON_RQ_QUEUED; 1975 } 1976 1977 void deactivate_task(struct rq *rq, struct task_struct *p, int flags) 1978 { 1979 p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING; 1980 1981 dequeue_task(rq, p, flags); 1982 } 1983 1984 static inline int __normal_prio(int policy, int rt_prio, int nice) 1985 { 1986 int prio; 1987 1988 if (dl_policy(policy)) 1989 prio = MAX_DL_PRIO - 1; 1990 else if (rt_policy(policy)) 1991 prio = MAX_RT_PRIO - 1 - rt_prio; 1992 else 1993 prio = NICE_TO_PRIO(nice); 1994 1995 return prio; 1996 } 1997 1998 /* 1999 * Calculate the expected normal priority: i.e. priority 2000 * without taking RT-inheritance into account. Might be 2001 * boosted by interactivity modifiers. Changes upon fork, 2002 * setprio syscalls, and whenever the interactivity 2003 * estimator recalculates. 2004 */ 2005 static inline int normal_prio(struct task_struct *p) 2006 { 2007 return __normal_prio(p->policy, p->rt_priority, PRIO_TO_NICE(p->static_prio)); 2008 } 2009 2010 /* 2011 * Calculate the current priority, i.e. the priority 2012 * taken into account by the scheduler. This value might 2013 * be boosted by RT tasks, or might be boosted by 2014 * interactivity modifiers. Will be RT if the task got 2015 * RT-boosted. If not then it returns p->normal_prio. 2016 */ 2017 static int effective_prio(struct task_struct *p) 2018 { 2019 p->normal_prio = normal_prio(p); 2020 /* 2021 * If we are RT tasks or we were boosted to RT priority, 2022 * keep the priority unchanged. Otherwise, update priority 2023 * to the normal priority: 2024 */ 2025 if (!rt_prio(p->prio)) 2026 return p->normal_prio; 2027 return p->prio; 2028 } 2029 2030 /** 2031 * task_curr - is this task currently executing on a CPU? 2032 * @p: the task in question. 2033 * 2034 * Return: 1 if the task is currently executing. 0 otherwise. 2035 */ 2036 inline int task_curr(const struct task_struct *p) 2037 { 2038 return cpu_curr(task_cpu(p)) == p; 2039 } 2040 2041 /* 2042 * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock, 2043 * use the balance_callback list if you want balancing. 2044 * 2045 * this means any call to check_class_changed() must be followed by a call to 2046 * balance_callback(). 2047 */ 2048 static inline void check_class_changed(struct rq *rq, struct task_struct *p, 2049 const struct sched_class *prev_class, 2050 int oldprio) 2051 { 2052 if (prev_class != p->sched_class) { 2053 if (prev_class->switched_from) 2054 prev_class->switched_from(rq, p); 2055 2056 p->sched_class->switched_to(rq, p); 2057 } else if (oldprio != p->prio || dl_task(p)) 2058 p->sched_class->prio_changed(rq, p, oldprio); 2059 } 2060 2061 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) 2062 { 2063 if (p->sched_class == rq->curr->sched_class) 2064 rq->curr->sched_class->check_preempt_curr(rq, p, flags); 2065 else if (p->sched_class > rq->curr->sched_class) 2066 resched_curr(rq); 2067 2068 /* 2069 * A queue event has occurred, and we're going to schedule. In 2070 * this case, we can save a useless back to back clock update. 2071 */ 2072 if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) 2073 rq_clock_skip_update(rq); 2074 } 2075 2076 #ifdef CONFIG_SMP 2077 2078 static void 2079 __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags); 2080 2081 static int __set_cpus_allowed_ptr(struct task_struct *p, 2082 const struct cpumask *new_mask, 2083 u32 flags); 2084 2085 static void migrate_disable_switch(struct rq *rq, struct task_struct *p) 2086 { 2087 if (likely(!p->migration_disabled)) 2088 return; 2089 2090 if (p->cpus_ptr != &p->cpus_mask) 2091 return; 2092 2093 /* 2094 * Violates locking rules! see comment in __do_set_cpus_allowed(). 2095 */ 2096 __do_set_cpus_allowed(p, cpumask_of(rq->cpu), SCA_MIGRATE_DISABLE); 2097 } 2098 2099 void migrate_disable(void) 2100 { 2101 struct task_struct *p = current; 2102 2103 if (p->migration_disabled) { 2104 p->migration_disabled++; 2105 return; 2106 } 2107 2108 preempt_disable(); 2109 this_rq()->nr_pinned++; 2110 p->migration_disabled = 1; 2111 preempt_enable(); 2112 } 2113 EXPORT_SYMBOL_GPL(migrate_disable); 2114 2115 void migrate_enable(void) 2116 { 2117 struct task_struct *p = current; 2118 2119 if (p->migration_disabled > 1) { 2120 p->migration_disabled--; 2121 return; 2122 } 2123 2124 /* 2125 * Ensure stop_task runs either before or after this, and that 2126 * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule(). 2127 */ 2128 preempt_disable(); 2129 if (p->cpus_ptr != &p->cpus_mask) 2130 __set_cpus_allowed_ptr(p, &p->cpus_mask, SCA_MIGRATE_ENABLE); 2131 /* 2132 * Mustn't clear migration_disabled() until cpus_ptr points back at the 2133 * regular cpus_mask, otherwise things that race (eg. 2134 * select_fallback_rq) get confused. 2135 */ 2136 barrier(); 2137 p->migration_disabled = 0; 2138 this_rq()->nr_pinned--; 2139 preempt_enable(); 2140 } 2141 EXPORT_SYMBOL_GPL(migrate_enable); 2142 2143 static inline bool rq_has_pinned_tasks(struct rq *rq) 2144 { 2145 return rq->nr_pinned; 2146 } 2147 2148 /* 2149 * Per-CPU kthreads are allowed to run on !active && online CPUs, see 2150 * __set_cpus_allowed_ptr() and select_fallback_rq(). 2151 */ 2152 static inline bool is_cpu_allowed(struct task_struct *p, int cpu) 2153 { 2154 /* When not in the task's cpumask, no point in looking further. */ 2155 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 2156 return false; 2157 2158 /* migrate_disabled() must be allowed to finish. */ 2159 if (is_migration_disabled(p)) 2160 return cpu_online(cpu); 2161 2162 /* Non kernel threads are not allowed during either online or offline. */ 2163 if (!(p->flags & PF_KTHREAD)) 2164 return cpu_active(cpu); 2165 2166 /* KTHREAD_IS_PER_CPU is always allowed. */ 2167 if (kthread_is_per_cpu(p)) 2168 return cpu_online(cpu); 2169 2170 /* Regular kernel threads don't get to stay during offline. */ 2171 if (cpu_dying(cpu)) 2172 return false; 2173 2174 /* But are allowed during online. */ 2175 return cpu_online(cpu); 2176 } 2177 2178 /* 2179 * This is how migration works: 2180 * 2181 * 1) we invoke migration_cpu_stop() on the target CPU using 2182 * stop_one_cpu(). 2183 * 2) stopper starts to run (implicitly forcing the migrated thread 2184 * off the CPU) 2185 * 3) it checks whether the migrated task is still in the wrong runqueue. 2186 * 4) if it's in the wrong runqueue then the migration thread removes 2187 * it and puts it into the right queue. 2188 * 5) stopper completes and stop_one_cpu() returns and the migration 2189 * is done. 2190 */ 2191 2192 /* 2193 * move_queued_task - move a queued task to new rq. 2194 * 2195 * Returns (locked) new rq. Old rq's lock is released. 2196 */ 2197 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, 2198 struct task_struct *p, int new_cpu) 2199 { 2200 lockdep_assert_rq_held(rq); 2201 2202 deactivate_task(rq, p, DEQUEUE_NOCLOCK); 2203 set_task_cpu(p, new_cpu); 2204 rq_unlock(rq, rf); 2205 2206 rq = cpu_rq(new_cpu); 2207 2208 rq_lock(rq, rf); 2209 BUG_ON(task_cpu(p) != new_cpu); 2210 activate_task(rq, p, 0); 2211 check_preempt_curr(rq, p, 0); 2212 2213 return rq; 2214 } 2215 2216 struct migration_arg { 2217 struct task_struct *task; 2218 int dest_cpu; 2219 struct set_affinity_pending *pending; 2220 }; 2221 2222 /* 2223 * @refs: number of wait_for_completion() 2224 * @stop_pending: is @stop_work in use 2225 */ 2226 struct set_affinity_pending { 2227 refcount_t refs; 2228 unsigned int stop_pending; 2229 struct completion done; 2230 struct cpu_stop_work stop_work; 2231 struct migration_arg arg; 2232 }; 2233 2234 /* 2235 * Move (not current) task off this CPU, onto the destination CPU. We're doing 2236 * this because either it can't run here any more (set_cpus_allowed() 2237 * away from this CPU, or CPU going down), or because we're 2238 * attempting to rebalance this task on exec (sched_exec). 2239 * 2240 * So we race with normal scheduler movements, but that's OK, as long 2241 * as the task is no longer on this CPU. 2242 */ 2243 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, 2244 struct task_struct *p, int dest_cpu) 2245 { 2246 /* Affinity changed (again). */ 2247 if (!is_cpu_allowed(p, dest_cpu)) 2248 return rq; 2249 2250 update_rq_clock(rq); 2251 rq = move_queued_task(rq, rf, p, dest_cpu); 2252 2253 return rq; 2254 } 2255 2256 /* 2257 * migration_cpu_stop - this will be executed by a highprio stopper thread 2258 * and performs thread migration by bumping thread off CPU then 2259 * 'pushing' onto another runqueue. 2260 */ 2261 static int migration_cpu_stop(void *data) 2262 { 2263 struct migration_arg *arg = data; 2264 struct set_affinity_pending *pending = arg->pending; 2265 struct task_struct *p = arg->task; 2266 struct rq *rq = this_rq(); 2267 bool complete = false; 2268 struct rq_flags rf; 2269 2270 /* 2271 * The original target CPU might have gone down and we might 2272 * be on another CPU but it doesn't matter. 2273 */ 2274 local_irq_save(rf.flags); 2275 /* 2276 * We need to explicitly wake pending tasks before running 2277 * __migrate_task() such that we will not miss enforcing cpus_ptr 2278 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. 2279 */ 2280 flush_smp_call_function_from_idle(); 2281 2282 raw_spin_lock(&p->pi_lock); 2283 rq_lock(rq, &rf); 2284 2285 /* 2286 * If we were passed a pending, then ->stop_pending was set, thus 2287 * p->migration_pending must have remained stable. 2288 */ 2289 WARN_ON_ONCE(pending && pending != p->migration_pending); 2290 2291 /* 2292 * If task_rq(p) != rq, it cannot be migrated here, because we're 2293 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because 2294 * we're holding p->pi_lock. 2295 */ 2296 if (task_rq(p) == rq) { 2297 if (is_migration_disabled(p)) 2298 goto out; 2299 2300 if (pending) { 2301 p->migration_pending = NULL; 2302 complete = true; 2303 2304 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) 2305 goto out; 2306 } 2307 2308 if (task_on_rq_queued(p)) 2309 rq = __migrate_task(rq, &rf, p, arg->dest_cpu); 2310 else 2311 p->wake_cpu = arg->dest_cpu; 2312 2313 /* 2314 * XXX __migrate_task() can fail, at which point we might end 2315 * up running on a dodgy CPU, AFAICT this can only happen 2316 * during CPU hotplug, at which point we'll get pushed out 2317 * anyway, so it's probably not a big deal. 2318 */ 2319 2320 } else if (pending) { 2321 /* 2322 * This happens when we get migrated between migrate_enable()'s 2323 * preempt_enable() and scheduling the stopper task. At that 2324 * point we're a regular task again and not current anymore. 2325 * 2326 * A !PREEMPT kernel has a giant hole here, which makes it far 2327 * more likely. 2328 */ 2329 2330 /* 2331 * The task moved before the stopper got to run. We're holding 2332 * ->pi_lock, so the allowed mask is stable - if it got 2333 * somewhere allowed, we're done. 2334 */ 2335 if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) { 2336 p->migration_pending = NULL; 2337 complete = true; 2338 goto out; 2339 } 2340 2341 /* 2342 * When migrate_enable() hits a rq mis-match we can't reliably 2343 * determine is_migration_disabled() and so have to chase after 2344 * it. 2345 */ 2346 WARN_ON_ONCE(!pending->stop_pending); 2347 task_rq_unlock(rq, p, &rf); 2348 stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop, 2349 &pending->arg, &pending->stop_work); 2350 return 0; 2351 } 2352 out: 2353 if (pending) 2354 pending->stop_pending = false; 2355 task_rq_unlock(rq, p, &rf); 2356 2357 if (complete) 2358 complete_all(&pending->done); 2359 2360 return 0; 2361 } 2362 2363 int push_cpu_stop(void *arg) 2364 { 2365 struct rq *lowest_rq = NULL, *rq = this_rq(); 2366 struct task_struct *p = arg; 2367 2368 raw_spin_lock_irq(&p->pi_lock); 2369 raw_spin_rq_lock(rq); 2370 2371 if (task_rq(p) != rq) 2372 goto out_unlock; 2373 2374 if (is_migration_disabled(p)) { 2375 p->migration_flags |= MDF_PUSH; 2376 goto out_unlock; 2377 } 2378 2379 p->migration_flags &= ~MDF_PUSH; 2380 2381 if (p->sched_class->find_lock_rq) 2382 lowest_rq = p->sched_class->find_lock_rq(p, rq); 2383 2384 if (!lowest_rq) 2385 goto out_unlock; 2386 2387 // XXX validate p is still the highest prio task 2388 if (task_rq(p) == rq) { 2389 deactivate_task(rq, p, 0); 2390 set_task_cpu(p, lowest_rq->cpu); 2391 activate_task(lowest_rq, p, 0); 2392 resched_curr(lowest_rq); 2393 } 2394 2395 double_unlock_balance(rq, lowest_rq); 2396 2397 out_unlock: 2398 rq->push_busy = false; 2399 raw_spin_rq_unlock(rq); 2400 raw_spin_unlock_irq(&p->pi_lock); 2401 2402 put_task_struct(p); 2403 return 0; 2404 } 2405 2406 /* 2407 * sched_class::set_cpus_allowed must do the below, but is not required to 2408 * actually call this function. 2409 */ 2410 void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags) 2411 { 2412 if (flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) { 2413 p->cpus_ptr = new_mask; 2414 return; 2415 } 2416 2417 cpumask_copy(&p->cpus_mask, new_mask); 2418 p->nr_cpus_allowed = cpumask_weight(new_mask); 2419 } 2420 2421 static void 2422 __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags) 2423 { 2424 struct rq *rq = task_rq(p); 2425 bool queued, running; 2426 2427 /* 2428 * This here violates the locking rules for affinity, since we're only 2429 * supposed to change these variables while holding both rq->lock and 2430 * p->pi_lock. 2431 * 2432 * HOWEVER, it magically works, because ttwu() is the only code that 2433 * accesses these variables under p->pi_lock and only does so after 2434 * smp_cond_load_acquire(&p->on_cpu, !VAL), and we're in __schedule() 2435 * before finish_task(). 2436 * 2437 * XXX do further audits, this smells like something putrid. 2438 */ 2439 if (flags & SCA_MIGRATE_DISABLE) 2440 SCHED_WARN_ON(!p->on_cpu); 2441 else 2442 lockdep_assert_held(&p->pi_lock); 2443 2444 queued = task_on_rq_queued(p); 2445 running = task_current(rq, p); 2446 2447 if (queued) { 2448 /* 2449 * Because __kthread_bind() calls this on blocked tasks without 2450 * holding rq->lock. 2451 */ 2452 lockdep_assert_rq_held(rq); 2453 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 2454 } 2455 if (running) 2456 put_prev_task(rq, p); 2457 2458 p->sched_class->set_cpus_allowed(p, new_mask, flags); 2459 2460 if (queued) 2461 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 2462 if (running) 2463 set_next_task(rq, p); 2464 } 2465 2466 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) 2467 { 2468 __do_set_cpus_allowed(p, new_mask, 0); 2469 } 2470 2471 /* 2472 * This function is wildly self concurrent; here be dragons. 2473 * 2474 * 2475 * When given a valid mask, __set_cpus_allowed_ptr() must block until the 2476 * designated task is enqueued on an allowed CPU. If that task is currently 2477 * running, we have to kick it out using the CPU stopper. 2478 * 2479 * Migrate-Disable comes along and tramples all over our nice sandcastle. 2480 * Consider: 2481 * 2482 * Initial conditions: P0->cpus_mask = [0, 1] 2483 * 2484 * P0@CPU0 P1 2485 * 2486 * migrate_disable(); 2487 * <preempted> 2488 * set_cpus_allowed_ptr(P0, [1]); 2489 * 2490 * P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes 2491 * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region). 2492 * This means we need the following scheme: 2493 * 2494 * P0@CPU0 P1 2495 * 2496 * migrate_disable(); 2497 * <preempted> 2498 * set_cpus_allowed_ptr(P0, [1]); 2499 * <blocks> 2500 * <resumes> 2501 * migrate_enable(); 2502 * __set_cpus_allowed_ptr(); 2503 * <wakes local stopper> 2504 * `--> <woken on migration completion> 2505 * 2506 * Now the fun stuff: there may be several P1-like tasks, i.e. multiple 2507 * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any 2508 * task p are serialized by p->pi_lock, which we can leverage: the one that 2509 * should come into effect at the end of the Migrate-Disable region is the last 2510 * one. This means we only need to track a single cpumask (i.e. p->cpus_mask), 2511 * but we still need to properly signal those waiting tasks at the appropriate 2512 * moment. 2513 * 2514 * This is implemented using struct set_affinity_pending. The first 2515 * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will 2516 * setup an instance of that struct and install it on the targeted task_struct. 2517 * Any and all further callers will reuse that instance. Those then wait for 2518 * a completion signaled at the tail of the CPU stopper callback (1), triggered 2519 * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()). 2520 * 2521 * 2522 * (1) In the cases covered above. There is one more where the completion is 2523 * signaled within affine_move_task() itself: when a subsequent affinity request 2524 * occurs after the stopper bailed out due to the targeted task still being 2525 * Migrate-Disable. Consider: 2526 * 2527 * Initial conditions: P0->cpus_mask = [0, 1] 2528 * 2529 * CPU0 P1 P2 2530 * <P0> 2531 * migrate_disable(); 2532 * <preempted> 2533 * set_cpus_allowed_ptr(P0, [1]); 2534 * <blocks> 2535 * <migration/0> 2536 * migration_cpu_stop() 2537 * is_migration_disabled() 2538 * <bails> 2539 * set_cpus_allowed_ptr(P0, [0, 1]); 2540 * <signal completion> 2541 * <awakes> 2542 * 2543 * Note that the above is safe vs a concurrent migrate_enable(), as any 2544 * pending affinity completion is preceded by an uninstallation of 2545 * p->migration_pending done with p->pi_lock held. 2546 */ 2547 static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf, 2548 int dest_cpu, unsigned int flags) 2549 { 2550 struct set_affinity_pending my_pending = { }, *pending = NULL; 2551 bool stop_pending, complete = false; 2552 2553 /* Can the task run on the task's current CPU? If so, we're done */ 2554 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) { 2555 struct task_struct *push_task = NULL; 2556 2557 if ((flags & SCA_MIGRATE_ENABLE) && 2558 (p->migration_flags & MDF_PUSH) && !rq->push_busy) { 2559 rq->push_busy = true; 2560 push_task = get_task_struct(p); 2561 } 2562 2563 /* 2564 * If there are pending waiters, but no pending stop_work, 2565 * then complete now. 2566 */ 2567 pending = p->migration_pending; 2568 if (pending && !pending->stop_pending) { 2569 p->migration_pending = NULL; 2570 complete = true; 2571 } 2572 2573 task_rq_unlock(rq, p, rf); 2574 2575 if (push_task) { 2576 stop_one_cpu_nowait(rq->cpu, push_cpu_stop, 2577 p, &rq->push_work); 2578 } 2579 2580 if (complete) 2581 complete_all(&pending->done); 2582 2583 return 0; 2584 } 2585 2586 if (!(flags & SCA_MIGRATE_ENABLE)) { 2587 /* serialized by p->pi_lock */ 2588 if (!p->migration_pending) { 2589 /* Install the request */ 2590 refcount_set(&my_pending.refs, 1); 2591 init_completion(&my_pending.done); 2592 my_pending.arg = (struct migration_arg) { 2593 .task = p, 2594 .dest_cpu = dest_cpu, 2595 .pending = &my_pending, 2596 }; 2597 2598 p->migration_pending = &my_pending; 2599 } else { 2600 pending = p->migration_pending; 2601 refcount_inc(&pending->refs); 2602 /* 2603 * Affinity has changed, but we've already installed a 2604 * pending. migration_cpu_stop() *must* see this, else 2605 * we risk a completion of the pending despite having a 2606 * task on a disallowed CPU. 2607 * 2608 * Serialized by p->pi_lock, so this is safe. 2609 */ 2610 pending->arg.dest_cpu = dest_cpu; 2611 } 2612 } 2613 pending = p->migration_pending; 2614 /* 2615 * - !MIGRATE_ENABLE: 2616 * we'll have installed a pending if there wasn't one already. 2617 * 2618 * - MIGRATE_ENABLE: 2619 * we're here because the current CPU isn't matching anymore, 2620 * the only way that can happen is because of a concurrent 2621 * set_cpus_allowed_ptr() call, which should then still be 2622 * pending completion. 2623 * 2624 * Either way, we really should have a @pending here. 2625 */ 2626 if (WARN_ON_ONCE(!pending)) { 2627 task_rq_unlock(rq, p, rf); 2628 return -EINVAL; 2629 } 2630 2631 if (task_running(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { 2632 /* 2633 * MIGRATE_ENABLE gets here because 'p == current', but for 2634 * anything else we cannot do is_migration_disabled(), punt 2635 * and have the stopper function handle it all race-free. 2636 */ 2637 stop_pending = pending->stop_pending; 2638 if (!stop_pending) 2639 pending->stop_pending = true; 2640 2641 if (flags & SCA_MIGRATE_ENABLE) 2642 p->migration_flags &= ~MDF_PUSH; 2643 2644 task_rq_unlock(rq, p, rf); 2645 2646 if (!stop_pending) { 2647 stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, 2648 &pending->arg, &pending->stop_work); 2649 } 2650 2651 if (flags & SCA_MIGRATE_ENABLE) 2652 return 0; 2653 } else { 2654 2655 if (!is_migration_disabled(p)) { 2656 if (task_on_rq_queued(p)) 2657 rq = move_queued_task(rq, rf, p, dest_cpu); 2658 2659 if (!pending->stop_pending) { 2660 p->migration_pending = NULL; 2661 complete = true; 2662 } 2663 } 2664 task_rq_unlock(rq, p, rf); 2665 2666 if (complete) 2667 complete_all(&pending->done); 2668 } 2669 2670 wait_for_completion(&pending->done); 2671 2672 if (refcount_dec_and_test(&pending->refs)) 2673 wake_up_var(&pending->refs); /* No UaF, just an address */ 2674 2675 /* 2676 * Block the original owner of &pending until all subsequent callers 2677 * have seen the completion and decremented the refcount 2678 */ 2679 wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs)); 2680 2681 /* ARGH */ 2682 WARN_ON_ONCE(my_pending.stop_pending); 2683 2684 return 0; 2685 } 2686 2687 /* 2688 * Change a given task's CPU affinity. Migrate the thread to a 2689 * proper CPU and schedule it away if the CPU it's executing on 2690 * is removed from the allowed bitmask. 2691 * 2692 * NOTE: the caller must have a valid reference to the task, the 2693 * task must not exit() & deallocate itself prematurely. The 2694 * call is not atomic; no spinlocks may be held. 2695 */ 2696 static int __set_cpus_allowed_ptr(struct task_struct *p, 2697 const struct cpumask *new_mask, 2698 u32 flags) 2699 { 2700 const struct cpumask *cpu_valid_mask = cpu_active_mask; 2701 unsigned int dest_cpu; 2702 struct rq_flags rf; 2703 struct rq *rq; 2704 int ret = 0; 2705 2706 rq = task_rq_lock(p, &rf); 2707 update_rq_clock(rq); 2708 2709 if (p->flags & PF_KTHREAD || is_migration_disabled(p)) { 2710 /* 2711 * Kernel threads are allowed on online && !active CPUs, 2712 * however, during cpu-hot-unplug, even these might get pushed 2713 * away if not KTHREAD_IS_PER_CPU. 2714 * 2715 * Specifically, migration_disabled() tasks must not fail the 2716 * cpumask_any_and_distribute() pick below, esp. so on 2717 * SCA_MIGRATE_ENABLE, otherwise we'll not call 2718 * set_cpus_allowed_common() and actually reset p->cpus_ptr. 2719 */ 2720 cpu_valid_mask = cpu_online_mask; 2721 } 2722 2723 /* 2724 * Must re-check here, to close a race against __kthread_bind(), 2725 * sched_setaffinity() is not guaranteed to observe the flag. 2726 */ 2727 if ((flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) { 2728 ret = -EINVAL; 2729 goto out; 2730 } 2731 2732 if (!(flags & SCA_MIGRATE_ENABLE)) { 2733 if (cpumask_equal(&p->cpus_mask, new_mask)) 2734 goto out; 2735 2736 if (WARN_ON_ONCE(p == current && 2737 is_migration_disabled(p) && 2738 !cpumask_test_cpu(task_cpu(p), new_mask))) { 2739 ret = -EBUSY; 2740 goto out; 2741 } 2742 } 2743 2744 /* 2745 * Picking a ~random cpu helps in cases where we are changing affinity 2746 * for groups of tasks (ie. cpuset), so that load balancing is not 2747 * immediately required to distribute the tasks within their new mask. 2748 */ 2749 dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, new_mask); 2750 if (dest_cpu >= nr_cpu_ids) { 2751 ret = -EINVAL; 2752 goto out; 2753 } 2754 2755 __do_set_cpus_allowed(p, new_mask, flags); 2756 2757 return affine_move_task(rq, p, &rf, dest_cpu, flags); 2758 2759 out: 2760 task_rq_unlock(rq, p, &rf); 2761 2762 return ret; 2763 } 2764 2765 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) 2766 { 2767 return __set_cpus_allowed_ptr(p, new_mask, 0); 2768 } 2769 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); 2770 2771 void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 2772 { 2773 #ifdef CONFIG_SCHED_DEBUG 2774 unsigned int state = READ_ONCE(p->__state); 2775 2776 /* 2777 * We should never call set_task_cpu() on a blocked task, 2778 * ttwu() will sort out the placement. 2779 */ 2780 WARN_ON_ONCE(state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq); 2781 2782 /* 2783 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING, 2784 * because schedstat_wait_{start,end} rebase migrating task's wait_start 2785 * time relying on p->on_rq. 2786 */ 2787 WARN_ON_ONCE(state == TASK_RUNNING && 2788 p->sched_class == &fair_sched_class && 2789 (p->on_rq && !task_on_rq_migrating(p))); 2790 2791 #ifdef CONFIG_LOCKDEP 2792 /* 2793 * The caller should hold either p->pi_lock or rq->lock, when changing 2794 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. 2795 * 2796 * sched_move_task() holds both and thus holding either pins the cgroup, 2797 * see task_group(). 2798 * 2799 * Furthermore, all task_rq users should acquire both locks, see 2800 * task_rq_lock(). 2801 */ 2802 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || 2803 lockdep_is_held(__rq_lockp(task_rq(p))))); 2804 #endif 2805 /* 2806 * Clearly, migrating tasks to offline CPUs is a fairly daft thing. 2807 */ 2808 WARN_ON_ONCE(!cpu_online(new_cpu)); 2809 2810 WARN_ON_ONCE(is_migration_disabled(p)); 2811 #endif 2812 2813 trace_sched_migrate_task(p, new_cpu); 2814 2815 if (task_cpu(p) != new_cpu) { 2816 if (p->sched_class->migrate_task_rq) 2817 p->sched_class->migrate_task_rq(p, new_cpu); 2818 p->se.nr_migrations++; 2819 rseq_migrate(p); 2820 perf_event_task_migrate(p); 2821 } 2822 2823 __set_task_cpu(p, new_cpu); 2824 } 2825 2826 #ifdef CONFIG_NUMA_BALANCING 2827 static void __migrate_swap_task(struct task_struct *p, int cpu) 2828 { 2829 if (task_on_rq_queued(p)) { 2830 struct rq *src_rq, *dst_rq; 2831 struct rq_flags srf, drf; 2832 2833 src_rq = task_rq(p); 2834 dst_rq = cpu_rq(cpu); 2835 2836 rq_pin_lock(src_rq, &srf); 2837 rq_pin_lock(dst_rq, &drf); 2838 2839 deactivate_task(src_rq, p, 0); 2840 set_task_cpu(p, cpu); 2841 activate_task(dst_rq, p, 0); 2842 check_preempt_curr(dst_rq, p, 0); 2843 2844 rq_unpin_lock(dst_rq, &drf); 2845 rq_unpin_lock(src_rq, &srf); 2846 2847 } else { 2848 /* 2849 * Task isn't running anymore; make it appear like we migrated 2850 * it before it went to sleep. This means on wakeup we make the 2851 * previous CPU our target instead of where it really is. 2852 */ 2853 p->wake_cpu = cpu; 2854 } 2855 } 2856 2857 struct migration_swap_arg { 2858 struct task_struct *src_task, *dst_task; 2859 int src_cpu, dst_cpu; 2860 }; 2861 2862 static int migrate_swap_stop(void *data) 2863 { 2864 struct migration_swap_arg *arg = data; 2865 struct rq *src_rq, *dst_rq; 2866 int ret = -EAGAIN; 2867 2868 if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu)) 2869 return -EAGAIN; 2870 2871 src_rq = cpu_rq(arg->src_cpu); 2872 dst_rq = cpu_rq(arg->dst_cpu); 2873 2874 double_raw_lock(&arg->src_task->pi_lock, 2875 &arg->dst_task->pi_lock); 2876 double_rq_lock(src_rq, dst_rq); 2877 2878 if (task_cpu(arg->dst_task) != arg->dst_cpu) 2879 goto unlock; 2880 2881 if (task_cpu(arg->src_task) != arg->src_cpu) 2882 goto unlock; 2883 2884 if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr)) 2885 goto unlock; 2886 2887 if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr)) 2888 goto unlock; 2889 2890 __migrate_swap_task(arg->src_task, arg->dst_cpu); 2891 __migrate_swap_task(arg->dst_task, arg->src_cpu); 2892 2893 ret = 0; 2894 2895 unlock: 2896 double_rq_unlock(src_rq, dst_rq); 2897 raw_spin_unlock(&arg->dst_task->pi_lock); 2898 raw_spin_unlock(&arg->src_task->pi_lock); 2899 2900 return ret; 2901 } 2902 2903 /* 2904 * Cross migrate two tasks 2905 */ 2906 int migrate_swap(struct task_struct *cur, struct task_struct *p, 2907 int target_cpu, int curr_cpu) 2908 { 2909 struct migration_swap_arg arg; 2910 int ret = -EINVAL; 2911 2912 arg = (struct migration_swap_arg){ 2913 .src_task = cur, 2914 .src_cpu = curr_cpu, 2915 .dst_task = p, 2916 .dst_cpu = target_cpu, 2917 }; 2918 2919 if (arg.src_cpu == arg.dst_cpu) 2920 goto out; 2921 2922 /* 2923 * These three tests are all lockless; this is OK since all of them 2924 * will be re-checked with proper locks held further down the line. 2925 */ 2926 if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) 2927 goto out; 2928 2929 if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr)) 2930 goto out; 2931 2932 if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr)) 2933 goto out; 2934 2935 trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); 2936 ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); 2937 2938 out: 2939 return ret; 2940 } 2941 #endif /* CONFIG_NUMA_BALANCING */ 2942 2943 /* 2944 * wait_task_inactive - wait for a thread to unschedule. 2945 * 2946 * If @match_state is nonzero, it's the @p->state value just checked and 2947 * not expected to change. If it changes, i.e. @p might have woken up, 2948 * then return zero. When we succeed in waiting for @p to be off its CPU, 2949 * we return a positive number (its total switch count). If a second call 2950 * a short while later returns the same number, the caller can be sure that 2951 * @p has remained unscheduled the whole time. 2952 * 2953 * The caller must ensure that the task *will* unschedule sometime soon, 2954 * else this function might spin for a *long* time. This function can't 2955 * be called with interrupts off, or it may introduce deadlock with 2956 * smp_call_function() if an IPI is sent by the same process we are 2957 * waiting to become inactive. 2958 */ 2959 unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state) 2960 { 2961 int running, queued; 2962 struct rq_flags rf; 2963 unsigned long ncsw; 2964 struct rq *rq; 2965 2966 for (;;) { 2967 /* 2968 * We do the initial early heuristics without holding 2969 * any task-queue locks at all. We'll only try to get 2970 * the runqueue lock when things look like they will 2971 * work out! 2972 */ 2973 rq = task_rq(p); 2974 2975 /* 2976 * If the task is actively running on another CPU 2977 * still, just relax and busy-wait without holding 2978 * any locks. 2979 * 2980 * NOTE! Since we don't hold any locks, it's not 2981 * even sure that "rq" stays as the right runqueue! 2982 * But we don't care, since "task_running()" will 2983 * return false if the runqueue has changed and p 2984 * is actually now running somewhere else! 2985 */ 2986 while (task_running(rq, p)) { 2987 if (match_state && unlikely(READ_ONCE(p->__state) != match_state)) 2988 return 0; 2989 cpu_relax(); 2990 } 2991 2992 /* 2993 * Ok, time to look more closely! We need the rq 2994 * lock now, to be *sure*. If we're wrong, we'll 2995 * just go back and repeat. 2996 */ 2997 rq = task_rq_lock(p, &rf); 2998 trace_sched_wait_task(p); 2999 running = task_running(rq, p); 3000 queued = task_on_rq_queued(p); 3001 ncsw = 0; 3002 if (!match_state || READ_ONCE(p->__state) == match_state) 3003 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ 3004 task_rq_unlock(rq, p, &rf); 3005 3006 /* 3007 * If it changed from the expected state, bail out now. 3008 */ 3009 if (unlikely(!ncsw)) 3010 break; 3011 3012 /* 3013 * Was it really running after all now that we 3014 * checked with the proper locks actually held? 3015 * 3016 * Oops. Go back and try again.. 3017 */ 3018 if (unlikely(running)) { 3019 cpu_relax(); 3020 continue; 3021 } 3022 3023 /* 3024 * It's not enough that it's not actively running, 3025 * it must be off the runqueue _entirely_, and not 3026 * preempted! 3027 * 3028 * So if it was still runnable (but just not actively 3029 * running right now), it's preempted, and we should 3030 * yield - it could be a while. 3031 */ 3032 if (unlikely(queued)) { 3033 ktime_t to = NSEC_PER_SEC / HZ; 3034 3035 set_current_state(TASK_UNINTERRUPTIBLE); 3036 schedule_hrtimeout(&to, HRTIMER_MODE_REL); 3037 continue; 3038 } 3039 3040 /* 3041 * Ahh, all good. It wasn't running, and it wasn't 3042 * runnable, which means that it will never become 3043 * running in the future either. We're all done! 3044 */ 3045 break; 3046 } 3047 3048 return ncsw; 3049 } 3050 3051 /*** 3052 * kick_process - kick a running thread to enter/exit the kernel 3053 * @p: the to-be-kicked thread 3054 * 3055 * Cause a process which is running on another CPU to enter 3056 * kernel-mode, without any delay. (to get signals handled.) 3057 * 3058 * NOTE: this function doesn't have to take the runqueue lock, 3059 * because all it wants to ensure is that the remote task enters 3060 * the kernel. If the IPI races and the task has been migrated 3061 * to another CPU then no harm is done and the purpose has been 3062 * achieved as well. 3063 */ 3064 void kick_process(struct task_struct *p) 3065 { 3066 int cpu; 3067 3068 preempt_disable(); 3069 cpu = task_cpu(p); 3070 if ((cpu != smp_processor_id()) && task_curr(p)) 3071 smp_send_reschedule(cpu); 3072 preempt_enable(); 3073 } 3074 EXPORT_SYMBOL_GPL(kick_process); 3075 3076 /* 3077 * ->cpus_ptr is protected by both rq->lock and p->pi_lock 3078 * 3079 * A few notes on cpu_active vs cpu_online: 3080 * 3081 * - cpu_active must be a subset of cpu_online 3082 * 3083 * - on CPU-up we allow per-CPU kthreads on the online && !active CPU, 3084 * see __set_cpus_allowed_ptr(). At this point the newly online 3085 * CPU isn't yet part of the sched domains, and balancing will not 3086 * see it. 3087 * 3088 * - on CPU-down we clear cpu_active() to mask the sched domains and 3089 * avoid the load balancer to place new tasks on the to be removed 3090 * CPU. Existing tasks will remain running there and will be taken 3091 * off. 3092 * 3093 * This means that fallback selection must not select !active CPUs. 3094 * And can assume that any active CPU must be online. Conversely 3095 * select_task_rq() below may allow selection of !active CPUs in order 3096 * to satisfy the above rules. 3097 */ 3098 static int select_fallback_rq(int cpu, struct task_struct *p) 3099 { 3100 int nid = cpu_to_node(cpu); 3101 const struct cpumask *nodemask = NULL; 3102 enum { cpuset, possible, fail } state = cpuset; 3103 int dest_cpu; 3104 3105 /* 3106 * If the node that the CPU is on has been offlined, cpu_to_node() 3107 * will return -1. There is no CPU on the node, and we should 3108 * select the CPU on the other node. 3109 */ 3110 if (nid != -1) { 3111 nodemask = cpumask_of_node(nid); 3112 3113 /* Look for allowed, online CPU in same node. */ 3114 for_each_cpu(dest_cpu, nodemask) { 3115 if (!cpu_active(dest_cpu)) 3116 continue; 3117 if (cpumask_test_cpu(dest_cpu, p->cpus_ptr)) 3118 return dest_cpu; 3119 } 3120 } 3121 3122 for (;;) { 3123 /* Any allowed, online CPU? */ 3124 for_each_cpu(dest_cpu, p->cpus_ptr) { 3125 if (!is_cpu_allowed(p, dest_cpu)) 3126 continue; 3127 3128 goto out; 3129 } 3130 3131 /* No more Mr. Nice Guy. */ 3132 switch (state) { 3133 case cpuset: 3134 if (IS_ENABLED(CONFIG_CPUSETS)) { 3135 cpuset_cpus_allowed_fallback(p); 3136 state = possible; 3137 break; 3138 } 3139 fallthrough; 3140 case possible: 3141 /* 3142 * XXX When called from select_task_rq() we only 3143 * hold p->pi_lock and again violate locking order. 3144 * 3145 * More yuck to audit. 3146 */ 3147 do_set_cpus_allowed(p, cpu_possible_mask); 3148 state = fail; 3149 break; 3150 3151 case fail: 3152 BUG(); 3153 break; 3154 } 3155 } 3156 3157 out: 3158 if (state != cpuset) { 3159 /* 3160 * Don't tell them about moving exiting tasks or 3161 * kernel threads (both mm NULL), since they never 3162 * leave kernel. 3163 */ 3164 if (p->mm && printk_ratelimit()) { 3165 printk_deferred("process %d (%s) no longer affine to cpu%d\n", 3166 task_pid_nr(p), p->comm, cpu); 3167 } 3168 } 3169 3170 return dest_cpu; 3171 } 3172 3173 /* 3174 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable. 3175 */ 3176 static inline 3177 int select_task_rq(struct task_struct *p, int cpu, int wake_flags) 3178 { 3179 lockdep_assert_held(&p->pi_lock); 3180 3181 if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) 3182 cpu = p->sched_class->select_task_rq(p, cpu, wake_flags); 3183 else 3184 cpu = cpumask_any(p->cpus_ptr); 3185 3186 /* 3187 * In order not to call set_task_cpu() on a blocking task we need 3188 * to rely on ttwu() to place the task on a valid ->cpus_ptr 3189 * CPU. 3190 * 3191 * Since this is common to all placement strategies, this lives here. 3192 * 3193 * [ this allows ->select_task() to simply return task_cpu(p) and 3194 * not worry about this generic constraint ] 3195 */ 3196 if (unlikely(!is_cpu_allowed(p, cpu))) 3197 cpu = select_fallback_rq(task_cpu(p), p); 3198 3199 return cpu; 3200 } 3201 3202 void sched_set_stop_task(int cpu, struct task_struct *stop) 3203 { 3204 static struct lock_class_key stop_pi_lock; 3205 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; 3206 struct task_struct *old_stop = cpu_rq(cpu)->stop; 3207 3208 if (stop) { 3209 /* 3210 * Make it appear like a SCHED_FIFO task, its something 3211 * userspace knows about and won't get confused about. 3212 * 3213 * Also, it will make PI more or less work without too 3214 * much confusion -- but then, stop work should not 3215 * rely on PI working anyway. 3216 */ 3217 sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); 3218 3219 stop->sched_class = &stop_sched_class; 3220 3221 /* 3222 * The PI code calls rt_mutex_setprio() with ->pi_lock held to 3223 * adjust the effective priority of a task. As a result, 3224 * rt_mutex_setprio() can trigger (RT) balancing operations, 3225 * which can then trigger wakeups of the stop thread to push 3226 * around the current task. 3227 * 3228 * The stop task itself will never be part of the PI-chain, it 3229 * never blocks, therefore that ->pi_lock recursion is safe. 3230 * Tell lockdep about this by placing the stop->pi_lock in its 3231 * own class. 3232 */ 3233 lockdep_set_class(&stop->pi_lock, &stop_pi_lock); 3234 } 3235 3236 cpu_rq(cpu)->stop = stop; 3237 3238 if (old_stop) { 3239 /* 3240 * Reset it back to a normal scheduling class so that 3241 * it can die in pieces. 3242 */ 3243 old_stop->sched_class = &rt_sched_class; 3244 } 3245 } 3246 3247 #else /* CONFIG_SMP */ 3248 3249 static inline int __set_cpus_allowed_ptr(struct task_struct *p, 3250 const struct cpumask *new_mask, 3251 u32 flags) 3252 { 3253 return set_cpus_allowed_ptr(p, new_mask); 3254 } 3255 3256 static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { } 3257 3258 static inline bool rq_has_pinned_tasks(struct rq *rq) 3259 { 3260 return false; 3261 } 3262 3263 #endif /* !CONFIG_SMP */ 3264 3265 static void 3266 ttwu_stat(struct task_struct *p, int cpu, int wake_flags) 3267 { 3268 struct rq *rq; 3269 3270 if (!schedstat_enabled()) 3271 return; 3272 3273 rq = this_rq(); 3274 3275 #ifdef CONFIG_SMP 3276 if (cpu == rq->cpu) { 3277 __schedstat_inc(rq->ttwu_local); 3278 __schedstat_inc(p->se.statistics.nr_wakeups_local); 3279 } else { 3280 struct sched_domain *sd; 3281 3282 __schedstat_inc(p->se.statistics.nr_wakeups_remote); 3283 rcu_read_lock(); 3284 for_each_domain(rq->cpu, sd) { 3285 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { 3286 __schedstat_inc(sd->ttwu_wake_remote); 3287 break; 3288 } 3289 } 3290 rcu_read_unlock(); 3291 } 3292 3293 if (wake_flags & WF_MIGRATED) 3294 __schedstat_inc(p->se.statistics.nr_wakeups_migrate); 3295 #endif /* CONFIG_SMP */ 3296 3297 __schedstat_inc(rq->ttwu_count); 3298 __schedstat_inc(p->se.statistics.nr_wakeups); 3299 3300 if (wake_flags & WF_SYNC) 3301 __schedstat_inc(p->se.statistics.nr_wakeups_sync); 3302 } 3303 3304 /* 3305 * Mark the task runnable and perform wakeup-preemption. 3306 */ 3307 static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags, 3308 struct rq_flags *rf) 3309 { 3310 check_preempt_curr(rq, p, wake_flags); 3311 WRITE_ONCE(p->__state, TASK_RUNNING); 3312 trace_sched_wakeup(p); 3313 3314 #ifdef CONFIG_SMP 3315 if (p->sched_class->task_woken) { 3316 /* 3317 * Our task @p is fully woken up and running; so it's safe to 3318 * drop the rq->lock, hereafter rq is only used for statistics. 3319 */ 3320 rq_unpin_lock(rq, rf); 3321 p->sched_class->task_woken(rq, p); 3322 rq_repin_lock(rq, rf); 3323 } 3324 3325 if (rq->idle_stamp) { 3326 u64 delta = rq_clock(rq) - rq->idle_stamp; 3327 u64 max = 2*rq->max_idle_balance_cost; 3328 3329 update_avg(&rq->avg_idle, delta); 3330 3331 if (rq->avg_idle > max) 3332 rq->avg_idle = max; 3333 3334 rq->wake_stamp = jiffies; 3335 rq->wake_avg_idle = rq->avg_idle / 2; 3336 3337 rq->idle_stamp = 0; 3338 } 3339 #endif 3340 } 3341 3342 static void 3343 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 3344 struct rq_flags *rf) 3345 { 3346 int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; 3347 3348 lockdep_assert_rq_held(rq); 3349 3350 if (p->sched_contributes_to_load) 3351 rq->nr_uninterruptible--; 3352 3353 #ifdef CONFIG_SMP 3354 if (wake_flags & WF_MIGRATED) 3355 en_flags |= ENQUEUE_MIGRATED; 3356 else 3357 #endif 3358 if (p->in_iowait) { 3359 delayacct_blkio_end(p); 3360 atomic_dec(&task_rq(p)->nr_iowait); 3361 } 3362 3363 activate_task(rq, p, en_flags); 3364 ttwu_do_wakeup(rq, p, wake_flags, rf); 3365 } 3366 3367 /* 3368 * Consider @p being inside a wait loop: 3369 * 3370 * for (;;) { 3371 * set_current_state(TASK_UNINTERRUPTIBLE); 3372 * 3373 * if (CONDITION) 3374 * break; 3375 * 3376 * schedule(); 3377 * } 3378 * __set_current_state(TASK_RUNNING); 3379 * 3380 * between set_current_state() and schedule(). In this case @p is still 3381 * runnable, so all that needs doing is change p->state back to TASK_RUNNING in 3382 * an atomic manner. 3383 * 3384 * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq 3385 * then schedule() must still happen and p->state can be changed to 3386 * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we 3387 * need to do a full wakeup with enqueue. 3388 * 3389 * Returns: %true when the wakeup is done, 3390 * %false otherwise. 3391 */ 3392 static int ttwu_runnable(struct task_struct *p, int wake_flags) 3393 { 3394 struct rq_flags rf; 3395 struct rq *rq; 3396 int ret = 0; 3397 3398 rq = __task_rq_lock(p, &rf); 3399 if (task_on_rq_queued(p)) { 3400 /* check_preempt_curr() may use rq clock */ 3401 update_rq_clock(rq); 3402 ttwu_do_wakeup(rq, p, wake_flags, &rf); 3403 ret = 1; 3404 } 3405 __task_rq_unlock(rq, &rf); 3406 3407 return ret; 3408 } 3409 3410 #ifdef CONFIG_SMP 3411 void sched_ttwu_pending(void *arg) 3412 { 3413 struct llist_node *llist = arg; 3414 struct rq *rq = this_rq(); 3415 struct task_struct *p, *t; 3416 struct rq_flags rf; 3417 3418 if (!llist) 3419 return; 3420 3421 /* 3422 * rq::ttwu_pending racy indication of out-standing wakeups. 3423 * Races such that false-negatives are possible, since they 3424 * are shorter lived that false-positives would be. 3425 */ 3426 WRITE_ONCE(rq->ttwu_pending, 0); 3427 3428 rq_lock_irqsave(rq, &rf); 3429 update_rq_clock(rq); 3430 3431 llist_for_each_entry_safe(p, t, llist, wake_entry.llist) { 3432 if (WARN_ON_ONCE(p->on_cpu)) 3433 smp_cond_load_acquire(&p->on_cpu, !VAL); 3434 3435 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) 3436 set_task_cpu(p, cpu_of(rq)); 3437 3438 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 3439 } 3440 3441 rq_unlock_irqrestore(rq, &rf); 3442 } 3443 3444 void send_call_function_single_ipi(int cpu) 3445 { 3446 struct rq *rq = cpu_rq(cpu); 3447 3448 if (!set_nr_if_polling(rq->idle)) 3449 arch_send_call_function_single_ipi(cpu); 3450 else 3451 trace_sched_wake_idle_without_ipi(cpu); 3452 } 3453 3454 /* 3455 * Queue a task on the target CPUs wake_list and wake the CPU via IPI if 3456 * necessary. The wakee CPU on receipt of the IPI will queue the task 3457 * via sched_ttwu_wakeup() for activation so the wakee incurs the cost 3458 * of the wakeup instead of the waker. 3459 */ 3460 static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3461 { 3462 struct rq *rq = cpu_rq(cpu); 3463 3464 p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); 3465 3466 WRITE_ONCE(rq->ttwu_pending, 1); 3467 __smp_call_single_queue(cpu, &p->wake_entry.llist); 3468 } 3469 3470 void wake_up_if_idle(int cpu) 3471 { 3472 struct rq *rq = cpu_rq(cpu); 3473 struct rq_flags rf; 3474 3475 rcu_read_lock(); 3476 3477 if (!is_idle_task(rcu_dereference(rq->curr))) 3478 goto out; 3479 3480 if (set_nr_if_polling(rq->idle)) { 3481 trace_sched_wake_idle_without_ipi(cpu); 3482 } else { 3483 rq_lock_irqsave(rq, &rf); 3484 if (is_idle_task(rq->curr)) 3485 smp_send_reschedule(cpu); 3486 /* Else CPU is not idle, do nothing here: */ 3487 rq_unlock_irqrestore(rq, &rf); 3488 } 3489 3490 out: 3491 rcu_read_unlock(); 3492 } 3493 3494 bool cpus_share_cache(int this_cpu, int that_cpu) 3495 { 3496 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 3497 } 3498 3499 static inline bool ttwu_queue_cond(int cpu, int wake_flags) 3500 { 3501 /* 3502 * Do not complicate things with the async wake_list while the CPU is 3503 * in hotplug state. 3504 */ 3505 if (!cpu_active(cpu)) 3506 return false; 3507 3508 /* 3509 * If the CPU does not share cache, then queue the task on the 3510 * remote rqs wakelist to avoid accessing remote data. 3511 */ 3512 if (!cpus_share_cache(smp_processor_id(), cpu)) 3513 return true; 3514 3515 /* 3516 * If the task is descheduling and the only running task on the 3517 * CPU then use the wakelist to offload the task activation to 3518 * the soon-to-be-idle CPU as the current CPU is likely busy. 3519 * nr_running is checked to avoid unnecessary task stacking. 3520 */ 3521 if ((wake_flags & WF_ON_CPU) && cpu_rq(cpu)->nr_running <= 1) 3522 return true; 3523 3524 return false; 3525 } 3526 3527 static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3528 { 3529 if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(cpu, wake_flags)) { 3530 if (WARN_ON_ONCE(cpu == smp_processor_id())) 3531 return false; 3532 3533 sched_clock_cpu(cpu); /* Sync clocks across CPUs */ 3534 __ttwu_queue_wakelist(p, cpu, wake_flags); 3535 return true; 3536 } 3537 3538 return false; 3539 } 3540 3541 #else /* !CONFIG_SMP */ 3542 3543 static inline bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3544 { 3545 return false; 3546 } 3547 3548 #endif /* CONFIG_SMP */ 3549 3550 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) 3551 { 3552 struct rq *rq = cpu_rq(cpu); 3553 struct rq_flags rf; 3554 3555 if (ttwu_queue_wakelist(p, cpu, wake_flags)) 3556 return; 3557 3558 rq_lock(rq, &rf); 3559 update_rq_clock(rq); 3560 ttwu_do_activate(rq, p, wake_flags, &rf); 3561 rq_unlock(rq, &rf); 3562 } 3563 3564 /* 3565 * Notes on Program-Order guarantees on SMP systems. 3566 * 3567 * MIGRATION 3568 * 3569 * The basic program-order guarantee on SMP systems is that when a task [t] 3570 * migrates, all its activity on its old CPU [c0] happens-before any subsequent 3571 * execution on its new CPU [c1]. 3572 * 3573 * For migration (of runnable tasks) this is provided by the following means: 3574 * 3575 * A) UNLOCK of the rq(c0)->lock scheduling out task t 3576 * B) migration for t is required to synchronize *both* rq(c0)->lock and 3577 * rq(c1)->lock (if not at the same time, then in that order). 3578 * C) LOCK of the rq(c1)->lock scheduling in task 3579 * 3580 * Release/acquire chaining guarantees that B happens after A and C after B. 3581 * Note: the CPU doing B need not be c0 or c1 3582 * 3583 * Example: 3584 * 3585 * CPU0 CPU1 CPU2 3586 * 3587 * LOCK rq(0)->lock 3588 * sched-out X 3589 * sched-in Y 3590 * UNLOCK rq(0)->lock 3591 * 3592 * LOCK rq(0)->lock // orders against CPU0 3593 * dequeue X 3594 * UNLOCK rq(0)->lock 3595 * 3596 * LOCK rq(1)->lock 3597 * enqueue X 3598 * UNLOCK rq(1)->lock 3599 * 3600 * LOCK rq(1)->lock // orders against CPU2 3601 * sched-out Z 3602 * sched-in X 3603 * UNLOCK rq(1)->lock 3604 * 3605 * 3606 * BLOCKING -- aka. SLEEP + WAKEUP 3607 * 3608 * For blocking we (obviously) need to provide the same guarantee as for 3609 * migration. However the means are completely different as there is no lock 3610 * chain to provide order. Instead we do: 3611 * 3612 * 1) smp_store_release(X->on_cpu, 0) -- finish_task() 3613 * 2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up() 3614 * 3615 * Example: 3616 * 3617 * CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule) 3618 * 3619 * LOCK rq(0)->lock LOCK X->pi_lock 3620 * dequeue X 3621 * sched-out X 3622 * smp_store_release(X->on_cpu, 0); 3623 * 3624 * smp_cond_load_acquire(&X->on_cpu, !VAL); 3625 * X->state = WAKING 3626 * set_task_cpu(X,2) 3627 * 3628 * LOCK rq(2)->lock 3629 * enqueue X 3630 * X->state = RUNNING 3631 * UNLOCK rq(2)->lock 3632 * 3633 * LOCK rq(2)->lock // orders against CPU1 3634 * sched-out Z 3635 * sched-in X 3636 * UNLOCK rq(2)->lock 3637 * 3638 * UNLOCK X->pi_lock 3639 * UNLOCK rq(0)->lock 3640 * 3641 * 3642 * However, for wakeups there is a second guarantee we must provide, namely we 3643 * must ensure that CONDITION=1 done by the caller can not be reordered with 3644 * accesses to the task state; see try_to_wake_up() and set_current_state(). 3645 */ 3646 3647 /** 3648 * try_to_wake_up - wake up a thread 3649 * @p: the thread to be awakened 3650 * @state: the mask of task states that can be woken 3651 * @wake_flags: wake modifier flags (WF_*) 3652 * 3653 * Conceptually does: 3654 * 3655 * If (@state & @p->state) @p->state = TASK_RUNNING. 3656 * 3657 * If the task was not queued/runnable, also place it back on a runqueue. 3658 * 3659 * This function is atomic against schedule() which would dequeue the task. 3660 * 3661 * It issues a full memory barrier before accessing @p->state, see the comment 3662 * with set_current_state(). 3663 * 3664 * Uses p->pi_lock to serialize against concurrent wake-ups. 3665 * 3666 * Relies on p->pi_lock stabilizing: 3667 * - p->sched_class 3668 * - p->cpus_ptr 3669 * - p->sched_task_group 3670 * in order to do migration, see its use of select_task_rq()/set_task_cpu(). 3671 * 3672 * Tries really hard to only take one task_rq(p)->lock for performance. 3673 * Takes rq->lock in: 3674 * - ttwu_runnable() -- old rq, unavoidable, see comment there; 3675 * - ttwu_queue() -- new rq, for enqueue of the task; 3676 * - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us. 3677 * 3678 * As a consequence we race really badly with just about everything. See the 3679 * many memory barriers and their comments for details. 3680 * 3681 * Return: %true if @p->state changes (an actual wakeup was done), 3682 * %false otherwise. 3683 */ 3684 static int 3685 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 3686 { 3687 unsigned long flags; 3688 int cpu, success = 0; 3689 3690 preempt_disable(); 3691 if (p == current) { 3692 /* 3693 * We're waking current, this means 'p->on_rq' and 'task_cpu(p) 3694 * == smp_processor_id()'. Together this means we can special 3695 * case the whole 'p->on_rq && ttwu_runnable()' case below 3696 * without taking any locks. 3697 * 3698 * In particular: 3699 * - we rely on Program-Order guarantees for all the ordering, 3700 * - we're serialized against set_special_state() by virtue of 3701 * it disabling IRQs (this allows not taking ->pi_lock). 3702 */ 3703 if (!(READ_ONCE(p->__state) & state)) 3704 goto out; 3705 3706 success = 1; 3707 trace_sched_waking(p); 3708 WRITE_ONCE(p->__state, TASK_RUNNING); 3709 trace_sched_wakeup(p); 3710 goto out; 3711 } 3712 3713 /* 3714 * If we are going to wake up a thread waiting for CONDITION we 3715 * need to ensure that CONDITION=1 done by the caller can not be 3716 * reordered with p->state check below. This pairs with smp_store_mb() 3717 * in set_current_state() that the waiting thread does. 3718 */ 3719 raw_spin_lock_irqsave(&p->pi_lock, flags); 3720 smp_mb__after_spinlock(); 3721 if (!(READ_ONCE(p->__state) & state)) 3722 goto unlock; 3723 3724 trace_sched_waking(p); 3725 3726 /* We're going to change ->state: */ 3727 success = 1; 3728 3729 /* 3730 * Ensure we load p->on_rq _after_ p->state, otherwise it would 3731 * be possible to, falsely, observe p->on_rq == 0 and get stuck 3732 * in smp_cond_load_acquire() below. 3733 * 3734 * sched_ttwu_pending() try_to_wake_up() 3735 * STORE p->on_rq = 1 LOAD p->state 3736 * UNLOCK rq->lock 3737 * 3738 * __schedule() (switch to task 'p') 3739 * LOCK rq->lock smp_rmb(); 3740 * smp_mb__after_spinlock(); 3741 * UNLOCK rq->lock 3742 * 3743 * [task p] 3744 * STORE p->state = UNINTERRUPTIBLE LOAD p->on_rq 3745 * 3746 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 3747 * __schedule(). See the comment for smp_mb__after_spinlock(). 3748 * 3749 * A similar smb_rmb() lives in try_invoke_on_locked_down_task(). 3750 */ 3751 smp_rmb(); 3752 if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags)) 3753 goto unlock; 3754 3755 #ifdef CONFIG_SMP 3756 /* 3757 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be 3758 * possible to, falsely, observe p->on_cpu == 0. 3759 * 3760 * One must be running (->on_cpu == 1) in order to remove oneself 3761 * from the runqueue. 3762 * 3763 * __schedule() (switch to task 'p') try_to_wake_up() 3764 * STORE p->on_cpu = 1 LOAD p->on_rq 3765 * UNLOCK rq->lock 3766 * 3767 * __schedule() (put 'p' to sleep) 3768 * LOCK rq->lock smp_rmb(); 3769 * smp_mb__after_spinlock(); 3770 * STORE p->on_rq = 0 LOAD p->on_cpu 3771 * 3772 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 3773 * __schedule(). See the comment for smp_mb__after_spinlock(). 3774 * 3775 * Form a control-dep-acquire with p->on_rq == 0 above, to ensure 3776 * schedule()'s deactivate_task() has 'happened' and p will no longer 3777 * care about it's own p->state. See the comment in __schedule(). 3778 */ 3779 smp_acquire__after_ctrl_dep(); 3780 3781 /* 3782 * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq 3783 * == 0), which means we need to do an enqueue, change p->state to 3784 * TASK_WAKING such that we can unlock p->pi_lock before doing the 3785 * enqueue, such as ttwu_queue_wakelist(). 3786 */ 3787 WRITE_ONCE(p->__state, TASK_WAKING); 3788 3789 /* 3790 * If the owning (remote) CPU is still in the middle of schedule() with 3791 * this task as prev, considering queueing p on the remote CPUs wake_list 3792 * which potentially sends an IPI instead of spinning on p->on_cpu to 3793 * let the waker make forward progress. This is safe because IRQs are 3794 * disabled and the IPI will deliver after on_cpu is cleared. 3795 * 3796 * Ensure we load task_cpu(p) after p->on_cpu: 3797 * 3798 * set_task_cpu(p, cpu); 3799 * STORE p->cpu = @cpu 3800 * __schedule() (switch to task 'p') 3801 * LOCK rq->lock 3802 * smp_mb__after_spin_lock() smp_cond_load_acquire(&p->on_cpu) 3803 * STORE p->on_cpu = 1 LOAD p->cpu 3804 * 3805 * to ensure we observe the correct CPU on which the task is currently 3806 * scheduling. 3807 */ 3808 if (smp_load_acquire(&p->on_cpu) && 3809 ttwu_queue_wakelist(p, task_cpu(p), wake_flags | WF_ON_CPU)) 3810 goto unlock; 3811 3812 /* 3813 * If the owning (remote) CPU is still in the middle of schedule() with 3814 * this task as prev, wait until it's done referencing the task. 3815 * 3816 * Pairs with the smp_store_release() in finish_task(). 3817 * 3818 * This ensures that tasks getting woken will be fully ordered against 3819 * their previous state and preserve Program Order. 3820 */ 3821 smp_cond_load_acquire(&p->on_cpu, !VAL); 3822 3823 cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU); 3824 if (task_cpu(p) != cpu) { 3825 if (p->in_iowait) { 3826 delayacct_blkio_end(p); 3827 atomic_dec(&task_rq(p)->nr_iowait); 3828 } 3829 3830 wake_flags |= WF_MIGRATED; 3831 psi_ttwu_dequeue(p); 3832 set_task_cpu(p, cpu); 3833 } 3834 #else 3835 cpu = task_cpu(p); 3836 #endif /* CONFIG_SMP */ 3837 3838 ttwu_queue(p, cpu, wake_flags); 3839 unlock: 3840 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 3841 out: 3842 if (success) 3843 ttwu_stat(p, task_cpu(p), wake_flags); 3844 preempt_enable(); 3845 3846 return success; 3847 } 3848 3849 /** 3850 * try_invoke_on_locked_down_task - Invoke a function on task in fixed state 3851 * @p: Process for which the function is to be invoked, can be @current. 3852 * @func: Function to invoke. 3853 * @arg: Argument to function. 3854 * 3855 * If the specified task can be quickly locked into a definite state 3856 * (either sleeping or on a given runqueue), arrange to keep it in that 3857 * state while invoking @func(@arg). This function can use ->on_rq and 3858 * task_curr() to work out what the state is, if required. Given that 3859 * @func can be invoked with a runqueue lock held, it had better be quite 3860 * lightweight. 3861 * 3862 * Returns: 3863 * @false if the task slipped out from under the locks. 3864 * @true if the task was locked onto a runqueue or is sleeping. 3865 * However, @func can override this by returning @false. 3866 */ 3867 bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg) 3868 { 3869 struct rq_flags rf; 3870 bool ret = false; 3871 struct rq *rq; 3872 3873 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 3874 if (p->on_rq) { 3875 rq = __task_rq_lock(p, &rf); 3876 if (task_rq(p) == rq) 3877 ret = func(p, arg); 3878 rq_unlock(rq, &rf); 3879 } else { 3880 switch (READ_ONCE(p->__state)) { 3881 case TASK_RUNNING: 3882 case TASK_WAKING: 3883 break; 3884 default: 3885 smp_rmb(); // See smp_rmb() comment in try_to_wake_up(). 3886 if (!p->on_rq) 3887 ret = func(p, arg); 3888 } 3889 } 3890 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); 3891 return ret; 3892 } 3893 3894 /** 3895 * wake_up_process - Wake up a specific process 3896 * @p: The process to be woken up. 3897 * 3898 * Attempt to wake up the nominated process and move it to the set of runnable 3899 * processes. 3900 * 3901 * Return: 1 if the process was woken up, 0 if it was already running. 3902 * 3903 * This function executes a full memory barrier before accessing the task state. 3904 */ 3905 int wake_up_process(struct task_struct *p) 3906 { 3907 return try_to_wake_up(p, TASK_NORMAL, 0); 3908 } 3909 EXPORT_SYMBOL(wake_up_process); 3910 3911 int wake_up_state(struct task_struct *p, unsigned int state) 3912 { 3913 return try_to_wake_up(p, state, 0); 3914 } 3915 3916 /* 3917 * Perform scheduler related setup for a newly forked process p. 3918 * p is forked by current. 3919 * 3920 * __sched_fork() is basic setup used by init_idle() too: 3921 */ 3922 static void __sched_fork(unsigned long clone_flags, struct task_struct *p) 3923 { 3924 p->on_rq = 0; 3925 3926 p->se.on_rq = 0; 3927 p->se.exec_start = 0; 3928 p->se.sum_exec_runtime = 0; 3929 p->se.prev_sum_exec_runtime = 0; 3930 p->se.nr_migrations = 0; 3931 p->se.vruntime = 0; 3932 INIT_LIST_HEAD(&p->se.group_node); 3933 3934 #ifdef CONFIG_FAIR_GROUP_SCHED 3935 p->se.cfs_rq = NULL; 3936 #endif 3937 3938 #ifdef CONFIG_SCHEDSTATS 3939 /* Even if schedstat is disabled, there should not be garbage */ 3940 memset(&p->se.statistics, 0, sizeof(p->se.statistics)); 3941 #endif 3942 3943 RB_CLEAR_NODE(&p->dl.rb_node); 3944 init_dl_task_timer(&p->dl); 3945 init_dl_inactive_task_timer(&p->dl); 3946 __dl_clear_params(p); 3947 3948 INIT_LIST_HEAD(&p->rt.run_list); 3949 p->rt.timeout = 0; 3950 p->rt.time_slice = sched_rr_timeslice; 3951 p->rt.on_rq = 0; 3952 p->rt.on_list = 0; 3953 3954 #ifdef CONFIG_PREEMPT_NOTIFIERS 3955 INIT_HLIST_HEAD(&p->preempt_notifiers); 3956 #endif 3957 3958 #ifdef CONFIG_COMPACTION 3959 p->capture_control = NULL; 3960 #endif 3961 init_numa_balancing(clone_flags, p); 3962 #ifdef CONFIG_SMP 3963 p->wake_entry.u_flags = CSD_TYPE_TTWU; 3964 p->migration_pending = NULL; 3965 #endif 3966 } 3967 3968 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); 3969 3970 #ifdef CONFIG_NUMA_BALANCING 3971 3972 void set_numabalancing_state(bool enabled) 3973 { 3974 if (enabled) 3975 static_branch_enable(&sched_numa_balancing); 3976 else 3977 static_branch_disable(&sched_numa_balancing); 3978 } 3979 3980 #ifdef CONFIG_PROC_SYSCTL 3981 int sysctl_numa_balancing(struct ctl_table *table, int write, 3982 void *buffer, size_t *lenp, loff_t *ppos) 3983 { 3984 struct ctl_table t; 3985 int err; 3986 int state = static_branch_likely(&sched_numa_balancing); 3987 3988 if (write && !capable(CAP_SYS_ADMIN)) 3989 return -EPERM; 3990 3991 t = *table; 3992 t.data = &state; 3993 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 3994 if (err < 0) 3995 return err; 3996 if (write) 3997 set_numabalancing_state(state); 3998 return err; 3999 } 4000 #endif 4001 #endif 4002 4003 #ifdef CONFIG_SCHEDSTATS 4004 4005 DEFINE_STATIC_KEY_FALSE(sched_schedstats); 4006 4007 static void set_schedstats(bool enabled) 4008 { 4009 if (enabled) 4010 static_branch_enable(&sched_schedstats); 4011 else 4012 static_branch_disable(&sched_schedstats); 4013 } 4014 4015 void force_schedstat_enabled(void) 4016 { 4017 if (!schedstat_enabled()) { 4018 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n"); 4019 static_branch_enable(&sched_schedstats); 4020 } 4021 } 4022 4023 static int __init setup_schedstats(char *str) 4024 { 4025 int ret = 0; 4026 if (!str) 4027 goto out; 4028 4029 if (!strcmp(str, "enable")) { 4030 set_schedstats(true); 4031 ret = 1; 4032 } else if (!strcmp(str, "disable")) { 4033 set_schedstats(false); 4034 ret = 1; 4035 } 4036 out: 4037 if (!ret) 4038 pr_warn("Unable to parse schedstats=\n"); 4039 4040 return ret; 4041 } 4042 __setup("schedstats=", setup_schedstats); 4043 4044 #ifdef CONFIG_PROC_SYSCTL 4045 int sysctl_schedstats(struct ctl_table *table, int write, void *buffer, 4046 size_t *lenp, loff_t *ppos) 4047 { 4048 struct ctl_table t; 4049 int err; 4050 int state = static_branch_likely(&sched_schedstats); 4051 4052 if (write && !capable(CAP_SYS_ADMIN)) 4053 return -EPERM; 4054 4055 t = *table; 4056 t.data = &state; 4057 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 4058 if (err < 0) 4059 return err; 4060 if (write) 4061 set_schedstats(state); 4062 return err; 4063 } 4064 #endif /* CONFIG_PROC_SYSCTL */ 4065 #endif /* CONFIG_SCHEDSTATS */ 4066 4067 /* 4068 * fork()/clone()-time setup: 4069 */ 4070 int sched_fork(unsigned long clone_flags, struct task_struct *p) 4071 { 4072 unsigned long flags; 4073 4074 __sched_fork(clone_flags, p); 4075 /* 4076 * We mark the process as NEW here. This guarantees that 4077 * nobody will actually run it, and a signal or other external 4078 * event cannot wake it up and insert it on the runqueue either. 4079 */ 4080 p->__state = TASK_NEW; 4081 4082 /* 4083 * Make sure we do not leak PI boosting priority to the child. 4084 */ 4085 p->prio = current->normal_prio; 4086 4087 uclamp_fork(p); 4088 4089 /* 4090 * Revert to default priority/policy on fork if requested. 4091 */ 4092 if (unlikely(p->sched_reset_on_fork)) { 4093 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 4094 p->policy = SCHED_NORMAL; 4095 p->static_prio = NICE_TO_PRIO(0); 4096 p->rt_priority = 0; 4097 } else if (PRIO_TO_NICE(p->static_prio) < 0) 4098 p->static_prio = NICE_TO_PRIO(0); 4099 4100 p->prio = p->normal_prio = p->static_prio; 4101 set_load_weight(p, false); 4102 4103 /* 4104 * We don't need the reset flag anymore after the fork. It has 4105 * fulfilled its duty: 4106 */ 4107 p->sched_reset_on_fork = 0; 4108 } 4109 4110 if (dl_prio(p->prio)) 4111 return -EAGAIN; 4112 else if (rt_prio(p->prio)) 4113 p->sched_class = &rt_sched_class; 4114 else 4115 p->sched_class = &fair_sched_class; 4116 4117 init_entity_runnable_average(&p->se); 4118 4119 /* 4120 * The child is not yet in the pid-hash so no cgroup attach races, 4121 * and the cgroup is pinned to this child due to cgroup_fork() 4122 * is ran before sched_fork(). 4123 * 4124 * Silence PROVE_RCU. 4125 */ 4126 raw_spin_lock_irqsave(&p->pi_lock, flags); 4127 rseq_migrate(p); 4128 /* 4129 * We're setting the CPU for the first time, we don't migrate, 4130 * so use __set_task_cpu(). 4131 */ 4132 __set_task_cpu(p, smp_processor_id()); 4133 if (p->sched_class->task_fork) 4134 p->sched_class->task_fork(p); 4135 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 4136 4137 #ifdef CONFIG_SCHED_INFO 4138 if (likely(sched_info_on())) 4139 memset(&p->sched_info, 0, sizeof(p->sched_info)); 4140 #endif 4141 #if defined(CONFIG_SMP) 4142 p->on_cpu = 0; 4143 #endif 4144 init_task_preempt_count(p); 4145 #ifdef CONFIG_SMP 4146 plist_node_init(&p->pushable_tasks, MAX_PRIO); 4147 RB_CLEAR_NODE(&p->pushable_dl_tasks); 4148 #endif 4149 return 0; 4150 } 4151 4152 void sched_post_fork(struct task_struct *p) 4153 { 4154 uclamp_post_fork(p); 4155 } 4156 4157 unsigned long to_ratio(u64 period, u64 runtime) 4158 { 4159 if (runtime == RUNTIME_INF) 4160 return BW_UNIT; 4161 4162 /* 4163 * Doing this here saves a lot of checks in all 4164 * the calling paths, and returning zero seems 4165 * safe for them anyway. 4166 */ 4167 if (period == 0) 4168 return 0; 4169 4170 return div64_u64(runtime << BW_SHIFT, period); 4171 } 4172 4173 /* 4174 * wake_up_new_task - wake up a newly created task for the first time. 4175 * 4176 * This function will do some initial scheduler statistics housekeeping 4177 * that must be done for every newly created context, then puts the task 4178 * on the runqueue and wakes it. 4179 */ 4180 void wake_up_new_task(struct task_struct *p) 4181 { 4182 struct rq_flags rf; 4183 struct rq *rq; 4184 4185 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 4186 WRITE_ONCE(p->__state, TASK_RUNNING); 4187 #ifdef CONFIG_SMP 4188 /* 4189 * Fork balancing, do it here and not earlier because: 4190 * - cpus_ptr can change in the fork path 4191 * - any previously selected CPU might disappear through hotplug 4192 * 4193 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq, 4194 * as we're not fully set-up yet. 4195 */ 4196 p->recent_used_cpu = task_cpu(p); 4197 rseq_migrate(p); 4198 __set_task_cpu(p, select_task_rq(p, task_cpu(p), WF_FORK)); 4199 #endif 4200 rq = __task_rq_lock(p, &rf); 4201 update_rq_clock(rq); 4202 post_init_entity_util_avg(p); 4203 4204 activate_task(rq, p, ENQUEUE_NOCLOCK); 4205 trace_sched_wakeup_new(p); 4206 check_preempt_curr(rq, p, WF_FORK); 4207 #ifdef CONFIG_SMP 4208 if (p->sched_class->task_woken) { 4209 /* 4210 * Nothing relies on rq->lock after this, so it's fine to 4211 * drop it. 4212 */ 4213 rq_unpin_lock(rq, &rf); 4214 p->sched_class->task_woken(rq, p); 4215 rq_repin_lock(rq, &rf); 4216 } 4217 #endif 4218 task_rq_unlock(rq, p, &rf); 4219 } 4220 4221 #ifdef CONFIG_PREEMPT_NOTIFIERS 4222 4223 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key); 4224 4225 void preempt_notifier_inc(void) 4226 { 4227 static_branch_inc(&preempt_notifier_key); 4228 } 4229 EXPORT_SYMBOL_GPL(preempt_notifier_inc); 4230 4231 void preempt_notifier_dec(void) 4232 { 4233 static_branch_dec(&preempt_notifier_key); 4234 } 4235 EXPORT_SYMBOL_GPL(preempt_notifier_dec); 4236 4237 /** 4238 * preempt_notifier_register - tell me when current is being preempted & rescheduled 4239 * @notifier: notifier struct to register 4240 */ 4241 void preempt_notifier_register(struct preempt_notifier *notifier) 4242 { 4243 if (!static_branch_unlikely(&preempt_notifier_key)) 4244 WARN(1, "registering preempt_notifier while notifiers disabled\n"); 4245 4246 hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); 4247 } 4248 EXPORT_SYMBOL_GPL(preempt_notifier_register); 4249 4250 /** 4251 * preempt_notifier_unregister - no longer interested in preemption notifications 4252 * @notifier: notifier struct to unregister 4253 * 4254 * This is *not* safe to call from within a preemption notifier. 4255 */ 4256 void preempt_notifier_unregister(struct preempt_notifier *notifier) 4257 { 4258 hlist_del(¬ifier->link); 4259 } 4260 EXPORT_SYMBOL_GPL(preempt_notifier_unregister); 4261 4262 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) 4263 { 4264 struct preempt_notifier *notifier; 4265 4266 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4267 notifier->ops->sched_in(notifier, raw_smp_processor_id()); 4268 } 4269 4270 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4271 { 4272 if (static_branch_unlikely(&preempt_notifier_key)) 4273 __fire_sched_in_preempt_notifiers(curr); 4274 } 4275 4276 static void 4277 __fire_sched_out_preempt_notifiers(struct task_struct *curr, 4278 struct task_struct *next) 4279 { 4280 struct preempt_notifier *notifier; 4281 4282 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4283 notifier->ops->sched_out(notifier, next); 4284 } 4285 4286 static __always_inline void 4287 fire_sched_out_preempt_notifiers(struct task_struct *curr, 4288 struct task_struct *next) 4289 { 4290 if (static_branch_unlikely(&preempt_notifier_key)) 4291 __fire_sched_out_preempt_notifiers(curr, next); 4292 } 4293 4294 #else /* !CONFIG_PREEMPT_NOTIFIERS */ 4295 4296 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4297 { 4298 } 4299 4300 static inline void 4301 fire_sched_out_preempt_notifiers(struct task_struct *curr, 4302 struct task_struct *next) 4303 { 4304 } 4305 4306 #endif /* CONFIG_PREEMPT_NOTIFIERS */ 4307 4308 static inline void prepare_task(struct task_struct *next) 4309 { 4310 #ifdef CONFIG_SMP 4311 /* 4312 * Claim the task as running, we do this before switching to it 4313 * such that any running task will have this set. 4314 * 4315 * See the ttwu() WF_ON_CPU case and its ordering comment. 4316 */ 4317 WRITE_ONCE(next->on_cpu, 1); 4318 #endif 4319 } 4320 4321 static inline void finish_task(struct task_struct *prev) 4322 { 4323 #ifdef CONFIG_SMP 4324 /* 4325 * This must be the very last reference to @prev from this CPU. After 4326 * p->on_cpu is cleared, the task can be moved to a different CPU. We 4327 * must ensure this doesn't happen until the switch is completely 4328 * finished. 4329 * 4330 * In particular, the load of prev->state in finish_task_switch() must 4331 * happen before this. 4332 * 4333 * Pairs with the smp_cond_load_acquire() in try_to_wake_up(). 4334 */ 4335 smp_store_release(&prev->on_cpu, 0); 4336 #endif 4337 } 4338 4339 #ifdef CONFIG_SMP 4340 4341 static void do_balance_callbacks(struct rq *rq, struct callback_head *head) 4342 { 4343 void (*func)(struct rq *rq); 4344 struct callback_head *next; 4345 4346 lockdep_assert_rq_held(rq); 4347 4348 while (head) { 4349 func = (void (*)(struct rq *))head->func; 4350 next = head->next; 4351 head->next = NULL; 4352 head = next; 4353 4354 func(rq); 4355 } 4356 } 4357 4358 static void balance_push(struct rq *rq); 4359 4360 struct callback_head balance_push_callback = { 4361 .next = NULL, 4362 .func = (void (*)(struct callback_head *))balance_push, 4363 }; 4364 4365 static inline struct callback_head *splice_balance_callbacks(struct rq *rq) 4366 { 4367 struct callback_head *head = rq->balance_callback; 4368 4369 lockdep_assert_rq_held(rq); 4370 if (head) 4371 rq->balance_callback = NULL; 4372 4373 return head; 4374 } 4375 4376 static void __balance_callbacks(struct rq *rq) 4377 { 4378 do_balance_callbacks(rq, splice_balance_callbacks(rq)); 4379 } 4380 4381 static inline void balance_callbacks(struct rq *rq, struct callback_head *head) 4382 { 4383 unsigned long flags; 4384 4385 if (unlikely(head)) { 4386 raw_spin_rq_lock_irqsave(rq, flags); 4387 do_balance_callbacks(rq, head); 4388 raw_spin_rq_unlock_irqrestore(rq, flags); 4389 } 4390 } 4391 4392 #else 4393 4394 static inline void __balance_callbacks(struct rq *rq) 4395 { 4396 } 4397 4398 static inline struct callback_head *splice_balance_callbacks(struct rq *rq) 4399 { 4400 return NULL; 4401 } 4402 4403 static inline void balance_callbacks(struct rq *rq, struct callback_head *head) 4404 { 4405 } 4406 4407 #endif 4408 4409 static inline void 4410 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) 4411 { 4412 /* 4413 * Since the runqueue lock will be released by the next 4414 * task (which is an invalid locking op but in the case 4415 * of the scheduler it's an obvious special-case), so we 4416 * do an early lockdep release here: 4417 */ 4418 rq_unpin_lock(rq, rf); 4419 spin_release(&__rq_lockp(rq)->dep_map, _THIS_IP_); 4420 #ifdef CONFIG_DEBUG_SPINLOCK 4421 /* this is a valid case when another task releases the spinlock */ 4422 rq_lockp(rq)->owner = next; 4423 #endif 4424 } 4425 4426 static inline void finish_lock_switch(struct rq *rq) 4427 { 4428 /* 4429 * If we are tracking spinlock dependencies then we have to 4430 * fix up the runqueue lock - which gets 'carried over' from 4431 * prev into current: 4432 */ 4433 spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_); 4434 __balance_callbacks(rq); 4435 raw_spin_rq_unlock_irq(rq); 4436 } 4437 4438 /* 4439 * NOP if the arch has not defined these: 4440 */ 4441 4442 #ifndef prepare_arch_switch 4443 # define prepare_arch_switch(next) do { } while (0) 4444 #endif 4445 4446 #ifndef finish_arch_post_lock_switch 4447 # define finish_arch_post_lock_switch() do { } while (0) 4448 #endif 4449 4450 static inline void kmap_local_sched_out(void) 4451 { 4452 #ifdef CONFIG_KMAP_LOCAL 4453 if (unlikely(current->kmap_ctrl.idx)) 4454 __kmap_local_sched_out(); 4455 #endif 4456 } 4457 4458 static inline void kmap_local_sched_in(void) 4459 { 4460 #ifdef CONFIG_KMAP_LOCAL 4461 if (unlikely(current->kmap_ctrl.idx)) 4462 __kmap_local_sched_in(); 4463 #endif 4464 } 4465 4466 /** 4467 * prepare_task_switch - prepare to switch tasks 4468 * @rq: the runqueue preparing to switch 4469 * @prev: the current task that is being switched out 4470 * @next: the task we are going to switch to. 4471 * 4472 * This is called with the rq lock held and interrupts off. It must 4473 * be paired with a subsequent finish_task_switch after the context 4474 * switch. 4475 * 4476 * prepare_task_switch sets up locking and calls architecture specific 4477 * hooks. 4478 */ 4479 static inline void 4480 prepare_task_switch(struct rq *rq, struct task_struct *prev, 4481 struct task_struct *next) 4482 { 4483 kcov_prepare_switch(prev); 4484 sched_info_switch(rq, prev, next); 4485 perf_event_task_sched_out(prev, next); 4486 rseq_preempt(prev); 4487 fire_sched_out_preempt_notifiers(prev, next); 4488 kmap_local_sched_out(); 4489 prepare_task(next); 4490 prepare_arch_switch(next); 4491 } 4492 4493 /** 4494 * finish_task_switch - clean up after a task-switch 4495 * @prev: the thread we just switched away from. 4496 * 4497 * finish_task_switch must be called after the context switch, paired 4498 * with a prepare_task_switch call before the context switch. 4499 * finish_task_switch will reconcile locking set up by prepare_task_switch, 4500 * and do any other architecture-specific cleanup actions. 4501 * 4502 * Note that we may have delayed dropping an mm in context_switch(). If 4503 * so, we finish that here outside of the runqueue lock. (Doing it 4504 * with the lock held can cause deadlocks; see schedule() for 4505 * details.) 4506 * 4507 * The context switch have flipped the stack from under us and restored the 4508 * local variables which were saved when this task called schedule() in the 4509 * past. prev == current is still correct but we need to recalculate this_rq 4510 * because prev may have moved to another CPU. 4511 */ 4512 static struct rq *finish_task_switch(struct task_struct *prev) 4513 __releases(rq->lock) 4514 { 4515 struct rq *rq = this_rq(); 4516 struct mm_struct *mm = rq->prev_mm; 4517 long prev_state; 4518 4519 /* 4520 * The previous task will have left us with a preempt_count of 2 4521 * because it left us after: 4522 * 4523 * schedule() 4524 * preempt_disable(); // 1 4525 * __schedule() 4526 * raw_spin_lock_irq(&rq->lock) // 2 4527 * 4528 * Also, see FORK_PREEMPT_COUNT. 4529 */ 4530 if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 4531 "corrupted preempt_count: %s/%d/0x%x\n", 4532 current->comm, current->pid, preempt_count())) 4533 preempt_count_set(FORK_PREEMPT_COUNT); 4534 4535 rq->prev_mm = NULL; 4536 4537 /* 4538 * A task struct has one reference for the use as "current". 4539 * If a task dies, then it sets TASK_DEAD in tsk->state and calls 4540 * schedule one last time. The schedule call will never return, and 4541 * the scheduled task must drop that reference. 4542 * 4543 * We must observe prev->state before clearing prev->on_cpu (in 4544 * finish_task), otherwise a concurrent wakeup can get prev 4545 * running on another CPU and we could rave with its RUNNING -> DEAD 4546 * transition, resulting in a double drop. 4547 */ 4548 prev_state = READ_ONCE(prev->__state); 4549 vtime_task_switch(prev); 4550 perf_event_task_sched_in(prev, current); 4551 finish_task(prev); 4552 tick_nohz_task_switch(); 4553 finish_lock_switch(rq); 4554 finish_arch_post_lock_switch(); 4555 kcov_finish_switch(current); 4556 /* 4557 * kmap_local_sched_out() is invoked with rq::lock held and 4558 * interrupts disabled. There is no requirement for that, but the 4559 * sched out code does not have an interrupt enabled section. 4560 * Restoring the maps on sched in does not require interrupts being 4561 * disabled either. 4562 */ 4563 kmap_local_sched_in(); 4564 4565 fire_sched_in_preempt_notifiers(current); 4566 /* 4567 * When switching through a kernel thread, the loop in 4568 * membarrier_{private,global}_expedited() may have observed that 4569 * kernel thread and not issued an IPI. It is therefore possible to 4570 * schedule between user->kernel->user threads without passing though 4571 * switch_mm(). Membarrier requires a barrier after storing to 4572 * rq->curr, before returning to userspace, so provide them here: 4573 * 4574 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly 4575 * provided by mmdrop(), 4576 * - a sync_core for SYNC_CORE. 4577 */ 4578 if (mm) { 4579 membarrier_mm_sync_core_before_usermode(mm); 4580 mmdrop(mm); 4581 } 4582 if (unlikely(prev_state == TASK_DEAD)) { 4583 if (prev->sched_class->task_dead) 4584 prev->sched_class->task_dead(prev); 4585 4586 /* 4587 * Remove function-return probe instances associated with this 4588 * task and put them back on the free list. 4589 */ 4590 kprobe_flush_task(prev); 4591 4592 /* Task is done with its stack. */ 4593 put_task_stack(prev); 4594 4595 put_task_struct_rcu_user(prev); 4596 } 4597 4598 return rq; 4599 } 4600 4601 /** 4602 * schedule_tail - first thing a freshly forked thread must call. 4603 * @prev: the thread we just switched away from. 4604 */ 4605 asmlinkage __visible void schedule_tail(struct task_struct *prev) 4606 __releases(rq->lock) 4607 { 4608 /* 4609 * New tasks start with FORK_PREEMPT_COUNT, see there and 4610 * finish_task_switch() for details. 4611 * 4612 * finish_task_switch() will drop rq->lock() and lower preempt_count 4613 * and the preempt_enable() will end up enabling preemption (on 4614 * PREEMPT_COUNT kernels). 4615 */ 4616 4617 finish_task_switch(prev); 4618 preempt_enable(); 4619 4620 if (current->set_child_tid) 4621 put_user(task_pid_vnr(current), current->set_child_tid); 4622 4623 calculate_sigpending(); 4624 } 4625 4626 /* 4627 * context_switch - switch to the new MM and the new thread's register state. 4628 */ 4629 static __always_inline struct rq * 4630 context_switch(struct rq *rq, struct task_struct *prev, 4631 struct task_struct *next, struct rq_flags *rf) 4632 { 4633 prepare_task_switch(rq, prev, next); 4634 4635 /* 4636 * For paravirt, this is coupled with an exit in switch_to to 4637 * combine the page table reload and the switch backend into 4638 * one hypercall. 4639 */ 4640 arch_start_context_switch(prev); 4641 4642 /* 4643 * kernel -> kernel lazy + transfer active 4644 * user -> kernel lazy + mmgrab() active 4645 * 4646 * kernel -> user switch + mmdrop() active 4647 * user -> user switch 4648 */ 4649 if (!next->mm) { // to kernel 4650 enter_lazy_tlb(prev->active_mm, next); 4651 4652 next->active_mm = prev->active_mm; 4653 if (prev->mm) // from user 4654 mmgrab(prev->active_mm); 4655 else 4656 prev->active_mm = NULL; 4657 } else { // to user 4658 membarrier_switch_mm(rq, prev->active_mm, next->mm); 4659 /* 4660 * sys_membarrier() requires an smp_mb() between setting 4661 * rq->curr / membarrier_switch_mm() and returning to userspace. 4662 * 4663 * The below provides this either through switch_mm(), or in 4664 * case 'prev->active_mm == next->mm' through 4665 * finish_task_switch()'s mmdrop(). 4666 */ 4667 switch_mm_irqs_off(prev->active_mm, next->mm, next); 4668 4669 if (!prev->mm) { // from kernel 4670 /* will mmdrop() in finish_task_switch(). */ 4671 rq->prev_mm = prev->active_mm; 4672 prev->active_mm = NULL; 4673 } 4674 } 4675 4676 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 4677 4678 prepare_lock_switch(rq, next, rf); 4679 4680 /* Here we just switch the register state and the stack. */ 4681 switch_to(prev, next, prev); 4682 barrier(); 4683 4684 return finish_task_switch(prev); 4685 } 4686 4687 /* 4688 * nr_running and nr_context_switches: 4689 * 4690 * externally visible scheduler statistics: current number of runnable 4691 * threads, total number of context switches performed since bootup. 4692 */ 4693 unsigned int nr_running(void) 4694 { 4695 unsigned int i, sum = 0; 4696 4697 for_each_online_cpu(i) 4698 sum += cpu_rq(i)->nr_running; 4699 4700 return sum; 4701 } 4702 4703 /* 4704 * Check if only the current task is running on the CPU. 4705 * 4706 * Caution: this function does not check that the caller has disabled 4707 * preemption, thus the result might have a time-of-check-to-time-of-use 4708 * race. The caller is responsible to use it correctly, for example: 4709 * 4710 * - from a non-preemptible section (of course) 4711 * 4712 * - from a thread that is bound to a single CPU 4713 * 4714 * - in a loop with very short iterations (e.g. a polling loop) 4715 */ 4716 bool single_task_running(void) 4717 { 4718 return raw_rq()->nr_running == 1; 4719 } 4720 EXPORT_SYMBOL(single_task_running); 4721 4722 unsigned long long nr_context_switches(void) 4723 { 4724 int i; 4725 unsigned long long sum = 0; 4726 4727 for_each_possible_cpu(i) 4728 sum += cpu_rq(i)->nr_switches; 4729 4730 return sum; 4731 } 4732 4733 /* 4734 * Consumers of these two interfaces, like for example the cpuidle menu 4735 * governor, are using nonsensical data. Preferring shallow idle state selection 4736 * for a CPU that has IO-wait which might not even end up running the task when 4737 * it does become runnable. 4738 */ 4739 4740 unsigned int nr_iowait_cpu(int cpu) 4741 { 4742 return atomic_read(&cpu_rq(cpu)->nr_iowait); 4743 } 4744 4745 /* 4746 * IO-wait accounting, and how it's mostly bollocks (on SMP). 4747 * 4748 * The idea behind IO-wait account is to account the idle time that we could 4749 * have spend running if it were not for IO. That is, if we were to improve the 4750 * storage performance, we'd have a proportional reduction in IO-wait time. 4751 * 4752 * This all works nicely on UP, where, when a task blocks on IO, we account 4753 * idle time as IO-wait, because if the storage were faster, it could've been 4754 * running and we'd not be idle. 4755 * 4756 * This has been extended to SMP, by doing the same for each CPU. This however 4757 * is broken. 4758 * 4759 * Imagine for instance the case where two tasks block on one CPU, only the one 4760 * CPU will have IO-wait accounted, while the other has regular idle. Even 4761 * though, if the storage were faster, both could've ran at the same time, 4762 * utilising both CPUs. 4763 * 4764 * This means, that when looking globally, the current IO-wait accounting on 4765 * SMP is a lower bound, by reason of under accounting. 4766 * 4767 * Worse, since the numbers are provided per CPU, they are sometimes 4768 * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly 4769 * associated with any one particular CPU, it can wake to another CPU than it 4770 * blocked on. This means the per CPU IO-wait number is meaningless. 4771 * 4772 * Task CPU affinities can make all that even more 'interesting'. 4773 */ 4774 4775 unsigned int nr_iowait(void) 4776 { 4777 unsigned int i, sum = 0; 4778 4779 for_each_possible_cpu(i) 4780 sum += nr_iowait_cpu(i); 4781 4782 return sum; 4783 } 4784 4785 #ifdef CONFIG_SMP 4786 4787 /* 4788 * sched_exec - execve() is a valuable balancing opportunity, because at 4789 * this point the task has the smallest effective memory and cache footprint. 4790 */ 4791 void sched_exec(void) 4792 { 4793 struct task_struct *p = current; 4794 unsigned long flags; 4795 int dest_cpu; 4796 4797 raw_spin_lock_irqsave(&p->pi_lock, flags); 4798 dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC); 4799 if (dest_cpu == smp_processor_id()) 4800 goto unlock; 4801 4802 if (likely(cpu_active(dest_cpu))) { 4803 struct migration_arg arg = { p, dest_cpu }; 4804 4805 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 4806 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); 4807 return; 4808 } 4809 unlock: 4810 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 4811 } 4812 4813 #endif 4814 4815 DEFINE_PER_CPU(struct kernel_stat, kstat); 4816 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); 4817 4818 EXPORT_PER_CPU_SYMBOL(kstat); 4819 EXPORT_PER_CPU_SYMBOL(kernel_cpustat); 4820 4821 /* 4822 * The function fair_sched_class.update_curr accesses the struct curr 4823 * and its field curr->exec_start; when called from task_sched_runtime(), 4824 * we observe a high rate of cache misses in practice. 4825 * Prefetching this data results in improved performance. 4826 */ 4827 static inline void prefetch_curr_exec_start(struct task_struct *p) 4828 { 4829 #ifdef CONFIG_FAIR_GROUP_SCHED 4830 struct sched_entity *curr = (&p->se)->cfs_rq->curr; 4831 #else 4832 struct sched_entity *curr = (&task_rq(p)->cfs)->curr; 4833 #endif 4834 prefetch(curr); 4835 prefetch(&curr->exec_start); 4836 } 4837 4838 /* 4839 * Return accounted runtime for the task. 4840 * In case the task is currently running, return the runtime plus current's 4841 * pending runtime that have not been accounted yet. 4842 */ 4843 unsigned long long task_sched_runtime(struct task_struct *p) 4844 { 4845 struct rq_flags rf; 4846 struct rq *rq; 4847 u64 ns; 4848 4849 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP) 4850 /* 4851 * 64-bit doesn't need locks to atomically read a 64-bit value. 4852 * So we have a optimization chance when the task's delta_exec is 0. 4853 * Reading ->on_cpu is racy, but this is ok. 4854 * 4855 * If we race with it leaving CPU, we'll take a lock. So we're correct. 4856 * If we race with it entering CPU, unaccounted time is 0. This is 4857 * indistinguishable from the read occurring a few cycles earlier. 4858 * If we see ->on_cpu without ->on_rq, the task is leaving, and has 4859 * been accounted, so we're correct here as well. 4860 */ 4861 if (!p->on_cpu || !task_on_rq_queued(p)) 4862 return p->se.sum_exec_runtime; 4863 #endif 4864 4865 rq = task_rq_lock(p, &rf); 4866 /* 4867 * Must be ->curr _and_ ->on_rq. If dequeued, we would 4868 * project cycles that may never be accounted to this 4869 * thread, breaking clock_gettime(). 4870 */ 4871 if (task_current(rq, p) && task_on_rq_queued(p)) { 4872 prefetch_curr_exec_start(p); 4873 update_rq_clock(rq); 4874 p->sched_class->update_curr(rq); 4875 } 4876 ns = p->se.sum_exec_runtime; 4877 task_rq_unlock(rq, p, &rf); 4878 4879 return ns; 4880 } 4881 4882 #ifdef CONFIG_SCHED_DEBUG 4883 static u64 cpu_resched_latency(struct rq *rq) 4884 { 4885 int latency_warn_ms = READ_ONCE(sysctl_resched_latency_warn_ms); 4886 u64 resched_latency, now = rq_clock(rq); 4887 static bool warned_once; 4888 4889 if (sysctl_resched_latency_warn_once && warned_once) 4890 return 0; 4891 4892 if (!need_resched() || !latency_warn_ms) 4893 return 0; 4894 4895 if (system_state == SYSTEM_BOOTING) 4896 return 0; 4897 4898 if (!rq->last_seen_need_resched_ns) { 4899 rq->last_seen_need_resched_ns = now; 4900 rq->ticks_without_resched = 0; 4901 return 0; 4902 } 4903 4904 rq->ticks_without_resched++; 4905 resched_latency = now - rq->last_seen_need_resched_ns; 4906 if (resched_latency <= latency_warn_ms * NSEC_PER_MSEC) 4907 return 0; 4908 4909 warned_once = true; 4910 4911 return resched_latency; 4912 } 4913 4914 static int __init setup_resched_latency_warn_ms(char *str) 4915 { 4916 long val; 4917 4918 if ((kstrtol(str, 0, &val))) { 4919 pr_warn("Unable to set resched_latency_warn_ms\n"); 4920 return 1; 4921 } 4922 4923 sysctl_resched_latency_warn_ms = val; 4924 return 1; 4925 } 4926 __setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms); 4927 #else 4928 static inline u64 cpu_resched_latency(struct rq *rq) { return 0; } 4929 #endif /* CONFIG_SCHED_DEBUG */ 4930 4931 /* 4932 * This function gets called by the timer code, with HZ frequency. 4933 * We call it with interrupts disabled. 4934 */ 4935 void scheduler_tick(void) 4936 { 4937 int cpu = smp_processor_id(); 4938 struct rq *rq = cpu_rq(cpu); 4939 struct task_struct *curr = rq->curr; 4940 struct rq_flags rf; 4941 unsigned long thermal_pressure; 4942 u64 resched_latency; 4943 4944 arch_scale_freq_tick(); 4945 sched_clock_tick(); 4946 4947 rq_lock(rq, &rf); 4948 4949 update_rq_clock(rq); 4950 thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); 4951 update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure); 4952 curr->sched_class->task_tick(rq, curr, 0); 4953 if (sched_feat(LATENCY_WARN)) 4954 resched_latency = cpu_resched_latency(rq); 4955 calc_global_load_tick(rq); 4956 4957 rq_unlock(rq, &rf); 4958 4959 if (sched_feat(LATENCY_WARN) && resched_latency) 4960 resched_latency_warn(cpu, resched_latency); 4961 4962 perf_event_task_tick(); 4963 4964 #ifdef CONFIG_SMP 4965 rq->idle_balance = idle_cpu(cpu); 4966 trigger_load_balance(rq); 4967 #endif 4968 } 4969 4970 #ifdef CONFIG_NO_HZ_FULL 4971 4972 struct tick_work { 4973 int cpu; 4974 atomic_t state; 4975 struct delayed_work work; 4976 }; 4977 /* Values for ->state, see diagram below. */ 4978 #define TICK_SCHED_REMOTE_OFFLINE 0 4979 #define TICK_SCHED_REMOTE_OFFLINING 1 4980 #define TICK_SCHED_REMOTE_RUNNING 2 4981 4982 /* 4983 * State diagram for ->state: 4984 * 4985 * 4986 * TICK_SCHED_REMOTE_OFFLINE 4987 * | ^ 4988 * | | 4989 * | | sched_tick_remote() 4990 * | | 4991 * | | 4992 * +--TICK_SCHED_REMOTE_OFFLINING 4993 * | ^ 4994 * | | 4995 * sched_tick_start() | | sched_tick_stop() 4996 * | | 4997 * V | 4998 * TICK_SCHED_REMOTE_RUNNING 4999 * 5000 * 5001 * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote() 5002 * and sched_tick_start() are happy to leave the state in RUNNING. 5003 */ 5004 5005 static struct tick_work __percpu *tick_work_cpu; 5006 5007 static void sched_tick_remote(struct work_struct *work) 5008 { 5009 struct delayed_work *dwork = to_delayed_work(work); 5010 struct tick_work *twork = container_of(dwork, struct tick_work, work); 5011 int cpu = twork->cpu; 5012 struct rq *rq = cpu_rq(cpu); 5013 struct task_struct *curr; 5014 struct rq_flags rf; 5015 u64 delta; 5016 int os; 5017 5018 /* 5019 * Handle the tick only if it appears the remote CPU is running in full 5020 * dynticks mode. The check is racy by nature, but missing a tick or 5021 * having one too much is no big deal because the scheduler tick updates 5022 * statistics and checks timeslices in a time-independent way, regardless 5023 * of when exactly it is running. 5024 */ 5025 if (!tick_nohz_tick_stopped_cpu(cpu)) 5026 goto out_requeue; 5027 5028 rq_lock_irq(rq, &rf); 5029 curr = rq->curr; 5030 if (cpu_is_offline(cpu)) 5031 goto out_unlock; 5032 5033 update_rq_clock(rq); 5034 5035 if (!is_idle_task(curr)) { 5036 /* 5037 * Make sure the next tick runs within a reasonable 5038 * amount of time. 5039 */ 5040 delta = rq_clock_task(rq) - curr->se.exec_start; 5041 WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3); 5042 } 5043 curr->sched_class->task_tick(rq, curr, 0); 5044 5045 calc_load_nohz_remote(rq); 5046 out_unlock: 5047 rq_unlock_irq(rq, &rf); 5048 out_requeue: 5049 5050 /* 5051 * Run the remote tick once per second (1Hz). This arbitrary 5052 * frequency is large enough to avoid overload but short enough 5053 * to keep scheduler internal stats reasonably up to date. But 5054 * first update state to reflect hotplug activity if required. 5055 */ 5056 os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING); 5057 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE); 5058 if (os == TICK_SCHED_REMOTE_RUNNING) 5059 queue_delayed_work(system_unbound_wq, dwork, HZ); 5060 } 5061 5062 static void sched_tick_start(int cpu) 5063 { 5064 int os; 5065 struct tick_work *twork; 5066 5067 if (housekeeping_cpu(cpu, HK_FLAG_TICK)) 5068 return; 5069 5070 WARN_ON_ONCE(!tick_work_cpu); 5071 5072 twork = per_cpu_ptr(tick_work_cpu, cpu); 5073 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING); 5074 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING); 5075 if (os == TICK_SCHED_REMOTE_OFFLINE) { 5076 twork->cpu = cpu; 5077 INIT_DELAYED_WORK(&twork->work, sched_tick_remote); 5078 queue_delayed_work(system_unbound_wq, &twork->work, HZ); 5079 } 5080 } 5081 5082 #ifdef CONFIG_HOTPLUG_CPU 5083 static void sched_tick_stop(int cpu) 5084 { 5085 struct tick_work *twork; 5086 int os; 5087 5088 if (housekeeping_cpu(cpu, HK_FLAG_TICK)) 5089 return; 5090 5091 WARN_ON_ONCE(!tick_work_cpu); 5092 5093 twork = per_cpu_ptr(tick_work_cpu, cpu); 5094 /* There cannot be competing actions, but don't rely on stop-machine. */ 5095 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING); 5096 WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING); 5097 /* Don't cancel, as this would mess up the state machine. */ 5098 } 5099 #endif /* CONFIG_HOTPLUG_CPU */ 5100 5101 int __init sched_tick_offload_init(void) 5102 { 5103 tick_work_cpu = alloc_percpu(struct tick_work); 5104 BUG_ON(!tick_work_cpu); 5105 return 0; 5106 } 5107 5108 #else /* !CONFIG_NO_HZ_FULL */ 5109 static inline void sched_tick_start(int cpu) { } 5110 static inline void sched_tick_stop(int cpu) { } 5111 #endif 5112 5113 #if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \ 5114 defined(CONFIG_TRACE_PREEMPT_TOGGLE)) 5115 /* 5116 * If the value passed in is equal to the current preempt count 5117 * then we just disabled preemption. Start timing the latency. 5118 */ 5119 static inline void preempt_latency_start(int val) 5120 { 5121 if (preempt_count() == val) { 5122 unsigned long ip = get_lock_parent_ip(); 5123 #ifdef CONFIG_DEBUG_PREEMPT 5124 current->preempt_disable_ip = ip; 5125 #endif 5126 trace_preempt_off(CALLER_ADDR0, ip); 5127 } 5128 } 5129 5130 void preempt_count_add(int val) 5131 { 5132 #ifdef CONFIG_DEBUG_PREEMPT 5133 /* 5134 * Underflow? 5135 */ 5136 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) 5137 return; 5138 #endif 5139 __preempt_count_add(val); 5140 #ifdef CONFIG_DEBUG_PREEMPT 5141 /* 5142 * Spinlock count overflowing soon? 5143 */ 5144 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= 5145 PREEMPT_MASK - 10); 5146 #endif 5147 preempt_latency_start(val); 5148 } 5149 EXPORT_SYMBOL(preempt_count_add); 5150 NOKPROBE_SYMBOL(preempt_count_add); 5151 5152 /* 5153 * If the value passed in equals to the current preempt count 5154 * then we just enabled preemption. Stop timing the latency. 5155 */ 5156 static inline void preempt_latency_stop(int val) 5157 { 5158 if (preempt_count() == val) 5159 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); 5160 } 5161 5162 void preempt_count_sub(int val) 5163 { 5164 #ifdef CONFIG_DEBUG_PREEMPT 5165 /* 5166 * Underflow? 5167 */ 5168 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 5169 return; 5170 /* 5171 * Is the spinlock portion underflowing? 5172 */ 5173 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && 5174 !(preempt_count() & PREEMPT_MASK))) 5175 return; 5176 #endif 5177 5178 preempt_latency_stop(val); 5179 __preempt_count_sub(val); 5180 } 5181 EXPORT_SYMBOL(preempt_count_sub); 5182 NOKPROBE_SYMBOL(preempt_count_sub); 5183 5184 #else 5185 static inline void preempt_latency_start(int val) { } 5186 static inline void preempt_latency_stop(int val) { } 5187 #endif 5188 5189 static inline unsigned long get_preempt_disable_ip(struct task_struct *p) 5190 { 5191 #ifdef CONFIG_DEBUG_PREEMPT 5192 return p->preempt_disable_ip; 5193 #else 5194 return 0; 5195 #endif 5196 } 5197 5198 /* 5199 * Print scheduling while atomic bug: 5200 */ 5201 static noinline void __schedule_bug(struct task_struct *prev) 5202 { 5203 /* Save this before calling printk(), since that will clobber it */ 5204 unsigned long preempt_disable_ip = get_preempt_disable_ip(current); 5205 5206 if (oops_in_progress) 5207 return; 5208 5209 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", 5210 prev->comm, prev->pid, preempt_count()); 5211 5212 debug_show_held_locks(prev); 5213 print_modules(); 5214 if (irqs_disabled()) 5215 print_irqtrace_events(prev); 5216 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) 5217 && in_atomic_preempt_off()) { 5218 pr_err("Preemption disabled at:"); 5219 print_ip_sym(KERN_ERR, preempt_disable_ip); 5220 } 5221 if (panic_on_warn) 5222 panic("scheduling while atomic\n"); 5223 5224 dump_stack(); 5225 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5226 } 5227 5228 /* 5229 * Various schedule()-time debugging checks and statistics: 5230 */ 5231 static inline void schedule_debug(struct task_struct *prev, bool preempt) 5232 { 5233 #ifdef CONFIG_SCHED_STACK_END_CHECK 5234 if (task_stack_end_corrupted(prev)) 5235 panic("corrupted stack end detected inside scheduler\n"); 5236 5237 if (task_scs_end_corrupted(prev)) 5238 panic("corrupted shadow stack detected inside scheduler\n"); 5239 #endif 5240 5241 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 5242 if (!preempt && READ_ONCE(prev->__state) && prev->non_block_count) { 5243 printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n", 5244 prev->comm, prev->pid, prev->non_block_count); 5245 dump_stack(); 5246 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5247 } 5248 #endif 5249 5250 if (unlikely(in_atomic_preempt_off())) { 5251 __schedule_bug(prev); 5252 preempt_count_set(PREEMPT_DISABLED); 5253 } 5254 rcu_sleep_check(); 5255 SCHED_WARN_ON(ct_state() == CONTEXT_USER); 5256 5257 profile_hit(SCHED_PROFILING, __builtin_return_address(0)); 5258 5259 schedstat_inc(this_rq()->sched_count); 5260 } 5261 5262 static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, 5263 struct rq_flags *rf) 5264 { 5265 #ifdef CONFIG_SMP 5266 const struct sched_class *class; 5267 /* 5268 * We must do the balancing pass before put_prev_task(), such 5269 * that when we release the rq->lock the task is in the same 5270 * state as before we took rq->lock. 5271 * 5272 * We can terminate the balance pass as soon as we know there is 5273 * a runnable task of @class priority or higher. 5274 */ 5275 for_class_range(class, prev->sched_class, &idle_sched_class) { 5276 if (class->balance(rq, prev, rf)) 5277 break; 5278 } 5279 #endif 5280 5281 put_prev_task(rq, prev); 5282 } 5283 5284 /* 5285 * Pick up the highest-prio task: 5286 */ 5287 static inline struct task_struct * 5288 __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 5289 { 5290 const struct sched_class *class; 5291 struct task_struct *p; 5292 5293 /* 5294 * Optimization: we know that if all tasks are in the fair class we can 5295 * call that function directly, but only if the @prev task wasn't of a 5296 * higher scheduling class, because otherwise those lose the 5297 * opportunity to pull in more work from other CPUs. 5298 */ 5299 if (likely(prev->sched_class <= &fair_sched_class && 5300 rq->nr_running == rq->cfs.h_nr_running)) { 5301 5302 p = pick_next_task_fair(rq, prev, rf); 5303 if (unlikely(p == RETRY_TASK)) 5304 goto restart; 5305 5306 /* Assume the next prioritized class is idle_sched_class */ 5307 if (!p) { 5308 put_prev_task(rq, prev); 5309 p = pick_next_task_idle(rq); 5310 } 5311 5312 return p; 5313 } 5314 5315 restart: 5316 put_prev_task_balance(rq, prev, rf); 5317 5318 for_each_class(class) { 5319 p = class->pick_next_task(rq); 5320 if (p) 5321 return p; 5322 } 5323 5324 /* The idle class should always have a runnable task: */ 5325 BUG(); 5326 } 5327 5328 #ifdef CONFIG_SCHED_CORE 5329 static inline bool is_task_rq_idle(struct task_struct *t) 5330 { 5331 return (task_rq(t)->idle == t); 5332 } 5333 5334 static inline bool cookie_equals(struct task_struct *a, unsigned long cookie) 5335 { 5336 return is_task_rq_idle(a) || (a->core_cookie == cookie); 5337 } 5338 5339 static inline bool cookie_match(struct task_struct *a, struct task_struct *b) 5340 { 5341 if (is_task_rq_idle(a) || is_task_rq_idle(b)) 5342 return true; 5343 5344 return a->core_cookie == b->core_cookie; 5345 } 5346 5347 // XXX fairness/fwd progress conditions 5348 /* 5349 * Returns 5350 * - NULL if there is no runnable task for this class. 5351 * - the highest priority task for this runqueue if it matches 5352 * rq->core->core_cookie or its priority is greater than max. 5353 * - Else returns idle_task. 5354 */ 5355 static struct task_struct * 5356 pick_task(struct rq *rq, const struct sched_class *class, struct task_struct *max, bool in_fi) 5357 { 5358 struct task_struct *class_pick, *cookie_pick; 5359 unsigned long cookie = rq->core->core_cookie; 5360 5361 class_pick = class->pick_task(rq); 5362 if (!class_pick) 5363 return NULL; 5364 5365 if (!cookie) { 5366 /* 5367 * If class_pick is tagged, return it only if it has 5368 * higher priority than max. 5369 */ 5370 if (max && class_pick->core_cookie && 5371 prio_less(class_pick, max, in_fi)) 5372 return idle_sched_class.pick_task(rq); 5373 5374 return class_pick; 5375 } 5376 5377 /* 5378 * If class_pick is idle or matches cookie, return early. 5379 */ 5380 if (cookie_equals(class_pick, cookie)) 5381 return class_pick; 5382 5383 cookie_pick = sched_core_find(rq, cookie); 5384 5385 /* 5386 * If class > max && class > cookie, it is the highest priority task on 5387 * the core (so far) and it must be selected, otherwise we must go with 5388 * the cookie pick in order to satisfy the constraint. 5389 */ 5390 if (prio_less(cookie_pick, class_pick, in_fi) && 5391 (!max || prio_less(max, class_pick, in_fi))) 5392 return class_pick; 5393 5394 return cookie_pick; 5395 } 5396 5397 extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi); 5398 5399 static struct task_struct * 5400 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 5401 { 5402 struct task_struct *next, *max = NULL; 5403 const struct sched_class *class; 5404 const struct cpumask *smt_mask; 5405 bool fi_before = false; 5406 int i, j, cpu, occ = 0; 5407 bool need_sync; 5408 5409 if (!sched_core_enabled(rq)) 5410 return __pick_next_task(rq, prev, rf); 5411 5412 cpu = cpu_of(rq); 5413 5414 /* Stopper task is switching into idle, no need core-wide selection. */ 5415 if (cpu_is_offline(cpu)) { 5416 /* 5417 * Reset core_pick so that we don't enter the fastpath when 5418 * coming online. core_pick would already be migrated to 5419 * another cpu during offline. 5420 */ 5421 rq->core_pick = NULL; 5422 return __pick_next_task(rq, prev, rf); 5423 } 5424 5425 /* 5426 * If there were no {en,de}queues since we picked (IOW, the task 5427 * pointers are all still valid), and we haven't scheduled the last 5428 * pick yet, do so now. 5429 * 5430 * rq->core_pick can be NULL if no selection was made for a CPU because 5431 * it was either offline or went offline during a sibling's core-wide 5432 * selection. In this case, do a core-wide selection. 5433 */ 5434 if (rq->core->core_pick_seq == rq->core->core_task_seq && 5435 rq->core->core_pick_seq != rq->core_sched_seq && 5436 rq->core_pick) { 5437 WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); 5438 5439 next = rq->core_pick; 5440 if (next != prev) { 5441 put_prev_task(rq, prev); 5442 set_next_task(rq, next); 5443 } 5444 5445 rq->core_pick = NULL; 5446 return next; 5447 } 5448 5449 put_prev_task_balance(rq, prev, rf); 5450 5451 smt_mask = cpu_smt_mask(cpu); 5452 need_sync = !!rq->core->core_cookie; 5453 5454 /* reset state */ 5455 rq->core->core_cookie = 0UL; 5456 if (rq->core->core_forceidle) { 5457 need_sync = true; 5458 fi_before = true; 5459 rq->core->core_forceidle = false; 5460 } 5461 5462 /* 5463 * core->core_task_seq, core->core_pick_seq, rq->core_sched_seq 5464 * 5465 * @task_seq guards the task state ({en,de}queues) 5466 * @pick_seq is the @task_seq we did a selection on 5467 * @sched_seq is the @pick_seq we scheduled 5468 * 5469 * However, preemptions can cause multiple picks on the same task set. 5470 * 'Fix' this by also increasing @task_seq for every pick. 5471 */ 5472 rq->core->core_task_seq++; 5473 5474 /* 5475 * Optimize for common case where this CPU has no cookies 5476 * and there are no cookied tasks running on siblings. 5477 */ 5478 if (!need_sync) { 5479 for_each_class(class) { 5480 next = class->pick_task(rq); 5481 if (next) 5482 break; 5483 } 5484 5485 if (!next->core_cookie) { 5486 rq->core_pick = NULL; 5487 /* 5488 * For robustness, update the min_vruntime_fi for 5489 * unconstrained picks as well. 5490 */ 5491 WARN_ON_ONCE(fi_before); 5492 task_vruntime_update(rq, next, false); 5493 goto done; 5494 } 5495 } 5496 5497 for_each_cpu(i, smt_mask) { 5498 struct rq *rq_i = cpu_rq(i); 5499 5500 rq_i->core_pick = NULL; 5501 5502 if (i != cpu) 5503 update_rq_clock(rq_i); 5504 } 5505 5506 /* 5507 * Try and select tasks for each sibling in descending sched_class 5508 * order. 5509 */ 5510 for_each_class(class) { 5511 again: 5512 for_each_cpu_wrap(i, smt_mask, cpu) { 5513 struct rq *rq_i = cpu_rq(i); 5514 struct task_struct *p; 5515 5516 if (rq_i->core_pick) 5517 continue; 5518 5519 /* 5520 * If this sibling doesn't yet have a suitable task to 5521 * run; ask for the most eligible task, given the 5522 * highest priority task already selected for this 5523 * core. 5524 */ 5525 p = pick_task(rq_i, class, max, fi_before); 5526 if (!p) 5527 continue; 5528 5529 if (!is_task_rq_idle(p)) 5530 occ++; 5531 5532 rq_i->core_pick = p; 5533 if (rq_i->idle == p && rq_i->nr_running) { 5534 rq->core->core_forceidle = true; 5535 if (!fi_before) 5536 rq->core->core_forceidle_seq++; 5537 } 5538 5539 /* 5540 * If this new candidate is of higher priority than the 5541 * previous; and they're incompatible; we need to wipe 5542 * the slate and start over. pick_task makes sure that 5543 * p's priority is more than max if it doesn't match 5544 * max's cookie. 5545 * 5546 * NOTE: this is a linear max-filter and is thus bounded 5547 * in execution time. 5548 */ 5549 if (!max || !cookie_match(max, p)) { 5550 struct task_struct *old_max = max; 5551 5552 rq->core->core_cookie = p->core_cookie; 5553 max = p; 5554 5555 if (old_max) { 5556 rq->core->core_forceidle = false; 5557 for_each_cpu(j, smt_mask) { 5558 if (j == i) 5559 continue; 5560 5561 cpu_rq(j)->core_pick = NULL; 5562 } 5563 occ = 1; 5564 goto again; 5565 } 5566 } 5567 } 5568 } 5569 5570 rq->core->core_pick_seq = rq->core->core_task_seq; 5571 next = rq->core_pick; 5572 rq->core_sched_seq = rq->core->core_pick_seq; 5573 5574 /* Something should have been selected for current CPU */ 5575 WARN_ON_ONCE(!next); 5576 5577 /* 5578 * Reschedule siblings 5579 * 5580 * NOTE: L1TF -- at this point we're no longer running the old task and 5581 * sending an IPI (below) ensures the sibling will no longer be running 5582 * their task. This ensures there is no inter-sibling overlap between 5583 * non-matching user state. 5584 */ 5585 for_each_cpu(i, smt_mask) { 5586 struct rq *rq_i = cpu_rq(i); 5587 5588 /* 5589 * An online sibling might have gone offline before a task 5590 * could be picked for it, or it might be offline but later 5591 * happen to come online, but its too late and nothing was 5592 * picked for it. That's Ok - it will pick tasks for itself, 5593 * so ignore it. 5594 */ 5595 if (!rq_i->core_pick) 5596 continue; 5597 5598 /* 5599 * Update for new !FI->FI transitions, or if continuing to be in !FI: 5600 * fi_before fi update? 5601 * 0 0 1 5602 * 0 1 1 5603 * 1 0 1 5604 * 1 1 0 5605 */ 5606 if (!(fi_before && rq->core->core_forceidle)) 5607 task_vruntime_update(rq_i, rq_i->core_pick, rq->core->core_forceidle); 5608 5609 rq_i->core_pick->core_occupation = occ; 5610 5611 if (i == cpu) { 5612 rq_i->core_pick = NULL; 5613 continue; 5614 } 5615 5616 /* Did we break L1TF mitigation requirements? */ 5617 WARN_ON_ONCE(!cookie_match(next, rq_i->core_pick)); 5618 5619 if (rq_i->curr == rq_i->core_pick) { 5620 rq_i->core_pick = NULL; 5621 continue; 5622 } 5623 5624 resched_curr(rq_i); 5625 } 5626 5627 done: 5628 set_next_task(rq, next); 5629 return next; 5630 } 5631 5632 static bool try_steal_cookie(int this, int that) 5633 { 5634 struct rq *dst = cpu_rq(this), *src = cpu_rq(that); 5635 struct task_struct *p; 5636 unsigned long cookie; 5637 bool success = false; 5638 5639 local_irq_disable(); 5640 double_rq_lock(dst, src); 5641 5642 cookie = dst->core->core_cookie; 5643 if (!cookie) 5644 goto unlock; 5645 5646 if (dst->curr != dst->idle) 5647 goto unlock; 5648 5649 p = sched_core_find(src, cookie); 5650 if (p == src->idle) 5651 goto unlock; 5652 5653 do { 5654 if (p == src->core_pick || p == src->curr) 5655 goto next; 5656 5657 if (!cpumask_test_cpu(this, &p->cpus_mask)) 5658 goto next; 5659 5660 if (p->core_occupation > dst->idle->core_occupation) 5661 goto next; 5662 5663 p->on_rq = TASK_ON_RQ_MIGRATING; 5664 deactivate_task(src, p, 0); 5665 set_task_cpu(p, this); 5666 activate_task(dst, p, 0); 5667 p->on_rq = TASK_ON_RQ_QUEUED; 5668 5669 resched_curr(dst); 5670 5671 success = true; 5672 break; 5673 5674 next: 5675 p = sched_core_next(p, cookie); 5676 } while (p); 5677 5678 unlock: 5679 double_rq_unlock(dst, src); 5680 local_irq_enable(); 5681 5682 return success; 5683 } 5684 5685 static bool steal_cookie_task(int cpu, struct sched_domain *sd) 5686 { 5687 int i; 5688 5689 for_each_cpu_wrap(i, sched_domain_span(sd), cpu) { 5690 if (i == cpu) 5691 continue; 5692 5693 if (need_resched()) 5694 break; 5695 5696 if (try_steal_cookie(cpu, i)) 5697 return true; 5698 } 5699 5700 return false; 5701 } 5702 5703 static void sched_core_balance(struct rq *rq) 5704 { 5705 struct sched_domain *sd; 5706 int cpu = cpu_of(rq); 5707 5708 preempt_disable(); 5709 rcu_read_lock(); 5710 raw_spin_rq_unlock_irq(rq); 5711 for_each_domain(cpu, sd) { 5712 if (need_resched()) 5713 break; 5714 5715 if (steal_cookie_task(cpu, sd)) 5716 break; 5717 } 5718 raw_spin_rq_lock_irq(rq); 5719 rcu_read_unlock(); 5720 preempt_enable(); 5721 } 5722 5723 static DEFINE_PER_CPU(struct callback_head, core_balance_head); 5724 5725 void queue_core_balance(struct rq *rq) 5726 { 5727 if (!sched_core_enabled(rq)) 5728 return; 5729 5730 if (!rq->core->core_cookie) 5731 return; 5732 5733 if (!rq->nr_running) /* not forced idle */ 5734 return; 5735 5736 queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance); 5737 } 5738 5739 static inline void sched_core_cpu_starting(unsigned int cpu) 5740 { 5741 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 5742 struct rq *rq, *core_rq = NULL; 5743 int i; 5744 5745 core_rq = cpu_rq(cpu)->core; 5746 5747 if (!core_rq) { 5748 for_each_cpu(i, smt_mask) { 5749 rq = cpu_rq(i); 5750 if (rq->core && rq->core == rq) 5751 core_rq = rq; 5752 } 5753 5754 if (!core_rq) 5755 core_rq = cpu_rq(cpu); 5756 5757 for_each_cpu(i, smt_mask) { 5758 rq = cpu_rq(i); 5759 5760 WARN_ON_ONCE(rq->core && rq->core != core_rq); 5761 rq->core = core_rq; 5762 } 5763 } 5764 } 5765 #else /* !CONFIG_SCHED_CORE */ 5766 5767 static inline void sched_core_cpu_starting(unsigned int cpu) {} 5768 5769 static struct task_struct * 5770 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 5771 { 5772 return __pick_next_task(rq, prev, rf); 5773 } 5774 5775 #endif /* CONFIG_SCHED_CORE */ 5776 5777 /* 5778 * __schedule() is the main scheduler function. 5779 * 5780 * The main means of driving the scheduler and thus entering this function are: 5781 * 5782 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. 5783 * 5784 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return 5785 * paths. For example, see arch/x86/entry_64.S. 5786 * 5787 * To drive preemption between tasks, the scheduler sets the flag in timer 5788 * interrupt handler scheduler_tick(). 5789 * 5790 * 3. Wakeups don't really cause entry into schedule(). They add a 5791 * task to the run-queue and that's it. 5792 * 5793 * Now, if the new task added to the run-queue preempts the current 5794 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets 5795 * called on the nearest possible occasion: 5796 * 5797 * - If the kernel is preemptible (CONFIG_PREEMPTION=y): 5798 * 5799 * - in syscall or exception context, at the next outmost 5800 * preempt_enable(). (this might be as soon as the wake_up()'s 5801 * spin_unlock()!) 5802 * 5803 * - in IRQ context, return from interrupt-handler to 5804 * preemptible context 5805 * 5806 * - If the kernel is not preemptible (CONFIG_PREEMPTION is not set) 5807 * then at the next: 5808 * 5809 * - cond_resched() call 5810 * - explicit schedule() call 5811 * - return from syscall or exception to user-space 5812 * - return from interrupt-handler to user-space 5813 * 5814 * WARNING: must be called with preemption disabled! 5815 */ 5816 static void __sched notrace __schedule(bool preempt) 5817 { 5818 struct task_struct *prev, *next; 5819 unsigned long *switch_count; 5820 unsigned long prev_state; 5821 struct rq_flags rf; 5822 struct rq *rq; 5823 int cpu; 5824 5825 cpu = smp_processor_id(); 5826 rq = cpu_rq(cpu); 5827 prev = rq->curr; 5828 5829 schedule_debug(prev, preempt); 5830 5831 if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) 5832 hrtick_clear(rq); 5833 5834 local_irq_disable(); 5835 rcu_note_context_switch(preempt); 5836 5837 /* 5838 * Make sure that signal_pending_state()->signal_pending() below 5839 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) 5840 * done by the caller to avoid the race with signal_wake_up(): 5841 * 5842 * __set_current_state(@state) signal_wake_up() 5843 * schedule() set_tsk_thread_flag(p, TIF_SIGPENDING) 5844 * wake_up_state(p, state) 5845 * LOCK rq->lock LOCK p->pi_state 5846 * smp_mb__after_spinlock() smp_mb__after_spinlock() 5847 * if (signal_pending_state()) if (p->state & @state) 5848 * 5849 * Also, the membarrier system call requires a full memory barrier 5850 * after coming from user-space, before storing to rq->curr. 5851 */ 5852 rq_lock(rq, &rf); 5853 smp_mb__after_spinlock(); 5854 5855 /* Promote REQ to ACT */ 5856 rq->clock_update_flags <<= 1; 5857 update_rq_clock(rq); 5858 5859 switch_count = &prev->nivcsw; 5860 5861 /* 5862 * We must load prev->state once (task_struct::state is volatile), such 5863 * that: 5864 * 5865 * - we form a control dependency vs deactivate_task() below. 5866 * - ptrace_{,un}freeze_traced() can change ->state underneath us. 5867 */ 5868 prev_state = READ_ONCE(prev->__state); 5869 if (!preempt && prev_state) { 5870 if (signal_pending_state(prev_state, prev)) { 5871 WRITE_ONCE(prev->__state, TASK_RUNNING); 5872 } else { 5873 prev->sched_contributes_to_load = 5874 (prev_state & TASK_UNINTERRUPTIBLE) && 5875 !(prev_state & TASK_NOLOAD) && 5876 !(prev->flags & PF_FROZEN); 5877 5878 if (prev->sched_contributes_to_load) 5879 rq->nr_uninterruptible++; 5880 5881 /* 5882 * __schedule() ttwu() 5883 * prev_state = prev->state; if (p->on_rq && ...) 5884 * if (prev_state) goto out; 5885 * p->on_rq = 0; smp_acquire__after_ctrl_dep(); 5886 * p->state = TASK_WAKING 5887 * 5888 * Where __schedule() and ttwu() have matching control dependencies. 5889 * 5890 * After this, schedule() must not care about p->state any more. 5891 */ 5892 deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK); 5893 5894 if (prev->in_iowait) { 5895 atomic_inc(&rq->nr_iowait); 5896 delayacct_blkio_start(); 5897 } 5898 } 5899 switch_count = &prev->nvcsw; 5900 } 5901 5902 next = pick_next_task(rq, prev, &rf); 5903 clear_tsk_need_resched(prev); 5904 clear_preempt_need_resched(); 5905 #ifdef CONFIG_SCHED_DEBUG 5906 rq->last_seen_need_resched_ns = 0; 5907 #endif 5908 5909 if (likely(prev != next)) { 5910 rq->nr_switches++; 5911 /* 5912 * RCU users of rcu_dereference(rq->curr) may not see 5913 * changes to task_struct made by pick_next_task(). 5914 */ 5915 RCU_INIT_POINTER(rq->curr, next); 5916 /* 5917 * The membarrier system call requires each architecture 5918 * to have a full memory barrier after updating 5919 * rq->curr, before returning to user-space. 5920 * 5921 * Here are the schemes providing that barrier on the 5922 * various architectures: 5923 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC. 5924 * switch_mm() rely on membarrier_arch_switch_mm() on PowerPC. 5925 * - finish_lock_switch() for weakly-ordered 5926 * architectures where spin_unlock is a full barrier, 5927 * - switch_to() for arm64 (weakly-ordered, spin_unlock 5928 * is a RELEASE barrier), 5929 */ 5930 ++*switch_count; 5931 5932 migrate_disable_switch(rq, prev); 5933 psi_sched_switch(prev, next, !task_on_rq_queued(prev)); 5934 5935 trace_sched_switch(preempt, prev, next); 5936 5937 /* Also unlocks the rq: */ 5938 rq = context_switch(rq, prev, next, &rf); 5939 } else { 5940 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 5941 5942 rq_unpin_lock(rq, &rf); 5943 __balance_callbacks(rq); 5944 raw_spin_rq_unlock_irq(rq); 5945 } 5946 } 5947 5948 void __noreturn do_task_dead(void) 5949 { 5950 /* Causes final put_task_struct in finish_task_switch(): */ 5951 set_special_state(TASK_DEAD); 5952 5953 /* Tell freezer to ignore us: */ 5954 current->flags |= PF_NOFREEZE; 5955 5956 __schedule(false); 5957 BUG(); 5958 5959 /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */ 5960 for (;;) 5961 cpu_relax(); 5962 } 5963 5964 static inline void sched_submit_work(struct task_struct *tsk) 5965 { 5966 unsigned int task_flags; 5967 5968 if (task_is_running(tsk)) 5969 return; 5970 5971 task_flags = tsk->flags; 5972 /* 5973 * If a worker went to sleep, notify and ask workqueue whether 5974 * it wants to wake up a task to maintain concurrency. 5975 * As this function is called inside the schedule() context, 5976 * we disable preemption to avoid it calling schedule() again 5977 * in the possible wakeup of a kworker and because wq_worker_sleeping() 5978 * requires it. 5979 */ 5980 if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) { 5981 preempt_disable(); 5982 if (task_flags & PF_WQ_WORKER) 5983 wq_worker_sleeping(tsk); 5984 else 5985 io_wq_worker_sleeping(tsk); 5986 preempt_enable_no_resched(); 5987 } 5988 5989 if (tsk_is_pi_blocked(tsk)) 5990 return; 5991 5992 /* 5993 * If we are going to sleep and we have plugged IO queued, 5994 * make sure to submit it to avoid deadlocks. 5995 */ 5996 if (blk_needs_flush_plug(tsk)) 5997 blk_schedule_flush_plug(tsk); 5998 } 5999 6000 static void sched_update_worker(struct task_struct *tsk) 6001 { 6002 if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { 6003 if (tsk->flags & PF_WQ_WORKER) 6004 wq_worker_running(tsk); 6005 else 6006 io_wq_worker_running(tsk); 6007 } 6008 } 6009 6010 asmlinkage __visible void __sched schedule(void) 6011 { 6012 struct task_struct *tsk = current; 6013 6014 sched_submit_work(tsk); 6015 do { 6016 preempt_disable(); 6017 __schedule(false); 6018 sched_preempt_enable_no_resched(); 6019 } while (need_resched()); 6020 sched_update_worker(tsk); 6021 } 6022 EXPORT_SYMBOL(schedule); 6023 6024 /* 6025 * synchronize_rcu_tasks() makes sure that no task is stuck in preempted 6026 * state (have scheduled out non-voluntarily) by making sure that all 6027 * tasks have either left the run queue or have gone into user space. 6028 * As idle tasks do not do either, they must not ever be preempted 6029 * (schedule out non-voluntarily). 6030 * 6031 * schedule_idle() is similar to schedule_preempt_disable() except that it 6032 * never enables preemption because it does not call sched_submit_work(). 6033 */ 6034 void __sched schedule_idle(void) 6035 { 6036 /* 6037 * As this skips calling sched_submit_work(), which the idle task does 6038 * regardless because that function is a nop when the task is in a 6039 * TASK_RUNNING state, make sure this isn't used someplace that the 6040 * current task can be in any other state. Note, idle is always in the 6041 * TASK_RUNNING state. 6042 */ 6043 WARN_ON_ONCE(current->__state); 6044 do { 6045 __schedule(false); 6046 } while (need_resched()); 6047 } 6048 6049 #if defined(CONFIG_CONTEXT_TRACKING) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK) 6050 asmlinkage __visible void __sched schedule_user(void) 6051 { 6052 /* 6053 * If we come here after a random call to set_need_resched(), 6054 * or we have been woken up remotely but the IPI has not yet arrived, 6055 * we haven't yet exited the RCU idle mode. Do it here manually until 6056 * we find a better solution. 6057 * 6058 * NB: There are buggy callers of this function. Ideally we 6059 * should warn if prev_state != CONTEXT_USER, but that will trigger 6060 * too frequently to make sense yet. 6061 */ 6062 enum ctx_state prev_state = exception_enter(); 6063 schedule(); 6064 exception_exit(prev_state); 6065 } 6066 #endif 6067 6068 /** 6069 * schedule_preempt_disabled - called with preemption disabled 6070 * 6071 * Returns with preemption disabled. Note: preempt_count must be 1 6072 */ 6073 void __sched schedule_preempt_disabled(void) 6074 { 6075 sched_preempt_enable_no_resched(); 6076 schedule(); 6077 preempt_disable(); 6078 } 6079 6080 static void __sched notrace preempt_schedule_common(void) 6081 { 6082 do { 6083 /* 6084 * Because the function tracer can trace preempt_count_sub() 6085 * and it also uses preempt_enable/disable_notrace(), if 6086 * NEED_RESCHED is set, the preempt_enable_notrace() called 6087 * by the function tracer will call this function again and 6088 * cause infinite recursion. 6089 * 6090 * Preemption must be disabled here before the function 6091 * tracer can trace. Break up preempt_disable() into two 6092 * calls. One to disable preemption without fear of being 6093 * traced. The other to still record the preemption latency, 6094 * which can also be traced by the function tracer. 6095 */ 6096 preempt_disable_notrace(); 6097 preempt_latency_start(1); 6098 __schedule(true); 6099 preempt_latency_stop(1); 6100 preempt_enable_no_resched_notrace(); 6101 6102 /* 6103 * Check again in case we missed a preemption opportunity 6104 * between schedule and now. 6105 */ 6106 } while (need_resched()); 6107 } 6108 6109 #ifdef CONFIG_PREEMPTION 6110 /* 6111 * This is the entry point to schedule() from in-kernel preemption 6112 * off of preempt_enable. 6113 */ 6114 asmlinkage __visible void __sched notrace preempt_schedule(void) 6115 { 6116 /* 6117 * If there is a non-zero preempt_count or interrupts are disabled, 6118 * we do not want to preempt the current task. Just return.. 6119 */ 6120 if (likely(!preemptible())) 6121 return; 6122 6123 preempt_schedule_common(); 6124 } 6125 NOKPROBE_SYMBOL(preempt_schedule); 6126 EXPORT_SYMBOL(preempt_schedule); 6127 6128 #ifdef CONFIG_PREEMPT_DYNAMIC 6129 DEFINE_STATIC_CALL(preempt_schedule, __preempt_schedule_func); 6130 EXPORT_STATIC_CALL_TRAMP(preempt_schedule); 6131 #endif 6132 6133 6134 /** 6135 * preempt_schedule_notrace - preempt_schedule called by tracing 6136 * 6137 * The tracing infrastructure uses preempt_enable_notrace to prevent 6138 * recursion and tracing preempt enabling caused by the tracing 6139 * infrastructure itself. But as tracing can happen in areas coming 6140 * from userspace or just about to enter userspace, a preempt enable 6141 * can occur before user_exit() is called. This will cause the scheduler 6142 * to be called when the system is still in usermode. 6143 * 6144 * To prevent this, the preempt_enable_notrace will use this function 6145 * instead of preempt_schedule() to exit user context if needed before 6146 * calling the scheduler. 6147 */ 6148 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) 6149 { 6150 enum ctx_state prev_ctx; 6151 6152 if (likely(!preemptible())) 6153 return; 6154 6155 do { 6156 /* 6157 * Because the function tracer can trace preempt_count_sub() 6158 * and it also uses preempt_enable/disable_notrace(), if 6159 * NEED_RESCHED is set, the preempt_enable_notrace() called 6160 * by the function tracer will call this function again and 6161 * cause infinite recursion. 6162 * 6163 * Preemption must be disabled here before the function 6164 * tracer can trace. Break up preempt_disable() into two 6165 * calls. One to disable preemption without fear of being 6166 * traced. The other to still record the preemption latency, 6167 * which can also be traced by the function tracer. 6168 */ 6169 preempt_disable_notrace(); 6170 preempt_latency_start(1); 6171 /* 6172 * Needs preempt disabled in case user_exit() is traced 6173 * and the tracer calls preempt_enable_notrace() causing 6174 * an infinite recursion. 6175 */ 6176 prev_ctx = exception_enter(); 6177 __schedule(true); 6178 exception_exit(prev_ctx); 6179 6180 preempt_latency_stop(1); 6181 preempt_enable_no_resched_notrace(); 6182 } while (need_resched()); 6183 } 6184 EXPORT_SYMBOL_GPL(preempt_schedule_notrace); 6185 6186 #ifdef CONFIG_PREEMPT_DYNAMIC 6187 DEFINE_STATIC_CALL(preempt_schedule_notrace, __preempt_schedule_notrace_func); 6188 EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); 6189 #endif 6190 6191 #endif /* CONFIG_PREEMPTION */ 6192 6193 #ifdef CONFIG_PREEMPT_DYNAMIC 6194 6195 #include <linux/entry-common.h> 6196 6197 /* 6198 * SC:cond_resched 6199 * SC:might_resched 6200 * SC:preempt_schedule 6201 * SC:preempt_schedule_notrace 6202 * SC:irqentry_exit_cond_resched 6203 * 6204 * 6205 * NONE: 6206 * cond_resched <- __cond_resched 6207 * might_resched <- RET0 6208 * preempt_schedule <- NOP 6209 * preempt_schedule_notrace <- NOP 6210 * irqentry_exit_cond_resched <- NOP 6211 * 6212 * VOLUNTARY: 6213 * cond_resched <- __cond_resched 6214 * might_resched <- __cond_resched 6215 * preempt_schedule <- NOP 6216 * preempt_schedule_notrace <- NOP 6217 * irqentry_exit_cond_resched <- NOP 6218 * 6219 * FULL: 6220 * cond_resched <- RET0 6221 * might_resched <- RET0 6222 * preempt_schedule <- preempt_schedule 6223 * preempt_schedule_notrace <- preempt_schedule_notrace 6224 * irqentry_exit_cond_resched <- irqentry_exit_cond_resched 6225 */ 6226 6227 enum { 6228 preempt_dynamic_none = 0, 6229 preempt_dynamic_voluntary, 6230 preempt_dynamic_full, 6231 }; 6232 6233 int preempt_dynamic_mode = preempt_dynamic_full; 6234 6235 int sched_dynamic_mode(const char *str) 6236 { 6237 if (!strcmp(str, "none")) 6238 return preempt_dynamic_none; 6239 6240 if (!strcmp(str, "voluntary")) 6241 return preempt_dynamic_voluntary; 6242 6243 if (!strcmp(str, "full")) 6244 return preempt_dynamic_full; 6245 6246 return -EINVAL; 6247 } 6248 6249 void sched_dynamic_update(int mode) 6250 { 6251 /* 6252 * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in 6253 * the ZERO state, which is invalid. 6254 */ 6255 static_call_update(cond_resched, __cond_resched); 6256 static_call_update(might_resched, __cond_resched); 6257 static_call_update(preempt_schedule, __preempt_schedule_func); 6258 static_call_update(preempt_schedule_notrace, __preempt_schedule_notrace_func); 6259 static_call_update(irqentry_exit_cond_resched, irqentry_exit_cond_resched); 6260 6261 switch (mode) { 6262 case preempt_dynamic_none: 6263 static_call_update(cond_resched, __cond_resched); 6264 static_call_update(might_resched, (void *)&__static_call_return0); 6265 static_call_update(preempt_schedule, NULL); 6266 static_call_update(preempt_schedule_notrace, NULL); 6267 static_call_update(irqentry_exit_cond_resched, NULL); 6268 pr_info("Dynamic Preempt: none\n"); 6269 break; 6270 6271 case preempt_dynamic_voluntary: 6272 static_call_update(cond_resched, __cond_resched); 6273 static_call_update(might_resched, __cond_resched); 6274 static_call_update(preempt_schedule, NULL); 6275 static_call_update(preempt_schedule_notrace, NULL); 6276 static_call_update(irqentry_exit_cond_resched, NULL); 6277 pr_info("Dynamic Preempt: voluntary\n"); 6278 break; 6279 6280 case preempt_dynamic_full: 6281 static_call_update(cond_resched, (void *)&__static_call_return0); 6282 static_call_update(might_resched, (void *)&__static_call_return0); 6283 static_call_update(preempt_schedule, __preempt_schedule_func); 6284 static_call_update(preempt_schedule_notrace, __preempt_schedule_notrace_func); 6285 static_call_update(irqentry_exit_cond_resched, irqentry_exit_cond_resched); 6286 pr_info("Dynamic Preempt: full\n"); 6287 break; 6288 } 6289 6290 preempt_dynamic_mode = mode; 6291 } 6292 6293 static int __init setup_preempt_mode(char *str) 6294 { 6295 int mode = sched_dynamic_mode(str); 6296 if (mode < 0) { 6297 pr_warn("Dynamic Preempt: unsupported mode: %s\n", str); 6298 return 1; 6299 } 6300 6301 sched_dynamic_update(mode); 6302 return 0; 6303 } 6304 __setup("preempt=", setup_preempt_mode); 6305 6306 #endif /* CONFIG_PREEMPT_DYNAMIC */ 6307 6308 /* 6309 * This is the entry point to schedule() from kernel preemption 6310 * off of irq context. 6311 * Note, that this is called and return with irqs disabled. This will 6312 * protect us against recursive calling from irq. 6313 */ 6314 asmlinkage __visible void __sched preempt_schedule_irq(void) 6315 { 6316 enum ctx_state prev_state; 6317 6318 /* Catch callers which need to be fixed */ 6319 BUG_ON(preempt_count() || !irqs_disabled()); 6320 6321 prev_state = exception_enter(); 6322 6323 do { 6324 preempt_disable(); 6325 local_irq_enable(); 6326 __schedule(true); 6327 local_irq_disable(); 6328 sched_preempt_enable_no_resched(); 6329 } while (need_resched()); 6330 6331 exception_exit(prev_state); 6332 } 6333 6334 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, 6335 void *key) 6336 { 6337 WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC); 6338 return try_to_wake_up(curr->private, mode, wake_flags); 6339 } 6340 EXPORT_SYMBOL(default_wake_function); 6341 6342 static void __setscheduler_prio(struct task_struct *p, int prio) 6343 { 6344 if (dl_prio(prio)) 6345 p->sched_class = &dl_sched_class; 6346 else if (rt_prio(prio)) 6347 p->sched_class = &rt_sched_class; 6348 else 6349 p->sched_class = &fair_sched_class; 6350 6351 p->prio = prio; 6352 } 6353 6354 #ifdef CONFIG_RT_MUTEXES 6355 6356 static inline int __rt_effective_prio(struct task_struct *pi_task, int prio) 6357 { 6358 if (pi_task) 6359 prio = min(prio, pi_task->prio); 6360 6361 return prio; 6362 } 6363 6364 static inline int rt_effective_prio(struct task_struct *p, int prio) 6365 { 6366 struct task_struct *pi_task = rt_mutex_get_top_task(p); 6367 6368 return __rt_effective_prio(pi_task, prio); 6369 } 6370 6371 /* 6372 * rt_mutex_setprio - set the current priority of a task 6373 * @p: task to boost 6374 * @pi_task: donor task 6375 * 6376 * This function changes the 'effective' priority of a task. It does 6377 * not touch ->normal_prio like __setscheduler(). 6378 * 6379 * Used by the rt_mutex code to implement priority inheritance 6380 * logic. Call site only calls if the priority of the task changed. 6381 */ 6382 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) 6383 { 6384 int prio, oldprio, queued, running, queue_flag = 6385 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 6386 const struct sched_class *prev_class; 6387 struct rq_flags rf; 6388 struct rq *rq; 6389 6390 /* XXX used to be waiter->prio, not waiter->task->prio */ 6391 prio = __rt_effective_prio(pi_task, p->normal_prio); 6392 6393 /* 6394 * If nothing changed; bail early. 6395 */ 6396 if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) 6397 return; 6398 6399 rq = __task_rq_lock(p, &rf); 6400 update_rq_clock(rq); 6401 /* 6402 * Set under pi_lock && rq->lock, such that the value can be used under 6403 * either lock. 6404 * 6405 * Note that there is loads of tricky to make this pointer cache work 6406 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to 6407 * ensure a task is de-boosted (pi_task is set to NULL) before the 6408 * task is allowed to run again (and can exit). This ensures the pointer 6409 * points to a blocked task -- which guarantees the task is present. 6410 */ 6411 p->pi_top_task = pi_task; 6412 6413 /* 6414 * For FIFO/RR we only need to set prio, if that matches we're done. 6415 */ 6416 if (prio == p->prio && !dl_prio(prio)) 6417 goto out_unlock; 6418 6419 /* 6420 * Idle task boosting is a nono in general. There is one 6421 * exception, when PREEMPT_RT and NOHZ is active: 6422 * 6423 * The idle task calls get_next_timer_interrupt() and holds 6424 * the timer wheel base->lock on the CPU and another CPU wants 6425 * to access the timer (probably to cancel it). We can safely 6426 * ignore the boosting request, as the idle CPU runs this code 6427 * with interrupts disabled and will complete the lock 6428 * protected section without being interrupted. So there is no 6429 * real need to boost. 6430 */ 6431 if (unlikely(p == rq->idle)) { 6432 WARN_ON(p != rq->curr); 6433 WARN_ON(p->pi_blocked_on); 6434 goto out_unlock; 6435 } 6436 6437 trace_sched_pi_setprio(p, pi_task); 6438 oldprio = p->prio; 6439 6440 if (oldprio == prio) 6441 queue_flag &= ~DEQUEUE_MOVE; 6442 6443 prev_class = p->sched_class; 6444 queued = task_on_rq_queued(p); 6445 running = task_current(rq, p); 6446 if (queued) 6447 dequeue_task(rq, p, queue_flag); 6448 if (running) 6449 put_prev_task(rq, p); 6450 6451 /* 6452 * Boosting condition are: 6453 * 1. -rt task is running and holds mutex A 6454 * --> -dl task blocks on mutex A 6455 * 6456 * 2. -dl task is running and holds mutex A 6457 * --> -dl task blocks on mutex A and could preempt the 6458 * running task 6459 */ 6460 if (dl_prio(prio)) { 6461 if (!dl_prio(p->normal_prio) || 6462 (pi_task && dl_prio(pi_task->prio) && 6463 dl_entity_preempt(&pi_task->dl, &p->dl))) { 6464 p->dl.pi_se = pi_task->dl.pi_se; 6465 queue_flag |= ENQUEUE_REPLENISH; 6466 } else { 6467 p->dl.pi_se = &p->dl; 6468 } 6469 } else if (rt_prio(prio)) { 6470 if (dl_prio(oldprio)) 6471 p->dl.pi_se = &p->dl; 6472 if (oldprio < prio) 6473 queue_flag |= ENQUEUE_HEAD; 6474 } else { 6475 if (dl_prio(oldprio)) 6476 p->dl.pi_se = &p->dl; 6477 if (rt_prio(oldprio)) 6478 p->rt.timeout = 0; 6479 } 6480 6481 __setscheduler_prio(p, prio); 6482 6483 if (queued) 6484 enqueue_task(rq, p, queue_flag); 6485 if (running) 6486 set_next_task(rq, p); 6487 6488 check_class_changed(rq, p, prev_class, oldprio); 6489 out_unlock: 6490 /* Avoid rq from going away on us: */ 6491 preempt_disable(); 6492 6493 rq_unpin_lock(rq, &rf); 6494 __balance_callbacks(rq); 6495 raw_spin_rq_unlock(rq); 6496 6497 preempt_enable(); 6498 } 6499 #else 6500 static inline int rt_effective_prio(struct task_struct *p, int prio) 6501 { 6502 return prio; 6503 } 6504 #endif 6505 6506 void set_user_nice(struct task_struct *p, long nice) 6507 { 6508 bool queued, running; 6509 int old_prio; 6510 struct rq_flags rf; 6511 struct rq *rq; 6512 6513 if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) 6514 return; 6515 /* 6516 * We have to be careful, if called from sys_setpriority(), 6517 * the task might be in the middle of scheduling on another CPU. 6518 */ 6519 rq = task_rq_lock(p, &rf); 6520 update_rq_clock(rq); 6521 6522 /* 6523 * The RT priorities are set via sched_setscheduler(), but we still 6524 * allow the 'normal' nice value to be set - but as expected 6525 * it won't have any effect on scheduling until the task is 6526 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: 6527 */ 6528 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 6529 p->static_prio = NICE_TO_PRIO(nice); 6530 goto out_unlock; 6531 } 6532 queued = task_on_rq_queued(p); 6533 running = task_current(rq, p); 6534 if (queued) 6535 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 6536 if (running) 6537 put_prev_task(rq, p); 6538 6539 p->static_prio = NICE_TO_PRIO(nice); 6540 set_load_weight(p, true); 6541 old_prio = p->prio; 6542 p->prio = effective_prio(p); 6543 6544 if (queued) 6545 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 6546 if (running) 6547 set_next_task(rq, p); 6548 6549 /* 6550 * If the task increased its priority or is running and 6551 * lowered its priority, then reschedule its CPU: 6552 */ 6553 p->sched_class->prio_changed(rq, p, old_prio); 6554 6555 out_unlock: 6556 task_rq_unlock(rq, p, &rf); 6557 } 6558 EXPORT_SYMBOL(set_user_nice); 6559 6560 /* 6561 * can_nice - check if a task can reduce its nice value 6562 * @p: task 6563 * @nice: nice value 6564 */ 6565 int can_nice(const struct task_struct *p, const int nice) 6566 { 6567 /* Convert nice value [19,-20] to rlimit style value [1,40]: */ 6568 int nice_rlim = nice_to_rlimit(nice); 6569 6570 return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || 6571 capable(CAP_SYS_NICE)); 6572 } 6573 6574 #ifdef __ARCH_WANT_SYS_NICE 6575 6576 /* 6577 * sys_nice - change the priority of the current process. 6578 * @increment: priority increment 6579 * 6580 * sys_setpriority is a more generic, but much slower function that 6581 * does similar things. 6582 */ 6583 SYSCALL_DEFINE1(nice, int, increment) 6584 { 6585 long nice, retval; 6586 6587 /* 6588 * Setpriority might change our priority at the same moment. 6589 * We don't have to worry. Conceptually one call occurs first 6590 * and we have a single winner. 6591 */ 6592 increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); 6593 nice = task_nice(current) + increment; 6594 6595 nice = clamp_val(nice, MIN_NICE, MAX_NICE); 6596 if (increment < 0 && !can_nice(current, nice)) 6597 return -EPERM; 6598 6599 retval = security_task_setnice(current, nice); 6600 if (retval) 6601 return retval; 6602 6603 set_user_nice(current, nice); 6604 return 0; 6605 } 6606 6607 #endif 6608 6609 /** 6610 * task_prio - return the priority value of a given task. 6611 * @p: the task in question. 6612 * 6613 * Return: The priority value as seen by users in /proc. 6614 * 6615 * sched policy return value kernel prio user prio/nice 6616 * 6617 * normal, batch, idle [0 ... 39] [100 ... 139] 0/[-20 ... 19] 6618 * fifo, rr [-2 ... -100] [98 ... 0] [1 ... 99] 6619 * deadline -101 -1 0 6620 */ 6621 int task_prio(const struct task_struct *p) 6622 { 6623 return p->prio - MAX_RT_PRIO; 6624 } 6625 6626 /** 6627 * idle_cpu - is a given CPU idle currently? 6628 * @cpu: the processor in question. 6629 * 6630 * Return: 1 if the CPU is currently idle. 0 otherwise. 6631 */ 6632 int idle_cpu(int cpu) 6633 { 6634 struct rq *rq = cpu_rq(cpu); 6635 6636 if (rq->curr != rq->idle) 6637 return 0; 6638 6639 if (rq->nr_running) 6640 return 0; 6641 6642 #ifdef CONFIG_SMP 6643 if (rq->ttwu_pending) 6644 return 0; 6645 #endif 6646 6647 return 1; 6648 } 6649 6650 /** 6651 * available_idle_cpu - is a given CPU idle for enqueuing work. 6652 * @cpu: the CPU in question. 6653 * 6654 * Return: 1 if the CPU is currently idle. 0 otherwise. 6655 */ 6656 int available_idle_cpu(int cpu) 6657 { 6658 if (!idle_cpu(cpu)) 6659 return 0; 6660 6661 if (vcpu_is_preempted(cpu)) 6662 return 0; 6663 6664 return 1; 6665 } 6666 6667 /** 6668 * idle_task - return the idle task for a given CPU. 6669 * @cpu: the processor in question. 6670 * 6671 * Return: The idle task for the CPU @cpu. 6672 */ 6673 struct task_struct *idle_task(int cpu) 6674 { 6675 return cpu_rq(cpu)->idle; 6676 } 6677 6678 #ifdef CONFIG_SMP 6679 /* 6680 * This function computes an effective utilization for the given CPU, to be 6681 * used for frequency selection given the linear relation: f = u * f_max. 6682 * 6683 * The scheduler tracks the following metrics: 6684 * 6685 * cpu_util_{cfs,rt,dl,irq}() 6686 * cpu_bw_dl() 6687 * 6688 * Where the cfs,rt and dl util numbers are tracked with the same metric and 6689 * synchronized windows and are thus directly comparable. 6690 * 6691 * The cfs,rt,dl utilization are the running times measured with rq->clock_task 6692 * which excludes things like IRQ and steal-time. These latter are then accrued 6693 * in the irq utilization. 6694 * 6695 * The DL bandwidth number otoh is not a measured metric but a value computed 6696 * based on the task model parameters and gives the minimal utilization 6697 * required to meet deadlines. 6698 */ 6699 unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, 6700 unsigned long max, enum cpu_util_type type, 6701 struct task_struct *p) 6702 { 6703 unsigned long dl_util, util, irq; 6704 struct rq *rq = cpu_rq(cpu); 6705 6706 if (!uclamp_is_used() && 6707 type == FREQUENCY_UTIL && rt_rq_is_runnable(&rq->rt)) { 6708 return max; 6709 } 6710 6711 /* 6712 * Early check to see if IRQ/steal time saturates the CPU, can be 6713 * because of inaccuracies in how we track these -- see 6714 * update_irq_load_avg(). 6715 */ 6716 irq = cpu_util_irq(rq); 6717 if (unlikely(irq >= max)) 6718 return max; 6719 6720 /* 6721 * Because the time spend on RT/DL tasks is visible as 'lost' time to 6722 * CFS tasks and we use the same metric to track the effective 6723 * utilization (PELT windows are synchronized) we can directly add them 6724 * to obtain the CPU's actual utilization. 6725 * 6726 * CFS and RT utilization can be boosted or capped, depending on 6727 * utilization clamp constraints requested by currently RUNNABLE 6728 * tasks. 6729 * When there are no CFS RUNNABLE tasks, clamps are released and 6730 * frequency will be gracefully reduced with the utilization decay. 6731 */ 6732 util = util_cfs + cpu_util_rt(rq); 6733 if (type == FREQUENCY_UTIL) 6734 util = uclamp_rq_util_with(rq, util, p); 6735 6736 dl_util = cpu_util_dl(rq); 6737 6738 /* 6739 * For frequency selection we do not make cpu_util_dl() a permanent part 6740 * of this sum because we want to use cpu_bw_dl() later on, but we need 6741 * to check if the CFS+RT+DL sum is saturated (ie. no idle time) such 6742 * that we select f_max when there is no idle time. 6743 * 6744 * NOTE: numerical errors or stop class might cause us to not quite hit 6745 * saturation when we should -- something for later. 6746 */ 6747 if (util + dl_util >= max) 6748 return max; 6749 6750 /* 6751 * OTOH, for energy computation we need the estimated running time, so 6752 * include util_dl and ignore dl_bw. 6753 */ 6754 if (type == ENERGY_UTIL) 6755 util += dl_util; 6756 6757 /* 6758 * There is still idle time; further improve the number by using the 6759 * irq metric. Because IRQ/steal time is hidden from the task clock we 6760 * need to scale the task numbers: 6761 * 6762 * max - irq 6763 * U' = irq + --------- * U 6764 * max 6765 */ 6766 util = scale_irq_capacity(util, irq, max); 6767 util += irq; 6768 6769 /* 6770 * Bandwidth required by DEADLINE must always be granted while, for 6771 * FAIR and RT, we use blocked utilization of IDLE CPUs as a mechanism 6772 * to gracefully reduce the frequency when no tasks show up for longer 6773 * periods of time. 6774 * 6775 * Ideally we would like to set bw_dl as min/guaranteed freq and util + 6776 * bw_dl as requested freq. However, cpufreq is not yet ready for such 6777 * an interface. So, we only do the latter for now. 6778 */ 6779 if (type == FREQUENCY_UTIL) 6780 util += cpu_bw_dl(rq); 6781 6782 return min(max, util); 6783 } 6784 6785 unsigned long sched_cpu_util(int cpu, unsigned long max) 6786 { 6787 return effective_cpu_util(cpu, cpu_util_cfs(cpu_rq(cpu)), max, 6788 ENERGY_UTIL, NULL); 6789 } 6790 #endif /* CONFIG_SMP */ 6791 6792 /** 6793 * find_process_by_pid - find a process with a matching PID value. 6794 * @pid: the pid in question. 6795 * 6796 * The task of @pid, if found. %NULL otherwise. 6797 */ 6798 static struct task_struct *find_process_by_pid(pid_t pid) 6799 { 6800 return pid ? find_task_by_vpid(pid) : current; 6801 } 6802 6803 /* 6804 * sched_setparam() passes in -1 for its policy, to let the functions 6805 * it calls know not to change it. 6806 */ 6807 #define SETPARAM_POLICY -1 6808 6809 static void __setscheduler_params(struct task_struct *p, 6810 const struct sched_attr *attr) 6811 { 6812 int policy = attr->sched_policy; 6813 6814 if (policy == SETPARAM_POLICY) 6815 policy = p->policy; 6816 6817 p->policy = policy; 6818 6819 if (dl_policy(policy)) 6820 __setparam_dl(p, attr); 6821 else if (fair_policy(policy)) 6822 p->static_prio = NICE_TO_PRIO(attr->sched_nice); 6823 6824 /* 6825 * __sched_setscheduler() ensures attr->sched_priority == 0 when 6826 * !rt_policy. Always setting this ensures that things like 6827 * getparam()/getattr() don't report silly values for !rt tasks. 6828 */ 6829 p->rt_priority = attr->sched_priority; 6830 p->normal_prio = normal_prio(p); 6831 set_load_weight(p, true); 6832 } 6833 6834 /* 6835 * Check the target process has a UID that matches the current process's: 6836 */ 6837 static bool check_same_owner(struct task_struct *p) 6838 { 6839 const struct cred *cred = current_cred(), *pcred; 6840 bool match; 6841 6842 rcu_read_lock(); 6843 pcred = __task_cred(p); 6844 match = (uid_eq(cred->euid, pcred->euid) || 6845 uid_eq(cred->euid, pcred->uid)); 6846 rcu_read_unlock(); 6847 return match; 6848 } 6849 6850 static int __sched_setscheduler(struct task_struct *p, 6851 const struct sched_attr *attr, 6852 bool user, bool pi) 6853 { 6854 int oldpolicy = -1, policy = attr->sched_policy; 6855 int retval, oldprio, newprio, queued, running; 6856 const struct sched_class *prev_class; 6857 struct callback_head *head; 6858 struct rq_flags rf; 6859 int reset_on_fork; 6860 int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 6861 struct rq *rq; 6862 6863 /* The pi code expects interrupts enabled */ 6864 BUG_ON(pi && in_interrupt()); 6865 recheck: 6866 /* Double check policy once rq lock held: */ 6867 if (policy < 0) { 6868 reset_on_fork = p->sched_reset_on_fork; 6869 policy = oldpolicy = p->policy; 6870 } else { 6871 reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); 6872 6873 if (!valid_policy(policy)) 6874 return -EINVAL; 6875 } 6876 6877 if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV)) 6878 return -EINVAL; 6879 6880 /* 6881 * Valid priorities for SCHED_FIFO and SCHED_RR are 6882 * 1..MAX_RT_PRIO-1, valid priority for SCHED_NORMAL, 6883 * SCHED_BATCH and SCHED_IDLE is 0. 6884 */ 6885 if (attr->sched_priority > MAX_RT_PRIO-1) 6886 return -EINVAL; 6887 if ((dl_policy(policy) && !__checkparam_dl(attr)) || 6888 (rt_policy(policy) != (attr->sched_priority != 0))) 6889 return -EINVAL; 6890 6891 /* 6892 * Allow unprivileged RT tasks to decrease priority: 6893 */ 6894 if (user && !capable(CAP_SYS_NICE)) { 6895 if (fair_policy(policy)) { 6896 if (attr->sched_nice < task_nice(p) && 6897 !can_nice(p, attr->sched_nice)) 6898 return -EPERM; 6899 } 6900 6901 if (rt_policy(policy)) { 6902 unsigned long rlim_rtprio = 6903 task_rlimit(p, RLIMIT_RTPRIO); 6904 6905 /* Can't set/change the rt policy: */ 6906 if (policy != p->policy && !rlim_rtprio) 6907 return -EPERM; 6908 6909 /* Can't increase priority: */ 6910 if (attr->sched_priority > p->rt_priority && 6911 attr->sched_priority > rlim_rtprio) 6912 return -EPERM; 6913 } 6914 6915 /* 6916 * Can't set/change SCHED_DEADLINE policy at all for now 6917 * (safest behavior); in the future we would like to allow 6918 * unprivileged DL tasks to increase their relative deadline 6919 * or reduce their runtime (both ways reducing utilization) 6920 */ 6921 if (dl_policy(policy)) 6922 return -EPERM; 6923 6924 /* 6925 * Treat SCHED_IDLE as nice 20. Only allow a switch to 6926 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. 6927 */ 6928 if (task_has_idle_policy(p) && !idle_policy(policy)) { 6929 if (!can_nice(p, task_nice(p))) 6930 return -EPERM; 6931 } 6932 6933 /* Can't change other user's priorities: */ 6934 if (!check_same_owner(p)) 6935 return -EPERM; 6936 6937 /* Normal users shall not reset the sched_reset_on_fork flag: */ 6938 if (p->sched_reset_on_fork && !reset_on_fork) 6939 return -EPERM; 6940 } 6941 6942 if (user) { 6943 if (attr->sched_flags & SCHED_FLAG_SUGOV) 6944 return -EINVAL; 6945 6946 retval = security_task_setscheduler(p); 6947 if (retval) 6948 return retval; 6949 } 6950 6951 /* Update task specific "requested" clamps */ 6952 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) { 6953 retval = uclamp_validate(p, attr); 6954 if (retval) 6955 return retval; 6956 } 6957 6958 if (pi) 6959 cpuset_read_lock(); 6960 6961 /* 6962 * Make sure no PI-waiters arrive (or leave) while we are 6963 * changing the priority of the task: 6964 * 6965 * To be able to change p->policy safely, the appropriate 6966 * runqueue lock must be held. 6967 */ 6968 rq = task_rq_lock(p, &rf); 6969 update_rq_clock(rq); 6970 6971 /* 6972 * Changing the policy of the stop threads its a very bad idea: 6973 */ 6974 if (p == rq->stop) { 6975 retval = -EINVAL; 6976 goto unlock; 6977 } 6978 6979 /* 6980 * If not changing anything there's no need to proceed further, 6981 * but store a possible modification of reset_on_fork. 6982 */ 6983 if (unlikely(policy == p->policy)) { 6984 if (fair_policy(policy) && attr->sched_nice != task_nice(p)) 6985 goto change; 6986 if (rt_policy(policy) && attr->sched_priority != p->rt_priority) 6987 goto change; 6988 if (dl_policy(policy) && dl_param_changed(p, attr)) 6989 goto change; 6990 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) 6991 goto change; 6992 6993 p->sched_reset_on_fork = reset_on_fork; 6994 retval = 0; 6995 goto unlock; 6996 } 6997 change: 6998 6999 if (user) { 7000 #ifdef CONFIG_RT_GROUP_SCHED 7001 /* 7002 * Do not allow realtime tasks into groups that have no runtime 7003 * assigned. 7004 */ 7005 if (rt_bandwidth_enabled() && rt_policy(policy) && 7006 task_group(p)->rt_bandwidth.rt_runtime == 0 && 7007 !task_group_is_autogroup(task_group(p))) { 7008 retval = -EPERM; 7009 goto unlock; 7010 } 7011 #endif 7012 #ifdef CONFIG_SMP 7013 if (dl_bandwidth_enabled() && dl_policy(policy) && 7014 !(attr->sched_flags & SCHED_FLAG_SUGOV)) { 7015 cpumask_t *span = rq->rd->span; 7016 7017 /* 7018 * Don't allow tasks with an affinity mask smaller than 7019 * the entire root_domain to become SCHED_DEADLINE. We 7020 * will also fail if there's no bandwidth available. 7021 */ 7022 if (!cpumask_subset(span, p->cpus_ptr) || 7023 rq->rd->dl_bw.bw == 0) { 7024 retval = -EPERM; 7025 goto unlock; 7026 } 7027 } 7028 #endif 7029 } 7030 7031 /* Re-check policy now with rq lock held: */ 7032 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { 7033 policy = oldpolicy = -1; 7034 task_rq_unlock(rq, p, &rf); 7035 if (pi) 7036 cpuset_read_unlock(); 7037 goto recheck; 7038 } 7039 7040 /* 7041 * If setscheduling to SCHED_DEADLINE (or changing the parameters 7042 * of a SCHED_DEADLINE task) we need to check if enough bandwidth 7043 * is available. 7044 */ 7045 if ((dl_policy(policy) || dl_task(p)) && sched_dl_overflow(p, policy, attr)) { 7046 retval = -EBUSY; 7047 goto unlock; 7048 } 7049 7050 p->sched_reset_on_fork = reset_on_fork; 7051 oldprio = p->prio; 7052 7053 newprio = __normal_prio(policy, attr->sched_priority, attr->sched_nice); 7054 if (pi) { 7055 /* 7056 * Take priority boosted tasks into account. If the new 7057 * effective priority is unchanged, we just store the new 7058 * normal parameters and do not touch the scheduler class and 7059 * the runqueue. This will be done when the task deboost 7060 * itself. 7061 */ 7062 newprio = rt_effective_prio(p, newprio); 7063 if (newprio == oldprio) 7064 queue_flags &= ~DEQUEUE_MOVE; 7065 } 7066 7067 queued = task_on_rq_queued(p); 7068 running = task_current(rq, p); 7069 if (queued) 7070 dequeue_task(rq, p, queue_flags); 7071 if (running) 7072 put_prev_task(rq, p); 7073 7074 prev_class = p->sched_class; 7075 7076 if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { 7077 __setscheduler_params(p, attr); 7078 __setscheduler_prio(p, newprio); 7079 } 7080 __setscheduler_uclamp(p, attr); 7081 7082 if (queued) { 7083 /* 7084 * We enqueue to tail when the priority of a task is 7085 * increased (user space view). 7086 */ 7087 if (oldprio < p->prio) 7088 queue_flags |= ENQUEUE_HEAD; 7089 7090 enqueue_task(rq, p, queue_flags); 7091 } 7092 if (running) 7093 set_next_task(rq, p); 7094 7095 check_class_changed(rq, p, prev_class, oldprio); 7096 7097 /* Avoid rq from going away on us: */ 7098 preempt_disable(); 7099 head = splice_balance_callbacks(rq); 7100 task_rq_unlock(rq, p, &rf); 7101 7102 if (pi) { 7103 cpuset_read_unlock(); 7104 rt_mutex_adjust_pi(p); 7105 } 7106 7107 /* Run balance callbacks after we've adjusted the PI chain: */ 7108 balance_callbacks(rq, head); 7109 preempt_enable(); 7110 7111 return 0; 7112 7113 unlock: 7114 task_rq_unlock(rq, p, &rf); 7115 if (pi) 7116 cpuset_read_unlock(); 7117 return retval; 7118 } 7119 7120 static int _sched_setscheduler(struct task_struct *p, int policy, 7121 const struct sched_param *param, bool check) 7122 { 7123 struct sched_attr attr = { 7124 .sched_policy = policy, 7125 .sched_priority = param->sched_priority, 7126 .sched_nice = PRIO_TO_NICE(p->static_prio), 7127 }; 7128 7129 /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ 7130 if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { 7131 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 7132 policy &= ~SCHED_RESET_ON_FORK; 7133 attr.sched_policy = policy; 7134 } 7135 7136 return __sched_setscheduler(p, &attr, check, true); 7137 } 7138 /** 7139 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. 7140 * @p: the task in question. 7141 * @policy: new policy. 7142 * @param: structure containing the new RT priority. 7143 * 7144 * Use sched_set_fifo(), read its comment. 7145 * 7146 * Return: 0 on success. An error code otherwise. 7147 * 7148 * NOTE that the task may be already dead. 7149 */ 7150 int sched_setscheduler(struct task_struct *p, int policy, 7151 const struct sched_param *param) 7152 { 7153 return _sched_setscheduler(p, policy, param, true); 7154 } 7155 7156 int sched_setattr(struct task_struct *p, const struct sched_attr *attr) 7157 { 7158 return __sched_setscheduler(p, attr, true, true); 7159 } 7160 7161 int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr) 7162 { 7163 return __sched_setscheduler(p, attr, false, true); 7164 } 7165 EXPORT_SYMBOL_GPL(sched_setattr_nocheck); 7166 7167 /** 7168 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. 7169 * @p: the task in question. 7170 * @policy: new policy. 7171 * @param: structure containing the new RT priority. 7172 * 7173 * Just like sched_setscheduler, only don't bother checking if the 7174 * current context has permission. For example, this is needed in 7175 * stop_machine(): we create temporary high priority worker threads, 7176 * but our caller might not have that capability. 7177 * 7178 * Return: 0 on success. An error code otherwise. 7179 */ 7180 int sched_setscheduler_nocheck(struct task_struct *p, int policy, 7181 const struct sched_param *param) 7182 { 7183 return _sched_setscheduler(p, policy, param, false); 7184 } 7185 7186 /* 7187 * SCHED_FIFO is a broken scheduler model; that is, it is fundamentally 7188 * incapable of resource management, which is the one thing an OS really should 7189 * be doing. 7190 * 7191 * This is of course the reason it is limited to privileged users only. 7192 * 7193 * Worse still; it is fundamentally impossible to compose static priority 7194 * workloads. You cannot take two correctly working static prio workloads 7195 * and smash them together and still expect them to work. 7196 * 7197 * For this reason 'all' FIFO tasks the kernel creates are basically at: 7198 * 7199 * MAX_RT_PRIO / 2 7200 * 7201 * The administrator _MUST_ configure the system, the kernel simply doesn't 7202 * know enough information to make a sensible choice. 7203 */ 7204 void sched_set_fifo(struct task_struct *p) 7205 { 7206 struct sched_param sp = { .sched_priority = MAX_RT_PRIO / 2 }; 7207 WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); 7208 } 7209 EXPORT_SYMBOL_GPL(sched_set_fifo); 7210 7211 /* 7212 * For when you don't much care about FIFO, but want to be above SCHED_NORMAL. 7213 */ 7214 void sched_set_fifo_low(struct task_struct *p) 7215 { 7216 struct sched_param sp = { .sched_priority = 1 }; 7217 WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); 7218 } 7219 EXPORT_SYMBOL_GPL(sched_set_fifo_low); 7220 7221 void sched_set_normal(struct task_struct *p, int nice) 7222 { 7223 struct sched_attr attr = { 7224 .sched_policy = SCHED_NORMAL, 7225 .sched_nice = nice, 7226 }; 7227 WARN_ON_ONCE(sched_setattr_nocheck(p, &attr) != 0); 7228 } 7229 EXPORT_SYMBOL_GPL(sched_set_normal); 7230 7231 static int 7232 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) 7233 { 7234 struct sched_param lparam; 7235 struct task_struct *p; 7236 int retval; 7237 7238 if (!param || pid < 0) 7239 return -EINVAL; 7240 if (copy_from_user(&lparam, param, sizeof(struct sched_param))) 7241 return -EFAULT; 7242 7243 rcu_read_lock(); 7244 retval = -ESRCH; 7245 p = find_process_by_pid(pid); 7246 if (likely(p)) 7247 get_task_struct(p); 7248 rcu_read_unlock(); 7249 7250 if (likely(p)) { 7251 retval = sched_setscheduler(p, policy, &lparam); 7252 put_task_struct(p); 7253 } 7254 7255 return retval; 7256 } 7257 7258 /* 7259 * Mimics kernel/events/core.c perf_copy_attr(). 7260 */ 7261 static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *attr) 7262 { 7263 u32 size; 7264 int ret; 7265 7266 /* Zero the full structure, so that a short copy will be nice: */ 7267 memset(attr, 0, sizeof(*attr)); 7268 7269 ret = get_user(size, &uattr->size); 7270 if (ret) 7271 return ret; 7272 7273 /* ABI compatibility quirk: */ 7274 if (!size) 7275 size = SCHED_ATTR_SIZE_VER0; 7276 if (size < SCHED_ATTR_SIZE_VER0 || size > PAGE_SIZE) 7277 goto err_size; 7278 7279 ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size); 7280 if (ret) { 7281 if (ret == -E2BIG) 7282 goto err_size; 7283 return ret; 7284 } 7285 7286 if ((attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) && 7287 size < SCHED_ATTR_SIZE_VER1) 7288 return -EINVAL; 7289 7290 /* 7291 * XXX: Do we want to be lenient like existing syscalls; or do we want 7292 * to be strict and return an error on out-of-bounds values? 7293 */ 7294 attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); 7295 7296 return 0; 7297 7298 err_size: 7299 put_user(sizeof(*attr), &uattr->size); 7300 return -E2BIG; 7301 } 7302 7303 /** 7304 * sys_sched_setscheduler - set/change the scheduler policy and RT priority 7305 * @pid: the pid in question. 7306 * @policy: new policy. 7307 * @param: structure containing the new RT priority. 7308 * 7309 * Return: 0 on success. An error code otherwise. 7310 */ 7311 SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param) 7312 { 7313 if (policy < 0) 7314 return -EINVAL; 7315 7316 return do_sched_setscheduler(pid, policy, param); 7317 } 7318 7319 /** 7320 * sys_sched_setparam - set/change the RT priority of a thread 7321 * @pid: the pid in question. 7322 * @param: structure containing the new RT priority. 7323 * 7324 * Return: 0 on success. An error code otherwise. 7325 */ 7326 SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) 7327 { 7328 return do_sched_setscheduler(pid, SETPARAM_POLICY, param); 7329 } 7330 7331 /** 7332 * sys_sched_setattr - same as above, but with extended sched_attr 7333 * @pid: the pid in question. 7334 * @uattr: structure containing the extended parameters. 7335 * @flags: for future extension. 7336 */ 7337 SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, 7338 unsigned int, flags) 7339 { 7340 struct sched_attr attr; 7341 struct task_struct *p; 7342 int retval; 7343 7344 if (!uattr || pid < 0 || flags) 7345 return -EINVAL; 7346 7347 retval = sched_copy_attr(uattr, &attr); 7348 if (retval) 7349 return retval; 7350 7351 if ((int)attr.sched_policy < 0) 7352 return -EINVAL; 7353 if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY) 7354 attr.sched_policy = SETPARAM_POLICY; 7355 7356 rcu_read_lock(); 7357 retval = -ESRCH; 7358 p = find_process_by_pid(pid); 7359 if (likely(p)) 7360 get_task_struct(p); 7361 rcu_read_unlock(); 7362 7363 if (likely(p)) { 7364 retval = sched_setattr(p, &attr); 7365 put_task_struct(p); 7366 } 7367 7368 return retval; 7369 } 7370 7371 /** 7372 * sys_sched_getscheduler - get the policy (scheduling class) of a thread 7373 * @pid: the pid in question. 7374 * 7375 * Return: On success, the policy of the thread. Otherwise, a negative error 7376 * code. 7377 */ 7378 SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) 7379 { 7380 struct task_struct *p; 7381 int retval; 7382 7383 if (pid < 0) 7384 return -EINVAL; 7385 7386 retval = -ESRCH; 7387 rcu_read_lock(); 7388 p = find_process_by_pid(pid); 7389 if (p) { 7390 retval = security_task_getscheduler(p); 7391 if (!retval) 7392 retval = p->policy 7393 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); 7394 } 7395 rcu_read_unlock(); 7396 return retval; 7397 } 7398 7399 /** 7400 * sys_sched_getparam - get the RT priority of a thread 7401 * @pid: the pid in question. 7402 * @param: structure containing the RT priority. 7403 * 7404 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error 7405 * code. 7406 */ 7407 SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) 7408 { 7409 struct sched_param lp = { .sched_priority = 0 }; 7410 struct task_struct *p; 7411 int retval; 7412 7413 if (!param || pid < 0) 7414 return -EINVAL; 7415 7416 rcu_read_lock(); 7417 p = find_process_by_pid(pid); 7418 retval = -ESRCH; 7419 if (!p) 7420 goto out_unlock; 7421 7422 retval = security_task_getscheduler(p); 7423 if (retval) 7424 goto out_unlock; 7425 7426 if (task_has_rt_policy(p)) 7427 lp.sched_priority = p->rt_priority; 7428 rcu_read_unlock(); 7429 7430 /* 7431 * This one might sleep, we cannot do it with a spinlock held ... 7432 */ 7433 retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; 7434 7435 return retval; 7436 7437 out_unlock: 7438 rcu_read_unlock(); 7439 return retval; 7440 } 7441 7442 /* 7443 * Copy the kernel size attribute structure (which might be larger 7444 * than what user-space knows about) to user-space. 7445 * 7446 * Note that all cases are valid: user-space buffer can be larger or 7447 * smaller than the kernel-space buffer. The usual case is that both 7448 * have the same size. 7449 */ 7450 static int 7451 sched_attr_copy_to_user(struct sched_attr __user *uattr, 7452 struct sched_attr *kattr, 7453 unsigned int usize) 7454 { 7455 unsigned int ksize = sizeof(*kattr); 7456 7457 if (!access_ok(uattr, usize)) 7458 return -EFAULT; 7459 7460 /* 7461 * sched_getattr() ABI forwards and backwards compatibility: 7462 * 7463 * If usize == ksize then we just copy everything to user-space and all is good. 7464 * 7465 * If usize < ksize then we only copy as much as user-space has space for, 7466 * this keeps ABI compatibility as well. We skip the rest. 7467 * 7468 * If usize > ksize then user-space is using a newer version of the ABI, 7469 * which part the kernel doesn't know about. Just ignore it - tooling can 7470 * detect the kernel's knowledge of attributes from the attr->size value 7471 * which is set to ksize in this case. 7472 */ 7473 kattr->size = min(usize, ksize); 7474 7475 if (copy_to_user(uattr, kattr, kattr->size)) 7476 return -EFAULT; 7477 7478 return 0; 7479 } 7480 7481 /** 7482 * sys_sched_getattr - similar to sched_getparam, but with sched_attr 7483 * @pid: the pid in question. 7484 * @uattr: structure containing the extended parameters. 7485 * @usize: sizeof(attr) for fwd/bwd comp. 7486 * @flags: for future extension. 7487 */ 7488 SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, 7489 unsigned int, usize, unsigned int, flags) 7490 { 7491 struct sched_attr kattr = { }; 7492 struct task_struct *p; 7493 int retval; 7494 7495 if (!uattr || pid < 0 || usize > PAGE_SIZE || 7496 usize < SCHED_ATTR_SIZE_VER0 || flags) 7497 return -EINVAL; 7498 7499 rcu_read_lock(); 7500 p = find_process_by_pid(pid); 7501 retval = -ESRCH; 7502 if (!p) 7503 goto out_unlock; 7504 7505 retval = security_task_getscheduler(p); 7506 if (retval) 7507 goto out_unlock; 7508 7509 kattr.sched_policy = p->policy; 7510 if (p->sched_reset_on_fork) 7511 kattr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 7512 if (task_has_dl_policy(p)) 7513 __getparam_dl(p, &kattr); 7514 else if (task_has_rt_policy(p)) 7515 kattr.sched_priority = p->rt_priority; 7516 else 7517 kattr.sched_nice = task_nice(p); 7518 7519 #ifdef CONFIG_UCLAMP_TASK 7520 /* 7521 * This could race with another potential updater, but this is fine 7522 * because it'll correctly read the old or the new value. We don't need 7523 * to guarantee who wins the race as long as it doesn't return garbage. 7524 */ 7525 kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN].value; 7526 kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX].value; 7527 #endif 7528 7529 rcu_read_unlock(); 7530 7531 return sched_attr_copy_to_user(uattr, &kattr, usize); 7532 7533 out_unlock: 7534 rcu_read_unlock(); 7535 return retval; 7536 } 7537 7538 long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) 7539 { 7540 cpumask_var_t cpus_allowed, new_mask; 7541 struct task_struct *p; 7542 int retval; 7543 7544 rcu_read_lock(); 7545 7546 p = find_process_by_pid(pid); 7547 if (!p) { 7548 rcu_read_unlock(); 7549 return -ESRCH; 7550 } 7551 7552 /* Prevent p going away */ 7553 get_task_struct(p); 7554 rcu_read_unlock(); 7555 7556 if (p->flags & PF_NO_SETAFFINITY) { 7557 retval = -EINVAL; 7558 goto out_put_task; 7559 } 7560 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { 7561 retval = -ENOMEM; 7562 goto out_put_task; 7563 } 7564 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { 7565 retval = -ENOMEM; 7566 goto out_free_cpus_allowed; 7567 } 7568 retval = -EPERM; 7569 if (!check_same_owner(p)) { 7570 rcu_read_lock(); 7571 if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { 7572 rcu_read_unlock(); 7573 goto out_free_new_mask; 7574 } 7575 rcu_read_unlock(); 7576 } 7577 7578 retval = security_task_setscheduler(p); 7579 if (retval) 7580 goto out_free_new_mask; 7581 7582 7583 cpuset_cpus_allowed(p, cpus_allowed); 7584 cpumask_and(new_mask, in_mask, cpus_allowed); 7585 7586 /* 7587 * Since bandwidth control happens on root_domain basis, 7588 * if admission test is enabled, we only admit -deadline 7589 * tasks allowed to run on all the CPUs in the task's 7590 * root_domain. 7591 */ 7592 #ifdef CONFIG_SMP 7593 if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { 7594 rcu_read_lock(); 7595 if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) { 7596 retval = -EBUSY; 7597 rcu_read_unlock(); 7598 goto out_free_new_mask; 7599 } 7600 rcu_read_unlock(); 7601 } 7602 #endif 7603 again: 7604 retval = __set_cpus_allowed_ptr(p, new_mask, SCA_CHECK); 7605 7606 if (!retval) { 7607 cpuset_cpus_allowed(p, cpus_allowed); 7608 if (!cpumask_subset(new_mask, cpus_allowed)) { 7609 /* 7610 * We must have raced with a concurrent cpuset 7611 * update. Just reset the cpus_allowed to the 7612 * cpuset's cpus_allowed 7613 */ 7614 cpumask_copy(new_mask, cpus_allowed); 7615 goto again; 7616 } 7617 } 7618 out_free_new_mask: 7619 free_cpumask_var(new_mask); 7620 out_free_cpus_allowed: 7621 free_cpumask_var(cpus_allowed); 7622 out_put_task: 7623 put_task_struct(p); 7624 return retval; 7625 } 7626 7627 static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 7628 struct cpumask *new_mask) 7629 { 7630 if (len < cpumask_size()) 7631 cpumask_clear(new_mask); 7632 else if (len > cpumask_size()) 7633 len = cpumask_size(); 7634 7635 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 7636 } 7637 7638 /** 7639 * sys_sched_setaffinity - set the CPU affinity of a process 7640 * @pid: pid of the process 7641 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 7642 * @user_mask_ptr: user-space pointer to the new CPU mask 7643 * 7644 * Return: 0 on success. An error code otherwise. 7645 */ 7646 SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, 7647 unsigned long __user *, user_mask_ptr) 7648 { 7649 cpumask_var_t new_mask; 7650 int retval; 7651 7652 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) 7653 return -ENOMEM; 7654 7655 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); 7656 if (retval == 0) 7657 retval = sched_setaffinity(pid, new_mask); 7658 free_cpumask_var(new_mask); 7659 return retval; 7660 } 7661 7662 long sched_getaffinity(pid_t pid, struct cpumask *mask) 7663 { 7664 struct task_struct *p; 7665 unsigned long flags; 7666 int retval; 7667 7668 rcu_read_lock(); 7669 7670 retval = -ESRCH; 7671 p = find_process_by_pid(pid); 7672 if (!p) 7673 goto out_unlock; 7674 7675 retval = security_task_getscheduler(p); 7676 if (retval) 7677 goto out_unlock; 7678 7679 raw_spin_lock_irqsave(&p->pi_lock, flags); 7680 cpumask_and(mask, &p->cpus_mask, cpu_active_mask); 7681 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 7682 7683 out_unlock: 7684 rcu_read_unlock(); 7685 7686 return retval; 7687 } 7688 7689 /** 7690 * sys_sched_getaffinity - get the CPU affinity of a process 7691 * @pid: pid of the process 7692 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 7693 * @user_mask_ptr: user-space pointer to hold the current CPU mask 7694 * 7695 * Return: size of CPU mask copied to user_mask_ptr on success. An 7696 * error code otherwise. 7697 */ 7698 SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, 7699 unsigned long __user *, user_mask_ptr) 7700 { 7701 int ret; 7702 cpumask_var_t mask; 7703 7704 if ((len * BITS_PER_BYTE) < nr_cpu_ids) 7705 return -EINVAL; 7706 if (len & (sizeof(unsigned long)-1)) 7707 return -EINVAL; 7708 7709 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 7710 return -ENOMEM; 7711 7712 ret = sched_getaffinity(pid, mask); 7713 if (ret == 0) { 7714 unsigned int retlen = min(len, cpumask_size()); 7715 7716 if (copy_to_user(user_mask_ptr, mask, retlen)) 7717 ret = -EFAULT; 7718 else 7719 ret = retlen; 7720 } 7721 free_cpumask_var(mask); 7722 7723 return ret; 7724 } 7725 7726 static void do_sched_yield(void) 7727 { 7728 struct rq_flags rf; 7729 struct rq *rq; 7730 7731 rq = this_rq_lock_irq(&rf); 7732 7733 schedstat_inc(rq->yld_count); 7734 current->sched_class->yield_task(rq); 7735 7736 preempt_disable(); 7737 rq_unlock_irq(rq, &rf); 7738 sched_preempt_enable_no_resched(); 7739 7740 schedule(); 7741 } 7742 7743 /** 7744 * sys_sched_yield - yield the current processor to other threads. 7745 * 7746 * This function yields the current CPU to other tasks. If there are no 7747 * other threads running on this CPU then this function will return. 7748 * 7749 * Return: 0. 7750 */ 7751 SYSCALL_DEFINE0(sched_yield) 7752 { 7753 do_sched_yield(); 7754 return 0; 7755 } 7756 7757 #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) 7758 int __sched __cond_resched(void) 7759 { 7760 if (should_resched(0)) { 7761 preempt_schedule_common(); 7762 return 1; 7763 } 7764 #ifndef CONFIG_PREEMPT_RCU 7765 rcu_all_qs(); 7766 #endif 7767 return 0; 7768 } 7769 EXPORT_SYMBOL(__cond_resched); 7770 #endif 7771 7772 #ifdef CONFIG_PREEMPT_DYNAMIC 7773 DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); 7774 EXPORT_STATIC_CALL_TRAMP(cond_resched); 7775 7776 DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); 7777 EXPORT_STATIC_CALL_TRAMP(might_resched); 7778 #endif 7779 7780 /* 7781 * __cond_resched_lock() - if a reschedule is pending, drop the given lock, 7782 * call schedule, and on return reacquire the lock. 7783 * 7784 * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level 7785 * operations here to prevent schedule() from being called twice (once via 7786 * spin_unlock(), once by hand). 7787 */ 7788 int __cond_resched_lock(spinlock_t *lock) 7789 { 7790 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7791 int ret = 0; 7792 7793 lockdep_assert_held(lock); 7794 7795 if (spin_needbreak(lock) || resched) { 7796 spin_unlock(lock); 7797 if (resched) 7798 preempt_schedule_common(); 7799 else 7800 cpu_relax(); 7801 ret = 1; 7802 spin_lock(lock); 7803 } 7804 return ret; 7805 } 7806 EXPORT_SYMBOL(__cond_resched_lock); 7807 7808 int __cond_resched_rwlock_read(rwlock_t *lock) 7809 { 7810 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7811 int ret = 0; 7812 7813 lockdep_assert_held_read(lock); 7814 7815 if (rwlock_needbreak(lock) || resched) { 7816 read_unlock(lock); 7817 if (resched) 7818 preempt_schedule_common(); 7819 else 7820 cpu_relax(); 7821 ret = 1; 7822 read_lock(lock); 7823 } 7824 return ret; 7825 } 7826 EXPORT_SYMBOL(__cond_resched_rwlock_read); 7827 7828 int __cond_resched_rwlock_write(rwlock_t *lock) 7829 { 7830 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7831 int ret = 0; 7832 7833 lockdep_assert_held_write(lock); 7834 7835 if (rwlock_needbreak(lock) || resched) { 7836 write_unlock(lock); 7837 if (resched) 7838 preempt_schedule_common(); 7839 else 7840 cpu_relax(); 7841 ret = 1; 7842 write_lock(lock); 7843 } 7844 return ret; 7845 } 7846 EXPORT_SYMBOL(__cond_resched_rwlock_write); 7847 7848 /** 7849 * yield - yield the current processor to other threads. 7850 * 7851 * Do not ever use this function, there's a 99% chance you're doing it wrong. 7852 * 7853 * The scheduler is at all times free to pick the calling task as the most 7854 * eligible task to run, if removing the yield() call from your code breaks 7855 * it, it's already broken. 7856 * 7857 * Typical broken usage is: 7858 * 7859 * while (!event) 7860 * yield(); 7861 * 7862 * where one assumes that yield() will let 'the other' process run that will 7863 * make event true. If the current task is a SCHED_FIFO task that will never 7864 * happen. Never use yield() as a progress guarantee!! 7865 * 7866 * If you want to use yield() to wait for something, use wait_event(). 7867 * If you want to use yield() to be 'nice' for others, use cond_resched(). 7868 * If you still want to use yield(), do not! 7869 */ 7870 void __sched yield(void) 7871 { 7872 set_current_state(TASK_RUNNING); 7873 do_sched_yield(); 7874 } 7875 EXPORT_SYMBOL(yield); 7876 7877 /** 7878 * yield_to - yield the current processor to another thread in 7879 * your thread group, or accelerate that thread toward the 7880 * processor it's on. 7881 * @p: target task 7882 * @preempt: whether task preemption is allowed or not 7883 * 7884 * It's the caller's job to ensure that the target task struct 7885 * can't go away on us before we can do any checks. 7886 * 7887 * Return: 7888 * true (>0) if we indeed boosted the target task. 7889 * false (0) if we failed to boost the target. 7890 * -ESRCH if there's no task to yield to. 7891 */ 7892 int __sched yield_to(struct task_struct *p, bool preempt) 7893 { 7894 struct task_struct *curr = current; 7895 struct rq *rq, *p_rq; 7896 unsigned long flags; 7897 int yielded = 0; 7898 7899 local_irq_save(flags); 7900 rq = this_rq(); 7901 7902 again: 7903 p_rq = task_rq(p); 7904 /* 7905 * If we're the only runnable task on the rq and target rq also 7906 * has only one task, there's absolutely no point in yielding. 7907 */ 7908 if (rq->nr_running == 1 && p_rq->nr_running == 1) { 7909 yielded = -ESRCH; 7910 goto out_irq; 7911 } 7912 7913 double_rq_lock(rq, p_rq); 7914 if (task_rq(p) != p_rq) { 7915 double_rq_unlock(rq, p_rq); 7916 goto again; 7917 } 7918 7919 if (!curr->sched_class->yield_to_task) 7920 goto out_unlock; 7921 7922 if (curr->sched_class != p->sched_class) 7923 goto out_unlock; 7924 7925 if (task_running(p_rq, p) || !task_is_running(p)) 7926 goto out_unlock; 7927 7928 yielded = curr->sched_class->yield_to_task(rq, p); 7929 if (yielded) { 7930 schedstat_inc(rq->yld_count); 7931 /* 7932 * Make p's CPU reschedule; pick_next_entity takes care of 7933 * fairness. 7934 */ 7935 if (preempt && rq != p_rq) 7936 resched_curr(p_rq); 7937 } 7938 7939 out_unlock: 7940 double_rq_unlock(rq, p_rq); 7941 out_irq: 7942 local_irq_restore(flags); 7943 7944 if (yielded > 0) 7945 schedule(); 7946 7947 return yielded; 7948 } 7949 EXPORT_SYMBOL_GPL(yield_to); 7950 7951 int io_schedule_prepare(void) 7952 { 7953 int old_iowait = current->in_iowait; 7954 7955 current->in_iowait = 1; 7956 blk_schedule_flush_plug(current); 7957 7958 return old_iowait; 7959 } 7960 7961 void io_schedule_finish(int token) 7962 { 7963 current->in_iowait = token; 7964 } 7965 7966 /* 7967 * This task is about to go to sleep on IO. Increment rq->nr_iowait so 7968 * that process accounting knows that this is a task in IO wait state. 7969 */ 7970 long __sched io_schedule_timeout(long timeout) 7971 { 7972 int token; 7973 long ret; 7974 7975 token = io_schedule_prepare(); 7976 ret = schedule_timeout(timeout); 7977 io_schedule_finish(token); 7978 7979 return ret; 7980 } 7981 EXPORT_SYMBOL(io_schedule_timeout); 7982 7983 void __sched io_schedule(void) 7984 { 7985 int token; 7986 7987 token = io_schedule_prepare(); 7988 schedule(); 7989 io_schedule_finish(token); 7990 } 7991 EXPORT_SYMBOL(io_schedule); 7992 7993 /** 7994 * sys_sched_get_priority_max - return maximum RT priority. 7995 * @policy: scheduling class. 7996 * 7997 * Return: On success, this syscall returns the maximum 7998 * rt_priority that can be used by a given scheduling class. 7999 * On failure, a negative error code is returned. 8000 */ 8001 SYSCALL_DEFINE1(sched_get_priority_max, int, policy) 8002 { 8003 int ret = -EINVAL; 8004 8005 switch (policy) { 8006 case SCHED_FIFO: 8007 case SCHED_RR: 8008 ret = MAX_RT_PRIO-1; 8009 break; 8010 case SCHED_DEADLINE: 8011 case SCHED_NORMAL: 8012 case SCHED_BATCH: 8013 case SCHED_IDLE: 8014 ret = 0; 8015 break; 8016 } 8017 return ret; 8018 } 8019 8020 /** 8021 * sys_sched_get_priority_min - return minimum RT priority. 8022 * @policy: scheduling class. 8023 * 8024 * Return: On success, this syscall returns the minimum 8025 * rt_priority that can be used by a given scheduling class. 8026 * On failure, a negative error code is returned. 8027 */ 8028 SYSCALL_DEFINE1(sched_get_priority_min, int, policy) 8029 { 8030 int ret = -EINVAL; 8031 8032 switch (policy) { 8033 case SCHED_FIFO: 8034 case SCHED_RR: 8035 ret = 1; 8036 break; 8037 case SCHED_DEADLINE: 8038 case SCHED_NORMAL: 8039 case SCHED_BATCH: 8040 case SCHED_IDLE: 8041 ret = 0; 8042 } 8043 return ret; 8044 } 8045 8046 static int sched_rr_get_interval(pid_t pid, struct timespec64 *t) 8047 { 8048 struct task_struct *p; 8049 unsigned int time_slice; 8050 struct rq_flags rf; 8051 struct rq *rq; 8052 int retval; 8053 8054 if (pid < 0) 8055 return -EINVAL; 8056 8057 retval = -ESRCH; 8058 rcu_read_lock(); 8059 p = find_process_by_pid(pid); 8060 if (!p) 8061 goto out_unlock; 8062 8063 retval = security_task_getscheduler(p); 8064 if (retval) 8065 goto out_unlock; 8066 8067 rq = task_rq_lock(p, &rf); 8068 time_slice = 0; 8069 if (p->sched_class->get_rr_interval) 8070 time_slice = p->sched_class->get_rr_interval(rq, p); 8071 task_rq_unlock(rq, p, &rf); 8072 8073 rcu_read_unlock(); 8074 jiffies_to_timespec64(time_slice, t); 8075 return 0; 8076 8077 out_unlock: 8078 rcu_read_unlock(); 8079 return retval; 8080 } 8081 8082 /** 8083 * sys_sched_rr_get_interval - return the default timeslice of a process. 8084 * @pid: pid of the process. 8085 * @interval: userspace pointer to the timeslice value. 8086 * 8087 * this syscall writes the default timeslice value of a given process 8088 * into the user-space timespec buffer. A value of '0' means infinity. 8089 * 8090 * Return: On success, 0 and the timeslice is in @interval. Otherwise, 8091 * an error code. 8092 */ 8093 SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, 8094 struct __kernel_timespec __user *, interval) 8095 { 8096 struct timespec64 t; 8097 int retval = sched_rr_get_interval(pid, &t); 8098 8099 if (retval == 0) 8100 retval = put_timespec64(&t, interval); 8101 8102 return retval; 8103 } 8104 8105 #ifdef CONFIG_COMPAT_32BIT_TIME 8106 SYSCALL_DEFINE2(sched_rr_get_interval_time32, pid_t, pid, 8107 struct old_timespec32 __user *, interval) 8108 { 8109 struct timespec64 t; 8110 int retval = sched_rr_get_interval(pid, &t); 8111 8112 if (retval == 0) 8113 retval = put_old_timespec32(&t, interval); 8114 return retval; 8115 } 8116 #endif 8117 8118 void sched_show_task(struct task_struct *p) 8119 { 8120 unsigned long free = 0; 8121 int ppid; 8122 8123 if (!try_get_task_stack(p)) 8124 return; 8125 8126 pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p)); 8127 8128 if (task_is_running(p)) 8129 pr_cont(" running task "); 8130 #ifdef CONFIG_DEBUG_STACK_USAGE 8131 free = stack_not_used(p); 8132 #endif 8133 ppid = 0; 8134 rcu_read_lock(); 8135 if (pid_alive(p)) 8136 ppid = task_pid_nr(rcu_dereference(p->real_parent)); 8137 rcu_read_unlock(); 8138 pr_cont(" stack:%5lu pid:%5d ppid:%6d flags:0x%08lx\n", 8139 free, task_pid_nr(p), ppid, 8140 (unsigned long)task_thread_info(p)->flags); 8141 8142 print_worker_info(KERN_INFO, p); 8143 print_stop_info(KERN_INFO, p); 8144 show_stack(p, NULL, KERN_INFO); 8145 put_task_stack(p); 8146 } 8147 EXPORT_SYMBOL_GPL(sched_show_task); 8148 8149 static inline bool 8150 state_filter_match(unsigned long state_filter, struct task_struct *p) 8151 { 8152 unsigned int state = READ_ONCE(p->__state); 8153 8154 /* no filter, everything matches */ 8155 if (!state_filter) 8156 return true; 8157 8158 /* filter, but doesn't match */ 8159 if (!(state & state_filter)) 8160 return false; 8161 8162 /* 8163 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows 8164 * TASK_KILLABLE). 8165 */ 8166 if (state_filter == TASK_UNINTERRUPTIBLE && state == TASK_IDLE) 8167 return false; 8168 8169 return true; 8170 } 8171 8172 8173 void show_state_filter(unsigned int state_filter) 8174 { 8175 struct task_struct *g, *p; 8176 8177 rcu_read_lock(); 8178 for_each_process_thread(g, p) { 8179 /* 8180 * reset the NMI-timeout, listing all files on a slow 8181 * console might take a lot of time: 8182 * Also, reset softlockup watchdogs on all CPUs, because 8183 * another CPU might be blocked waiting for us to process 8184 * an IPI. 8185 */ 8186 touch_nmi_watchdog(); 8187 touch_all_softlockup_watchdogs(); 8188 if (state_filter_match(state_filter, p)) 8189 sched_show_task(p); 8190 } 8191 8192 #ifdef CONFIG_SCHED_DEBUG 8193 if (!state_filter) 8194 sysrq_sched_debug_show(); 8195 #endif 8196 rcu_read_unlock(); 8197 /* 8198 * Only show locks if all tasks are dumped: 8199 */ 8200 if (!state_filter) 8201 debug_show_all_locks(); 8202 } 8203 8204 /** 8205 * init_idle - set up an idle thread for a given CPU 8206 * @idle: task in question 8207 * @cpu: CPU the idle task belongs to 8208 * 8209 * NOTE: this function does not set the idle thread's NEED_RESCHED 8210 * flag, to make booting more robust. 8211 */ 8212 void __init init_idle(struct task_struct *idle, int cpu) 8213 { 8214 struct rq *rq = cpu_rq(cpu); 8215 unsigned long flags; 8216 8217 __sched_fork(0, idle); 8218 8219 /* 8220 * The idle task doesn't need the kthread struct to function, but it 8221 * is dressed up as a per-CPU kthread and thus needs to play the part 8222 * if we want to avoid special-casing it in code that deals with per-CPU 8223 * kthreads. 8224 */ 8225 set_kthread_struct(idle); 8226 8227 raw_spin_lock_irqsave(&idle->pi_lock, flags); 8228 raw_spin_rq_lock(rq); 8229 8230 idle->__state = TASK_RUNNING; 8231 idle->se.exec_start = sched_clock(); 8232 /* 8233 * PF_KTHREAD should already be set at this point; regardless, make it 8234 * look like a proper per-CPU kthread. 8235 */ 8236 idle->flags |= PF_IDLE | PF_KTHREAD | PF_NO_SETAFFINITY; 8237 kthread_set_per_cpu(idle, cpu); 8238 8239 scs_task_reset(idle); 8240 kasan_unpoison_task_stack(idle); 8241 8242 #ifdef CONFIG_SMP 8243 /* 8244 * It's possible that init_idle() gets called multiple times on a task, 8245 * in that case do_set_cpus_allowed() will not do the right thing. 8246 * 8247 * And since this is boot we can forgo the serialization. 8248 */ 8249 set_cpus_allowed_common(idle, cpumask_of(cpu), 0); 8250 #endif 8251 /* 8252 * We're having a chicken and egg problem, even though we are 8253 * holding rq->lock, the CPU isn't yet set to this CPU so the 8254 * lockdep check in task_group() will fail. 8255 * 8256 * Similar case to sched_fork(). / Alternatively we could 8257 * use task_rq_lock() here and obtain the other rq->lock. 8258 * 8259 * Silence PROVE_RCU 8260 */ 8261 rcu_read_lock(); 8262 __set_task_cpu(idle, cpu); 8263 rcu_read_unlock(); 8264 8265 rq->idle = idle; 8266 rcu_assign_pointer(rq->curr, idle); 8267 idle->on_rq = TASK_ON_RQ_QUEUED; 8268 #ifdef CONFIG_SMP 8269 idle->on_cpu = 1; 8270 #endif 8271 raw_spin_rq_unlock(rq); 8272 raw_spin_unlock_irqrestore(&idle->pi_lock, flags); 8273 8274 /* Set the preempt count _outside_ the spinlocks! */ 8275 init_idle_preempt_count(idle, cpu); 8276 8277 /* 8278 * The idle tasks have their own, simple scheduling class: 8279 */ 8280 idle->sched_class = &idle_sched_class; 8281 ftrace_graph_init_idle_task(idle, cpu); 8282 vtime_init_idle(idle, cpu); 8283 #ifdef CONFIG_SMP 8284 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); 8285 #endif 8286 } 8287 8288 #ifdef CONFIG_SMP 8289 8290 int cpuset_cpumask_can_shrink(const struct cpumask *cur, 8291 const struct cpumask *trial) 8292 { 8293 int ret = 1; 8294 8295 if (!cpumask_weight(cur)) 8296 return ret; 8297 8298 ret = dl_cpuset_cpumask_can_shrink(cur, trial); 8299 8300 return ret; 8301 } 8302 8303 int task_can_attach(struct task_struct *p, 8304 const struct cpumask *cs_cpus_allowed) 8305 { 8306 int ret = 0; 8307 8308 /* 8309 * Kthreads which disallow setaffinity shouldn't be moved 8310 * to a new cpuset; we don't want to change their CPU 8311 * affinity and isolating such threads by their set of 8312 * allowed nodes is unnecessary. Thus, cpusets are not 8313 * applicable for such threads. This prevents checking for 8314 * success of set_cpus_allowed_ptr() on all attached tasks 8315 * before cpus_mask may be changed. 8316 */ 8317 if (p->flags & PF_NO_SETAFFINITY) { 8318 ret = -EINVAL; 8319 goto out; 8320 } 8321 8322 if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span, 8323 cs_cpus_allowed)) 8324 ret = dl_task_can_attach(p, cs_cpus_allowed); 8325 8326 out: 8327 return ret; 8328 } 8329 8330 bool sched_smp_initialized __read_mostly; 8331 8332 #ifdef CONFIG_NUMA_BALANCING 8333 /* Migrate current task p to target_cpu */ 8334 int migrate_task_to(struct task_struct *p, int target_cpu) 8335 { 8336 struct migration_arg arg = { p, target_cpu }; 8337 int curr_cpu = task_cpu(p); 8338 8339 if (curr_cpu == target_cpu) 8340 return 0; 8341 8342 if (!cpumask_test_cpu(target_cpu, p->cpus_ptr)) 8343 return -EINVAL; 8344 8345 /* TODO: This is not properly updating schedstats */ 8346 8347 trace_sched_move_numa(p, curr_cpu, target_cpu); 8348 return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); 8349 } 8350 8351 /* 8352 * Requeue a task on a given node and accurately track the number of NUMA 8353 * tasks on the runqueues 8354 */ 8355 void sched_setnuma(struct task_struct *p, int nid) 8356 { 8357 bool queued, running; 8358 struct rq_flags rf; 8359 struct rq *rq; 8360 8361 rq = task_rq_lock(p, &rf); 8362 queued = task_on_rq_queued(p); 8363 running = task_current(rq, p); 8364 8365 if (queued) 8366 dequeue_task(rq, p, DEQUEUE_SAVE); 8367 if (running) 8368 put_prev_task(rq, p); 8369 8370 p->numa_preferred_nid = nid; 8371 8372 if (queued) 8373 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 8374 if (running) 8375 set_next_task(rq, p); 8376 task_rq_unlock(rq, p, &rf); 8377 } 8378 #endif /* CONFIG_NUMA_BALANCING */ 8379 8380 #ifdef CONFIG_HOTPLUG_CPU 8381 /* 8382 * Ensure that the idle task is using init_mm right before its CPU goes 8383 * offline. 8384 */ 8385 void idle_task_exit(void) 8386 { 8387 struct mm_struct *mm = current->active_mm; 8388 8389 BUG_ON(cpu_online(smp_processor_id())); 8390 BUG_ON(current != this_rq()->idle); 8391 8392 if (mm != &init_mm) { 8393 switch_mm(mm, &init_mm, current); 8394 finish_arch_post_lock_switch(); 8395 } 8396 8397 /* finish_cpu(), as ran on the BP, will clean up the active_mm state */ 8398 } 8399 8400 static int __balance_push_cpu_stop(void *arg) 8401 { 8402 struct task_struct *p = arg; 8403 struct rq *rq = this_rq(); 8404 struct rq_flags rf; 8405 int cpu; 8406 8407 raw_spin_lock_irq(&p->pi_lock); 8408 rq_lock(rq, &rf); 8409 8410 update_rq_clock(rq); 8411 8412 if (task_rq(p) == rq && task_on_rq_queued(p)) { 8413 cpu = select_fallback_rq(rq->cpu, p); 8414 rq = __migrate_task(rq, &rf, p, cpu); 8415 } 8416 8417 rq_unlock(rq, &rf); 8418 raw_spin_unlock_irq(&p->pi_lock); 8419 8420 put_task_struct(p); 8421 8422 return 0; 8423 } 8424 8425 static DEFINE_PER_CPU(struct cpu_stop_work, push_work); 8426 8427 /* 8428 * Ensure we only run per-cpu kthreads once the CPU goes !active. 8429 * 8430 * This is enabled below SCHED_AP_ACTIVE; when !cpu_active(), but only 8431 * effective when the hotplug motion is down. 8432 */ 8433 static void balance_push(struct rq *rq) 8434 { 8435 struct task_struct *push_task = rq->curr; 8436 8437 lockdep_assert_rq_held(rq); 8438 SCHED_WARN_ON(rq->cpu != smp_processor_id()); 8439 8440 /* 8441 * Ensure the thing is persistent until balance_push_set(.on = false); 8442 */ 8443 rq->balance_callback = &balance_push_callback; 8444 8445 /* 8446 * Only active while going offline. 8447 */ 8448 if (!cpu_dying(rq->cpu)) 8449 return; 8450 8451 /* 8452 * Both the cpu-hotplug and stop task are in this case and are 8453 * required to complete the hotplug process. 8454 */ 8455 if (kthread_is_per_cpu(push_task) || 8456 is_migration_disabled(push_task)) { 8457 8458 /* 8459 * If this is the idle task on the outgoing CPU try to wake 8460 * up the hotplug control thread which might wait for the 8461 * last task to vanish. The rcuwait_active() check is 8462 * accurate here because the waiter is pinned on this CPU 8463 * and can't obviously be running in parallel. 8464 * 8465 * On RT kernels this also has to check whether there are 8466 * pinned and scheduled out tasks on the runqueue. They 8467 * need to leave the migrate disabled section first. 8468 */ 8469 if (!rq->nr_running && !rq_has_pinned_tasks(rq) && 8470 rcuwait_active(&rq->hotplug_wait)) { 8471 raw_spin_rq_unlock(rq); 8472 rcuwait_wake_up(&rq->hotplug_wait); 8473 raw_spin_rq_lock(rq); 8474 } 8475 return; 8476 } 8477 8478 get_task_struct(push_task); 8479 /* 8480 * Temporarily drop rq->lock such that we can wake-up the stop task. 8481 * Both preemption and IRQs are still disabled. 8482 */ 8483 raw_spin_rq_unlock(rq); 8484 stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task, 8485 this_cpu_ptr(&push_work)); 8486 /* 8487 * At this point need_resched() is true and we'll take the loop in 8488 * schedule(). The next pick is obviously going to be the stop task 8489 * which kthread_is_per_cpu() and will push this task away. 8490 */ 8491 raw_spin_rq_lock(rq); 8492 } 8493 8494 static void balance_push_set(int cpu, bool on) 8495 { 8496 struct rq *rq = cpu_rq(cpu); 8497 struct rq_flags rf; 8498 8499 rq_lock_irqsave(rq, &rf); 8500 if (on) { 8501 WARN_ON_ONCE(rq->balance_callback); 8502 rq->balance_callback = &balance_push_callback; 8503 } else if (rq->balance_callback == &balance_push_callback) { 8504 rq->balance_callback = NULL; 8505 } 8506 rq_unlock_irqrestore(rq, &rf); 8507 } 8508 8509 /* 8510 * Invoked from a CPUs hotplug control thread after the CPU has been marked 8511 * inactive. All tasks which are not per CPU kernel threads are either 8512 * pushed off this CPU now via balance_push() or placed on a different CPU 8513 * during wakeup. Wait until the CPU is quiescent. 8514 */ 8515 static void balance_hotplug_wait(void) 8516 { 8517 struct rq *rq = this_rq(); 8518 8519 rcuwait_wait_event(&rq->hotplug_wait, 8520 rq->nr_running == 1 && !rq_has_pinned_tasks(rq), 8521 TASK_UNINTERRUPTIBLE); 8522 } 8523 8524 #else 8525 8526 static inline void balance_push(struct rq *rq) 8527 { 8528 } 8529 8530 static inline void balance_push_set(int cpu, bool on) 8531 { 8532 } 8533 8534 static inline void balance_hotplug_wait(void) 8535 { 8536 } 8537 8538 #endif /* CONFIG_HOTPLUG_CPU */ 8539 8540 void set_rq_online(struct rq *rq) 8541 { 8542 if (!rq->online) { 8543 const struct sched_class *class; 8544 8545 cpumask_set_cpu(rq->cpu, rq->rd->online); 8546 rq->online = 1; 8547 8548 for_each_class(class) { 8549 if (class->rq_online) 8550 class->rq_online(rq); 8551 } 8552 } 8553 } 8554 8555 void set_rq_offline(struct rq *rq) 8556 { 8557 if (rq->online) { 8558 const struct sched_class *class; 8559 8560 for_each_class(class) { 8561 if (class->rq_offline) 8562 class->rq_offline(rq); 8563 } 8564 8565 cpumask_clear_cpu(rq->cpu, rq->rd->online); 8566 rq->online = 0; 8567 } 8568 } 8569 8570 /* 8571 * used to mark begin/end of suspend/resume: 8572 */ 8573 static int num_cpus_frozen; 8574 8575 /* 8576 * Update cpusets according to cpu_active mask. If cpusets are 8577 * disabled, cpuset_update_active_cpus() becomes a simple wrapper 8578 * around partition_sched_domains(). 8579 * 8580 * If we come here as part of a suspend/resume, don't touch cpusets because we 8581 * want to restore it back to its original state upon resume anyway. 8582 */ 8583 static void cpuset_cpu_active(void) 8584 { 8585 if (cpuhp_tasks_frozen) { 8586 /* 8587 * num_cpus_frozen tracks how many CPUs are involved in suspend 8588 * resume sequence. As long as this is not the last online 8589 * operation in the resume sequence, just build a single sched 8590 * domain, ignoring cpusets. 8591 */ 8592 partition_sched_domains(1, NULL, NULL); 8593 if (--num_cpus_frozen) 8594 return; 8595 /* 8596 * This is the last CPU online operation. So fall through and 8597 * restore the original sched domains by considering the 8598 * cpuset configurations. 8599 */ 8600 cpuset_force_rebuild(); 8601 } 8602 cpuset_update_active_cpus(); 8603 } 8604 8605 static int cpuset_cpu_inactive(unsigned int cpu) 8606 { 8607 if (!cpuhp_tasks_frozen) { 8608 if (dl_cpu_busy(cpu)) 8609 return -EBUSY; 8610 cpuset_update_active_cpus(); 8611 } else { 8612 num_cpus_frozen++; 8613 partition_sched_domains(1, NULL, NULL); 8614 } 8615 return 0; 8616 } 8617 8618 int sched_cpu_activate(unsigned int cpu) 8619 { 8620 struct rq *rq = cpu_rq(cpu); 8621 struct rq_flags rf; 8622 8623 /* 8624 * Clear the balance_push callback and prepare to schedule 8625 * regular tasks. 8626 */ 8627 balance_push_set(cpu, false); 8628 8629 #ifdef CONFIG_SCHED_SMT 8630 /* 8631 * When going up, increment the number of cores with SMT present. 8632 */ 8633 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 8634 static_branch_inc_cpuslocked(&sched_smt_present); 8635 #endif 8636 set_cpu_active(cpu, true); 8637 8638 if (sched_smp_initialized) { 8639 sched_domains_numa_masks_set(cpu); 8640 cpuset_cpu_active(); 8641 } 8642 8643 /* 8644 * Put the rq online, if not already. This happens: 8645 * 8646 * 1) In the early boot process, because we build the real domains 8647 * after all CPUs have been brought up. 8648 * 8649 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the 8650 * domains. 8651 */ 8652 rq_lock_irqsave(rq, &rf); 8653 if (rq->rd) { 8654 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 8655 set_rq_online(rq); 8656 } 8657 rq_unlock_irqrestore(rq, &rf); 8658 8659 return 0; 8660 } 8661 8662 int sched_cpu_deactivate(unsigned int cpu) 8663 { 8664 struct rq *rq = cpu_rq(cpu); 8665 struct rq_flags rf; 8666 int ret; 8667 8668 /* 8669 * Remove CPU from nohz.idle_cpus_mask to prevent participating in 8670 * load balancing when not active 8671 */ 8672 nohz_balance_exit_idle(rq); 8673 8674 set_cpu_active(cpu, false); 8675 8676 /* 8677 * From this point forward, this CPU will refuse to run any task that 8678 * is not: migrate_disable() or KTHREAD_IS_PER_CPU, and will actively 8679 * push those tasks away until this gets cleared, see 8680 * sched_cpu_dying(). 8681 */ 8682 balance_push_set(cpu, true); 8683 8684 /* 8685 * We've cleared cpu_active_mask / set balance_push, wait for all 8686 * preempt-disabled and RCU users of this state to go away such that 8687 * all new such users will observe it. 8688 * 8689 * Specifically, we rely on ttwu to no longer target this CPU, see 8690 * ttwu_queue_cond() and is_cpu_allowed(). 8691 * 8692 * Do sync before park smpboot threads to take care the rcu boost case. 8693 */ 8694 synchronize_rcu(); 8695 8696 rq_lock_irqsave(rq, &rf); 8697 if (rq->rd) { 8698 update_rq_clock(rq); 8699 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 8700 set_rq_offline(rq); 8701 } 8702 rq_unlock_irqrestore(rq, &rf); 8703 8704 #ifdef CONFIG_SCHED_SMT 8705 /* 8706 * When going down, decrement the number of cores with SMT present. 8707 */ 8708 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 8709 static_branch_dec_cpuslocked(&sched_smt_present); 8710 #endif 8711 8712 if (!sched_smp_initialized) 8713 return 0; 8714 8715 ret = cpuset_cpu_inactive(cpu); 8716 if (ret) { 8717 balance_push_set(cpu, false); 8718 set_cpu_active(cpu, true); 8719 return ret; 8720 } 8721 sched_domains_numa_masks_clear(cpu); 8722 return 0; 8723 } 8724 8725 static void sched_rq_cpu_starting(unsigned int cpu) 8726 { 8727 struct rq *rq = cpu_rq(cpu); 8728 8729 rq->calc_load_update = calc_load_update; 8730 update_max_interval(); 8731 } 8732 8733 int sched_cpu_starting(unsigned int cpu) 8734 { 8735 sched_core_cpu_starting(cpu); 8736 sched_rq_cpu_starting(cpu); 8737 sched_tick_start(cpu); 8738 return 0; 8739 } 8740 8741 #ifdef CONFIG_HOTPLUG_CPU 8742 8743 /* 8744 * Invoked immediately before the stopper thread is invoked to bring the 8745 * CPU down completely. At this point all per CPU kthreads except the 8746 * hotplug thread (current) and the stopper thread (inactive) have been 8747 * either parked or have been unbound from the outgoing CPU. Ensure that 8748 * any of those which might be on the way out are gone. 8749 * 8750 * If after this point a bound task is being woken on this CPU then the 8751 * responsible hotplug callback has failed to do it's job. 8752 * sched_cpu_dying() will catch it with the appropriate fireworks. 8753 */ 8754 int sched_cpu_wait_empty(unsigned int cpu) 8755 { 8756 balance_hotplug_wait(); 8757 return 0; 8758 } 8759 8760 /* 8761 * Since this CPU is going 'away' for a while, fold any nr_active delta we 8762 * might have. Called from the CPU stopper task after ensuring that the 8763 * stopper is the last running task on the CPU, so nr_active count is 8764 * stable. We need to take the teardown thread which is calling this into 8765 * account, so we hand in adjust = 1 to the load calculation. 8766 * 8767 * Also see the comment "Global load-average calculations". 8768 */ 8769 static void calc_load_migrate(struct rq *rq) 8770 { 8771 long delta = calc_load_fold_active(rq, 1); 8772 8773 if (delta) 8774 atomic_long_add(delta, &calc_load_tasks); 8775 } 8776 8777 static void dump_rq_tasks(struct rq *rq, const char *loglvl) 8778 { 8779 struct task_struct *g, *p; 8780 int cpu = cpu_of(rq); 8781 8782 lockdep_assert_rq_held(rq); 8783 8784 printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running); 8785 for_each_process_thread(g, p) { 8786 if (task_cpu(p) != cpu) 8787 continue; 8788 8789 if (!task_on_rq_queued(p)) 8790 continue; 8791 8792 printk("%s\tpid: %d, name: %s\n", loglvl, p->pid, p->comm); 8793 } 8794 } 8795 8796 int sched_cpu_dying(unsigned int cpu) 8797 { 8798 struct rq *rq = cpu_rq(cpu); 8799 struct rq_flags rf; 8800 8801 /* Handle pending wakeups and then migrate everything off */ 8802 sched_tick_stop(cpu); 8803 8804 rq_lock_irqsave(rq, &rf); 8805 if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) { 8806 WARN(true, "Dying CPU not properly vacated!"); 8807 dump_rq_tasks(rq, KERN_WARNING); 8808 } 8809 rq_unlock_irqrestore(rq, &rf); 8810 8811 calc_load_migrate(rq); 8812 update_max_interval(); 8813 hrtick_clear(rq); 8814 return 0; 8815 } 8816 #endif 8817 8818 void __init sched_init_smp(void) 8819 { 8820 sched_init_numa(); 8821 8822 /* 8823 * There's no userspace yet to cause hotplug operations; hence all the 8824 * CPU masks are stable and all blatant races in the below code cannot 8825 * happen. 8826 */ 8827 mutex_lock(&sched_domains_mutex); 8828 sched_init_domains(cpu_active_mask); 8829 mutex_unlock(&sched_domains_mutex); 8830 8831 /* Move init over to a non-isolated CPU */ 8832 if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_DOMAIN)) < 0) 8833 BUG(); 8834 current->flags &= ~PF_NO_SETAFFINITY; 8835 sched_init_granularity(); 8836 8837 init_sched_rt_class(); 8838 init_sched_dl_class(); 8839 8840 sched_smp_initialized = true; 8841 } 8842 8843 static int __init migration_init(void) 8844 { 8845 sched_cpu_starting(smp_processor_id()); 8846 return 0; 8847 } 8848 early_initcall(migration_init); 8849 8850 #else 8851 void __init sched_init_smp(void) 8852 { 8853 sched_init_granularity(); 8854 } 8855 #endif /* CONFIG_SMP */ 8856 8857 int in_sched_functions(unsigned long addr) 8858 { 8859 return in_lock_functions(addr) || 8860 (addr >= (unsigned long)__sched_text_start 8861 && addr < (unsigned long)__sched_text_end); 8862 } 8863 8864 #ifdef CONFIG_CGROUP_SCHED 8865 /* 8866 * Default task group. 8867 * Every task in system belongs to this group at bootup. 8868 */ 8869 struct task_group root_task_group; 8870 LIST_HEAD(task_groups); 8871 8872 /* Cacheline aligned slab cache for task_group */ 8873 static struct kmem_cache *task_group_cache __read_mostly; 8874 #endif 8875 8876 DECLARE_PER_CPU(cpumask_var_t, load_balance_mask); 8877 DECLARE_PER_CPU(cpumask_var_t, select_idle_mask); 8878 8879 void __init sched_init(void) 8880 { 8881 unsigned long ptr = 0; 8882 int i; 8883 8884 /* Make sure the linker didn't screw up */ 8885 BUG_ON(&idle_sched_class + 1 != &fair_sched_class || 8886 &fair_sched_class + 1 != &rt_sched_class || 8887 &rt_sched_class + 1 != &dl_sched_class); 8888 #ifdef CONFIG_SMP 8889 BUG_ON(&dl_sched_class + 1 != &stop_sched_class); 8890 #endif 8891 8892 wait_bit_init(); 8893 8894 #ifdef CONFIG_FAIR_GROUP_SCHED 8895 ptr += 2 * nr_cpu_ids * sizeof(void **); 8896 #endif 8897 #ifdef CONFIG_RT_GROUP_SCHED 8898 ptr += 2 * nr_cpu_ids * sizeof(void **); 8899 #endif 8900 if (ptr) { 8901 ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); 8902 8903 #ifdef CONFIG_FAIR_GROUP_SCHED 8904 root_task_group.se = (struct sched_entity **)ptr; 8905 ptr += nr_cpu_ids * sizeof(void **); 8906 8907 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 8908 ptr += nr_cpu_ids * sizeof(void **); 8909 8910 root_task_group.shares = ROOT_TASK_GROUP_LOAD; 8911 init_cfs_bandwidth(&root_task_group.cfs_bandwidth); 8912 #endif /* CONFIG_FAIR_GROUP_SCHED */ 8913 #ifdef CONFIG_RT_GROUP_SCHED 8914 root_task_group.rt_se = (struct sched_rt_entity **)ptr; 8915 ptr += nr_cpu_ids * sizeof(void **); 8916 8917 root_task_group.rt_rq = (struct rt_rq **)ptr; 8918 ptr += nr_cpu_ids * sizeof(void **); 8919 8920 #endif /* CONFIG_RT_GROUP_SCHED */ 8921 } 8922 #ifdef CONFIG_CPUMASK_OFFSTACK 8923 for_each_possible_cpu(i) { 8924 per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node( 8925 cpumask_size(), GFP_KERNEL, cpu_to_node(i)); 8926 per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node( 8927 cpumask_size(), GFP_KERNEL, cpu_to_node(i)); 8928 } 8929 #endif /* CONFIG_CPUMASK_OFFSTACK */ 8930 8931 init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime()); 8932 init_dl_bandwidth(&def_dl_bandwidth, global_rt_period(), global_rt_runtime()); 8933 8934 #ifdef CONFIG_SMP 8935 init_defrootdomain(); 8936 #endif 8937 8938 #ifdef CONFIG_RT_GROUP_SCHED 8939 init_rt_bandwidth(&root_task_group.rt_bandwidth, 8940 global_rt_period(), global_rt_runtime()); 8941 #endif /* CONFIG_RT_GROUP_SCHED */ 8942 8943 #ifdef CONFIG_CGROUP_SCHED 8944 task_group_cache = KMEM_CACHE(task_group, 0); 8945 8946 list_add(&root_task_group.list, &task_groups); 8947 INIT_LIST_HEAD(&root_task_group.children); 8948 INIT_LIST_HEAD(&root_task_group.siblings); 8949 autogroup_init(&init_task); 8950 #endif /* CONFIG_CGROUP_SCHED */ 8951 8952 for_each_possible_cpu(i) { 8953 struct rq *rq; 8954 8955 rq = cpu_rq(i); 8956 raw_spin_lock_init(&rq->__lock); 8957 rq->nr_running = 0; 8958 rq->calc_load_active = 0; 8959 rq->calc_load_update = jiffies + LOAD_FREQ; 8960 init_cfs_rq(&rq->cfs); 8961 init_rt_rq(&rq->rt); 8962 init_dl_rq(&rq->dl); 8963 #ifdef CONFIG_FAIR_GROUP_SCHED 8964 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); 8965 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 8966 /* 8967 * How much CPU bandwidth does root_task_group get? 8968 * 8969 * In case of task-groups formed thr' the cgroup filesystem, it 8970 * gets 100% of the CPU resources in the system. This overall 8971 * system CPU resource is divided among the tasks of 8972 * root_task_group and its child task-groups in a fair manner, 8973 * based on each entity's (task or task-group's) weight 8974 * (se->load.weight). 8975 * 8976 * In other words, if root_task_group has 10 tasks of weight 8977 * 1024) and two child groups A0 and A1 (of weight 1024 each), 8978 * then A0's share of the CPU resource is: 8979 * 8980 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% 8981 * 8982 * We achieve this by letting root_task_group's tasks sit 8983 * directly in rq->cfs (i.e root_task_group->se[] = NULL). 8984 */ 8985 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); 8986 #endif /* CONFIG_FAIR_GROUP_SCHED */ 8987 8988 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; 8989 #ifdef CONFIG_RT_GROUP_SCHED 8990 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); 8991 #endif 8992 #ifdef CONFIG_SMP 8993 rq->sd = NULL; 8994 rq->rd = NULL; 8995 rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; 8996 rq->balance_callback = &balance_push_callback; 8997 rq->active_balance = 0; 8998 rq->next_balance = jiffies; 8999 rq->push_cpu = 0; 9000 rq->cpu = i; 9001 rq->online = 0; 9002 rq->idle_stamp = 0; 9003 rq->avg_idle = 2*sysctl_sched_migration_cost; 9004 rq->wake_stamp = jiffies; 9005 rq->wake_avg_idle = rq->avg_idle; 9006 rq->max_idle_balance_cost = sysctl_sched_migration_cost; 9007 9008 INIT_LIST_HEAD(&rq->cfs_tasks); 9009 9010 rq_attach_root(rq, &def_root_domain); 9011 #ifdef CONFIG_NO_HZ_COMMON 9012 rq->last_blocked_load_update_tick = jiffies; 9013 atomic_set(&rq->nohz_flags, 0); 9014 9015 INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq); 9016 #endif 9017 #ifdef CONFIG_HOTPLUG_CPU 9018 rcuwait_init(&rq->hotplug_wait); 9019 #endif 9020 #endif /* CONFIG_SMP */ 9021 hrtick_rq_init(rq); 9022 atomic_set(&rq->nr_iowait, 0); 9023 9024 #ifdef CONFIG_SCHED_CORE 9025 rq->core = NULL; 9026 rq->core_pick = NULL; 9027 rq->core_enabled = 0; 9028 rq->core_tree = RB_ROOT; 9029 rq->core_forceidle = false; 9030 9031 rq->core_cookie = 0UL; 9032 #endif 9033 } 9034 9035 set_load_weight(&init_task, false); 9036 9037 /* 9038 * The boot idle thread does lazy MMU switching as well: 9039 */ 9040 mmgrab(&init_mm); 9041 enter_lazy_tlb(&init_mm, current); 9042 9043 /* 9044 * Make us the idle thread. Technically, schedule() should not be 9045 * called from this thread, however somewhere below it might be, 9046 * but because we are the idle thread, we just pick up running again 9047 * when this runqueue becomes "idle". 9048 */ 9049 init_idle(current, smp_processor_id()); 9050 9051 calc_load_update = jiffies + LOAD_FREQ; 9052 9053 #ifdef CONFIG_SMP 9054 idle_thread_set_boot_cpu(); 9055 balance_push_set(smp_processor_id(), false); 9056 #endif 9057 init_sched_fair_class(); 9058 9059 psi_init(); 9060 9061 init_uclamp(); 9062 9063 scheduler_running = 1; 9064 } 9065 9066 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 9067 static inline int preempt_count_equals(int preempt_offset) 9068 { 9069 int nested = preempt_count() + rcu_preempt_depth(); 9070 9071 return (nested == preempt_offset); 9072 } 9073 9074 void __might_sleep(const char *file, int line, int preempt_offset) 9075 { 9076 unsigned int state = get_current_state(); 9077 /* 9078 * Blocking primitives will set (and therefore destroy) current->state, 9079 * since we will exit with TASK_RUNNING make sure we enter with it, 9080 * otherwise we will destroy state. 9081 */ 9082 WARN_ONCE(state != TASK_RUNNING && current->task_state_change, 9083 "do not call blocking ops when !TASK_RUNNING; " 9084 "state=%x set at [<%p>] %pS\n", state, 9085 (void *)current->task_state_change, 9086 (void *)current->task_state_change); 9087 9088 ___might_sleep(file, line, preempt_offset); 9089 } 9090 EXPORT_SYMBOL(__might_sleep); 9091 9092 void ___might_sleep(const char *file, int line, int preempt_offset) 9093 { 9094 /* Ratelimiting timestamp: */ 9095 static unsigned long prev_jiffy; 9096 9097 unsigned long preempt_disable_ip; 9098 9099 /* WARN_ON_ONCE() by default, no rate limit required: */ 9100 rcu_sleep_check(); 9101 9102 if ((preempt_count_equals(preempt_offset) && !irqs_disabled() && 9103 !is_idle_task(current) && !current->non_block_count) || 9104 system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || 9105 oops_in_progress) 9106 return; 9107 9108 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 9109 return; 9110 prev_jiffy = jiffies; 9111 9112 /* Save this before calling printk(), since that will clobber it: */ 9113 preempt_disable_ip = get_preempt_disable_ip(current); 9114 9115 printk(KERN_ERR 9116 "BUG: sleeping function called from invalid context at %s:%d\n", 9117 file, line); 9118 printk(KERN_ERR 9119 "in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n", 9120 in_atomic(), irqs_disabled(), current->non_block_count, 9121 current->pid, current->comm); 9122 9123 if (task_stack_end_corrupted(current)) 9124 printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); 9125 9126 debug_show_held_locks(current); 9127 if (irqs_disabled()) 9128 print_irqtrace_events(current); 9129 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) 9130 && !preempt_count_equals(preempt_offset)) { 9131 pr_err("Preemption disabled at:"); 9132 print_ip_sym(KERN_ERR, preempt_disable_ip); 9133 } 9134 dump_stack(); 9135 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 9136 } 9137 EXPORT_SYMBOL(___might_sleep); 9138 9139 void __cant_sleep(const char *file, int line, int preempt_offset) 9140 { 9141 static unsigned long prev_jiffy; 9142 9143 if (irqs_disabled()) 9144 return; 9145 9146 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 9147 return; 9148 9149 if (preempt_count() > preempt_offset) 9150 return; 9151 9152 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 9153 return; 9154 prev_jiffy = jiffies; 9155 9156 printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line); 9157 printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", 9158 in_atomic(), irqs_disabled(), 9159 current->pid, current->comm); 9160 9161 debug_show_held_locks(current); 9162 dump_stack(); 9163 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 9164 } 9165 EXPORT_SYMBOL_GPL(__cant_sleep); 9166 9167 #ifdef CONFIG_SMP 9168 void __cant_migrate(const char *file, int line) 9169 { 9170 static unsigned long prev_jiffy; 9171 9172 if (irqs_disabled()) 9173 return; 9174 9175 if (is_migration_disabled(current)) 9176 return; 9177 9178 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 9179 return; 9180 9181 if (preempt_count() > 0) 9182 return; 9183 9184 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 9185 return; 9186 prev_jiffy = jiffies; 9187 9188 pr_err("BUG: assuming non migratable context at %s:%d\n", file, line); 9189 pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n", 9190 in_atomic(), irqs_disabled(), is_migration_disabled(current), 9191 current->pid, current->comm); 9192 9193 debug_show_held_locks(current); 9194 dump_stack(); 9195 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 9196 } 9197 EXPORT_SYMBOL_GPL(__cant_migrate); 9198 #endif 9199 #endif 9200 9201 #ifdef CONFIG_MAGIC_SYSRQ 9202 void normalize_rt_tasks(void) 9203 { 9204 struct task_struct *g, *p; 9205 struct sched_attr attr = { 9206 .sched_policy = SCHED_NORMAL, 9207 }; 9208 9209 read_lock(&tasklist_lock); 9210 for_each_process_thread(g, p) { 9211 /* 9212 * Only normalize user tasks: 9213 */ 9214 if (p->flags & PF_KTHREAD) 9215 continue; 9216 9217 p->se.exec_start = 0; 9218 schedstat_set(p->se.statistics.wait_start, 0); 9219 schedstat_set(p->se.statistics.sleep_start, 0); 9220 schedstat_set(p->se.statistics.block_start, 0); 9221 9222 if (!dl_task(p) && !rt_task(p)) { 9223 /* 9224 * Renice negative nice level userspace 9225 * tasks back to 0: 9226 */ 9227 if (task_nice(p) < 0) 9228 set_user_nice(p, 0); 9229 continue; 9230 } 9231 9232 __sched_setscheduler(p, &attr, false, false); 9233 } 9234 read_unlock(&tasklist_lock); 9235 } 9236 9237 #endif /* CONFIG_MAGIC_SYSRQ */ 9238 9239 #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) 9240 /* 9241 * These functions are only useful for the IA64 MCA handling, or kdb. 9242 * 9243 * They can only be called when the whole system has been 9244 * stopped - every CPU needs to be quiescent, and no scheduling 9245 * activity can take place. Using them for anything else would 9246 * be a serious bug, and as a result, they aren't even visible 9247 * under any other configuration. 9248 */ 9249 9250 /** 9251 * curr_task - return the current task for a given CPU. 9252 * @cpu: the processor in question. 9253 * 9254 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 9255 * 9256 * Return: The current task for @cpu. 9257 */ 9258 struct task_struct *curr_task(int cpu) 9259 { 9260 return cpu_curr(cpu); 9261 } 9262 9263 #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ 9264 9265 #ifdef CONFIG_IA64 9266 /** 9267 * ia64_set_curr_task - set the current task for a given CPU. 9268 * @cpu: the processor in question. 9269 * @p: the task pointer to set. 9270 * 9271 * Description: This function must only be used when non-maskable interrupts 9272 * are serviced on a separate stack. It allows the architecture to switch the 9273 * notion of the current task on a CPU in a non-blocking manner. This function 9274 * must be called with all CPU's synchronized, and interrupts disabled, the 9275 * and caller must save the original value of the current task (see 9276 * curr_task() above) and restore that value before reenabling interrupts and 9277 * re-starting the system. 9278 * 9279 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 9280 */ 9281 void ia64_set_curr_task(int cpu, struct task_struct *p) 9282 { 9283 cpu_curr(cpu) = p; 9284 } 9285 9286 #endif 9287 9288 #ifdef CONFIG_CGROUP_SCHED 9289 /* task_group_lock serializes the addition/removal of task groups */ 9290 static DEFINE_SPINLOCK(task_group_lock); 9291 9292 static inline void alloc_uclamp_sched_group(struct task_group *tg, 9293 struct task_group *parent) 9294 { 9295 #ifdef CONFIG_UCLAMP_TASK_GROUP 9296 enum uclamp_id clamp_id; 9297 9298 for_each_clamp_id(clamp_id) { 9299 uclamp_se_set(&tg->uclamp_req[clamp_id], 9300 uclamp_none(clamp_id), false); 9301 tg->uclamp[clamp_id] = parent->uclamp[clamp_id]; 9302 } 9303 #endif 9304 } 9305 9306 static void sched_free_group(struct task_group *tg) 9307 { 9308 free_fair_sched_group(tg); 9309 free_rt_sched_group(tg); 9310 autogroup_free(tg); 9311 kmem_cache_free(task_group_cache, tg); 9312 } 9313 9314 /* allocate runqueue etc for a new task group */ 9315 struct task_group *sched_create_group(struct task_group *parent) 9316 { 9317 struct task_group *tg; 9318 9319 tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO); 9320 if (!tg) 9321 return ERR_PTR(-ENOMEM); 9322 9323 if (!alloc_fair_sched_group(tg, parent)) 9324 goto err; 9325 9326 if (!alloc_rt_sched_group(tg, parent)) 9327 goto err; 9328 9329 alloc_uclamp_sched_group(tg, parent); 9330 9331 return tg; 9332 9333 err: 9334 sched_free_group(tg); 9335 return ERR_PTR(-ENOMEM); 9336 } 9337 9338 void sched_online_group(struct task_group *tg, struct task_group *parent) 9339 { 9340 unsigned long flags; 9341 9342 spin_lock_irqsave(&task_group_lock, flags); 9343 list_add_rcu(&tg->list, &task_groups); 9344 9345 /* Root should already exist: */ 9346 WARN_ON(!parent); 9347 9348 tg->parent = parent; 9349 INIT_LIST_HEAD(&tg->children); 9350 list_add_rcu(&tg->siblings, &parent->children); 9351 spin_unlock_irqrestore(&task_group_lock, flags); 9352 9353 online_fair_sched_group(tg); 9354 } 9355 9356 /* rcu callback to free various structures associated with a task group */ 9357 static void sched_free_group_rcu(struct rcu_head *rhp) 9358 { 9359 /* Now it should be safe to free those cfs_rqs: */ 9360 sched_free_group(container_of(rhp, struct task_group, rcu)); 9361 } 9362 9363 void sched_destroy_group(struct task_group *tg) 9364 { 9365 /* Wait for possible concurrent references to cfs_rqs complete: */ 9366 call_rcu(&tg->rcu, sched_free_group_rcu); 9367 } 9368 9369 void sched_offline_group(struct task_group *tg) 9370 { 9371 unsigned long flags; 9372 9373 /* End participation in shares distribution: */ 9374 unregister_fair_sched_group(tg); 9375 9376 spin_lock_irqsave(&task_group_lock, flags); 9377 list_del_rcu(&tg->list); 9378 list_del_rcu(&tg->siblings); 9379 spin_unlock_irqrestore(&task_group_lock, flags); 9380 } 9381 9382 static void sched_change_group(struct task_struct *tsk, int type) 9383 { 9384 struct task_group *tg; 9385 9386 /* 9387 * All callers are synchronized by task_rq_lock(); we do not use RCU 9388 * which is pointless here. Thus, we pass "true" to task_css_check() 9389 * to prevent lockdep warnings. 9390 */ 9391 tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), 9392 struct task_group, css); 9393 tg = autogroup_task_group(tsk, tg); 9394 tsk->sched_task_group = tg; 9395 9396 #ifdef CONFIG_FAIR_GROUP_SCHED 9397 if (tsk->sched_class->task_change_group) 9398 tsk->sched_class->task_change_group(tsk, type); 9399 else 9400 #endif 9401 set_task_rq(tsk, task_cpu(tsk)); 9402 } 9403 9404 /* 9405 * Change task's runqueue when it moves between groups. 9406 * 9407 * The caller of this function should have put the task in its new group by 9408 * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect 9409 * its new group. 9410 */ 9411 void sched_move_task(struct task_struct *tsk) 9412 { 9413 int queued, running, queue_flags = 9414 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 9415 struct rq_flags rf; 9416 struct rq *rq; 9417 9418 rq = task_rq_lock(tsk, &rf); 9419 update_rq_clock(rq); 9420 9421 running = task_current(rq, tsk); 9422 queued = task_on_rq_queued(tsk); 9423 9424 if (queued) 9425 dequeue_task(rq, tsk, queue_flags); 9426 if (running) 9427 put_prev_task(rq, tsk); 9428 9429 sched_change_group(tsk, TASK_MOVE_GROUP); 9430 9431 if (queued) 9432 enqueue_task(rq, tsk, queue_flags); 9433 if (running) { 9434 set_next_task(rq, tsk); 9435 /* 9436 * After changing group, the running task may have joined a 9437 * throttled one but it's still the running task. Trigger a 9438 * resched to make sure that task can still run. 9439 */ 9440 resched_curr(rq); 9441 } 9442 9443 task_rq_unlock(rq, tsk, &rf); 9444 } 9445 9446 static inline struct task_group *css_tg(struct cgroup_subsys_state *css) 9447 { 9448 return css ? container_of(css, struct task_group, css) : NULL; 9449 } 9450 9451 static struct cgroup_subsys_state * 9452 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 9453 { 9454 struct task_group *parent = css_tg(parent_css); 9455 struct task_group *tg; 9456 9457 if (!parent) { 9458 /* This is early initialization for the top cgroup */ 9459 return &root_task_group.css; 9460 } 9461 9462 tg = sched_create_group(parent); 9463 if (IS_ERR(tg)) 9464 return ERR_PTR(-ENOMEM); 9465 9466 return &tg->css; 9467 } 9468 9469 /* Expose task group only after completing cgroup initialization */ 9470 static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) 9471 { 9472 struct task_group *tg = css_tg(css); 9473 struct task_group *parent = css_tg(css->parent); 9474 9475 if (parent) 9476 sched_online_group(tg, parent); 9477 9478 #ifdef CONFIG_UCLAMP_TASK_GROUP 9479 /* Propagate the effective uclamp value for the new group */ 9480 mutex_lock(&uclamp_mutex); 9481 rcu_read_lock(); 9482 cpu_util_update_eff(css); 9483 rcu_read_unlock(); 9484 mutex_unlock(&uclamp_mutex); 9485 #endif 9486 9487 return 0; 9488 } 9489 9490 static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) 9491 { 9492 struct task_group *tg = css_tg(css); 9493 9494 sched_offline_group(tg); 9495 } 9496 9497 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) 9498 { 9499 struct task_group *tg = css_tg(css); 9500 9501 /* 9502 * Relies on the RCU grace period between css_released() and this. 9503 */ 9504 sched_free_group(tg); 9505 } 9506 9507 /* 9508 * This is called before wake_up_new_task(), therefore we really only 9509 * have to set its group bits, all the other stuff does not apply. 9510 */ 9511 static void cpu_cgroup_fork(struct task_struct *task) 9512 { 9513 struct rq_flags rf; 9514 struct rq *rq; 9515 9516 rq = task_rq_lock(task, &rf); 9517 9518 update_rq_clock(rq); 9519 sched_change_group(task, TASK_SET_GROUP); 9520 9521 task_rq_unlock(rq, task, &rf); 9522 } 9523 9524 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) 9525 { 9526 struct task_struct *task; 9527 struct cgroup_subsys_state *css; 9528 int ret = 0; 9529 9530 cgroup_taskset_for_each(task, css, tset) { 9531 #ifdef CONFIG_RT_GROUP_SCHED 9532 if (!sched_rt_can_attach(css_tg(css), task)) 9533 return -EINVAL; 9534 #endif 9535 /* 9536 * Serialize against wake_up_new_task() such that if it's 9537 * running, we're sure to observe its full state. 9538 */ 9539 raw_spin_lock_irq(&task->pi_lock); 9540 /* 9541 * Avoid calling sched_move_task() before wake_up_new_task() 9542 * has happened. This would lead to problems with PELT, due to 9543 * move wanting to detach+attach while we're not attached yet. 9544 */ 9545 if (READ_ONCE(task->__state) == TASK_NEW) 9546 ret = -EINVAL; 9547 raw_spin_unlock_irq(&task->pi_lock); 9548 9549 if (ret) 9550 break; 9551 } 9552 return ret; 9553 } 9554 9555 static void cpu_cgroup_attach(struct cgroup_taskset *tset) 9556 { 9557 struct task_struct *task; 9558 struct cgroup_subsys_state *css; 9559 9560 cgroup_taskset_for_each(task, css, tset) 9561 sched_move_task(task); 9562 } 9563 9564 #ifdef CONFIG_UCLAMP_TASK_GROUP 9565 static void cpu_util_update_eff(struct cgroup_subsys_state *css) 9566 { 9567 struct cgroup_subsys_state *top_css = css; 9568 struct uclamp_se *uc_parent = NULL; 9569 struct uclamp_se *uc_se = NULL; 9570 unsigned int eff[UCLAMP_CNT]; 9571 enum uclamp_id clamp_id; 9572 unsigned int clamps; 9573 9574 lockdep_assert_held(&uclamp_mutex); 9575 SCHED_WARN_ON(!rcu_read_lock_held()); 9576 9577 css_for_each_descendant_pre(css, top_css) { 9578 uc_parent = css_tg(css)->parent 9579 ? css_tg(css)->parent->uclamp : NULL; 9580 9581 for_each_clamp_id(clamp_id) { 9582 /* Assume effective clamps matches requested clamps */ 9583 eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value; 9584 /* Cap effective clamps with parent's effective clamps */ 9585 if (uc_parent && 9586 eff[clamp_id] > uc_parent[clamp_id].value) { 9587 eff[clamp_id] = uc_parent[clamp_id].value; 9588 } 9589 } 9590 /* Ensure protection is always capped by limit */ 9591 eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]); 9592 9593 /* Propagate most restrictive effective clamps */ 9594 clamps = 0x0; 9595 uc_se = css_tg(css)->uclamp; 9596 for_each_clamp_id(clamp_id) { 9597 if (eff[clamp_id] == uc_se[clamp_id].value) 9598 continue; 9599 uc_se[clamp_id].value = eff[clamp_id]; 9600 uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]); 9601 clamps |= (0x1 << clamp_id); 9602 } 9603 if (!clamps) { 9604 css = css_rightmost_descendant(css); 9605 continue; 9606 } 9607 9608 /* Immediately update descendants RUNNABLE tasks */ 9609 uclamp_update_active_tasks(css); 9610 } 9611 } 9612 9613 /* 9614 * Integer 10^N with a given N exponent by casting to integer the literal "1eN" 9615 * C expression. Since there is no way to convert a macro argument (N) into a 9616 * character constant, use two levels of macros. 9617 */ 9618 #define _POW10(exp) ((unsigned int)1e##exp) 9619 #define POW10(exp) _POW10(exp) 9620 9621 struct uclamp_request { 9622 #define UCLAMP_PERCENT_SHIFT 2 9623 #define UCLAMP_PERCENT_SCALE (100 * POW10(UCLAMP_PERCENT_SHIFT)) 9624 s64 percent; 9625 u64 util; 9626 int ret; 9627 }; 9628 9629 static inline struct uclamp_request 9630 capacity_from_percent(char *buf) 9631 { 9632 struct uclamp_request req = { 9633 .percent = UCLAMP_PERCENT_SCALE, 9634 .util = SCHED_CAPACITY_SCALE, 9635 .ret = 0, 9636 }; 9637 9638 buf = strim(buf); 9639 if (strcmp(buf, "max")) { 9640 req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT, 9641 &req.percent); 9642 if (req.ret) 9643 return req; 9644 if ((u64)req.percent > UCLAMP_PERCENT_SCALE) { 9645 req.ret = -ERANGE; 9646 return req; 9647 } 9648 9649 req.util = req.percent << SCHED_CAPACITY_SHIFT; 9650 req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE); 9651 } 9652 9653 return req; 9654 } 9655 9656 static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf, 9657 size_t nbytes, loff_t off, 9658 enum uclamp_id clamp_id) 9659 { 9660 struct uclamp_request req; 9661 struct task_group *tg; 9662 9663 req = capacity_from_percent(buf); 9664 if (req.ret) 9665 return req.ret; 9666 9667 static_branch_enable(&sched_uclamp_used); 9668 9669 mutex_lock(&uclamp_mutex); 9670 rcu_read_lock(); 9671 9672 tg = css_tg(of_css(of)); 9673 if (tg->uclamp_req[clamp_id].value != req.util) 9674 uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false); 9675 9676 /* 9677 * Because of not recoverable conversion rounding we keep track of the 9678 * exact requested value 9679 */ 9680 tg->uclamp_pct[clamp_id] = req.percent; 9681 9682 /* Update effective clamps to track the most restrictive value */ 9683 cpu_util_update_eff(of_css(of)); 9684 9685 rcu_read_unlock(); 9686 mutex_unlock(&uclamp_mutex); 9687 9688 return nbytes; 9689 } 9690 9691 static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of, 9692 char *buf, size_t nbytes, 9693 loff_t off) 9694 { 9695 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN); 9696 } 9697 9698 static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of, 9699 char *buf, size_t nbytes, 9700 loff_t off) 9701 { 9702 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX); 9703 } 9704 9705 static inline void cpu_uclamp_print(struct seq_file *sf, 9706 enum uclamp_id clamp_id) 9707 { 9708 struct task_group *tg; 9709 u64 util_clamp; 9710 u64 percent; 9711 u32 rem; 9712 9713 rcu_read_lock(); 9714 tg = css_tg(seq_css(sf)); 9715 util_clamp = tg->uclamp_req[clamp_id].value; 9716 rcu_read_unlock(); 9717 9718 if (util_clamp == SCHED_CAPACITY_SCALE) { 9719 seq_puts(sf, "max\n"); 9720 return; 9721 } 9722 9723 percent = tg->uclamp_pct[clamp_id]; 9724 percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem); 9725 seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem); 9726 } 9727 9728 static int cpu_uclamp_min_show(struct seq_file *sf, void *v) 9729 { 9730 cpu_uclamp_print(sf, UCLAMP_MIN); 9731 return 0; 9732 } 9733 9734 static int cpu_uclamp_max_show(struct seq_file *sf, void *v) 9735 { 9736 cpu_uclamp_print(sf, UCLAMP_MAX); 9737 return 0; 9738 } 9739 #endif /* CONFIG_UCLAMP_TASK_GROUP */ 9740 9741 #ifdef CONFIG_FAIR_GROUP_SCHED 9742 static int cpu_shares_write_u64(struct cgroup_subsys_state *css, 9743 struct cftype *cftype, u64 shareval) 9744 { 9745 if (shareval > scale_load_down(ULONG_MAX)) 9746 shareval = MAX_SHARES; 9747 return sched_group_set_shares(css_tg(css), scale_load(shareval)); 9748 } 9749 9750 static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, 9751 struct cftype *cft) 9752 { 9753 struct task_group *tg = css_tg(css); 9754 9755 return (u64) scale_load_down(tg->shares); 9756 } 9757 9758 #ifdef CONFIG_CFS_BANDWIDTH 9759 static DEFINE_MUTEX(cfs_constraints_mutex); 9760 9761 const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ 9762 static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ 9763 /* More than 203 days if BW_SHIFT equals 20. */ 9764 static const u64 max_cfs_runtime = MAX_BW * NSEC_PER_USEC; 9765 9766 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); 9767 9768 static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota, 9769 u64 burst) 9770 { 9771 int i, ret = 0, runtime_enabled, runtime_was_enabled; 9772 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9773 9774 if (tg == &root_task_group) 9775 return -EINVAL; 9776 9777 /* 9778 * Ensure we have at some amount of bandwidth every period. This is 9779 * to prevent reaching a state of large arrears when throttled via 9780 * entity_tick() resulting in prolonged exit starvation. 9781 */ 9782 if (quota < min_cfs_quota_period || period < min_cfs_quota_period) 9783 return -EINVAL; 9784 9785 /* 9786 * Likewise, bound things on the other side by preventing insane quota 9787 * periods. This also allows us to normalize in computing quota 9788 * feasibility. 9789 */ 9790 if (period > max_cfs_quota_period) 9791 return -EINVAL; 9792 9793 /* 9794 * Bound quota to defend quota against overflow during bandwidth shift. 9795 */ 9796 if (quota != RUNTIME_INF && quota > max_cfs_runtime) 9797 return -EINVAL; 9798 9799 if (quota != RUNTIME_INF && (burst > quota || 9800 burst + quota > max_cfs_runtime)) 9801 return -EINVAL; 9802 9803 /* 9804 * Prevent race between setting of cfs_rq->runtime_enabled and 9805 * unthrottle_offline_cfs_rqs(). 9806 */ 9807 get_online_cpus(); 9808 mutex_lock(&cfs_constraints_mutex); 9809 ret = __cfs_schedulable(tg, period, quota); 9810 if (ret) 9811 goto out_unlock; 9812 9813 runtime_enabled = quota != RUNTIME_INF; 9814 runtime_was_enabled = cfs_b->quota != RUNTIME_INF; 9815 /* 9816 * If we need to toggle cfs_bandwidth_used, off->on must occur 9817 * before making related changes, and on->off must occur afterwards 9818 */ 9819 if (runtime_enabled && !runtime_was_enabled) 9820 cfs_bandwidth_usage_inc(); 9821 raw_spin_lock_irq(&cfs_b->lock); 9822 cfs_b->period = ns_to_ktime(period); 9823 cfs_b->quota = quota; 9824 cfs_b->burst = burst; 9825 9826 __refill_cfs_bandwidth_runtime(cfs_b); 9827 9828 /* Restart the period timer (if active) to handle new period expiry: */ 9829 if (runtime_enabled) 9830 start_cfs_bandwidth(cfs_b); 9831 9832 raw_spin_unlock_irq(&cfs_b->lock); 9833 9834 for_each_online_cpu(i) { 9835 struct cfs_rq *cfs_rq = tg->cfs_rq[i]; 9836 struct rq *rq = cfs_rq->rq; 9837 struct rq_flags rf; 9838 9839 rq_lock_irq(rq, &rf); 9840 cfs_rq->runtime_enabled = runtime_enabled; 9841 cfs_rq->runtime_remaining = 0; 9842 9843 if (cfs_rq->throttled) 9844 unthrottle_cfs_rq(cfs_rq); 9845 rq_unlock_irq(rq, &rf); 9846 } 9847 if (runtime_was_enabled && !runtime_enabled) 9848 cfs_bandwidth_usage_dec(); 9849 out_unlock: 9850 mutex_unlock(&cfs_constraints_mutex); 9851 put_online_cpus(); 9852 9853 return ret; 9854 } 9855 9856 static int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) 9857 { 9858 u64 quota, period, burst; 9859 9860 period = ktime_to_ns(tg->cfs_bandwidth.period); 9861 burst = tg->cfs_bandwidth.burst; 9862 if (cfs_quota_us < 0) 9863 quota = RUNTIME_INF; 9864 else if ((u64)cfs_quota_us <= U64_MAX / NSEC_PER_USEC) 9865 quota = (u64)cfs_quota_us * NSEC_PER_USEC; 9866 else 9867 return -EINVAL; 9868 9869 return tg_set_cfs_bandwidth(tg, period, quota, burst); 9870 } 9871 9872 static long tg_get_cfs_quota(struct task_group *tg) 9873 { 9874 u64 quota_us; 9875 9876 if (tg->cfs_bandwidth.quota == RUNTIME_INF) 9877 return -1; 9878 9879 quota_us = tg->cfs_bandwidth.quota; 9880 do_div(quota_us, NSEC_PER_USEC); 9881 9882 return quota_us; 9883 } 9884 9885 static int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) 9886 { 9887 u64 quota, period, burst; 9888 9889 if ((u64)cfs_period_us > U64_MAX / NSEC_PER_USEC) 9890 return -EINVAL; 9891 9892 period = (u64)cfs_period_us * NSEC_PER_USEC; 9893 quota = tg->cfs_bandwidth.quota; 9894 burst = tg->cfs_bandwidth.burst; 9895 9896 return tg_set_cfs_bandwidth(tg, period, quota, burst); 9897 } 9898 9899 static long tg_get_cfs_period(struct task_group *tg) 9900 { 9901 u64 cfs_period_us; 9902 9903 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); 9904 do_div(cfs_period_us, NSEC_PER_USEC); 9905 9906 return cfs_period_us; 9907 } 9908 9909 static int tg_set_cfs_burst(struct task_group *tg, long cfs_burst_us) 9910 { 9911 u64 quota, period, burst; 9912 9913 if ((u64)cfs_burst_us > U64_MAX / NSEC_PER_USEC) 9914 return -EINVAL; 9915 9916 burst = (u64)cfs_burst_us * NSEC_PER_USEC; 9917 period = ktime_to_ns(tg->cfs_bandwidth.period); 9918 quota = tg->cfs_bandwidth.quota; 9919 9920 return tg_set_cfs_bandwidth(tg, period, quota, burst); 9921 } 9922 9923 static long tg_get_cfs_burst(struct task_group *tg) 9924 { 9925 u64 burst_us; 9926 9927 burst_us = tg->cfs_bandwidth.burst; 9928 do_div(burst_us, NSEC_PER_USEC); 9929 9930 return burst_us; 9931 } 9932 9933 static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, 9934 struct cftype *cft) 9935 { 9936 return tg_get_cfs_quota(css_tg(css)); 9937 } 9938 9939 static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css, 9940 struct cftype *cftype, s64 cfs_quota_us) 9941 { 9942 return tg_set_cfs_quota(css_tg(css), cfs_quota_us); 9943 } 9944 9945 static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css, 9946 struct cftype *cft) 9947 { 9948 return tg_get_cfs_period(css_tg(css)); 9949 } 9950 9951 static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css, 9952 struct cftype *cftype, u64 cfs_period_us) 9953 { 9954 return tg_set_cfs_period(css_tg(css), cfs_period_us); 9955 } 9956 9957 static u64 cpu_cfs_burst_read_u64(struct cgroup_subsys_state *css, 9958 struct cftype *cft) 9959 { 9960 return tg_get_cfs_burst(css_tg(css)); 9961 } 9962 9963 static int cpu_cfs_burst_write_u64(struct cgroup_subsys_state *css, 9964 struct cftype *cftype, u64 cfs_burst_us) 9965 { 9966 return tg_set_cfs_burst(css_tg(css), cfs_burst_us); 9967 } 9968 9969 struct cfs_schedulable_data { 9970 struct task_group *tg; 9971 u64 period, quota; 9972 }; 9973 9974 /* 9975 * normalize group quota/period to be quota/max_period 9976 * note: units are usecs 9977 */ 9978 static u64 normalize_cfs_quota(struct task_group *tg, 9979 struct cfs_schedulable_data *d) 9980 { 9981 u64 quota, period; 9982 9983 if (tg == d->tg) { 9984 period = d->period; 9985 quota = d->quota; 9986 } else { 9987 period = tg_get_cfs_period(tg); 9988 quota = tg_get_cfs_quota(tg); 9989 } 9990 9991 /* note: these should typically be equivalent */ 9992 if (quota == RUNTIME_INF || quota == -1) 9993 return RUNTIME_INF; 9994 9995 return to_ratio(period, quota); 9996 } 9997 9998 static int tg_cfs_schedulable_down(struct task_group *tg, void *data) 9999 { 10000 struct cfs_schedulable_data *d = data; 10001 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 10002 s64 quota = 0, parent_quota = -1; 10003 10004 if (!tg->parent) { 10005 quota = RUNTIME_INF; 10006 } else { 10007 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; 10008 10009 quota = normalize_cfs_quota(tg, d); 10010 parent_quota = parent_b->hierarchical_quota; 10011 10012 /* 10013 * Ensure max(child_quota) <= parent_quota. On cgroup2, 10014 * always take the min. On cgroup1, only inherit when no 10015 * limit is set: 10016 */ 10017 if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) { 10018 quota = min(quota, parent_quota); 10019 } else { 10020 if (quota == RUNTIME_INF) 10021 quota = parent_quota; 10022 else if (parent_quota != RUNTIME_INF && quota > parent_quota) 10023 return -EINVAL; 10024 } 10025 } 10026 cfs_b->hierarchical_quota = quota; 10027 10028 return 0; 10029 } 10030 10031 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) 10032 { 10033 int ret; 10034 struct cfs_schedulable_data data = { 10035 .tg = tg, 10036 .period = period, 10037 .quota = quota, 10038 }; 10039 10040 if (quota != RUNTIME_INF) { 10041 do_div(data.period, NSEC_PER_USEC); 10042 do_div(data.quota, NSEC_PER_USEC); 10043 } 10044 10045 rcu_read_lock(); 10046 ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); 10047 rcu_read_unlock(); 10048 10049 return ret; 10050 } 10051 10052 static int cpu_cfs_stat_show(struct seq_file *sf, void *v) 10053 { 10054 struct task_group *tg = css_tg(seq_css(sf)); 10055 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 10056 10057 seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); 10058 seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); 10059 seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); 10060 10061 if (schedstat_enabled() && tg != &root_task_group) { 10062 u64 ws = 0; 10063 int i; 10064 10065 for_each_possible_cpu(i) 10066 ws += schedstat_val(tg->se[i]->statistics.wait_sum); 10067 10068 seq_printf(sf, "wait_sum %llu\n", ws); 10069 } 10070 10071 return 0; 10072 } 10073 #endif /* CONFIG_CFS_BANDWIDTH */ 10074 #endif /* CONFIG_FAIR_GROUP_SCHED */ 10075 10076 #ifdef CONFIG_RT_GROUP_SCHED 10077 static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, 10078 struct cftype *cft, s64 val) 10079 { 10080 return sched_group_set_rt_runtime(css_tg(css), val); 10081 } 10082 10083 static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, 10084 struct cftype *cft) 10085 { 10086 return sched_group_rt_runtime(css_tg(css)); 10087 } 10088 10089 static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, 10090 struct cftype *cftype, u64 rt_period_us) 10091 { 10092 return sched_group_set_rt_period(css_tg(css), rt_period_us); 10093 } 10094 10095 static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, 10096 struct cftype *cft) 10097 { 10098 return sched_group_rt_period(css_tg(css)); 10099 } 10100 #endif /* CONFIG_RT_GROUP_SCHED */ 10101 10102 static struct cftype cpu_legacy_files[] = { 10103 #ifdef CONFIG_FAIR_GROUP_SCHED 10104 { 10105 .name = "shares", 10106 .read_u64 = cpu_shares_read_u64, 10107 .write_u64 = cpu_shares_write_u64, 10108 }, 10109 #endif 10110 #ifdef CONFIG_CFS_BANDWIDTH 10111 { 10112 .name = "cfs_quota_us", 10113 .read_s64 = cpu_cfs_quota_read_s64, 10114 .write_s64 = cpu_cfs_quota_write_s64, 10115 }, 10116 { 10117 .name = "cfs_period_us", 10118 .read_u64 = cpu_cfs_period_read_u64, 10119 .write_u64 = cpu_cfs_period_write_u64, 10120 }, 10121 { 10122 .name = "cfs_burst_us", 10123 .read_u64 = cpu_cfs_burst_read_u64, 10124 .write_u64 = cpu_cfs_burst_write_u64, 10125 }, 10126 { 10127 .name = "stat", 10128 .seq_show = cpu_cfs_stat_show, 10129 }, 10130 #endif 10131 #ifdef CONFIG_RT_GROUP_SCHED 10132 { 10133 .name = "rt_runtime_us", 10134 .read_s64 = cpu_rt_runtime_read, 10135 .write_s64 = cpu_rt_runtime_write, 10136 }, 10137 { 10138 .name = "rt_period_us", 10139 .read_u64 = cpu_rt_period_read_uint, 10140 .write_u64 = cpu_rt_period_write_uint, 10141 }, 10142 #endif 10143 #ifdef CONFIG_UCLAMP_TASK_GROUP 10144 { 10145 .name = "uclamp.min", 10146 .flags = CFTYPE_NOT_ON_ROOT, 10147 .seq_show = cpu_uclamp_min_show, 10148 .write = cpu_uclamp_min_write, 10149 }, 10150 { 10151 .name = "uclamp.max", 10152 .flags = CFTYPE_NOT_ON_ROOT, 10153 .seq_show = cpu_uclamp_max_show, 10154 .write = cpu_uclamp_max_write, 10155 }, 10156 #endif 10157 { } /* Terminate */ 10158 }; 10159 10160 static int cpu_extra_stat_show(struct seq_file *sf, 10161 struct cgroup_subsys_state *css) 10162 { 10163 #ifdef CONFIG_CFS_BANDWIDTH 10164 { 10165 struct task_group *tg = css_tg(css); 10166 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 10167 u64 throttled_usec; 10168 10169 throttled_usec = cfs_b->throttled_time; 10170 do_div(throttled_usec, NSEC_PER_USEC); 10171 10172 seq_printf(sf, "nr_periods %d\n" 10173 "nr_throttled %d\n" 10174 "throttled_usec %llu\n", 10175 cfs_b->nr_periods, cfs_b->nr_throttled, 10176 throttled_usec); 10177 } 10178 #endif 10179 return 0; 10180 } 10181 10182 #ifdef CONFIG_FAIR_GROUP_SCHED 10183 static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, 10184 struct cftype *cft) 10185 { 10186 struct task_group *tg = css_tg(css); 10187 u64 weight = scale_load_down(tg->shares); 10188 10189 return DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024); 10190 } 10191 10192 static int cpu_weight_write_u64(struct cgroup_subsys_state *css, 10193 struct cftype *cft, u64 weight) 10194 { 10195 /* 10196 * cgroup weight knobs should use the common MIN, DFL and MAX 10197 * values which are 1, 100 and 10000 respectively. While it loses 10198 * a bit of range on both ends, it maps pretty well onto the shares 10199 * value used by scheduler and the round-trip conversions preserve 10200 * the original value over the entire range. 10201 */ 10202 if (weight < CGROUP_WEIGHT_MIN || weight > CGROUP_WEIGHT_MAX) 10203 return -ERANGE; 10204 10205 weight = DIV_ROUND_CLOSEST_ULL(weight * 1024, CGROUP_WEIGHT_DFL); 10206 10207 return sched_group_set_shares(css_tg(css), scale_load(weight)); 10208 } 10209 10210 static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css, 10211 struct cftype *cft) 10212 { 10213 unsigned long weight = scale_load_down(css_tg(css)->shares); 10214 int last_delta = INT_MAX; 10215 int prio, delta; 10216 10217 /* find the closest nice value to the current weight */ 10218 for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) { 10219 delta = abs(sched_prio_to_weight[prio] - weight); 10220 if (delta >= last_delta) 10221 break; 10222 last_delta = delta; 10223 } 10224 10225 return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO); 10226 } 10227 10228 static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css, 10229 struct cftype *cft, s64 nice) 10230 { 10231 unsigned long weight; 10232 int idx; 10233 10234 if (nice < MIN_NICE || nice > MAX_NICE) 10235 return -ERANGE; 10236 10237 idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO; 10238 idx = array_index_nospec(idx, 40); 10239 weight = sched_prio_to_weight[idx]; 10240 10241 return sched_group_set_shares(css_tg(css), scale_load(weight)); 10242 } 10243 #endif 10244 10245 static void __maybe_unused cpu_period_quota_print(struct seq_file *sf, 10246 long period, long quota) 10247 { 10248 if (quota < 0) 10249 seq_puts(sf, "max"); 10250 else 10251 seq_printf(sf, "%ld", quota); 10252 10253 seq_printf(sf, " %ld\n", period); 10254 } 10255 10256 /* caller should put the current value in *@periodp before calling */ 10257 static int __maybe_unused cpu_period_quota_parse(char *buf, 10258 u64 *periodp, u64 *quotap) 10259 { 10260 char tok[21]; /* U64_MAX */ 10261 10262 if (sscanf(buf, "%20s %llu", tok, periodp) < 1) 10263 return -EINVAL; 10264 10265 *periodp *= NSEC_PER_USEC; 10266 10267 if (sscanf(tok, "%llu", quotap)) 10268 *quotap *= NSEC_PER_USEC; 10269 else if (!strcmp(tok, "max")) 10270 *quotap = RUNTIME_INF; 10271 else 10272 return -EINVAL; 10273 10274 return 0; 10275 } 10276 10277 #ifdef CONFIG_CFS_BANDWIDTH 10278 static int cpu_max_show(struct seq_file *sf, void *v) 10279 { 10280 struct task_group *tg = css_tg(seq_css(sf)); 10281 10282 cpu_period_quota_print(sf, tg_get_cfs_period(tg), tg_get_cfs_quota(tg)); 10283 return 0; 10284 } 10285 10286 static ssize_t cpu_max_write(struct kernfs_open_file *of, 10287 char *buf, size_t nbytes, loff_t off) 10288 { 10289 struct task_group *tg = css_tg(of_css(of)); 10290 u64 period = tg_get_cfs_period(tg); 10291 u64 burst = tg_get_cfs_burst(tg); 10292 u64 quota; 10293 int ret; 10294 10295 ret = cpu_period_quota_parse(buf, &period, "a); 10296 if (!ret) 10297 ret = tg_set_cfs_bandwidth(tg, period, quota, burst); 10298 return ret ?: nbytes; 10299 } 10300 #endif 10301 10302 static struct cftype cpu_files[] = { 10303 #ifdef CONFIG_FAIR_GROUP_SCHED 10304 { 10305 .name = "weight", 10306 .flags = CFTYPE_NOT_ON_ROOT, 10307 .read_u64 = cpu_weight_read_u64, 10308 .write_u64 = cpu_weight_write_u64, 10309 }, 10310 { 10311 .name = "weight.nice", 10312 .flags = CFTYPE_NOT_ON_ROOT, 10313 .read_s64 = cpu_weight_nice_read_s64, 10314 .write_s64 = cpu_weight_nice_write_s64, 10315 }, 10316 #endif 10317 #ifdef CONFIG_CFS_BANDWIDTH 10318 { 10319 .name = "max", 10320 .flags = CFTYPE_NOT_ON_ROOT, 10321 .seq_show = cpu_max_show, 10322 .write = cpu_max_write, 10323 }, 10324 { 10325 .name = "max.burst", 10326 .flags = CFTYPE_NOT_ON_ROOT, 10327 .read_u64 = cpu_cfs_burst_read_u64, 10328 .write_u64 = cpu_cfs_burst_write_u64, 10329 }, 10330 #endif 10331 #ifdef CONFIG_UCLAMP_TASK_GROUP 10332 { 10333 .name = "uclamp.min", 10334 .flags = CFTYPE_NOT_ON_ROOT, 10335 .seq_show = cpu_uclamp_min_show, 10336 .write = cpu_uclamp_min_write, 10337 }, 10338 { 10339 .name = "uclamp.max", 10340 .flags = CFTYPE_NOT_ON_ROOT, 10341 .seq_show = cpu_uclamp_max_show, 10342 .write = cpu_uclamp_max_write, 10343 }, 10344 #endif 10345 { } /* terminate */ 10346 }; 10347 10348 struct cgroup_subsys cpu_cgrp_subsys = { 10349 .css_alloc = cpu_cgroup_css_alloc, 10350 .css_online = cpu_cgroup_css_online, 10351 .css_released = cpu_cgroup_css_released, 10352 .css_free = cpu_cgroup_css_free, 10353 .css_extra_stat_show = cpu_extra_stat_show, 10354 .fork = cpu_cgroup_fork, 10355 .can_attach = cpu_cgroup_can_attach, 10356 .attach = cpu_cgroup_attach, 10357 .legacy_cftypes = cpu_legacy_files, 10358 .dfl_cftypes = cpu_files, 10359 .early_init = true, 10360 .threaded = true, 10361 }; 10362 10363 #endif /* CONFIG_CGROUP_SCHED */ 10364 10365 void dump_cpu_task(int cpu) 10366 { 10367 pr_info("Task dump for CPU %d:\n", cpu); 10368 sched_show_task(cpu_curr(cpu)); 10369 } 10370 10371 /* 10372 * Nice levels are multiplicative, with a gentle 10% change for every 10373 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to 10374 * nice 1, it will get ~10% less CPU time than another CPU-bound task 10375 * that remained on nice 0. 10376 * 10377 * The "10% effect" is relative and cumulative: from _any_ nice level, 10378 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level 10379 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. 10380 * If a task goes up by ~10% and another task goes down by ~10% then 10381 * the relative distance between them is ~25%.) 10382 */ 10383 const int sched_prio_to_weight[40] = { 10384 /* -20 */ 88761, 71755, 56483, 46273, 36291, 10385 /* -15 */ 29154, 23254, 18705, 14949, 11916, 10386 /* -10 */ 9548, 7620, 6100, 4904, 3906, 10387 /* -5 */ 3121, 2501, 1991, 1586, 1277, 10388 /* 0 */ 1024, 820, 655, 526, 423, 10389 /* 5 */ 335, 272, 215, 172, 137, 10390 /* 10 */ 110, 87, 70, 56, 45, 10391 /* 15 */ 36, 29, 23, 18, 15, 10392 }; 10393 10394 /* 10395 * Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated. 10396 * 10397 * In cases where the weight does not change often, we can use the 10398 * precalculated inverse to speed up arithmetics by turning divisions 10399 * into multiplications: 10400 */ 10401 const u32 sched_prio_to_wmult[40] = { 10402 /* -20 */ 48388, 59856, 76040, 92818, 118348, 10403 /* -15 */ 147320, 184698, 229616, 287308, 360437, 10404 /* -10 */ 449829, 563644, 704093, 875809, 1099582, 10405 /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, 10406 /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, 10407 /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, 10408 /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, 10409 /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, 10410 }; 10411 10412 void call_trace_sched_update_nr_running(struct rq *rq, int count) 10413 { 10414 trace_sched_update_nr_running_tp(rq, count); 10415 } 10416