1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR 4 * policies) 5 */ 6 7 int sched_rr_timeslice = RR_TIMESLICE; 8 /* More than 4 hours if BW_SHIFT equals 20. */ 9 static const u64 max_rt_runtime = MAX_BW; 10 11 static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun); 12 13 struct rt_bandwidth def_rt_bandwidth; 14 15 /* 16 * period over which we measure -rt task CPU usage in us. 17 * default: 1s 18 */ 19 int sysctl_sched_rt_period = 1000000; 20 21 /* 22 * part of the period that we allow rt tasks to run in us. 23 * default: 0.95s 24 */ 25 int sysctl_sched_rt_runtime = 950000; 26 27 #ifdef CONFIG_SYSCTL 28 static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC * RR_TIMESLICE) / HZ; 29 static int sched_rt_handler(struct ctl_table *table, int write, void *buffer, 30 size_t *lenp, loff_t *ppos); 31 static int sched_rr_handler(struct ctl_table *table, int write, void *buffer, 32 size_t *lenp, loff_t *ppos); 33 static struct ctl_table sched_rt_sysctls[] = { 34 { 35 .procname = "sched_rt_period_us", 36 .data = &sysctl_sched_rt_period, 37 .maxlen = sizeof(int), 38 .mode = 0644, 39 .proc_handler = sched_rt_handler, 40 .extra1 = SYSCTL_ONE, 41 .extra2 = SYSCTL_INT_MAX, 42 }, 43 { 44 .procname = "sched_rt_runtime_us", 45 .data = &sysctl_sched_rt_runtime, 46 .maxlen = sizeof(int), 47 .mode = 0644, 48 .proc_handler = sched_rt_handler, 49 .extra1 = SYSCTL_NEG_ONE, 50 .extra2 = (void *)&sysctl_sched_rt_period, 51 }, 52 { 53 .procname = "sched_rr_timeslice_ms", 54 .data = &sysctl_sched_rr_timeslice, 55 .maxlen = sizeof(int), 56 .mode = 0644, 57 .proc_handler = sched_rr_handler, 58 }, 59 {} 60 }; 61 62 static int __init sched_rt_sysctl_init(void) 63 { 64 register_sysctl_init("kernel", sched_rt_sysctls); 65 return 0; 66 } 67 late_initcall(sched_rt_sysctl_init); 68 #endif 69 70 static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer) 71 { 72 struct rt_bandwidth *rt_b = 73 container_of(timer, struct rt_bandwidth, rt_period_timer); 74 int idle = 0; 75 int overrun; 76 77 raw_spin_lock(&rt_b->rt_runtime_lock); 78 for (;;) { 79 overrun = hrtimer_forward_now(timer, rt_b->rt_period); 80 if (!overrun) 81 break; 82 83 raw_spin_unlock(&rt_b->rt_runtime_lock); 84 idle = do_sched_rt_period_timer(rt_b, overrun); 85 raw_spin_lock(&rt_b->rt_runtime_lock); 86 } 87 if (idle) 88 rt_b->rt_period_active = 0; 89 raw_spin_unlock(&rt_b->rt_runtime_lock); 90 91 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 92 } 93 94 void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime) 95 { 96 rt_b->rt_period = ns_to_ktime(period); 97 rt_b->rt_runtime = runtime; 98 99 raw_spin_lock_init(&rt_b->rt_runtime_lock); 100 101 hrtimer_init(&rt_b->rt_period_timer, CLOCK_MONOTONIC, 102 HRTIMER_MODE_REL_HARD); 103 rt_b->rt_period_timer.function = sched_rt_period_timer; 104 } 105 106 static inline void do_start_rt_bandwidth(struct rt_bandwidth *rt_b) 107 { 108 raw_spin_lock(&rt_b->rt_runtime_lock); 109 if (!rt_b->rt_period_active) { 110 rt_b->rt_period_active = 1; 111 /* 112 * SCHED_DEADLINE updates the bandwidth, as a run away 113 * RT task with a DL task could hog a CPU. But DL does 114 * not reset the period. If a deadline task was running 115 * without an RT task running, it can cause RT tasks to 116 * throttle when they start up. Kick the timer right away 117 * to update the period. 118 */ 119 hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0)); 120 hrtimer_start_expires(&rt_b->rt_period_timer, 121 HRTIMER_MODE_ABS_PINNED_HARD); 122 } 123 raw_spin_unlock(&rt_b->rt_runtime_lock); 124 } 125 126 static void start_rt_bandwidth(struct rt_bandwidth *rt_b) 127 { 128 if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF) 129 return; 130 131 do_start_rt_bandwidth(rt_b); 132 } 133 134 void init_rt_rq(struct rt_rq *rt_rq) 135 { 136 struct rt_prio_array *array; 137 int i; 138 139 array = &rt_rq->active; 140 for (i = 0; i < MAX_RT_PRIO; i++) { 141 INIT_LIST_HEAD(array->queue + i); 142 __clear_bit(i, array->bitmap); 143 } 144 /* delimiter for bitsearch: */ 145 __set_bit(MAX_RT_PRIO, array->bitmap); 146 147 #if defined CONFIG_SMP 148 rt_rq->highest_prio.curr = MAX_RT_PRIO-1; 149 rt_rq->highest_prio.next = MAX_RT_PRIO-1; 150 rt_rq->overloaded = 0; 151 plist_head_init(&rt_rq->pushable_tasks); 152 #endif /* CONFIG_SMP */ 153 /* We start is dequeued state, because no RT tasks are queued */ 154 rt_rq->rt_queued = 0; 155 156 rt_rq->rt_time = 0; 157 rt_rq->rt_throttled = 0; 158 rt_rq->rt_runtime = 0; 159 raw_spin_lock_init(&rt_rq->rt_runtime_lock); 160 } 161 162 #ifdef CONFIG_RT_GROUP_SCHED 163 static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b) 164 { 165 hrtimer_cancel(&rt_b->rt_period_timer); 166 } 167 168 #define rt_entity_is_task(rt_se) (!(rt_se)->my_q) 169 170 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) 171 { 172 #ifdef CONFIG_SCHED_DEBUG 173 WARN_ON_ONCE(!rt_entity_is_task(rt_se)); 174 #endif 175 return container_of(rt_se, struct task_struct, rt); 176 } 177 178 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) 179 { 180 return rt_rq->rq; 181 } 182 183 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) 184 { 185 return rt_se->rt_rq; 186 } 187 188 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) 189 { 190 struct rt_rq *rt_rq = rt_se->rt_rq; 191 192 return rt_rq->rq; 193 } 194 195 void unregister_rt_sched_group(struct task_group *tg) 196 { 197 if (tg->rt_se) 198 destroy_rt_bandwidth(&tg->rt_bandwidth); 199 200 } 201 202 void free_rt_sched_group(struct task_group *tg) 203 { 204 int i; 205 206 for_each_possible_cpu(i) { 207 if (tg->rt_rq) 208 kfree(tg->rt_rq[i]); 209 if (tg->rt_se) 210 kfree(tg->rt_se[i]); 211 } 212 213 kfree(tg->rt_rq); 214 kfree(tg->rt_se); 215 } 216 217 void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq, 218 struct sched_rt_entity *rt_se, int cpu, 219 struct sched_rt_entity *parent) 220 { 221 struct rq *rq = cpu_rq(cpu); 222 223 rt_rq->highest_prio.curr = MAX_RT_PRIO-1; 224 rt_rq->rt_nr_boosted = 0; 225 rt_rq->rq = rq; 226 rt_rq->tg = tg; 227 228 tg->rt_rq[cpu] = rt_rq; 229 tg->rt_se[cpu] = rt_se; 230 231 if (!rt_se) 232 return; 233 234 if (!parent) 235 rt_se->rt_rq = &rq->rt; 236 else 237 rt_se->rt_rq = parent->my_q; 238 239 rt_se->my_q = rt_rq; 240 rt_se->parent = parent; 241 INIT_LIST_HEAD(&rt_se->run_list); 242 } 243 244 int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) 245 { 246 struct rt_rq *rt_rq; 247 struct sched_rt_entity *rt_se; 248 int i; 249 250 tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL); 251 if (!tg->rt_rq) 252 goto err; 253 tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL); 254 if (!tg->rt_se) 255 goto err; 256 257 init_rt_bandwidth(&tg->rt_bandwidth, 258 ktime_to_ns(def_rt_bandwidth.rt_period), 0); 259 260 for_each_possible_cpu(i) { 261 rt_rq = kzalloc_node(sizeof(struct rt_rq), 262 GFP_KERNEL, cpu_to_node(i)); 263 if (!rt_rq) 264 goto err; 265 266 rt_se = kzalloc_node(sizeof(struct sched_rt_entity), 267 GFP_KERNEL, cpu_to_node(i)); 268 if (!rt_se) 269 goto err_free_rq; 270 271 init_rt_rq(rt_rq); 272 rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime; 273 init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]); 274 } 275 276 return 1; 277 278 err_free_rq: 279 kfree(rt_rq); 280 err: 281 return 0; 282 } 283 284 #else /* CONFIG_RT_GROUP_SCHED */ 285 286 #define rt_entity_is_task(rt_se) (1) 287 288 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) 289 { 290 return container_of(rt_se, struct task_struct, rt); 291 } 292 293 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) 294 { 295 return container_of(rt_rq, struct rq, rt); 296 } 297 298 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) 299 { 300 struct task_struct *p = rt_task_of(rt_se); 301 302 return task_rq(p); 303 } 304 305 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) 306 { 307 struct rq *rq = rq_of_rt_se(rt_se); 308 309 return &rq->rt; 310 } 311 312 void unregister_rt_sched_group(struct task_group *tg) { } 313 314 void free_rt_sched_group(struct task_group *tg) { } 315 316 int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) 317 { 318 return 1; 319 } 320 #endif /* CONFIG_RT_GROUP_SCHED */ 321 322 #ifdef CONFIG_SMP 323 324 static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev) 325 { 326 /* Try to pull RT tasks here if we lower this rq's prio */ 327 return rq->online && rq->rt.highest_prio.curr > prev->prio; 328 } 329 330 static inline int rt_overloaded(struct rq *rq) 331 { 332 return atomic_read(&rq->rd->rto_count); 333 } 334 335 static inline void rt_set_overload(struct rq *rq) 336 { 337 if (!rq->online) 338 return; 339 340 cpumask_set_cpu(rq->cpu, rq->rd->rto_mask); 341 /* 342 * Make sure the mask is visible before we set 343 * the overload count. That is checked to determine 344 * if we should look at the mask. It would be a shame 345 * if we looked at the mask, but the mask was not 346 * updated yet. 347 * 348 * Matched by the barrier in pull_rt_task(). 349 */ 350 smp_wmb(); 351 atomic_inc(&rq->rd->rto_count); 352 } 353 354 static inline void rt_clear_overload(struct rq *rq) 355 { 356 if (!rq->online) 357 return; 358 359 /* the order here really doesn't matter */ 360 atomic_dec(&rq->rd->rto_count); 361 cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask); 362 } 363 364 static inline int has_pushable_tasks(struct rq *rq) 365 { 366 return !plist_head_empty(&rq->rt.pushable_tasks); 367 } 368 369 static DEFINE_PER_CPU(struct balance_callback, rt_push_head); 370 static DEFINE_PER_CPU(struct balance_callback, rt_pull_head); 371 372 static void push_rt_tasks(struct rq *); 373 static void pull_rt_task(struct rq *); 374 375 static inline void rt_queue_push_tasks(struct rq *rq) 376 { 377 if (!has_pushable_tasks(rq)) 378 return; 379 380 queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks); 381 } 382 383 static inline void rt_queue_pull_task(struct rq *rq) 384 { 385 queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task); 386 } 387 388 static void enqueue_pushable_task(struct rq *rq, struct task_struct *p) 389 { 390 plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 391 plist_node_init(&p->pushable_tasks, p->prio); 392 plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks); 393 394 /* Update the highest prio pushable task */ 395 if (p->prio < rq->rt.highest_prio.next) 396 rq->rt.highest_prio.next = p->prio; 397 398 if (!rq->rt.overloaded) { 399 rt_set_overload(rq); 400 rq->rt.overloaded = 1; 401 } 402 } 403 404 static void dequeue_pushable_task(struct rq *rq, struct task_struct *p) 405 { 406 plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 407 408 /* Update the new highest prio pushable task */ 409 if (has_pushable_tasks(rq)) { 410 p = plist_first_entry(&rq->rt.pushable_tasks, 411 struct task_struct, pushable_tasks); 412 rq->rt.highest_prio.next = p->prio; 413 } else { 414 rq->rt.highest_prio.next = MAX_RT_PRIO-1; 415 416 if (rq->rt.overloaded) { 417 rt_clear_overload(rq); 418 rq->rt.overloaded = 0; 419 } 420 } 421 } 422 423 #else 424 425 static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p) 426 { 427 } 428 429 static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p) 430 { 431 } 432 433 static inline void rt_queue_push_tasks(struct rq *rq) 434 { 435 } 436 #endif /* CONFIG_SMP */ 437 438 static void enqueue_top_rt_rq(struct rt_rq *rt_rq); 439 static void dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count); 440 441 static inline int on_rt_rq(struct sched_rt_entity *rt_se) 442 { 443 return rt_se->on_rq; 444 } 445 446 #ifdef CONFIG_UCLAMP_TASK 447 /* 448 * Verify the fitness of task @p to run on @cpu taking into account the uclamp 449 * settings. 450 * 451 * This check is only important for heterogeneous systems where uclamp_min value 452 * is higher than the capacity of a @cpu. For non-heterogeneous system this 453 * function will always return true. 454 * 455 * The function will return true if the capacity of the @cpu is >= the 456 * uclamp_min and false otherwise. 457 * 458 * Note that uclamp_min will be clamped to uclamp_max if uclamp_min 459 * > uclamp_max. 460 */ 461 static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu) 462 { 463 unsigned int min_cap; 464 unsigned int max_cap; 465 unsigned int cpu_cap; 466 467 /* Only heterogeneous systems can benefit from this check */ 468 if (!sched_asym_cpucap_active()) 469 return true; 470 471 min_cap = uclamp_eff_value(p, UCLAMP_MIN); 472 max_cap = uclamp_eff_value(p, UCLAMP_MAX); 473 474 cpu_cap = arch_scale_cpu_capacity(cpu); 475 476 return cpu_cap >= min(min_cap, max_cap); 477 } 478 #else 479 static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu) 480 { 481 return true; 482 } 483 #endif 484 485 #ifdef CONFIG_RT_GROUP_SCHED 486 487 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) 488 { 489 if (!rt_rq->tg) 490 return RUNTIME_INF; 491 492 return rt_rq->rt_runtime; 493 } 494 495 static inline u64 sched_rt_period(struct rt_rq *rt_rq) 496 { 497 return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period); 498 } 499 500 typedef struct task_group *rt_rq_iter_t; 501 502 static inline struct task_group *next_task_group(struct task_group *tg) 503 { 504 do { 505 tg = list_entry_rcu(tg->list.next, 506 typeof(struct task_group), list); 507 } while (&tg->list != &task_groups && task_group_is_autogroup(tg)); 508 509 if (&tg->list == &task_groups) 510 tg = NULL; 511 512 return tg; 513 } 514 515 #define for_each_rt_rq(rt_rq, iter, rq) \ 516 for (iter = container_of(&task_groups, typeof(*iter), list); \ 517 (iter = next_task_group(iter)) && \ 518 (rt_rq = iter->rt_rq[cpu_of(rq)]);) 519 520 #define for_each_sched_rt_entity(rt_se) \ 521 for (; rt_se; rt_se = rt_se->parent) 522 523 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) 524 { 525 return rt_se->my_q; 526 } 527 528 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags); 529 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags); 530 531 static void sched_rt_rq_enqueue(struct rt_rq *rt_rq) 532 { 533 struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr; 534 struct rq *rq = rq_of_rt_rq(rt_rq); 535 struct sched_rt_entity *rt_se; 536 537 int cpu = cpu_of(rq); 538 539 rt_se = rt_rq->tg->rt_se[cpu]; 540 541 if (rt_rq->rt_nr_running) { 542 if (!rt_se) 543 enqueue_top_rt_rq(rt_rq); 544 else if (!on_rt_rq(rt_se)) 545 enqueue_rt_entity(rt_se, 0); 546 547 if (rt_rq->highest_prio.curr < curr->prio) 548 resched_curr(rq); 549 } 550 } 551 552 static void sched_rt_rq_dequeue(struct rt_rq *rt_rq) 553 { 554 struct sched_rt_entity *rt_se; 555 int cpu = cpu_of(rq_of_rt_rq(rt_rq)); 556 557 rt_se = rt_rq->tg->rt_se[cpu]; 558 559 if (!rt_se) { 560 dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running); 561 /* Kick cpufreq (see the comment in kernel/sched/sched.h). */ 562 cpufreq_update_util(rq_of_rt_rq(rt_rq), 0); 563 } 564 else if (on_rt_rq(rt_se)) 565 dequeue_rt_entity(rt_se, 0); 566 } 567 568 static inline int rt_rq_throttled(struct rt_rq *rt_rq) 569 { 570 return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted; 571 } 572 573 static int rt_se_boosted(struct sched_rt_entity *rt_se) 574 { 575 struct rt_rq *rt_rq = group_rt_rq(rt_se); 576 struct task_struct *p; 577 578 if (rt_rq) 579 return !!rt_rq->rt_nr_boosted; 580 581 p = rt_task_of(rt_se); 582 return p->prio != p->normal_prio; 583 } 584 585 #ifdef CONFIG_SMP 586 static inline const struct cpumask *sched_rt_period_mask(void) 587 { 588 return this_rq()->rd->span; 589 } 590 #else 591 static inline const struct cpumask *sched_rt_period_mask(void) 592 { 593 return cpu_online_mask; 594 } 595 #endif 596 597 static inline 598 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu) 599 { 600 return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu]; 601 } 602 603 static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq) 604 { 605 return &rt_rq->tg->rt_bandwidth; 606 } 607 608 #else /* !CONFIG_RT_GROUP_SCHED */ 609 610 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) 611 { 612 return rt_rq->rt_runtime; 613 } 614 615 static inline u64 sched_rt_period(struct rt_rq *rt_rq) 616 { 617 return ktime_to_ns(def_rt_bandwidth.rt_period); 618 } 619 620 typedef struct rt_rq *rt_rq_iter_t; 621 622 #define for_each_rt_rq(rt_rq, iter, rq) \ 623 for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL) 624 625 #define for_each_sched_rt_entity(rt_se) \ 626 for (; rt_se; rt_se = NULL) 627 628 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) 629 { 630 return NULL; 631 } 632 633 static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq) 634 { 635 struct rq *rq = rq_of_rt_rq(rt_rq); 636 637 if (!rt_rq->rt_nr_running) 638 return; 639 640 enqueue_top_rt_rq(rt_rq); 641 resched_curr(rq); 642 } 643 644 static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq) 645 { 646 dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running); 647 } 648 649 static inline int rt_rq_throttled(struct rt_rq *rt_rq) 650 { 651 return rt_rq->rt_throttled; 652 } 653 654 static inline const struct cpumask *sched_rt_period_mask(void) 655 { 656 return cpu_online_mask; 657 } 658 659 static inline 660 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu) 661 { 662 return &cpu_rq(cpu)->rt; 663 } 664 665 static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq) 666 { 667 return &def_rt_bandwidth; 668 } 669 670 #endif /* CONFIG_RT_GROUP_SCHED */ 671 672 bool sched_rt_bandwidth_account(struct rt_rq *rt_rq) 673 { 674 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); 675 676 return (hrtimer_active(&rt_b->rt_period_timer) || 677 rt_rq->rt_time < rt_b->rt_runtime); 678 } 679 680 #ifdef CONFIG_SMP 681 /* 682 * We ran out of runtime, see if we can borrow some from our neighbours. 683 */ 684 static void do_balance_runtime(struct rt_rq *rt_rq) 685 { 686 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); 687 struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd; 688 int i, weight; 689 u64 rt_period; 690 691 weight = cpumask_weight(rd->span); 692 693 raw_spin_lock(&rt_b->rt_runtime_lock); 694 rt_period = ktime_to_ns(rt_b->rt_period); 695 for_each_cpu(i, rd->span) { 696 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); 697 s64 diff; 698 699 if (iter == rt_rq) 700 continue; 701 702 raw_spin_lock(&iter->rt_runtime_lock); 703 /* 704 * Either all rqs have inf runtime and there's nothing to steal 705 * or __disable_runtime() below sets a specific rq to inf to 706 * indicate its been disabled and disallow stealing. 707 */ 708 if (iter->rt_runtime == RUNTIME_INF) 709 goto next; 710 711 /* 712 * From runqueues with spare time, take 1/n part of their 713 * spare time, but no more than our period. 714 */ 715 diff = iter->rt_runtime - iter->rt_time; 716 if (diff > 0) { 717 diff = div_u64((u64)diff, weight); 718 if (rt_rq->rt_runtime + diff > rt_period) 719 diff = rt_period - rt_rq->rt_runtime; 720 iter->rt_runtime -= diff; 721 rt_rq->rt_runtime += diff; 722 if (rt_rq->rt_runtime == rt_period) { 723 raw_spin_unlock(&iter->rt_runtime_lock); 724 break; 725 } 726 } 727 next: 728 raw_spin_unlock(&iter->rt_runtime_lock); 729 } 730 raw_spin_unlock(&rt_b->rt_runtime_lock); 731 } 732 733 /* 734 * Ensure this RQ takes back all the runtime it lend to its neighbours. 735 */ 736 static void __disable_runtime(struct rq *rq) 737 { 738 struct root_domain *rd = rq->rd; 739 rt_rq_iter_t iter; 740 struct rt_rq *rt_rq; 741 742 if (unlikely(!scheduler_running)) 743 return; 744 745 for_each_rt_rq(rt_rq, iter, rq) { 746 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); 747 s64 want; 748 int i; 749 750 raw_spin_lock(&rt_b->rt_runtime_lock); 751 raw_spin_lock(&rt_rq->rt_runtime_lock); 752 /* 753 * Either we're all inf and nobody needs to borrow, or we're 754 * already disabled and thus have nothing to do, or we have 755 * exactly the right amount of runtime to take out. 756 */ 757 if (rt_rq->rt_runtime == RUNTIME_INF || 758 rt_rq->rt_runtime == rt_b->rt_runtime) 759 goto balanced; 760 raw_spin_unlock(&rt_rq->rt_runtime_lock); 761 762 /* 763 * Calculate the difference between what we started out with 764 * and what we current have, that's the amount of runtime 765 * we lend and now have to reclaim. 766 */ 767 want = rt_b->rt_runtime - rt_rq->rt_runtime; 768 769 /* 770 * Greedy reclaim, take back as much as we can. 771 */ 772 for_each_cpu(i, rd->span) { 773 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); 774 s64 diff; 775 776 /* 777 * Can't reclaim from ourselves or disabled runqueues. 778 */ 779 if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF) 780 continue; 781 782 raw_spin_lock(&iter->rt_runtime_lock); 783 if (want > 0) { 784 diff = min_t(s64, iter->rt_runtime, want); 785 iter->rt_runtime -= diff; 786 want -= diff; 787 } else { 788 iter->rt_runtime -= want; 789 want -= want; 790 } 791 raw_spin_unlock(&iter->rt_runtime_lock); 792 793 if (!want) 794 break; 795 } 796 797 raw_spin_lock(&rt_rq->rt_runtime_lock); 798 /* 799 * We cannot be left wanting - that would mean some runtime 800 * leaked out of the system. 801 */ 802 WARN_ON_ONCE(want); 803 balanced: 804 /* 805 * Disable all the borrow logic by pretending we have inf 806 * runtime - in which case borrowing doesn't make sense. 807 */ 808 rt_rq->rt_runtime = RUNTIME_INF; 809 rt_rq->rt_throttled = 0; 810 raw_spin_unlock(&rt_rq->rt_runtime_lock); 811 raw_spin_unlock(&rt_b->rt_runtime_lock); 812 813 /* Make rt_rq available for pick_next_task() */ 814 sched_rt_rq_enqueue(rt_rq); 815 } 816 } 817 818 static void __enable_runtime(struct rq *rq) 819 { 820 rt_rq_iter_t iter; 821 struct rt_rq *rt_rq; 822 823 if (unlikely(!scheduler_running)) 824 return; 825 826 /* 827 * Reset each runqueue's bandwidth settings 828 */ 829 for_each_rt_rq(rt_rq, iter, rq) { 830 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); 831 832 raw_spin_lock(&rt_b->rt_runtime_lock); 833 raw_spin_lock(&rt_rq->rt_runtime_lock); 834 rt_rq->rt_runtime = rt_b->rt_runtime; 835 rt_rq->rt_time = 0; 836 rt_rq->rt_throttled = 0; 837 raw_spin_unlock(&rt_rq->rt_runtime_lock); 838 raw_spin_unlock(&rt_b->rt_runtime_lock); 839 } 840 } 841 842 static void balance_runtime(struct rt_rq *rt_rq) 843 { 844 if (!sched_feat(RT_RUNTIME_SHARE)) 845 return; 846 847 if (rt_rq->rt_time > rt_rq->rt_runtime) { 848 raw_spin_unlock(&rt_rq->rt_runtime_lock); 849 do_balance_runtime(rt_rq); 850 raw_spin_lock(&rt_rq->rt_runtime_lock); 851 } 852 } 853 #else /* !CONFIG_SMP */ 854 static inline void balance_runtime(struct rt_rq *rt_rq) {} 855 #endif /* CONFIG_SMP */ 856 857 static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) 858 { 859 int i, idle = 1, throttled = 0; 860 const struct cpumask *span; 861 862 span = sched_rt_period_mask(); 863 #ifdef CONFIG_RT_GROUP_SCHED 864 /* 865 * FIXME: isolated CPUs should really leave the root task group, 866 * whether they are isolcpus or were isolated via cpusets, lest 867 * the timer run on a CPU which does not service all runqueues, 868 * potentially leaving other CPUs indefinitely throttled. If 869 * isolation is really required, the user will turn the throttle 870 * off to kill the perturbations it causes anyway. Meanwhile, 871 * this maintains functionality for boot and/or troubleshooting. 872 */ 873 if (rt_b == &root_task_group.rt_bandwidth) 874 span = cpu_online_mask; 875 #endif 876 for_each_cpu(i, span) { 877 int enqueue = 0; 878 struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i); 879 struct rq *rq = rq_of_rt_rq(rt_rq); 880 struct rq_flags rf; 881 int skip; 882 883 /* 884 * When span == cpu_online_mask, taking each rq->lock 885 * can be time-consuming. Try to avoid it when possible. 886 */ 887 raw_spin_lock(&rt_rq->rt_runtime_lock); 888 if (!sched_feat(RT_RUNTIME_SHARE) && rt_rq->rt_runtime != RUNTIME_INF) 889 rt_rq->rt_runtime = rt_b->rt_runtime; 890 skip = !rt_rq->rt_time && !rt_rq->rt_nr_running; 891 raw_spin_unlock(&rt_rq->rt_runtime_lock); 892 if (skip) 893 continue; 894 895 rq_lock(rq, &rf); 896 update_rq_clock(rq); 897 898 if (rt_rq->rt_time) { 899 u64 runtime; 900 901 raw_spin_lock(&rt_rq->rt_runtime_lock); 902 if (rt_rq->rt_throttled) 903 balance_runtime(rt_rq); 904 runtime = rt_rq->rt_runtime; 905 rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime); 906 if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) { 907 rt_rq->rt_throttled = 0; 908 enqueue = 1; 909 910 /* 911 * When we're idle and a woken (rt) task is 912 * throttled wakeup_preempt() will set 913 * skip_update and the time between the wakeup 914 * and this unthrottle will get accounted as 915 * 'runtime'. 916 */ 917 if (rt_rq->rt_nr_running && rq->curr == rq->idle) 918 rq_clock_cancel_skipupdate(rq); 919 } 920 if (rt_rq->rt_time || rt_rq->rt_nr_running) 921 idle = 0; 922 raw_spin_unlock(&rt_rq->rt_runtime_lock); 923 } else if (rt_rq->rt_nr_running) { 924 idle = 0; 925 if (!rt_rq_throttled(rt_rq)) 926 enqueue = 1; 927 } 928 if (rt_rq->rt_throttled) 929 throttled = 1; 930 931 if (enqueue) 932 sched_rt_rq_enqueue(rt_rq); 933 rq_unlock(rq, &rf); 934 } 935 936 if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)) 937 return 1; 938 939 return idle; 940 } 941 942 static inline int rt_se_prio(struct sched_rt_entity *rt_se) 943 { 944 #ifdef CONFIG_RT_GROUP_SCHED 945 struct rt_rq *rt_rq = group_rt_rq(rt_se); 946 947 if (rt_rq) 948 return rt_rq->highest_prio.curr; 949 #endif 950 951 return rt_task_of(rt_se)->prio; 952 } 953 954 static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) 955 { 956 u64 runtime = sched_rt_runtime(rt_rq); 957 958 if (rt_rq->rt_throttled) 959 return rt_rq_throttled(rt_rq); 960 961 if (runtime >= sched_rt_period(rt_rq)) 962 return 0; 963 964 balance_runtime(rt_rq); 965 runtime = sched_rt_runtime(rt_rq); 966 if (runtime == RUNTIME_INF) 967 return 0; 968 969 if (rt_rq->rt_time > runtime) { 970 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); 971 972 /* 973 * Don't actually throttle groups that have no runtime assigned 974 * but accrue some time due to boosting. 975 */ 976 if (likely(rt_b->rt_runtime)) { 977 rt_rq->rt_throttled = 1; 978 printk_deferred_once("sched: RT throttling activated\n"); 979 } else { 980 /* 981 * In case we did anyway, make it go away, 982 * replenishment is a joke, since it will replenish us 983 * with exactly 0 ns. 984 */ 985 rt_rq->rt_time = 0; 986 } 987 988 if (rt_rq_throttled(rt_rq)) { 989 sched_rt_rq_dequeue(rt_rq); 990 return 1; 991 } 992 } 993 994 return 0; 995 } 996 997 /* 998 * Update the current task's runtime statistics. Skip current tasks that 999 * are not in our scheduling class. 1000 */ 1001 static void update_curr_rt(struct rq *rq) 1002 { 1003 struct task_struct *curr = rq->curr; 1004 struct sched_rt_entity *rt_se = &curr->rt; 1005 u64 delta_exec; 1006 u64 now; 1007 1008 if (curr->sched_class != &rt_sched_class) 1009 return; 1010 1011 now = rq_clock_task(rq); 1012 delta_exec = now - curr->se.exec_start; 1013 if (unlikely((s64)delta_exec <= 0)) 1014 return; 1015 1016 schedstat_set(curr->stats.exec_max, 1017 max(curr->stats.exec_max, delta_exec)); 1018 1019 trace_sched_stat_runtime(curr, delta_exec, 0); 1020 1021 update_current_exec_runtime(curr, now, delta_exec); 1022 1023 if (!rt_bandwidth_enabled()) 1024 return; 1025 1026 for_each_sched_rt_entity(rt_se) { 1027 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); 1028 int exceeded; 1029 1030 if (sched_rt_runtime(rt_rq) != RUNTIME_INF) { 1031 raw_spin_lock(&rt_rq->rt_runtime_lock); 1032 rt_rq->rt_time += delta_exec; 1033 exceeded = sched_rt_runtime_exceeded(rt_rq); 1034 if (exceeded) 1035 resched_curr(rq); 1036 raw_spin_unlock(&rt_rq->rt_runtime_lock); 1037 if (exceeded) 1038 do_start_rt_bandwidth(sched_rt_bandwidth(rt_rq)); 1039 } 1040 } 1041 } 1042 1043 static void 1044 dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count) 1045 { 1046 struct rq *rq = rq_of_rt_rq(rt_rq); 1047 1048 BUG_ON(&rq->rt != rt_rq); 1049 1050 if (!rt_rq->rt_queued) 1051 return; 1052 1053 BUG_ON(!rq->nr_running); 1054 1055 sub_nr_running(rq, count); 1056 rt_rq->rt_queued = 0; 1057 1058 } 1059 1060 static void 1061 enqueue_top_rt_rq(struct rt_rq *rt_rq) 1062 { 1063 struct rq *rq = rq_of_rt_rq(rt_rq); 1064 1065 BUG_ON(&rq->rt != rt_rq); 1066 1067 if (rt_rq->rt_queued) 1068 return; 1069 1070 if (rt_rq_throttled(rt_rq)) 1071 return; 1072 1073 if (rt_rq->rt_nr_running) { 1074 add_nr_running(rq, rt_rq->rt_nr_running); 1075 rt_rq->rt_queued = 1; 1076 } 1077 1078 /* Kick cpufreq (see the comment in kernel/sched/sched.h). */ 1079 cpufreq_update_util(rq, 0); 1080 } 1081 1082 #if defined CONFIG_SMP 1083 1084 static void 1085 inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) 1086 { 1087 struct rq *rq = rq_of_rt_rq(rt_rq); 1088 1089 #ifdef CONFIG_RT_GROUP_SCHED 1090 /* 1091 * Change rq's cpupri only if rt_rq is the top queue. 1092 */ 1093 if (&rq->rt != rt_rq) 1094 return; 1095 #endif 1096 if (rq->online && prio < prev_prio) 1097 cpupri_set(&rq->rd->cpupri, rq->cpu, prio); 1098 } 1099 1100 static void 1101 dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) 1102 { 1103 struct rq *rq = rq_of_rt_rq(rt_rq); 1104 1105 #ifdef CONFIG_RT_GROUP_SCHED 1106 /* 1107 * Change rq's cpupri only if rt_rq is the top queue. 1108 */ 1109 if (&rq->rt != rt_rq) 1110 return; 1111 #endif 1112 if (rq->online && rt_rq->highest_prio.curr != prev_prio) 1113 cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr); 1114 } 1115 1116 #else /* CONFIG_SMP */ 1117 1118 static inline 1119 void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {} 1120 static inline 1121 void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {} 1122 1123 #endif /* CONFIG_SMP */ 1124 1125 #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED 1126 static void 1127 inc_rt_prio(struct rt_rq *rt_rq, int prio) 1128 { 1129 int prev_prio = rt_rq->highest_prio.curr; 1130 1131 if (prio < prev_prio) 1132 rt_rq->highest_prio.curr = prio; 1133 1134 inc_rt_prio_smp(rt_rq, prio, prev_prio); 1135 } 1136 1137 static void 1138 dec_rt_prio(struct rt_rq *rt_rq, int prio) 1139 { 1140 int prev_prio = rt_rq->highest_prio.curr; 1141 1142 if (rt_rq->rt_nr_running) { 1143 1144 WARN_ON(prio < prev_prio); 1145 1146 /* 1147 * This may have been our highest task, and therefore 1148 * we may have some recomputation to do 1149 */ 1150 if (prio == prev_prio) { 1151 struct rt_prio_array *array = &rt_rq->active; 1152 1153 rt_rq->highest_prio.curr = 1154 sched_find_first_bit(array->bitmap); 1155 } 1156 1157 } else { 1158 rt_rq->highest_prio.curr = MAX_RT_PRIO-1; 1159 } 1160 1161 dec_rt_prio_smp(rt_rq, prio, prev_prio); 1162 } 1163 1164 #else 1165 1166 static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {} 1167 static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {} 1168 1169 #endif /* CONFIG_SMP || CONFIG_RT_GROUP_SCHED */ 1170 1171 #ifdef CONFIG_RT_GROUP_SCHED 1172 1173 static void 1174 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 1175 { 1176 if (rt_se_boosted(rt_se)) 1177 rt_rq->rt_nr_boosted++; 1178 1179 if (rt_rq->tg) 1180 start_rt_bandwidth(&rt_rq->tg->rt_bandwidth); 1181 } 1182 1183 static void 1184 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 1185 { 1186 if (rt_se_boosted(rt_se)) 1187 rt_rq->rt_nr_boosted--; 1188 1189 WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted); 1190 } 1191 1192 #else /* CONFIG_RT_GROUP_SCHED */ 1193 1194 static void 1195 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 1196 { 1197 start_rt_bandwidth(&def_rt_bandwidth); 1198 } 1199 1200 static inline 1201 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {} 1202 1203 #endif /* CONFIG_RT_GROUP_SCHED */ 1204 1205 static inline 1206 unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se) 1207 { 1208 struct rt_rq *group_rq = group_rt_rq(rt_se); 1209 1210 if (group_rq) 1211 return group_rq->rt_nr_running; 1212 else 1213 return 1; 1214 } 1215 1216 static inline 1217 unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se) 1218 { 1219 struct rt_rq *group_rq = group_rt_rq(rt_se); 1220 struct task_struct *tsk; 1221 1222 if (group_rq) 1223 return group_rq->rr_nr_running; 1224 1225 tsk = rt_task_of(rt_se); 1226 1227 return (tsk->policy == SCHED_RR) ? 1 : 0; 1228 } 1229 1230 static inline 1231 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 1232 { 1233 int prio = rt_se_prio(rt_se); 1234 1235 WARN_ON(!rt_prio(prio)); 1236 rt_rq->rt_nr_running += rt_se_nr_running(rt_se); 1237 rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se); 1238 1239 inc_rt_prio(rt_rq, prio); 1240 inc_rt_group(rt_se, rt_rq); 1241 } 1242 1243 static inline 1244 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 1245 { 1246 WARN_ON(!rt_prio(rt_se_prio(rt_se))); 1247 WARN_ON(!rt_rq->rt_nr_running); 1248 rt_rq->rt_nr_running -= rt_se_nr_running(rt_se); 1249 rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se); 1250 1251 dec_rt_prio(rt_rq, rt_se_prio(rt_se)); 1252 dec_rt_group(rt_se, rt_rq); 1253 } 1254 1255 /* 1256 * Change rt_se->run_list location unless SAVE && !MOVE 1257 * 1258 * assumes ENQUEUE/DEQUEUE flags match 1259 */ 1260 static inline bool move_entity(unsigned int flags) 1261 { 1262 if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE) 1263 return false; 1264 1265 return true; 1266 } 1267 1268 static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array) 1269 { 1270 list_del_init(&rt_se->run_list); 1271 1272 if (list_empty(array->queue + rt_se_prio(rt_se))) 1273 __clear_bit(rt_se_prio(rt_se), array->bitmap); 1274 1275 rt_se->on_list = 0; 1276 } 1277 1278 static inline struct sched_statistics * 1279 __schedstats_from_rt_se(struct sched_rt_entity *rt_se) 1280 { 1281 #ifdef CONFIG_RT_GROUP_SCHED 1282 /* schedstats is not supported for rt group. */ 1283 if (!rt_entity_is_task(rt_se)) 1284 return NULL; 1285 #endif 1286 1287 return &rt_task_of(rt_se)->stats; 1288 } 1289 1290 static inline void 1291 update_stats_wait_start_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) 1292 { 1293 struct sched_statistics *stats; 1294 struct task_struct *p = NULL; 1295 1296 if (!schedstat_enabled()) 1297 return; 1298 1299 if (rt_entity_is_task(rt_se)) 1300 p = rt_task_of(rt_se); 1301 1302 stats = __schedstats_from_rt_se(rt_se); 1303 if (!stats) 1304 return; 1305 1306 __update_stats_wait_start(rq_of_rt_rq(rt_rq), p, stats); 1307 } 1308 1309 static inline void 1310 update_stats_enqueue_sleeper_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) 1311 { 1312 struct sched_statistics *stats; 1313 struct task_struct *p = NULL; 1314 1315 if (!schedstat_enabled()) 1316 return; 1317 1318 if (rt_entity_is_task(rt_se)) 1319 p = rt_task_of(rt_se); 1320 1321 stats = __schedstats_from_rt_se(rt_se); 1322 if (!stats) 1323 return; 1324 1325 __update_stats_enqueue_sleeper(rq_of_rt_rq(rt_rq), p, stats); 1326 } 1327 1328 static inline void 1329 update_stats_enqueue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, 1330 int flags) 1331 { 1332 if (!schedstat_enabled()) 1333 return; 1334 1335 if (flags & ENQUEUE_WAKEUP) 1336 update_stats_enqueue_sleeper_rt(rt_rq, rt_se); 1337 } 1338 1339 static inline void 1340 update_stats_wait_end_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) 1341 { 1342 struct sched_statistics *stats; 1343 struct task_struct *p = NULL; 1344 1345 if (!schedstat_enabled()) 1346 return; 1347 1348 if (rt_entity_is_task(rt_se)) 1349 p = rt_task_of(rt_se); 1350 1351 stats = __schedstats_from_rt_se(rt_se); 1352 if (!stats) 1353 return; 1354 1355 __update_stats_wait_end(rq_of_rt_rq(rt_rq), p, stats); 1356 } 1357 1358 static inline void 1359 update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, 1360 int flags) 1361 { 1362 struct task_struct *p = NULL; 1363 1364 if (!schedstat_enabled()) 1365 return; 1366 1367 if (rt_entity_is_task(rt_se)) 1368 p = rt_task_of(rt_se); 1369 1370 if ((flags & DEQUEUE_SLEEP) && p) { 1371 unsigned int state; 1372 1373 state = READ_ONCE(p->__state); 1374 if (state & TASK_INTERRUPTIBLE) 1375 __schedstat_set(p->stats.sleep_start, 1376 rq_clock(rq_of_rt_rq(rt_rq))); 1377 1378 if (state & TASK_UNINTERRUPTIBLE) 1379 __schedstat_set(p->stats.block_start, 1380 rq_clock(rq_of_rt_rq(rt_rq))); 1381 } 1382 } 1383 1384 static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) 1385 { 1386 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); 1387 struct rt_prio_array *array = &rt_rq->active; 1388 struct rt_rq *group_rq = group_rt_rq(rt_se); 1389 struct list_head *queue = array->queue + rt_se_prio(rt_se); 1390 1391 /* 1392 * Don't enqueue the group if its throttled, or when empty. 1393 * The latter is a consequence of the former when a child group 1394 * get throttled and the current group doesn't have any other 1395 * active members. 1396 */ 1397 if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) { 1398 if (rt_se->on_list) 1399 __delist_rt_entity(rt_se, array); 1400 return; 1401 } 1402 1403 if (move_entity(flags)) { 1404 WARN_ON_ONCE(rt_se->on_list); 1405 if (flags & ENQUEUE_HEAD) 1406 list_add(&rt_se->run_list, queue); 1407 else 1408 list_add_tail(&rt_se->run_list, queue); 1409 1410 __set_bit(rt_se_prio(rt_se), array->bitmap); 1411 rt_se->on_list = 1; 1412 } 1413 rt_se->on_rq = 1; 1414 1415 inc_rt_tasks(rt_se, rt_rq); 1416 } 1417 1418 static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) 1419 { 1420 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); 1421 struct rt_prio_array *array = &rt_rq->active; 1422 1423 if (move_entity(flags)) { 1424 WARN_ON_ONCE(!rt_se->on_list); 1425 __delist_rt_entity(rt_se, array); 1426 } 1427 rt_se->on_rq = 0; 1428 1429 dec_rt_tasks(rt_se, rt_rq); 1430 } 1431 1432 /* 1433 * Because the prio of an upper entry depends on the lower 1434 * entries, we must remove entries top - down. 1435 */ 1436 static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags) 1437 { 1438 struct sched_rt_entity *back = NULL; 1439 unsigned int rt_nr_running; 1440 1441 for_each_sched_rt_entity(rt_se) { 1442 rt_se->back = back; 1443 back = rt_se; 1444 } 1445 1446 rt_nr_running = rt_rq_of_se(back)->rt_nr_running; 1447 1448 for (rt_se = back; rt_se; rt_se = rt_se->back) { 1449 if (on_rt_rq(rt_se)) 1450 __dequeue_rt_entity(rt_se, flags); 1451 } 1452 1453 dequeue_top_rt_rq(rt_rq_of_se(back), rt_nr_running); 1454 } 1455 1456 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) 1457 { 1458 struct rq *rq = rq_of_rt_se(rt_se); 1459 1460 update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags); 1461 1462 dequeue_rt_stack(rt_se, flags); 1463 for_each_sched_rt_entity(rt_se) 1464 __enqueue_rt_entity(rt_se, flags); 1465 enqueue_top_rt_rq(&rq->rt); 1466 } 1467 1468 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) 1469 { 1470 struct rq *rq = rq_of_rt_se(rt_se); 1471 1472 update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags); 1473 1474 dequeue_rt_stack(rt_se, flags); 1475 1476 for_each_sched_rt_entity(rt_se) { 1477 struct rt_rq *rt_rq = group_rt_rq(rt_se); 1478 1479 if (rt_rq && rt_rq->rt_nr_running) 1480 __enqueue_rt_entity(rt_se, flags); 1481 } 1482 enqueue_top_rt_rq(&rq->rt); 1483 } 1484 1485 /* 1486 * Adding/removing a task to/from a priority array: 1487 */ 1488 static void 1489 enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags) 1490 { 1491 struct sched_rt_entity *rt_se = &p->rt; 1492 1493 if (flags & ENQUEUE_WAKEUP) 1494 rt_se->timeout = 0; 1495 1496 check_schedstat_required(); 1497 update_stats_wait_start_rt(rt_rq_of_se(rt_se), rt_se); 1498 1499 enqueue_rt_entity(rt_se, flags); 1500 1501 if (!task_current(rq, p) && p->nr_cpus_allowed > 1) 1502 enqueue_pushable_task(rq, p); 1503 } 1504 1505 static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) 1506 { 1507 struct sched_rt_entity *rt_se = &p->rt; 1508 1509 update_curr_rt(rq); 1510 dequeue_rt_entity(rt_se, flags); 1511 1512 dequeue_pushable_task(rq, p); 1513 } 1514 1515 /* 1516 * Put task to the head or the end of the run list without the overhead of 1517 * dequeue followed by enqueue. 1518 */ 1519 static void 1520 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head) 1521 { 1522 if (on_rt_rq(rt_se)) { 1523 struct rt_prio_array *array = &rt_rq->active; 1524 struct list_head *queue = array->queue + rt_se_prio(rt_se); 1525 1526 if (head) 1527 list_move(&rt_se->run_list, queue); 1528 else 1529 list_move_tail(&rt_se->run_list, queue); 1530 } 1531 } 1532 1533 static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head) 1534 { 1535 struct sched_rt_entity *rt_se = &p->rt; 1536 struct rt_rq *rt_rq; 1537 1538 for_each_sched_rt_entity(rt_se) { 1539 rt_rq = rt_rq_of_se(rt_se); 1540 requeue_rt_entity(rt_rq, rt_se, head); 1541 } 1542 } 1543 1544 static void yield_task_rt(struct rq *rq) 1545 { 1546 requeue_task_rt(rq, rq->curr, 0); 1547 } 1548 1549 #ifdef CONFIG_SMP 1550 static int find_lowest_rq(struct task_struct *task); 1551 1552 static int 1553 select_task_rq_rt(struct task_struct *p, int cpu, int flags) 1554 { 1555 struct task_struct *curr; 1556 struct rq *rq; 1557 bool test; 1558 1559 /* For anything but wake ups, just return the task_cpu */ 1560 if (!(flags & (WF_TTWU | WF_FORK))) 1561 goto out; 1562 1563 rq = cpu_rq(cpu); 1564 1565 rcu_read_lock(); 1566 curr = READ_ONCE(rq->curr); /* unlocked access */ 1567 1568 /* 1569 * If the current task on @p's runqueue is an RT task, then 1570 * try to see if we can wake this RT task up on another 1571 * runqueue. Otherwise simply start this RT task 1572 * on its current runqueue. 1573 * 1574 * We want to avoid overloading runqueues. If the woken 1575 * task is a higher priority, then it will stay on this CPU 1576 * and the lower prio task should be moved to another CPU. 1577 * Even though this will probably make the lower prio task 1578 * lose its cache, we do not want to bounce a higher task 1579 * around just because it gave up its CPU, perhaps for a 1580 * lock? 1581 * 1582 * For equal prio tasks, we just let the scheduler sort it out. 1583 * 1584 * Otherwise, just let it ride on the affined RQ and the 1585 * post-schedule router will push the preempted task away 1586 * 1587 * This test is optimistic, if we get it wrong the load-balancer 1588 * will have to sort it out. 1589 * 1590 * We take into account the capacity of the CPU to ensure it fits the 1591 * requirement of the task - which is only important on heterogeneous 1592 * systems like big.LITTLE. 1593 */ 1594 test = curr && 1595 unlikely(rt_task(curr)) && 1596 (curr->nr_cpus_allowed < 2 || curr->prio <= p->prio); 1597 1598 if (test || !rt_task_fits_capacity(p, cpu)) { 1599 int target = find_lowest_rq(p); 1600 1601 /* 1602 * Bail out if we were forcing a migration to find a better 1603 * fitting CPU but our search failed. 1604 */ 1605 if (!test && target != -1 && !rt_task_fits_capacity(p, target)) 1606 goto out_unlock; 1607 1608 /* 1609 * Don't bother moving it if the destination CPU is 1610 * not running a lower priority task. 1611 */ 1612 if (target != -1 && 1613 p->prio < cpu_rq(target)->rt.highest_prio.curr) 1614 cpu = target; 1615 } 1616 1617 out_unlock: 1618 rcu_read_unlock(); 1619 1620 out: 1621 return cpu; 1622 } 1623 1624 static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) 1625 { 1626 /* 1627 * Current can't be migrated, useless to reschedule, 1628 * let's hope p can move out. 1629 */ 1630 if (rq->curr->nr_cpus_allowed == 1 || 1631 !cpupri_find(&rq->rd->cpupri, rq->curr, NULL)) 1632 return; 1633 1634 /* 1635 * p is migratable, so let's not schedule it and 1636 * see if it is pushed or pulled somewhere else. 1637 */ 1638 if (p->nr_cpus_allowed != 1 && 1639 cpupri_find(&rq->rd->cpupri, p, NULL)) 1640 return; 1641 1642 /* 1643 * There appear to be other CPUs that can accept 1644 * the current task but none can run 'p', so lets reschedule 1645 * to try and push the current task away: 1646 */ 1647 requeue_task_rt(rq, p, 1); 1648 resched_curr(rq); 1649 } 1650 1651 static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf) 1652 { 1653 if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) { 1654 /* 1655 * This is OK, because current is on_cpu, which avoids it being 1656 * picked for load-balance and preemption/IRQs are still 1657 * disabled avoiding further scheduler activity on it and we've 1658 * not yet started the picking loop. 1659 */ 1660 rq_unpin_lock(rq, rf); 1661 pull_rt_task(rq); 1662 rq_repin_lock(rq, rf); 1663 } 1664 1665 return sched_stop_runnable(rq) || sched_dl_runnable(rq) || sched_rt_runnable(rq); 1666 } 1667 #endif /* CONFIG_SMP */ 1668 1669 /* 1670 * Preempt the current task with a newly woken task if needed: 1671 */ 1672 static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags) 1673 { 1674 if (p->prio < rq->curr->prio) { 1675 resched_curr(rq); 1676 return; 1677 } 1678 1679 #ifdef CONFIG_SMP 1680 /* 1681 * If: 1682 * 1683 * - the newly woken task is of equal priority to the current task 1684 * - the newly woken task is non-migratable while current is migratable 1685 * - current will be preempted on the next reschedule 1686 * 1687 * we should check to see if current can readily move to a different 1688 * cpu. If so, we will reschedule to allow the push logic to try 1689 * to move current somewhere else, making room for our non-migratable 1690 * task. 1691 */ 1692 if (p->prio == rq->curr->prio && !test_tsk_need_resched(rq->curr)) 1693 check_preempt_equal_prio(rq, p); 1694 #endif 1695 } 1696 1697 static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first) 1698 { 1699 struct sched_rt_entity *rt_se = &p->rt; 1700 struct rt_rq *rt_rq = &rq->rt; 1701 1702 p->se.exec_start = rq_clock_task(rq); 1703 if (on_rt_rq(&p->rt)) 1704 update_stats_wait_end_rt(rt_rq, rt_se); 1705 1706 /* The running task is never eligible for pushing */ 1707 dequeue_pushable_task(rq, p); 1708 1709 if (!first) 1710 return; 1711 1712 /* 1713 * If prev task was rt, put_prev_task() has already updated the 1714 * utilization. We only care of the case where we start to schedule a 1715 * rt task 1716 */ 1717 if (rq->curr->sched_class != &rt_sched_class) 1718 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0); 1719 1720 rt_queue_push_tasks(rq); 1721 } 1722 1723 static struct sched_rt_entity *pick_next_rt_entity(struct rt_rq *rt_rq) 1724 { 1725 struct rt_prio_array *array = &rt_rq->active; 1726 struct sched_rt_entity *next = NULL; 1727 struct list_head *queue; 1728 int idx; 1729 1730 idx = sched_find_first_bit(array->bitmap); 1731 BUG_ON(idx >= MAX_RT_PRIO); 1732 1733 queue = array->queue + idx; 1734 if (SCHED_WARN_ON(list_empty(queue))) 1735 return NULL; 1736 next = list_entry(queue->next, struct sched_rt_entity, run_list); 1737 1738 return next; 1739 } 1740 1741 static struct task_struct *_pick_next_task_rt(struct rq *rq) 1742 { 1743 struct sched_rt_entity *rt_se; 1744 struct rt_rq *rt_rq = &rq->rt; 1745 1746 do { 1747 rt_se = pick_next_rt_entity(rt_rq); 1748 if (unlikely(!rt_se)) 1749 return NULL; 1750 rt_rq = group_rt_rq(rt_se); 1751 } while (rt_rq); 1752 1753 return rt_task_of(rt_se); 1754 } 1755 1756 static struct task_struct *pick_task_rt(struct rq *rq) 1757 { 1758 struct task_struct *p; 1759 1760 if (!sched_rt_runnable(rq)) 1761 return NULL; 1762 1763 p = _pick_next_task_rt(rq); 1764 1765 return p; 1766 } 1767 1768 static struct task_struct *pick_next_task_rt(struct rq *rq) 1769 { 1770 struct task_struct *p = pick_task_rt(rq); 1771 1772 if (p) 1773 set_next_task_rt(rq, p, true); 1774 1775 return p; 1776 } 1777 1778 static void put_prev_task_rt(struct rq *rq, struct task_struct *p) 1779 { 1780 struct sched_rt_entity *rt_se = &p->rt; 1781 struct rt_rq *rt_rq = &rq->rt; 1782 1783 if (on_rt_rq(&p->rt)) 1784 update_stats_wait_start_rt(rt_rq, rt_se); 1785 1786 update_curr_rt(rq); 1787 1788 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1); 1789 1790 /* 1791 * The previous task needs to be made eligible for pushing 1792 * if it is still active 1793 */ 1794 if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1) 1795 enqueue_pushable_task(rq, p); 1796 } 1797 1798 #ifdef CONFIG_SMP 1799 1800 /* Only try algorithms three times */ 1801 #define RT_MAX_TRIES 3 1802 1803 static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) 1804 { 1805 if (!task_on_cpu(rq, p) && 1806 cpumask_test_cpu(cpu, &p->cpus_mask)) 1807 return 1; 1808 1809 return 0; 1810 } 1811 1812 /* 1813 * Return the highest pushable rq's task, which is suitable to be executed 1814 * on the CPU, NULL otherwise 1815 */ 1816 static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu) 1817 { 1818 struct plist_head *head = &rq->rt.pushable_tasks; 1819 struct task_struct *p; 1820 1821 if (!has_pushable_tasks(rq)) 1822 return NULL; 1823 1824 plist_for_each_entry(p, head, pushable_tasks) { 1825 if (pick_rt_task(rq, p, cpu)) 1826 return p; 1827 } 1828 1829 return NULL; 1830 } 1831 1832 static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); 1833 1834 static int find_lowest_rq(struct task_struct *task) 1835 { 1836 struct sched_domain *sd; 1837 struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask); 1838 int this_cpu = smp_processor_id(); 1839 int cpu = task_cpu(task); 1840 int ret; 1841 1842 /* Make sure the mask is initialized first */ 1843 if (unlikely(!lowest_mask)) 1844 return -1; 1845 1846 if (task->nr_cpus_allowed == 1) 1847 return -1; /* No other targets possible */ 1848 1849 /* 1850 * If we're on asym system ensure we consider the different capacities 1851 * of the CPUs when searching for the lowest_mask. 1852 */ 1853 if (sched_asym_cpucap_active()) { 1854 1855 ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri, 1856 task, lowest_mask, 1857 rt_task_fits_capacity); 1858 } else { 1859 1860 ret = cpupri_find(&task_rq(task)->rd->cpupri, 1861 task, lowest_mask); 1862 } 1863 1864 if (!ret) 1865 return -1; /* No targets found */ 1866 1867 /* 1868 * At this point we have built a mask of CPUs representing the 1869 * lowest priority tasks in the system. Now we want to elect 1870 * the best one based on our affinity and topology. 1871 * 1872 * We prioritize the last CPU that the task executed on since 1873 * it is most likely cache-hot in that location. 1874 */ 1875 if (cpumask_test_cpu(cpu, lowest_mask)) 1876 return cpu; 1877 1878 /* 1879 * Otherwise, we consult the sched_domains span maps to figure 1880 * out which CPU is logically closest to our hot cache data. 1881 */ 1882 if (!cpumask_test_cpu(this_cpu, lowest_mask)) 1883 this_cpu = -1; /* Skip this_cpu opt if not among lowest */ 1884 1885 rcu_read_lock(); 1886 for_each_domain(cpu, sd) { 1887 if (sd->flags & SD_WAKE_AFFINE) { 1888 int best_cpu; 1889 1890 /* 1891 * "this_cpu" is cheaper to preempt than a 1892 * remote processor. 1893 */ 1894 if (this_cpu != -1 && 1895 cpumask_test_cpu(this_cpu, sched_domain_span(sd))) { 1896 rcu_read_unlock(); 1897 return this_cpu; 1898 } 1899 1900 best_cpu = cpumask_any_and_distribute(lowest_mask, 1901 sched_domain_span(sd)); 1902 if (best_cpu < nr_cpu_ids) { 1903 rcu_read_unlock(); 1904 return best_cpu; 1905 } 1906 } 1907 } 1908 rcu_read_unlock(); 1909 1910 /* 1911 * And finally, if there were no matches within the domains 1912 * just give the caller *something* to work with from the compatible 1913 * locations. 1914 */ 1915 if (this_cpu != -1) 1916 return this_cpu; 1917 1918 cpu = cpumask_any_distribute(lowest_mask); 1919 if (cpu < nr_cpu_ids) 1920 return cpu; 1921 1922 return -1; 1923 } 1924 1925 /* Will lock the rq it finds */ 1926 static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) 1927 { 1928 struct rq *lowest_rq = NULL; 1929 int tries; 1930 int cpu; 1931 1932 for (tries = 0; tries < RT_MAX_TRIES; tries++) { 1933 cpu = find_lowest_rq(task); 1934 1935 if ((cpu == -1) || (cpu == rq->cpu)) 1936 break; 1937 1938 lowest_rq = cpu_rq(cpu); 1939 1940 if (lowest_rq->rt.highest_prio.curr <= task->prio) { 1941 /* 1942 * Target rq has tasks of equal or higher priority, 1943 * retrying does not release any lock and is unlikely 1944 * to yield a different result. 1945 */ 1946 lowest_rq = NULL; 1947 break; 1948 } 1949 1950 /* if the prio of this runqueue changed, try again */ 1951 if (double_lock_balance(rq, lowest_rq)) { 1952 /* 1953 * We had to unlock the run queue. In 1954 * the mean time, task could have 1955 * migrated already or had its affinity changed. 1956 * Also make sure that it wasn't scheduled on its rq. 1957 * It is possible the task was scheduled, set 1958 * "migrate_disabled" and then got preempted, so we must 1959 * check the task migration disable flag here too. 1960 */ 1961 if (unlikely(task_rq(task) != rq || 1962 !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) || 1963 task_on_cpu(rq, task) || 1964 !rt_task(task) || 1965 is_migration_disabled(task) || 1966 !task_on_rq_queued(task))) { 1967 1968 double_unlock_balance(rq, lowest_rq); 1969 lowest_rq = NULL; 1970 break; 1971 } 1972 } 1973 1974 /* If this rq is still suitable use it. */ 1975 if (lowest_rq->rt.highest_prio.curr > task->prio) 1976 break; 1977 1978 /* try again */ 1979 double_unlock_balance(rq, lowest_rq); 1980 lowest_rq = NULL; 1981 } 1982 1983 return lowest_rq; 1984 } 1985 1986 static struct task_struct *pick_next_pushable_task(struct rq *rq) 1987 { 1988 struct task_struct *p; 1989 1990 if (!has_pushable_tasks(rq)) 1991 return NULL; 1992 1993 p = plist_first_entry(&rq->rt.pushable_tasks, 1994 struct task_struct, pushable_tasks); 1995 1996 BUG_ON(rq->cpu != task_cpu(p)); 1997 BUG_ON(task_current(rq, p)); 1998 BUG_ON(p->nr_cpus_allowed <= 1); 1999 2000 BUG_ON(!task_on_rq_queued(p)); 2001 BUG_ON(!rt_task(p)); 2002 2003 return p; 2004 } 2005 2006 /* 2007 * If the current CPU has more than one RT task, see if the non 2008 * running task can migrate over to a CPU that is running a task 2009 * of lesser priority. 2010 */ 2011 static int push_rt_task(struct rq *rq, bool pull) 2012 { 2013 struct task_struct *next_task; 2014 struct rq *lowest_rq; 2015 int ret = 0; 2016 2017 if (!rq->rt.overloaded) 2018 return 0; 2019 2020 next_task = pick_next_pushable_task(rq); 2021 if (!next_task) 2022 return 0; 2023 2024 retry: 2025 /* 2026 * It's possible that the next_task slipped in of 2027 * higher priority than current. If that's the case 2028 * just reschedule current. 2029 */ 2030 if (unlikely(next_task->prio < rq->curr->prio)) { 2031 resched_curr(rq); 2032 return 0; 2033 } 2034 2035 if (is_migration_disabled(next_task)) { 2036 struct task_struct *push_task = NULL; 2037 int cpu; 2038 2039 if (!pull || rq->push_busy) 2040 return 0; 2041 2042 /* 2043 * Invoking find_lowest_rq() on anything but an RT task doesn't 2044 * make sense. Per the above priority check, curr has to 2045 * be of higher priority than next_task, so no need to 2046 * reschedule when bailing out. 2047 * 2048 * Note that the stoppers are masqueraded as SCHED_FIFO 2049 * (cf. sched_set_stop_task()), so we can't rely on rt_task(). 2050 */ 2051 if (rq->curr->sched_class != &rt_sched_class) 2052 return 0; 2053 2054 cpu = find_lowest_rq(rq->curr); 2055 if (cpu == -1 || cpu == rq->cpu) 2056 return 0; 2057 2058 /* 2059 * Given we found a CPU with lower priority than @next_task, 2060 * therefore it should be running. However we cannot migrate it 2061 * to this other CPU, instead attempt to push the current 2062 * running task on this CPU away. 2063 */ 2064 push_task = get_push_task(rq); 2065 if (push_task) { 2066 preempt_disable(); 2067 raw_spin_rq_unlock(rq); 2068 stop_one_cpu_nowait(rq->cpu, push_cpu_stop, 2069 push_task, &rq->push_work); 2070 preempt_enable(); 2071 raw_spin_rq_lock(rq); 2072 } 2073 2074 return 0; 2075 } 2076 2077 if (WARN_ON(next_task == rq->curr)) 2078 return 0; 2079 2080 /* We might release rq lock */ 2081 get_task_struct(next_task); 2082 2083 /* find_lock_lowest_rq locks the rq if found */ 2084 lowest_rq = find_lock_lowest_rq(next_task, rq); 2085 if (!lowest_rq) { 2086 struct task_struct *task; 2087 /* 2088 * find_lock_lowest_rq releases rq->lock 2089 * so it is possible that next_task has migrated. 2090 * 2091 * We need to make sure that the task is still on the same 2092 * run-queue and is also still the next task eligible for 2093 * pushing. 2094 */ 2095 task = pick_next_pushable_task(rq); 2096 if (task == next_task) { 2097 /* 2098 * The task hasn't migrated, and is still the next 2099 * eligible task, but we failed to find a run-queue 2100 * to push it to. Do not retry in this case, since 2101 * other CPUs will pull from us when ready. 2102 */ 2103 goto out; 2104 } 2105 2106 if (!task) 2107 /* No more tasks, just exit */ 2108 goto out; 2109 2110 /* 2111 * Something has shifted, try again. 2112 */ 2113 put_task_struct(next_task); 2114 next_task = task; 2115 goto retry; 2116 } 2117 2118 deactivate_task(rq, next_task, 0); 2119 set_task_cpu(next_task, lowest_rq->cpu); 2120 activate_task(lowest_rq, next_task, 0); 2121 resched_curr(lowest_rq); 2122 ret = 1; 2123 2124 double_unlock_balance(rq, lowest_rq); 2125 out: 2126 put_task_struct(next_task); 2127 2128 return ret; 2129 } 2130 2131 static void push_rt_tasks(struct rq *rq) 2132 { 2133 /* push_rt_task will return true if it moved an RT */ 2134 while (push_rt_task(rq, false)) 2135 ; 2136 } 2137 2138 #ifdef HAVE_RT_PUSH_IPI 2139 2140 /* 2141 * When a high priority task schedules out from a CPU and a lower priority 2142 * task is scheduled in, a check is made to see if there's any RT tasks 2143 * on other CPUs that are waiting to run because a higher priority RT task 2144 * is currently running on its CPU. In this case, the CPU with multiple RT 2145 * tasks queued on it (overloaded) needs to be notified that a CPU has opened 2146 * up that may be able to run one of its non-running queued RT tasks. 2147 * 2148 * All CPUs with overloaded RT tasks need to be notified as there is currently 2149 * no way to know which of these CPUs have the highest priority task waiting 2150 * to run. Instead of trying to take a spinlock on each of these CPUs, 2151 * which has shown to cause large latency when done on machines with many 2152 * CPUs, sending an IPI to the CPUs to have them push off the overloaded 2153 * RT tasks waiting to run. 2154 * 2155 * Just sending an IPI to each of the CPUs is also an issue, as on large 2156 * count CPU machines, this can cause an IPI storm on a CPU, especially 2157 * if its the only CPU with multiple RT tasks queued, and a large number 2158 * of CPUs scheduling a lower priority task at the same time. 2159 * 2160 * Each root domain has its own irq work function that can iterate over 2161 * all CPUs with RT overloaded tasks. Since all CPUs with overloaded RT 2162 * task must be checked if there's one or many CPUs that are lowering 2163 * their priority, there's a single irq work iterator that will try to 2164 * push off RT tasks that are waiting to run. 2165 * 2166 * When a CPU schedules a lower priority task, it will kick off the 2167 * irq work iterator that will jump to each CPU with overloaded RT tasks. 2168 * As it only takes the first CPU that schedules a lower priority task 2169 * to start the process, the rto_start variable is incremented and if 2170 * the atomic result is one, then that CPU will try to take the rto_lock. 2171 * This prevents high contention on the lock as the process handles all 2172 * CPUs scheduling lower priority tasks. 2173 * 2174 * All CPUs that are scheduling a lower priority task will increment the 2175 * rt_loop_next variable. This will make sure that the irq work iterator 2176 * checks all RT overloaded CPUs whenever a CPU schedules a new lower 2177 * priority task, even if the iterator is in the middle of a scan. Incrementing 2178 * the rt_loop_next will cause the iterator to perform another scan. 2179 * 2180 */ 2181 static int rto_next_cpu(struct root_domain *rd) 2182 { 2183 int next; 2184 int cpu; 2185 2186 /* 2187 * When starting the IPI RT pushing, the rto_cpu is set to -1, 2188 * rt_next_cpu() will simply return the first CPU found in 2189 * the rto_mask. 2190 * 2191 * If rto_next_cpu() is called with rto_cpu is a valid CPU, it 2192 * will return the next CPU found in the rto_mask. 2193 * 2194 * If there are no more CPUs left in the rto_mask, then a check is made 2195 * against rto_loop and rto_loop_next. rto_loop is only updated with 2196 * the rto_lock held, but any CPU may increment the rto_loop_next 2197 * without any locking. 2198 */ 2199 for (;;) { 2200 2201 /* When rto_cpu is -1 this acts like cpumask_first() */ 2202 cpu = cpumask_next(rd->rto_cpu, rd->rto_mask); 2203 2204 rd->rto_cpu = cpu; 2205 2206 if (cpu < nr_cpu_ids) 2207 return cpu; 2208 2209 rd->rto_cpu = -1; 2210 2211 /* 2212 * ACQUIRE ensures we see the @rto_mask changes 2213 * made prior to the @next value observed. 2214 * 2215 * Matches WMB in rt_set_overload(). 2216 */ 2217 next = atomic_read_acquire(&rd->rto_loop_next); 2218 2219 if (rd->rto_loop == next) 2220 break; 2221 2222 rd->rto_loop = next; 2223 } 2224 2225 return -1; 2226 } 2227 2228 static inline bool rto_start_trylock(atomic_t *v) 2229 { 2230 return !atomic_cmpxchg_acquire(v, 0, 1); 2231 } 2232 2233 static inline void rto_start_unlock(atomic_t *v) 2234 { 2235 atomic_set_release(v, 0); 2236 } 2237 2238 static void tell_cpu_to_push(struct rq *rq) 2239 { 2240 int cpu = -1; 2241 2242 /* Keep the loop going if the IPI is currently active */ 2243 atomic_inc(&rq->rd->rto_loop_next); 2244 2245 /* Only one CPU can initiate a loop at a time */ 2246 if (!rto_start_trylock(&rq->rd->rto_loop_start)) 2247 return; 2248 2249 raw_spin_lock(&rq->rd->rto_lock); 2250 2251 /* 2252 * The rto_cpu is updated under the lock, if it has a valid CPU 2253 * then the IPI is still running and will continue due to the 2254 * update to loop_next, and nothing needs to be done here. 2255 * Otherwise it is finishing up and an ipi needs to be sent. 2256 */ 2257 if (rq->rd->rto_cpu < 0) 2258 cpu = rto_next_cpu(rq->rd); 2259 2260 raw_spin_unlock(&rq->rd->rto_lock); 2261 2262 rto_start_unlock(&rq->rd->rto_loop_start); 2263 2264 if (cpu >= 0) { 2265 /* Make sure the rd does not get freed while pushing */ 2266 sched_get_rd(rq->rd); 2267 irq_work_queue_on(&rq->rd->rto_push_work, cpu); 2268 } 2269 } 2270 2271 /* Called from hardirq context */ 2272 void rto_push_irq_work_func(struct irq_work *work) 2273 { 2274 struct root_domain *rd = 2275 container_of(work, struct root_domain, rto_push_work); 2276 struct rq *rq; 2277 int cpu; 2278 2279 rq = this_rq(); 2280 2281 /* 2282 * We do not need to grab the lock to check for has_pushable_tasks. 2283 * When it gets updated, a check is made if a push is possible. 2284 */ 2285 if (has_pushable_tasks(rq)) { 2286 raw_spin_rq_lock(rq); 2287 while (push_rt_task(rq, true)) 2288 ; 2289 raw_spin_rq_unlock(rq); 2290 } 2291 2292 raw_spin_lock(&rd->rto_lock); 2293 2294 /* Pass the IPI to the next rt overloaded queue */ 2295 cpu = rto_next_cpu(rd); 2296 2297 raw_spin_unlock(&rd->rto_lock); 2298 2299 if (cpu < 0) { 2300 sched_put_rd(rd); 2301 return; 2302 } 2303 2304 /* Try the next RT overloaded CPU */ 2305 irq_work_queue_on(&rd->rto_push_work, cpu); 2306 } 2307 #endif /* HAVE_RT_PUSH_IPI */ 2308 2309 static void pull_rt_task(struct rq *this_rq) 2310 { 2311 int this_cpu = this_rq->cpu, cpu; 2312 bool resched = false; 2313 struct task_struct *p, *push_task; 2314 struct rq *src_rq; 2315 int rt_overload_count = rt_overloaded(this_rq); 2316 2317 if (likely(!rt_overload_count)) 2318 return; 2319 2320 /* 2321 * Match the barrier from rt_set_overloaded; this guarantees that if we 2322 * see overloaded we must also see the rto_mask bit. 2323 */ 2324 smp_rmb(); 2325 2326 /* If we are the only overloaded CPU do nothing */ 2327 if (rt_overload_count == 1 && 2328 cpumask_test_cpu(this_rq->cpu, this_rq->rd->rto_mask)) 2329 return; 2330 2331 #ifdef HAVE_RT_PUSH_IPI 2332 if (sched_feat(RT_PUSH_IPI)) { 2333 tell_cpu_to_push(this_rq); 2334 return; 2335 } 2336 #endif 2337 2338 for_each_cpu(cpu, this_rq->rd->rto_mask) { 2339 if (this_cpu == cpu) 2340 continue; 2341 2342 src_rq = cpu_rq(cpu); 2343 2344 /* 2345 * Don't bother taking the src_rq->lock if the next highest 2346 * task is known to be lower-priority than our current task. 2347 * This may look racy, but if this value is about to go 2348 * logically higher, the src_rq will push this task away. 2349 * And if its going logically lower, we do not care 2350 */ 2351 if (src_rq->rt.highest_prio.next >= 2352 this_rq->rt.highest_prio.curr) 2353 continue; 2354 2355 /* 2356 * We can potentially drop this_rq's lock in 2357 * double_lock_balance, and another CPU could 2358 * alter this_rq 2359 */ 2360 push_task = NULL; 2361 double_lock_balance(this_rq, src_rq); 2362 2363 /* 2364 * We can pull only a task, which is pushable 2365 * on its rq, and no others. 2366 */ 2367 p = pick_highest_pushable_task(src_rq, this_cpu); 2368 2369 /* 2370 * Do we have an RT task that preempts 2371 * the to-be-scheduled task? 2372 */ 2373 if (p && (p->prio < this_rq->rt.highest_prio.curr)) { 2374 WARN_ON(p == src_rq->curr); 2375 WARN_ON(!task_on_rq_queued(p)); 2376 2377 /* 2378 * There's a chance that p is higher in priority 2379 * than what's currently running on its CPU. 2380 * This is just that p is waking up and hasn't 2381 * had a chance to schedule. We only pull 2382 * p if it is lower in priority than the 2383 * current task on the run queue 2384 */ 2385 if (p->prio < src_rq->curr->prio) 2386 goto skip; 2387 2388 if (is_migration_disabled(p)) { 2389 push_task = get_push_task(src_rq); 2390 } else { 2391 deactivate_task(src_rq, p, 0); 2392 set_task_cpu(p, this_cpu); 2393 activate_task(this_rq, p, 0); 2394 resched = true; 2395 } 2396 /* 2397 * We continue with the search, just in 2398 * case there's an even higher prio task 2399 * in another runqueue. (low likelihood 2400 * but possible) 2401 */ 2402 } 2403 skip: 2404 double_unlock_balance(this_rq, src_rq); 2405 2406 if (push_task) { 2407 preempt_disable(); 2408 raw_spin_rq_unlock(this_rq); 2409 stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop, 2410 push_task, &src_rq->push_work); 2411 preempt_enable(); 2412 raw_spin_rq_lock(this_rq); 2413 } 2414 } 2415 2416 if (resched) 2417 resched_curr(this_rq); 2418 } 2419 2420 /* 2421 * If we are not running and we are not going to reschedule soon, we should 2422 * try to push tasks away now 2423 */ 2424 static void task_woken_rt(struct rq *rq, struct task_struct *p) 2425 { 2426 bool need_to_push = !task_on_cpu(rq, p) && 2427 !test_tsk_need_resched(rq->curr) && 2428 p->nr_cpus_allowed > 1 && 2429 (dl_task(rq->curr) || rt_task(rq->curr)) && 2430 (rq->curr->nr_cpus_allowed < 2 || 2431 rq->curr->prio <= p->prio); 2432 2433 if (need_to_push) 2434 push_rt_tasks(rq); 2435 } 2436 2437 /* Assumes rq->lock is held */ 2438 static void rq_online_rt(struct rq *rq) 2439 { 2440 if (rq->rt.overloaded) 2441 rt_set_overload(rq); 2442 2443 __enable_runtime(rq); 2444 2445 cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr); 2446 } 2447 2448 /* Assumes rq->lock is held */ 2449 static void rq_offline_rt(struct rq *rq) 2450 { 2451 if (rq->rt.overloaded) 2452 rt_clear_overload(rq); 2453 2454 __disable_runtime(rq); 2455 2456 cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID); 2457 } 2458 2459 /* 2460 * When switch from the rt queue, we bring ourselves to a position 2461 * that we might want to pull RT tasks from other runqueues. 2462 */ 2463 static void switched_from_rt(struct rq *rq, struct task_struct *p) 2464 { 2465 /* 2466 * If there are other RT tasks then we will reschedule 2467 * and the scheduling of the other RT tasks will handle 2468 * the balancing. But if we are the last RT task 2469 * we may need to handle the pulling of RT tasks 2470 * now. 2471 */ 2472 if (!task_on_rq_queued(p) || rq->rt.rt_nr_running) 2473 return; 2474 2475 rt_queue_pull_task(rq); 2476 } 2477 2478 void __init init_sched_rt_class(void) 2479 { 2480 unsigned int i; 2481 2482 for_each_possible_cpu(i) { 2483 zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i), 2484 GFP_KERNEL, cpu_to_node(i)); 2485 } 2486 } 2487 #endif /* CONFIG_SMP */ 2488 2489 /* 2490 * When switching a task to RT, we may overload the runqueue 2491 * with RT tasks. In this case we try to push them off to 2492 * other runqueues. 2493 */ 2494 static void switched_to_rt(struct rq *rq, struct task_struct *p) 2495 { 2496 /* 2497 * If we are running, update the avg_rt tracking, as the running time 2498 * will now on be accounted into the latter. 2499 */ 2500 if (task_current(rq, p)) { 2501 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0); 2502 return; 2503 } 2504 2505 /* 2506 * If we are not running we may need to preempt the current 2507 * running task. If that current running task is also an RT task 2508 * then see if we can move to another run queue. 2509 */ 2510 if (task_on_rq_queued(p)) { 2511 #ifdef CONFIG_SMP 2512 if (p->nr_cpus_allowed > 1 && rq->rt.overloaded) 2513 rt_queue_push_tasks(rq); 2514 #endif /* CONFIG_SMP */ 2515 if (p->prio < rq->curr->prio && cpu_online(cpu_of(rq))) 2516 resched_curr(rq); 2517 } 2518 } 2519 2520 /* 2521 * Priority of the task has changed. This may cause 2522 * us to initiate a push or pull. 2523 */ 2524 static void 2525 prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio) 2526 { 2527 if (!task_on_rq_queued(p)) 2528 return; 2529 2530 if (task_current(rq, p)) { 2531 #ifdef CONFIG_SMP 2532 /* 2533 * If our priority decreases while running, we 2534 * may need to pull tasks to this runqueue. 2535 */ 2536 if (oldprio < p->prio) 2537 rt_queue_pull_task(rq); 2538 2539 /* 2540 * If there's a higher priority task waiting to run 2541 * then reschedule. 2542 */ 2543 if (p->prio > rq->rt.highest_prio.curr) 2544 resched_curr(rq); 2545 #else 2546 /* For UP simply resched on drop of prio */ 2547 if (oldprio < p->prio) 2548 resched_curr(rq); 2549 #endif /* CONFIG_SMP */ 2550 } else { 2551 /* 2552 * This task is not running, but if it is 2553 * greater than the current running task 2554 * then reschedule. 2555 */ 2556 if (p->prio < rq->curr->prio) 2557 resched_curr(rq); 2558 } 2559 } 2560 2561 #ifdef CONFIG_POSIX_TIMERS 2562 static void watchdog(struct rq *rq, struct task_struct *p) 2563 { 2564 unsigned long soft, hard; 2565 2566 /* max may change after cur was read, this will be fixed next tick */ 2567 soft = task_rlimit(p, RLIMIT_RTTIME); 2568 hard = task_rlimit_max(p, RLIMIT_RTTIME); 2569 2570 if (soft != RLIM_INFINITY) { 2571 unsigned long next; 2572 2573 if (p->rt.watchdog_stamp != jiffies) { 2574 p->rt.timeout++; 2575 p->rt.watchdog_stamp = jiffies; 2576 } 2577 2578 next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ); 2579 if (p->rt.timeout > next) { 2580 posix_cputimers_rt_watchdog(&p->posix_cputimers, 2581 p->se.sum_exec_runtime); 2582 } 2583 } 2584 } 2585 #else 2586 static inline void watchdog(struct rq *rq, struct task_struct *p) { } 2587 #endif 2588 2589 /* 2590 * scheduler tick hitting a task of our scheduling class. 2591 * 2592 * NOTE: This function can be called remotely by the tick offload that 2593 * goes along full dynticks. Therefore no local assumption can be made 2594 * and everything must be accessed through the @rq and @curr passed in 2595 * parameters. 2596 */ 2597 static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued) 2598 { 2599 struct sched_rt_entity *rt_se = &p->rt; 2600 2601 update_curr_rt(rq); 2602 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1); 2603 2604 watchdog(rq, p); 2605 2606 /* 2607 * RR tasks need a special form of timeslice management. 2608 * FIFO tasks have no timeslices. 2609 */ 2610 if (p->policy != SCHED_RR) 2611 return; 2612 2613 if (--p->rt.time_slice) 2614 return; 2615 2616 p->rt.time_slice = sched_rr_timeslice; 2617 2618 /* 2619 * Requeue to the end of queue if we (and all of our ancestors) are not 2620 * the only element on the queue 2621 */ 2622 for_each_sched_rt_entity(rt_se) { 2623 if (rt_se->run_list.prev != rt_se->run_list.next) { 2624 requeue_task_rt(rq, p, 0); 2625 resched_curr(rq); 2626 return; 2627 } 2628 } 2629 } 2630 2631 static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task) 2632 { 2633 /* 2634 * Time slice is 0 for SCHED_FIFO tasks 2635 */ 2636 if (task->policy == SCHED_RR) 2637 return sched_rr_timeslice; 2638 else 2639 return 0; 2640 } 2641 2642 #ifdef CONFIG_SCHED_CORE 2643 static int task_is_throttled_rt(struct task_struct *p, int cpu) 2644 { 2645 struct rt_rq *rt_rq; 2646 2647 #ifdef CONFIG_RT_GROUP_SCHED 2648 rt_rq = task_group(p)->rt_rq[cpu]; 2649 #else 2650 rt_rq = &cpu_rq(cpu)->rt; 2651 #endif 2652 2653 return rt_rq_throttled(rt_rq); 2654 } 2655 #endif 2656 2657 DEFINE_SCHED_CLASS(rt) = { 2658 2659 .enqueue_task = enqueue_task_rt, 2660 .dequeue_task = dequeue_task_rt, 2661 .yield_task = yield_task_rt, 2662 2663 .wakeup_preempt = wakeup_preempt_rt, 2664 2665 .pick_next_task = pick_next_task_rt, 2666 .put_prev_task = put_prev_task_rt, 2667 .set_next_task = set_next_task_rt, 2668 2669 #ifdef CONFIG_SMP 2670 .balance = balance_rt, 2671 .pick_task = pick_task_rt, 2672 .select_task_rq = select_task_rq_rt, 2673 .set_cpus_allowed = set_cpus_allowed_common, 2674 .rq_online = rq_online_rt, 2675 .rq_offline = rq_offline_rt, 2676 .task_woken = task_woken_rt, 2677 .switched_from = switched_from_rt, 2678 .find_lock_rq = find_lock_lowest_rq, 2679 #endif 2680 2681 .task_tick = task_tick_rt, 2682 2683 .get_rr_interval = get_rr_interval_rt, 2684 2685 .prio_changed = prio_changed_rt, 2686 .switched_to = switched_to_rt, 2687 2688 .update_curr = update_curr_rt, 2689 2690 #ifdef CONFIG_SCHED_CORE 2691 .task_is_throttled = task_is_throttled_rt, 2692 #endif 2693 2694 #ifdef CONFIG_UCLAMP_TASK 2695 .uclamp_enabled = 1, 2696 #endif 2697 }; 2698 2699 #ifdef CONFIG_RT_GROUP_SCHED 2700 /* 2701 * Ensure that the real time constraints are schedulable. 2702 */ 2703 static DEFINE_MUTEX(rt_constraints_mutex); 2704 2705 static inline int tg_has_rt_tasks(struct task_group *tg) 2706 { 2707 struct task_struct *task; 2708 struct css_task_iter it; 2709 int ret = 0; 2710 2711 /* 2712 * Autogroups do not have RT tasks; see autogroup_create(). 2713 */ 2714 if (task_group_is_autogroup(tg)) 2715 return 0; 2716 2717 css_task_iter_start(&tg->css, 0, &it); 2718 while (!ret && (task = css_task_iter_next(&it))) 2719 ret |= rt_task(task); 2720 css_task_iter_end(&it); 2721 2722 return ret; 2723 } 2724 2725 struct rt_schedulable_data { 2726 struct task_group *tg; 2727 u64 rt_period; 2728 u64 rt_runtime; 2729 }; 2730 2731 static int tg_rt_schedulable(struct task_group *tg, void *data) 2732 { 2733 struct rt_schedulable_data *d = data; 2734 struct task_group *child; 2735 unsigned long total, sum = 0; 2736 u64 period, runtime; 2737 2738 period = ktime_to_ns(tg->rt_bandwidth.rt_period); 2739 runtime = tg->rt_bandwidth.rt_runtime; 2740 2741 if (tg == d->tg) { 2742 period = d->rt_period; 2743 runtime = d->rt_runtime; 2744 } 2745 2746 /* 2747 * Cannot have more runtime than the period. 2748 */ 2749 if (runtime > period && runtime != RUNTIME_INF) 2750 return -EINVAL; 2751 2752 /* 2753 * Ensure we don't starve existing RT tasks if runtime turns zero. 2754 */ 2755 if (rt_bandwidth_enabled() && !runtime && 2756 tg->rt_bandwidth.rt_runtime && tg_has_rt_tasks(tg)) 2757 return -EBUSY; 2758 2759 total = to_ratio(period, runtime); 2760 2761 /* 2762 * Nobody can have more than the global setting allows. 2763 */ 2764 if (total > to_ratio(global_rt_period(), global_rt_runtime())) 2765 return -EINVAL; 2766 2767 /* 2768 * The sum of our children's runtime should not exceed our own. 2769 */ 2770 list_for_each_entry_rcu(child, &tg->children, siblings) { 2771 period = ktime_to_ns(child->rt_bandwidth.rt_period); 2772 runtime = child->rt_bandwidth.rt_runtime; 2773 2774 if (child == d->tg) { 2775 period = d->rt_period; 2776 runtime = d->rt_runtime; 2777 } 2778 2779 sum += to_ratio(period, runtime); 2780 } 2781 2782 if (sum > total) 2783 return -EINVAL; 2784 2785 return 0; 2786 } 2787 2788 static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime) 2789 { 2790 int ret; 2791 2792 struct rt_schedulable_data data = { 2793 .tg = tg, 2794 .rt_period = period, 2795 .rt_runtime = runtime, 2796 }; 2797 2798 rcu_read_lock(); 2799 ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data); 2800 rcu_read_unlock(); 2801 2802 return ret; 2803 } 2804 2805 static int tg_set_rt_bandwidth(struct task_group *tg, 2806 u64 rt_period, u64 rt_runtime) 2807 { 2808 int i, err = 0; 2809 2810 /* 2811 * Disallowing the root group RT runtime is BAD, it would disallow the 2812 * kernel creating (and or operating) RT threads. 2813 */ 2814 if (tg == &root_task_group && rt_runtime == 0) 2815 return -EINVAL; 2816 2817 /* No period doesn't make any sense. */ 2818 if (rt_period == 0) 2819 return -EINVAL; 2820 2821 /* 2822 * Bound quota to defend quota against overflow during bandwidth shift. 2823 */ 2824 if (rt_runtime != RUNTIME_INF && rt_runtime > max_rt_runtime) 2825 return -EINVAL; 2826 2827 mutex_lock(&rt_constraints_mutex); 2828 err = __rt_schedulable(tg, rt_period, rt_runtime); 2829 if (err) 2830 goto unlock; 2831 2832 raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); 2833 tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); 2834 tg->rt_bandwidth.rt_runtime = rt_runtime; 2835 2836 for_each_possible_cpu(i) { 2837 struct rt_rq *rt_rq = tg->rt_rq[i]; 2838 2839 raw_spin_lock(&rt_rq->rt_runtime_lock); 2840 rt_rq->rt_runtime = rt_runtime; 2841 raw_spin_unlock(&rt_rq->rt_runtime_lock); 2842 } 2843 raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); 2844 unlock: 2845 mutex_unlock(&rt_constraints_mutex); 2846 2847 return err; 2848 } 2849 2850 int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us) 2851 { 2852 u64 rt_runtime, rt_period; 2853 2854 rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period); 2855 rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC; 2856 if (rt_runtime_us < 0) 2857 rt_runtime = RUNTIME_INF; 2858 else if ((u64)rt_runtime_us > U64_MAX / NSEC_PER_USEC) 2859 return -EINVAL; 2860 2861 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); 2862 } 2863 2864 long sched_group_rt_runtime(struct task_group *tg) 2865 { 2866 u64 rt_runtime_us; 2867 2868 if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF) 2869 return -1; 2870 2871 rt_runtime_us = tg->rt_bandwidth.rt_runtime; 2872 do_div(rt_runtime_us, NSEC_PER_USEC); 2873 return rt_runtime_us; 2874 } 2875 2876 int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us) 2877 { 2878 u64 rt_runtime, rt_period; 2879 2880 if (rt_period_us > U64_MAX / NSEC_PER_USEC) 2881 return -EINVAL; 2882 2883 rt_period = rt_period_us * NSEC_PER_USEC; 2884 rt_runtime = tg->rt_bandwidth.rt_runtime; 2885 2886 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); 2887 } 2888 2889 long sched_group_rt_period(struct task_group *tg) 2890 { 2891 u64 rt_period_us; 2892 2893 rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period); 2894 do_div(rt_period_us, NSEC_PER_USEC); 2895 return rt_period_us; 2896 } 2897 2898 #ifdef CONFIG_SYSCTL 2899 static int sched_rt_global_constraints(void) 2900 { 2901 int ret = 0; 2902 2903 mutex_lock(&rt_constraints_mutex); 2904 ret = __rt_schedulable(NULL, 0, 0); 2905 mutex_unlock(&rt_constraints_mutex); 2906 2907 return ret; 2908 } 2909 #endif /* CONFIG_SYSCTL */ 2910 2911 int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) 2912 { 2913 /* Don't accept realtime tasks when there is no way for them to run */ 2914 if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0) 2915 return 0; 2916 2917 return 1; 2918 } 2919 2920 #else /* !CONFIG_RT_GROUP_SCHED */ 2921 2922 #ifdef CONFIG_SYSCTL 2923 static int sched_rt_global_constraints(void) 2924 { 2925 unsigned long flags; 2926 int i; 2927 2928 raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); 2929 for_each_possible_cpu(i) { 2930 struct rt_rq *rt_rq = &cpu_rq(i)->rt; 2931 2932 raw_spin_lock(&rt_rq->rt_runtime_lock); 2933 rt_rq->rt_runtime = global_rt_runtime(); 2934 raw_spin_unlock(&rt_rq->rt_runtime_lock); 2935 } 2936 raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); 2937 2938 return 0; 2939 } 2940 #endif /* CONFIG_SYSCTL */ 2941 #endif /* CONFIG_RT_GROUP_SCHED */ 2942 2943 #ifdef CONFIG_SYSCTL 2944 static int sched_rt_global_validate(void) 2945 { 2946 if ((sysctl_sched_rt_runtime != RUNTIME_INF) && 2947 ((sysctl_sched_rt_runtime > sysctl_sched_rt_period) || 2948 ((u64)sysctl_sched_rt_runtime * 2949 NSEC_PER_USEC > max_rt_runtime))) 2950 return -EINVAL; 2951 2952 return 0; 2953 } 2954 2955 static void sched_rt_do_global(void) 2956 { 2957 unsigned long flags; 2958 2959 raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); 2960 def_rt_bandwidth.rt_runtime = global_rt_runtime(); 2961 def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period()); 2962 raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); 2963 } 2964 2965 static int sched_rt_handler(struct ctl_table *table, int write, void *buffer, 2966 size_t *lenp, loff_t *ppos) 2967 { 2968 int old_period, old_runtime; 2969 static DEFINE_MUTEX(mutex); 2970 int ret; 2971 2972 mutex_lock(&mutex); 2973 old_period = sysctl_sched_rt_period; 2974 old_runtime = sysctl_sched_rt_runtime; 2975 2976 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 2977 2978 if (!ret && write) { 2979 ret = sched_rt_global_validate(); 2980 if (ret) 2981 goto undo; 2982 2983 ret = sched_dl_global_validate(); 2984 if (ret) 2985 goto undo; 2986 2987 ret = sched_rt_global_constraints(); 2988 if (ret) 2989 goto undo; 2990 2991 sched_rt_do_global(); 2992 sched_dl_do_global(); 2993 } 2994 if (0) { 2995 undo: 2996 sysctl_sched_rt_period = old_period; 2997 sysctl_sched_rt_runtime = old_runtime; 2998 } 2999 mutex_unlock(&mutex); 3000 3001 return ret; 3002 } 3003 3004 static int sched_rr_handler(struct ctl_table *table, int write, void *buffer, 3005 size_t *lenp, loff_t *ppos) 3006 { 3007 int ret; 3008 static DEFINE_MUTEX(mutex); 3009 3010 mutex_lock(&mutex); 3011 ret = proc_dointvec(table, write, buffer, lenp, ppos); 3012 /* 3013 * Make sure that internally we keep jiffies. 3014 * Also, writing zero resets the timeslice to default: 3015 */ 3016 if (!ret && write) { 3017 sched_rr_timeslice = 3018 sysctl_sched_rr_timeslice <= 0 ? RR_TIMESLICE : 3019 msecs_to_jiffies(sysctl_sched_rr_timeslice); 3020 3021 if (sysctl_sched_rr_timeslice <= 0) 3022 sysctl_sched_rr_timeslice = jiffies_to_msecs(RR_TIMESLICE); 3023 } 3024 mutex_unlock(&mutex); 3025 3026 return ret; 3027 } 3028 #endif /* CONFIG_SYSCTL */ 3029 3030 #ifdef CONFIG_SCHED_DEBUG 3031 void print_rt_stats(struct seq_file *m, int cpu) 3032 { 3033 rt_rq_iter_t iter; 3034 struct rt_rq *rt_rq; 3035 3036 rcu_read_lock(); 3037 for_each_rt_rq(rt_rq, iter, cpu_rq(cpu)) 3038 print_rt_rq(m, cpu, rt_rq); 3039 rcu_read_unlock(); 3040 } 3041 #endif /* CONFIG_SCHED_DEBUG */ 3042