Lines Matching +full:container +full:- +full:rules
1 // SPDX-License-Identifier: GPL-2.0
62 return dl_se->dl_server; in dl_server()
78 struct rq *rq = dl_se->rq; in rq_of_dl_se()
88 return &rq_of_dl_se(dl_se)->dl; in dl_rq_of_se()
93 return !RB_EMPTY_NODE(&dl_se->rb_node); in on_dl_rq()
99 return dl_se->pi_se; in pi_of()
122 return &cpu_rq(i)->rd->dl_bw; in dl_bw_of()
127 struct root_domain *rd = cpu_rq(i)->rd; in dl_bw_cpus()
132 return cpumask_weight_and(rd->span, cpu_active_mask); in dl_bw_cpus()
147 * XXX Fix: If 'rq->rd == def_root_domain' perform AC against capacity
159 return __dl_bw_capacity(cpu_rq(i)->rd->span); in dl_bw_capacity()
165 struct root_domain *rd = cpu_rq(cpu)->rd; in dl_bw_visited()
167 if (rd->visit_cookie == cookie) in dl_bw_visited()
170 rd->visit_cookie = cookie; in dl_bw_visited()
182 for_each_cpu_and(i, rd->span, cpu_active_mask) { in __dl_update()
185 rq->dl.extra_bw += bw; in __dl_update()
192 dl_b->total_bw -= tsk_bw; in __dl_sub()
199 dl_b->total_bw += tsk_bw; in __dl_add()
200 __dl_update(dl_b, -((s32)tsk_bw / cpus)); in __dl_add()
206 return dl_b->bw != -1 && in __dl_overflow()
207 cap_scale(dl_b->bw, cap) < dl_b->total_bw - old_bw + new_bw; in __dl_overflow()
213 u64 old = dl_rq->running_bw; in __add_running_bw()
216 dl_rq->running_bw += dl_bw; in __add_running_bw()
217 WARN_ON_ONCE(dl_rq->running_bw < old); /* overflow */ in __add_running_bw()
218 WARN_ON_ONCE(dl_rq->running_bw > dl_rq->this_bw); in __add_running_bw()
226 u64 old = dl_rq->running_bw; in __sub_running_bw()
229 dl_rq->running_bw -= dl_bw; in __sub_running_bw()
230 WARN_ON_ONCE(dl_rq->running_bw > old); /* underflow */ in __sub_running_bw()
231 if (dl_rq->running_bw > old) in __sub_running_bw()
232 dl_rq->running_bw = 0; in __sub_running_bw()
240 u64 old = dl_rq->this_bw; in __add_rq_bw()
243 dl_rq->this_bw += dl_bw; in __add_rq_bw()
244 WARN_ON_ONCE(dl_rq->this_bw < old); /* overflow */ in __add_rq_bw()
250 u64 old = dl_rq->this_bw; in __sub_rq_bw()
253 dl_rq->this_bw -= dl_bw; in __sub_rq_bw()
254 WARN_ON_ONCE(dl_rq->this_bw > old); /* underflow */ in __sub_rq_bw()
255 if (dl_rq->this_bw > old) in __sub_rq_bw()
256 dl_rq->this_bw = 0; in __sub_rq_bw()
257 WARN_ON_ONCE(dl_rq->running_bw > dl_rq->this_bw); in __sub_rq_bw()
264 __add_rq_bw(dl_se->dl_bw, dl_rq); in add_rq_bw()
271 __sub_rq_bw(dl_se->dl_bw, dl_rq); in sub_rq_bw()
278 __add_running_bw(dl_se->dl_bw, dl_rq); in add_running_bw()
285 __sub_running_bw(dl_se->dl_bw, dl_rq); in sub_running_bw()
290 if (dl_se->dl_non_contending) { in dl_rq_change_utilization()
291 sub_running_bw(dl_se, &rq->dl); in dl_rq_change_utilization()
292 dl_se->dl_non_contending = 0; in dl_rq_change_utilization()
301 if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1) { in dl_rq_change_utilization()
306 __sub_rq_bw(dl_se->dl_bw, &rq->dl); in dl_rq_change_utilization()
307 __add_rq_bw(new_bw, &rq->dl); in dl_rq_change_utilization()
314 * If the timer callback was running (hrtimer_try_to_cancel == -1), in cancel_dl_timer()
324 cancel_dl_timer(dl_se, &dl_se->dl_timer); in cancel_replenish_timer()
330 cancel_dl_timer(dl_se, &dl_se->inactive_timer); in cancel_inactive_timer()
335 WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV); in dl_change_utilization()
340 dl_rq_change_utilization(task_rq(p), &p->dl, new_bw); in dl_change_utilization()
348 * Instead, we have to wait for the so called "0-lag time".
350 * If a task blocks before the "0-lag time", a timer (the inactive
365 * "ACTIVE non contending" task is a blocked task for which the "0-lag time"
366 * has not passed yet. An "INACTIVE" task is a task for which the "0-lag"
368 * +------------------+
370 * +------------------>+ contending |
372 * | +----+------+------+
375 * +--------+-------+ | |
376 * | | t >= 0-lag | | wakeup
377 * | INACTIVE |<---------------+ |
379 * +--------+-------+ | |
381 * | t < 0-lag | |
384 * | +----+------+------+
386 * +-------------------+ |
388 * fired +------------------+
391 * blocks, and checks if the 0-lag time already passed or
401 struct hrtimer *timer = &dl_se->inactive_timer; in task_non_contending()
403 struct dl_rq *dl_rq = &rq->dl; in task_non_contending()
407 * If this is a non-deadline task that has been boosted, in task_non_contending()
410 if (dl_se->dl_runtime == 0) in task_non_contending()
416 WARN_ON(dl_se->dl_non_contending); in task_non_contending()
418 zerolag_time = dl_se->deadline - in task_non_contending()
419 div64_long((dl_se->runtime * dl_se->dl_period), in task_non_contending()
420 dl_se->dl_runtime); in task_non_contending()
423 * Using relative times instead of the absolute "0-lag time" in task_non_contending()
426 zerolag_time -= rq_clock(rq); in task_non_contending()
429 * If the "0-lag time" already passed, decrease the active in task_non_contending()
432 if ((zerolag_time < 0) || hrtimer_active(&dl_se->inactive_timer)) { in task_non_contending()
441 if (!dl_task || READ_ONCE(p->__state) == TASK_DEAD) { in task_non_contending()
444 if (READ_ONCE(p->__state) == TASK_DEAD) in task_non_contending()
445 sub_rq_bw(dl_se, &rq->dl); in task_non_contending()
446 raw_spin_lock(&dl_b->lock); in task_non_contending()
447 __dl_sub(dl_b, dl_se->dl_bw, dl_bw_cpus(task_cpu(p))); in task_non_contending()
448 raw_spin_unlock(&dl_b->lock); in task_non_contending()
456 dl_se->dl_non_contending = 1; in task_non_contending()
468 * If this is a non-deadline task that has been boosted, in task_contending()
471 if (dl_se->dl_runtime == 0) in task_contending()
477 if (dl_se->dl_non_contending) { in task_contending()
478 dl_se->dl_non_contending = 0; in task_contending()
501 return rb_first_cached(&dl_rq->root) == &dl_se->rb_node; in is_leftmost()
508 raw_spin_lock_init(&dl_b->lock); in init_dl_bw()
510 dl_b->bw = -1; in init_dl_bw()
512 dl_b->bw = to_ratio(global_rt_period(), global_rt_runtime()); in init_dl_bw()
513 dl_b->total_bw = 0; in init_dl_bw()
518 dl_rq->root = RB_ROOT_CACHED; in init_dl_rq()
520 /* zero means no -deadline tasks */ in init_dl_rq()
521 dl_rq->earliest_dl.curr = dl_rq->earliest_dl.next = 0; in init_dl_rq()
523 dl_rq->overloaded = 0; in init_dl_rq()
524 dl_rq->pushable_dl_tasks_root = RB_ROOT_CACHED; in init_dl_rq()
526 dl_rq->running_bw = 0; in init_dl_rq()
527 dl_rq->this_bw = 0; in init_dl_rq()
533 return atomic_read(&rq->rd->dlo_count); in dl_overloaded()
538 if (!rq->online) in dl_set_overload()
541 cpumask_set_cpu(rq->cpu, rq->rd->dlo_mask); in dl_set_overload()
549 atomic_inc(&rq->rd->dlo_count); in dl_set_overload()
554 if (!rq->online) in dl_clear_overload()
557 atomic_dec(&rq->rd->dlo_count); in dl_clear_overload()
558 cpumask_clear_cpu(rq->cpu, rq->rd->dlo_mask); in dl_clear_overload()
566 return dl_entity_preempt(&__node_2_pdl(a)->dl, &__node_2_pdl(b)->dl); in __pushable_less()
571 return !RB_EMPTY_ROOT(&rq->dl.pushable_dl_tasks_root.rb_root); in has_pushable_dl_tasks()
575 * The list of pushable -deadline task is not a plist, like in
576 * sched_rt.c, it is an rb-tree with tasks ordered by deadline.
582 WARN_ON_ONCE(!RB_EMPTY_NODE(&p->pushable_dl_tasks)); in enqueue_pushable_dl_task()
584 leftmost = rb_add_cached(&p->pushable_dl_tasks, in enqueue_pushable_dl_task()
585 &rq->dl.pushable_dl_tasks_root, in enqueue_pushable_dl_task()
588 rq->dl.earliest_dl.next = p->dl.deadline; in enqueue_pushable_dl_task()
590 if (!rq->dl.overloaded) { in enqueue_pushable_dl_task()
592 rq->dl.overloaded = 1; in enqueue_pushable_dl_task()
598 struct dl_rq *dl_rq = &rq->dl; in dequeue_pushable_dl_task()
599 struct rb_root_cached *root = &dl_rq->pushable_dl_tasks_root; in dequeue_pushable_dl_task()
602 if (RB_EMPTY_NODE(&p->pushable_dl_tasks)) in dequeue_pushable_dl_task()
605 leftmost = rb_erase_cached(&p->pushable_dl_tasks, root); in dequeue_pushable_dl_task()
607 dl_rq->earliest_dl.next = __node_2_pdl(leftmost)->dl.deadline; in dequeue_pushable_dl_task()
609 RB_CLEAR_NODE(&p->pushable_dl_tasks); in dequeue_pushable_dl_task()
611 if (!has_pushable_dl_tasks(rq) && rq->dl.overloaded) { in dequeue_pushable_dl_task()
613 rq->dl.overloaded = 0; in dequeue_pushable_dl_task()
621 return rq->online && dl_task(prev); in need_pull_dl_task()
635 queue_balance_callback(rq, &per_cpu(dl_push_head, rq->cpu), push_dl_tasks); in deadline_queue_push_tasks()
640 queue_balance_callback(rq, &per_cpu(dl_pull_head, rq->cpu), pull_dl_task); in deadline_queue_pull_task()
658 cpu = cpumask_any_and(cpu_active_mask, p->cpus_ptr); in dl_task_offline_migration()
677 if (p->dl.dl_non_contending || p->dl.dl_throttled) { in dl_task_offline_migration()
684 sub_running_bw(&p->dl, &rq->dl); in dl_task_offline_migration()
685 sub_rq_bw(&p->dl, &rq->dl); in dl_task_offline_migration()
687 add_rq_bw(&p->dl, &later_rq->dl); in dl_task_offline_migration()
688 add_running_bw(&p->dl, &later_rq->dl); in dl_task_offline_migration()
690 sub_rq_bw(&p->dl, &rq->dl); in dl_task_offline_migration()
691 add_rq_bw(&p->dl, &later_rq->dl); in dl_task_offline_migration()
699 dl_b = &rq->rd->dl_bw; in dl_task_offline_migration()
700 raw_spin_lock(&dl_b->lock); in dl_task_offline_migration()
701 __dl_sub(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span)); in dl_task_offline_migration()
702 raw_spin_unlock(&dl_b->lock); in dl_task_offline_migration()
704 dl_b = &later_rq->rd->dl_bw; in dl_task_offline_migration()
705 raw_spin_lock(&dl_b->lock); in dl_task_offline_migration()
706 __dl_add(dl_b, p->dl.dl_bw, cpumask_weight(later_rq->rd->span)); in dl_task_offline_migration()
707 raw_spin_unlock(&dl_b->lock); in dl_task_offline_migration()
709 set_task_cpu(p, later_rq->cpu); in dl_task_offline_migration()
724 /* for non-boosted task, pi_of(dl_se) == dl_se */ in replenish_dl_new_period()
725 dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; in replenish_dl_new_period()
726 dl_se->runtime = pi_of(dl_se)->dl_runtime; in replenish_dl_new_period()
732 if (dl_se->dl_defer && !dl_se->dl_defer_running) { in replenish_dl_new_period()
733 dl_se->dl_throttled = 1; in replenish_dl_new_period()
734 dl_se->dl_defer_armed = 1; in replenish_dl_new_period()
741 * - the absolute deadline of the entity has to be placed at
743 * - the runtime of the entity has to be set to the maximum value.
745 * The capability of specifying such event is useful whenever a -deadline
758 WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline)); in setup_new_dl_entity()
765 if (dl_se->dl_throttled) in setup_new_dl_entity()
802 WARN_ON_ONCE(pi_of(dl_se)->dl_runtime <= 0); in replenish_dl_entity()
805 * This could be the case for a !-dl task that is boosted. in replenish_dl_entity()
814 if (dl_se->dl_deadline == 0 || in replenish_dl_entity()
815 (dl_se->dl_defer_armed && dl_entity_overflow(dl_se, rq_clock(rq)))) { in replenish_dl_entity()
816 dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; in replenish_dl_entity()
817 dl_se->runtime = pi_of(dl_se)->dl_runtime; in replenish_dl_entity()
820 if (dl_se->dl_yielded && dl_se->runtime > 0) in replenish_dl_entity()
821 dl_se->runtime = 0; in replenish_dl_entity()
829 while (dl_se->runtime <= 0) { in replenish_dl_entity()
830 dl_se->deadline += pi_of(dl_se)->dl_period; in replenish_dl_entity()
831 dl_se->runtime += pi_of(dl_se)->dl_runtime; in replenish_dl_entity()
836 * the future" with respect to rq->clock. If it's in replenish_dl_entity()
843 if (dl_time_before(dl_se->deadline, rq_clock(rq))) { in replenish_dl_entity()
848 if (dl_se->dl_yielded) in replenish_dl_entity()
849 dl_se->dl_yielded = 0; in replenish_dl_entity()
850 if (dl_se->dl_throttled) in replenish_dl_entity()
851 dl_se->dl_throttled = 0; in replenish_dl_entity()
857 if (dl_se->dl_defer_armed) { in replenish_dl_entity()
858 dl_se->dl_defer_armed = 0; in replenish_dl_entity()
867 if (dl_se->dl_defer && !dl_se->dl_defer_running && in replenish_dl_entity()
868 dl_time_before(rq_clock(dl_se->rq), dl_se->deadline - dl_se->runtime)) { in replenish_dl_entity()
872 * Set dl_se->dl_defer_armed and dl_throttled variables to in replenish_dl_entity()
876 dl_se->dl_defer_armed = 1; in replenish_dl_entity()
877 dl_se->dl_throttled = 1; in replenish_dl_entity()
884 hrtimer_try_to_cancel(&dl_se->dl_timer); in replenish_dl_entity()
885 dl_se->dl_defer_armed = 0; in replenish_dl_entity()
886 dl_se->dl_throttled = 0; in replenish_dl_entity()
893 * Here we check if --at time t-- an entity (which is probably being
897 * one of the CBS rules: when a task wakes up, if the residual runtime
904 * Documentation/scheduler/sched-deadline.rst for more information).
908 * runtime / (deadline - t) > dl_runtime / dl_deadline ,
929 * and (deadline - t), since t is rq->clock, is the time left in dl_entity_overflow()
938 left = (pi_of(dl_se)->dl_deadline >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); in dl_entity_overflow()
939 right = ((dl_se->deadline - t) >> DL_SCALE) * in dl_entity_overflow()
940 (pi_of(dl_se)->dl_runtime >> DL_SCALE); in dl_entity_overflow()
946 * Revised wakeup rule [1]: For self-suspending tasks, rather then
947 * re-initializing task's runtime and deadline, the revised wakeup
952 * runtime / (deadline - t) > dl_runtime / dl_deadline
955 * runtime = (dl_runtime / dl_deadline) * (deadline - t)
961 * bandwidth server revisited. SIGBED Rev. 11, 4 (January 2015), 19-24.
966 u64 laxity = dl_se->deadline - rq_clock(rq); in update_dl_revised_wakeup()
974 WARN_ON(dl_time_before(dl_se->deadline, rq_clock(rq))); in update_dl_revised_wakeup()
976 dl_se->runtime = (dl_se->dl_density * laxity) >> BW_SHIFT; in update_dl_revised_wakeup()
992 return dl_se->dl_deadline == dl_se->dl_period; in dl_is_implicit()
998 * different rules: 1) the original CBS; and 2) the Revisited CBS.
1029 if (dl_time_before(dl_se->deadline, rq_clock(rq)) || in update_dl_entity()
1033 !dl_time_before(dl_se->deadline, rq_clock(rq)) && in update_dl_entity()
1040 } else if (dl_server(dl_se) && dl_se->dl_defer) { in update_dl_entity()
1045 if (!dl_se->dl_defer_running) { in update_dl_entity()
1046 dl_se->dl_defer_armed = 1; in update_dl_entity()
1047 dl_se->dl_throttled = 1; in update_dl_entity()
1054 return dl_se->deadline - dl_se->dl_deadline + dl_se->dl_period; in dl_next_period()
1069 struct hrtimer *timer = &dl_se->dl_timer; in start_dl_timer()
1079 * that it is actually coming from rq->clock and not from in start_dl_timer()
1083 * (deadline - runtime). At that point, the CBS rule will decide in start_dl_timer()
1088 if (dl_se->dl_defer_armed) { in start_dl_timer()
1089 WARN_ON_ONCE(!dl_se->dl_throttled); in start_dl_timer()
1090 act = ns_to_ktime(dl_se->deadline - dl_se->runtime); in start_dl_timer()
1092 /* act = deadline - rel-deadline + period */ in start_dl_timer()
1097 delta = ktime_to_ns(now) - rq_clock(rq); in start_dl_timer()
1113 * harmless because we're holding task_rq()->lock, therefore the timer in start_dl_timer()
1134 * Nothing relies on rq->lock after this, so its safe to drop in __push_dl_task()
1135 * rq->lock. in __push_dl_task()
1154 if (!dl_se->dl_throttled || !dl_se->dl_runtime) in dl_server_timer()
1165 rq->donor->sched_class->update_curr(rq); in dl_server_timer()
1167 if (dl_se->dl_defer_idle) { in dl_server_timer()
1172 if (dl_se->dl_defer_armed) { in dl_server_timer()
1179 if (dl_time_before(rq_clock(dl_se->rq), in dl_server_timer()
1180 (dl_se->deadline - dl_se->runtime - dl_server_min_res))) { in dl_server_timer()
1183 fw = dl_se->deadline - rq_clock(dl_se->rq) - dl_se->runtime; in dl_server_timer()
1189 dl_se->dl_defer_running = 1; in dl_server_timer()
1194 if (!dl_task(dl_se->rq->curr) || dl_entity_preempt(dl_se, &dl_se->rq->curr->dl)) in dl_server_timer()
1249 if (!dl_se->dl_throttled) in dl_task_timer()
1256 * If the throttle happened during sched-out; like: in dl_task_timer()
1264 * prev->on_rq = 0; in dl_task_timer()
1267 * but do not enqueue -- wait for our wakeup to do that. in dl_task_timer()
1274 if (unlikely(!rq->online)) { in dl_task_timer()
1292 if (dl_task(rq->donor)) in dl_task_timer()
1313 struct hrtimer *timer = &dl_se->dl_timer; in init_dl_task_timer()
1340 if (dl_time_before(dl_se->deadline, rq_clock(rq)) && in dl_check_constrained_dl()
1344 dl_se->dl_throttled = 1; in dl_check_constrained_dl()
1345 if (dl_se->runtime > 0) in dl_check_constrained_dl()
1346 dl_se->runtime = 0; in dl_check_constrained_dl()
1353 return (dl_se->runtime <= 0); in dl_runtime_exceeded()
1358 * GRUB reclaiming algorithm, the runtime is not decreased as "dq = -dt",
1359 * but as "dq = -(max{u, (Umax - Uinact - Uextra)} / Umax) dt",
1361 * utilization, Uinact is the (per-runqueue) inactive utilization, computed
1365 * Since rq->dl.running_bw and rq->dl.this_bw contain utilizations multiplied
1367 * Since rq->dl.bw_ratio contains 1 / Umax multiplied by 2^RATIO_SHIFT, dl_bw
1368 * is multiplied by rq->dl.bw_ratio and shifted right by RATIO_SHIFT.
1370 * larger than 2^(64 - 20 - 8), which is more than 64 seconds. So, overflow is
1376 u64 u_inact = rq->dl.this_bw - rq->dl.running_bw; /* Utot - Uact */ in grub_reclaim()
1379 * Instead of computing max{u, (u_max - u_inact - u_extra)}, we in grub_reclaim()
1380 * compare u_inact + u_extra with u_max - u, because u_inact + u_extra in grub_reclaim()
1381 * can be larger than u_max. So, u_max - u_inact - u_extra would be in grub_reclaim()
1384 if (u_inact + rq->dl.extra_bw > rq->dl.max_bw - dl_se->dl_bw) in grub_reclaim()
1385 u_act = dl_se->dl_bw; in grub_reclaim()
1387 u_act = rq->dl.max_bw - u_inact - rq->dl.extra_bw; in grub_reclaim()
1389 u_act = (u_act * rq->dl.bw_ratio) >> RATIO_SHIFT; in grub_reclaim()
1398 * For tasks that participate in GRUB, we implement GRUB-PA: the in dl_scaled_delta_exec()
1404 if (unlikely(dl_se->flags & SCHED_FLAG_RECLAIM)) { in dl_scaled_delta_exec()
1423 bool idle = rq->curr == rq->idle; in update_curr_dl_se()
1427 if (unlikely(dl_se->dl_yielded)) in update_curr_dl_se()
1432 if (dl_server(dl_se) && dl_se->dl_throttled && !dl_se->dl_defer) in update_curr_dl_se()
1442 dl_se->runtime -= scaled_delta_exec; in update_curr_dl_se()
1444 if (dl_se->dl_defer_idle && !idle) in update_curr_dl_se()
1445 dl_se->dl_defer_idle = 0; in update_curr_dl_se()
1455 if (dl_se->dl_defer && dl_se->dl_throttled && dl_runtime_exceeded(dl_se)) { in update_curr_dl_se()
1457 * Non-servers would never get time accounted while throttled. in update_curr_dl_se()
1466 if (dl_se->dl_defer_idle && idle) { in update_curr_dl_se()
1468 * The timer is at the zero-laxity point, this means in update_curr_dl_se()
1472 * will be set for 'deadline - runtime'. Negative in update_curr_dl_se()
1475 dl_se->runtime = 0; in update_curr_dl_se()
1480 * If the server was previously activated - the starving condition in update_curr_dl_se()
1485 dl_se->dl_defer_running = 0; in update_curr_dl_se()
1487 hrtimer_try_to_cancel(&dl_se->dl_timer); in update_curr_dl_se()
1489 replenish_dl_new_period(dl_se, dl_se->rq); in update_curr_dl_se()
1492 dl_se->dl_defer_idle = 1; in update_curr_dl_se()
1506 if (dl_runtime_exceeded(dl_se) || dl_se->dl_yielded) { in update_curr_dl_se()
1507 dl_se->dl_throttled = 1; in update_curr_dl_se()
1511 (dl_se->flags & SCHED_FLAG_DL_OVERRUN)) in update_curr_dl_se()
1512 dl_se->dl_overrun = 1; in update_curr_dl_se()
1516 update_stats_dequeue_dl(&rq->dl, dl_se, 0); in update_curr_dl_se()
1529 if (!is_leftmost(dl_se, &rq->dl)) in update_curr_dl_se()
1534 * The fair server (sole dl_server) does not account for real-time in update_curr_dl_se()
1537 if (dl_se == &rq->fair_server) in update_curr_dl_se()
1542 * Because -- for now -- we share the rt bandwidth, we need to in update_curr_dl_se()
1549 * using deadline servers -- however there's a few nasties to figure in update_curr_dl_se()
1553 struct rt_rq *rt_rq = &rq->rt; in update_curr_dl_se()
1555 raw_spin_lock(&rt_rq->rt_runtime_lock); in update_curr_dl_se()
1562 rt_rq->rt_time += delta_exec; in update_curr_dl_se()
1563 raw_spin_unlock(&rt_rq->rt_runtime_lock); in update_curr_dl_se()
1569 * In the non-defer mode, the idle time is not accounted, as the
1578 if (dl_se->dl_server_active && dl_se->dl_runtime && dl_se->dl_defer) in dl_server_update_idle()
1579 update_curr_dl_se(dl_se->rq, dl_se, delta_exec); in dl_server_update_idle()
1585 if (dl_se->dl_server_active && dl_se->dl_runtime) in dl_server_update()
1586 update_curr_dl_se(dl_se->rq, dl_se, delta_exec); in dl_server_update()
1593 * +--------------------+
1595 * +-------------+ 4 +-----------+ 5 +------------------+
1596 * +-> | A:init | <--- | D:running | -----> | E:replenish-wait |
1597 * | +-------------+ +-----------+ +------------------+
1599 * | | 1 +----------+ | 3 |
1601 * | +--------------------------------+ 2 |
1602 * | | | ----+ |
1603 * | 8 | B:zero_laxity-wait | | |
1604 * | | | <---+ |
1605 * | +--------------------------------+ |
1607 * | | 7 | 2 +--------------------+
1609 * | +-------------+ |
1610 * +-- | C:idle-wait | -+
1611 * +-------------+
1613 * +---------+
1616 * [A] - init
1623 * [B] - zero_laxity-wait
1630 * [C] - idle-wait
1637 * [D] - running
1644 * [E] - replenish-wait
1652 * [1] A->B, A->D
1665 * // deplete server runtime from client-class
1666 * [2] B->B, C->B, E->B
1682 * [3] B->D
1697 * [4] D->A
1711 * [5] D->E
1720 * [6] E->D
1724 * fwd-period
1731 * [7] B->C, C->C
1748 * [8] C->A
1756 * "A:init" -> "B:zero_laxity-wait" [label="1:dl_server_start"]
1757 * "A:init" -> "D:running" [label="1:dl_server_start"]
1758 * "B:zero_laxity-wait" -> "B:zero_laxity-wait" [label="2:dl_server_update"]
1759 * "B:zero_laxity-wait" -> "C:idle-wait" [label="7:dl_server_update_idle"]
1760 * "B:zero_laxity-wait" -> "D:running" [label="3:dl_server_timer"]
1761 * "C:idle-wait" -> "A:init" [label="8:dl_server_timer"]
1762 * "C:idle-wait" -> "B:zero_laxity-wait" [label="2:dl_server_update"]
1763 * "C:idle-wait" -> "C:idle-wait" [label="7:dl_server_update_idle"]
1764 * "D:running" -> "A:init" [label="4:pick_task_dl"]
1765 * "D:running" -> "E:replenish-wait" [label="5:update_curr_dl_se"]
1766 * "E:replenish-wait" -> "B:zero_laxity-wait" [label="2:dl_server_update"]
1767 * "E:replenish-wait" -> "D:running" [label="6:dl_server_timer"]
1773 * - When there are fair tasks running the most likely loop is [2]->[2].
1776 * - When there is actual fair starvation; the timer fires and starts the
1780 * - When idle it will push the actication forward once, and then wait
1781 * for the timer to hit or a non-idle update to restart things.
1785 struct rq *rq = dl_se->rq; in dl_server_start()
1787 if (!dl_server(dl_se) || dl_se->dl_server_active) in dl_server_start()
1793 rq->donor->sched_class->update_curr(rq); in dl_server_start()
1798 dl_se->dl_server_active = 1; in dl_server_start()
1800 if (!dl_task(dl_se->rq->curr) || dl_entity_preempt(dl_se, &rq->curr->dl)) in dl_server_start()
1801 resched_curr(dl_se->rq); in dl_server_start()
1810 hrtimer_try_to_cancel(&dl_se->dl_timer); in dl_server_stop()
1811 dl_se->dl_defer_armed = 0; in dl_server_stop()
1812 dl_se->dl_throttled = 0; in dl_server_stop()
1813 dl_se->dl_defer_idle = 0; in dl_server_stop()
1814 dl_se->dl_server_active = 0; in dl_server_stop()
1820 dl_se->rq = rq; in dl_server_init()
1821 dl_se->server_pick_task = pick_task; in dl_server_init()
1838 dl_se = &rq->fair_server; in sched_init_dl_servers()
1844 dl_se->dl_server = 1; in sched_init_dl_servers()
1845 dl_se->dl_defer = 1; in sched_init_dl_servers()
1852 u64 new_bw = dl_se->dl_bw; in __dl_server_attach_root()
1857 guard(raw_spinlock)(&dl_b->lock); in __dl_server_attach_root()
1867 u64 old_bw = init ? 0 : to_ratio(dl_se->dl_period, dl_se->dl_runtime); in dl_server_apply_params()
1869 struct rq *rq = dl_se->rq; in dl_server_apply_params()
1877 guard(raw_spinlock)(&dl_b->lock); in dl_server_apply_params()
1883 return -EBUSY; in dl_server_apply_params()
1886 __add_rq_bw(new_bw, &rq->dl); in dl_server_apply_params()
1889 __dl_sub(dl_b, dl_se->dl_bw, cpus); in dl_server_apply_params()
1895 dl_se->dl_runtime = runtime; in dl_server_apply_params()
1896 dl_se->dl_deadline = period; in dl_server_apply_params()
1897 dl_se->dl_period = period; in dl_server_apply_params()
1899 dl_se->runtime = 0; in dl_server_apply_params()
1900 dl_se->deadline = 0; in dl_server_apply_params()
1902 dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); in dl_server_apply_params()
1903 dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime); in dl_server_apply_params()
1910 * a -deadline task and has not been removed from the dl_rq).
1914 struct task_struct *donor = rq->donor; in update_curr_dl()
1915 struct sched_dl_entity *dl_se = &donor->dl; in update_curr_dl()
1946 rq = dl_se->rq; in inactive_task_timer()
1956 if (!dl_task(p) || READ_ONCE(p->__state) == TASK_DEAD) { in inactive_task_timer()
1959 if (READ_ONCE(p->__state) == TASK_DEAD && dl_se->dl_non_contending) { in inactive_task_timer()
1960 sub_running_bw(&p->dl, dl_rq_of_se(&p->dl)); in inactive_task_timer()
1961 sub_rq_bw(&p->dl, dl_rq_of_se(&p->dl)); in inactive_task_timer()
1962 dl_se->dl_non_contending = 0; in inactive_task_timer()
1965 raw_spin_lock(&dl_b->lock); in inactive_task_timer()
1966 __dl_sub(dl_b, p->dl.dl_bw, dl_bw_cpus(task_cpu(p))); in inactive_task_timer()
1967 raw_spin_unlock(&dl_b->lock); in inactive_task_timer()
1974 if (dl_se->dl_non_contending == 0) in inactive_task_timer()
1977 sub_running_bw(dl_se, &rq->dl); in inactive_task_timer()
1978 dl_se->dl_non_contending = 0; in inactive_task_timer()
1993 struct hrtimer *timer = &dl_se->inactive_timer; in init_dl_inactive_task_timer()
2005 if (dl_rq->earliest_dl.curr == 0 || in inc_dl_deadline()
2006 dl_time_before(deadline, dl_rq->earliest_dl.curr)) { in inc_dl_deadline()
2007 if (dl_rq->earliest_dl.curr == 0) in inc_dl_deadline()
2008 cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_HIGHER); in inc_dl_deadline()
2009 dl_rq->earliest_dl.curr = deadline; in inc_dl_deadline()
2010 cpudl_set(&rq->rd->cpudl, rq->cpu, deadline); in inc_dl_deadline()
2022 if (!dl_rq->dl_nr_running) { in dec_dl_deadline()
2023 dl_rq->earliest_dl.curr = 0; in dec_dl_deadline()
2024 dl_rq->earliest_dl.next = 0; in dec_dl_deadline()
2025 cpudl_clear(&rq->rd->cpudl, rq->cpu, rq->online); in dec_dl_deadline()
2026 cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr); in dec_dl_deadline()
2028 struct rb_node *leftmost = rb_first_cached(&dl_rq->root); in dec_dl_deadline()
2031 dl_rq->earliest_dl.curr = entry->deadline; in dec_dl_deadline()
2032 cpudl_set(&rq->rd->cpudl, rq->cpu, entry->deadline); in dec_dl_deadline()
2039 u64 deadline = dl_se->deadline; in inc_dl_tasks()
2041 dl_rq->dl_nr_running++; in inc_dl_tasks()
2052 WARN_ON(!dl_rq->dl_nr_running); in dec_dl_tasks()
2053 dl_rq->dl_nr_running--; in dec_dl_tasks()
2058 dec_dl_deadline(dl_rq, dl_se->deadline); in dec_dl_tasks()
2063 return dl_time_before(__node_2_dle(a)->deadline, __node_2_dle(b)->deadline); in __dl_less()
2075 return &dl_task_of(dl_se)->stats; in __schedstats_from_dl_se()
2125 state = READ_ONCE(p->__state); in update_stats_dequeue_dl()
2127 __schedstat_set(p->stats.sleep_start, in update_stats_dequeue_dl()
2131 __schedstat_set(p->stats.block_start, in update_stats_dequeue_dl()
2140 WARN_ON_ONCE(!RB_EMPTY_NODE(&dl_se->rb_node)); in __enqueue_dl_entity()
2142 rb_add_cached(&dl_se->rb_node, &dl_rq->root, __dl_less); in __enqueue_dl_entity()
2151 if (RB_EMPTY_NODE(&dl_se->rb_node)) in __dequeue_dl_entity()
2154 rb_erase_cached(&dl_se->rb_node, &dl_rq->root); in __dequeue_dl_entity()
2156 RB_CLEAR_NODE(&dl_se->rb_node); in __dequeue_dl_entity()
2174 if (!dl_se->dl_throttled && !dl_is_implicit(dl_se)) in enqueue_dl_entity()
2191 * task's state - in GRUB parlance, "inactive" vs "active contending"). in enqueue_dl_entity()
2196 if (!dl_se->dl_defer && dl_se->dl_throttled && !(flags & ENQUEUE_REPLENISH)) { in enqueue_dl_entity()
2215 dl_time_before(dl_se->deadline, rq_clock(rq_of_dl_se(dl_se)))) { in enqueue_dl_entity()
2223 if (dl_se->dl_throttled && start_dl_timer(dl_se)) in enqueue_dl_entity()
2227 * We're about to enqueue, make sure we're not ->dl_throttled! in enqueue_dl_entity()
2232 if (dl_se->dl_throttled) { in enqueue_dl_entity()
2233 hrtimer_try_to_cancel(&dl_se->dl_timer); in enqueue_dl_entity()
2234 dl_se->dl_defer_armed = 0; in enqueue_dl_entity()
2235 dl_se->dl_throttled = 0; in enqueue_dl_entity()
2256 * (p->state == TASK_DEAD). We can handle the two cases in the same in dequeue_dl_entity()
2267 if (is_dl_boosted(&p->dl)) { in enqueue_task_dl()
2280 if (p->dl.dl_throttled) { in enqueue_task_dl()
2286 cancel_replenish_timer(&p->dl); in enqueue_task_dl()
2287 p->dl.dl_throttled = 0; in enqueue_task_dl()
2289 } else if (!dl_prio(p->normal_prio)) { in enqueue_task_dl()
2299 p->dl.dl_throttled = 0; in enqueue_task_dl()
2301 printk_deferred_once("sched: DL de-boosted task PID %d: REPLENISH flag missing\n", in enqueue_task_dl()
2308 update_stats_wait_start_dl(dl_rq_of_se(&p->dl), &p->dl); in enqueue_task_dl()
2310 if (p->on_rq == TASK_ON_RQ_MIGRATING) in enqueue_task_dl()
2313 enqueue_dl_entity(&p->dl, flags); in enqueue_task_dl()
2315 if (dl_server(&p->dl)) in enqueue_task_dl()
2321 if (!task_current(rq, p) && !p->dl.dl_throttled && p->nr_cpus_allowed > 1) in enqueue_task_dl()
2329 if (p->on_rq == TASK_ON_RQ_MIGRATING) in dequeue_task_dl()
2332 dequeue_dl_entity(&p->dl, flags); in dequeue_task_dl()
2333 if (!p->dl.dl_throttled && !dl_server(&p->dl)) in dequeue_task_dl()
2340 * Yield task semantic for -deadline tasks is:
2357 rq->donor->dl.dl_yielded = 1; in yield_task_dl()
2372 return (!rq->dl.dl_nr_running || in dl_task_is_earliest_deadline()
2373 dl_time_before(p->dl.deadline, in dl_task_is_earliest_deadline()
2374 rq->dl.earliest_dl.curr)); in dl_task_is_earliest_deadline()
2392 curr = READ_ONCE(rq->curr); /* unlocked access */ in select_task_rq_dl()
2393 donor = READ_ONCE(rq->donor); in select_task_rq_dl()
2396 * If we are dealing with a -deadline task, we must in select_task_rq_dl()
2405 (curr->nr_cpus_allowed < 2 || in select_task_rq_dl()
2406 !dl_entity_preempt(&p->dl, &donor->dl)) && in select_task_rq_dl()
2407 p->nr_cpus_allowed > 1; in select_task_rq_dl()
2419 if (target != -1 && in select_task_rq_dl()
2433 if (READ_ONCE(p->__state) != TASK_WAKING) in migrate_task_rq_dl()
2438 * Since p->state == TASK_WAKING, set_task_cpu() has been called in migrate_task_rq_dl()
2439 * from try_to_wake_up(). Hence, p->pi_lock is locked, but in migrate_task_rq_dl()
2440 * rq->lock is not... So, lock it in migrate_task_rq_dl()
2443 if (p->dl.dl_non_contending) { in migrate_task_rq_dl()
2445 sub_running_bw(&p->dl, &rq->dl); in migrate_task_rq_dl()
2446 p->dl.dl_non_contending = 0; in migrate_task_rq_dl()
2454 cancel_inactive_timer(&p->dl); in migrate_task_rq_dl()
2456 sub_rq_bw(&p->dl, &rq->dl); in migrate_task_rq_dl()
2466 if (rq->curr->nr_cpus_allowed == 1 || in check_preempt_equal_dl()
2467 !cpudl_find(&rq->rd->cpudl, rq->donor, NULL)) in check_preempt_equal_dl()
2474 if (p->nr_cpus_allowed != 1 && in check_preempt_equal_dl()
2475 cpudl_find(&rq->rd->cpudl, p, NULL)) in check_preempt_equal_dl()
2483 if (!on_dl_rq(&p->dl) && need_pull_dl_task(rq, p)) { in balance_dl()
2486 * picked for load-balance and preemption/IRQs are still in balance_dl()
2499 * Only called when both the current and waking task are -deadline
2505 if (dl_entity_preempt(&p->dl, &rq->donor->dl)) { in wakeup_preempt_dl()
2514 if ((p->dl.deadline == rq->donor->dl.deadline) && in wakeup_preempt_dl()
2515 !test_tsk_need_resched(rq->curr)) in wakeup_preempt_dl()
2522 hrtick_start(rq, dl_se->runtime); in start_hrtick_dl()
2532 struct sched_dl_entity *dl_se = &p->dl; in set_next_task_dl()
2533 struct dl_rq *dl_rq = &rq->dl; in set_next_task_dl()
2535 p->se.exec_start = rq_clock_task(rq); in set_next_task_dl()
2536 if (on_dl_rq(&p->dl)) in set_next_task_dl()
2545 if (rq->donor->sched_class != &dl_sched_class) in set_next_task_dl()
2551 start_hrtick_dl(rq, &p->dl); in set_next_task_dl()
2556 struct rb_node *left = rb_first_cached(&dl_rq->root); in pick_next_dl_entity()
2565 * __pick_next_task_dl - Helper to pick the next -deadline task to run.
2571 struct dl_rq *dl_rq = &rq->dl; in __pick_task_dl()
2582 p = dl_se->server_pick_task(dl_se, rf); in __pick_task_dl()
2587 rq->dl_server = dl_se; in __pick_task_dl()
2602 struct sched_dl_entity *dl_se = &p->dl; in put_prev_task_dl()
2603 struct dl_rq *dl_rq = &rq->dl; in put_prev_task_dl()
2605 if (on_dl_rq(&p->dl)) in put_prev_task_dl()
2615 if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1) in put_prev_task_dl()
2637 if (hrtick_enabled_dl(rq) && queued && p->dl.runtime > 0 && in task_tick_dl()
2638 is_leftmost(&p->dl, &rq->dl)) in task_tick_dl()
2639 start_hrtick_dl(rq, &p->dl); in task_tick_dl()
2665 next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root); in pick_earliest_pushable_dl_task()
2690 return -1; in find_later_rq()
2692 if (task->nr_cpus_allowed == 1) in find_later_rq()
2693 return -1; in find_later_rq()
2699 if (!cpudl_find(&task_rq(task)->rd->cpudl, task, later_mask)) in find_later_rq()
2700 return -1; in find_later_rq()
2712 * guess, since it is most likely cache-hot there. in find_later_rq()
2721 this_cpu = -1; in find_later_rq()
2725 if (sd->flags & SD_WAKE_AFFINE) { in find_later_rq()
2732 if (this_cpu != -1 && in find_later_rq()
2758 if (this_cpu != -1) in find_later_rq()
2765 return -1; in find_later_rq()
2775 p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root)); in pick_next_pushable_dl_task()
2777 WARN_ON_ONCE(rq->cpu != task_cpu(p)); in pick_next_pushable_dl_task()
2779 WARN_ON_ONCE(p->nr_cpus_allowed <= 1); in pick_next_pushable_dl_task()
2797 if ((cpu == -1) || (cpu == rq->cpu)) in find_lock_later_rq()
2815 * double_lock_balance had to release rq->lock, in the in find_lock_later_rq()
2826 * - the task is not on the rq anymore (it was in find_lock_later_rq()
2828 * - the task is not on CPU anymore in find_lock_later_rq()
2829 * - the task is still a dl task in find_lock_later_rq()
2830 * - the task is not queued on the rq anymore in find_lock_later_rq()
2831 * 4. For the non-throttled task (push_dl_task), the in find_lock_later_rq()
2836 !cpumask_test_cpu(later_rq->cpu, &task->cpus_mask) || in find_lock_later_rq()
2837 (task->dl.dl_throttled && in find_lock_later_rq()
2842 (!task->dl.dl_throttled && in find_lock_later_rq()
2852 * If the rq we found has no -deadline task, or in find_lock_later_rq()
2868 * See if the non running -deadline tasks on this rq
2884 * If next_task preempts rq->curr, and rq->curr in push_dl_task()
2888 if (dl_task(rq->donor) && in push_dl_task()
2889 dl_time_before(next_task->dl.deadline, rq->donor->dl.deadline) && in push_dl_task()
2890 rq->curr->nr_cpus_allowed > 1) { in push_dl_task()
2898 if (WARN_ON(next_task == rq->curr)) in push_dl_task()
2911 * find_lock_later_rq releases rq->lock and it is in push_dl_task()
2947 /* push_dl_task() will return true if it moved a -deadline task */ in push_dl_tasks()
2954 int this_cpu = this_rq->cpu, cpu; in pull_dl_task()
2969 for_each_cpu(cpu, this_rq->rd->dlo_mask) { in pull_dl_task()
2979 if (this_rq->dl.dl_nr_running && in pull_dl_task()
2980 dl_time_before(this_rq->dl.earliest_dl.curr, in pull_dl_task()
2981 src_rq->dl.earliest_dl.next)) in pull_dl_task()
2984 /* Might drop this_rq->lock */ in pull_dl_task()
2992 if (src_rq->dl.dl_nr_running <= 1) in pull_dl_task()
2999 * - it preempts our current (if there's one), in pull_dl_task()
3000 * - it will preempt the last one we pulled (if any). in pull_dl_task()
3002 if (p && dl_time_before(p->dl.deadline, dmin) && in pull_dl_task()
3004 WARN_ON(p == src_rq->curr); in pull_dl_task()
3011 if (dl_time_before(p->dl.deadline, in pull_dl_task()
3012 src_rq->donor->dl.deadline)) in pull_dl_task()
3019 dmin = p->dl.deadline; in pull_dl_task()
3031 stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop, in pull_dl_task()
3032 push_task, &src_rq->push_work); in pull_dl_task()
3049 !test_tsk_need_resched(rq->curr) && in task_woken_dl()
3050 p->nr_cpus_allowed > 1 && in task_woken_dl()
3051 dl_task(rq->donor) && in task_woken_dl()
3052 (rq->curr->nr_cpus_allowed < 2 || in task_woken_dl()
3053 !dl_entity_preempt(&p->dl, &rq->donor->dl))) { in task_woken_dl()
3067 src_rd = rq->rd; in set_cpus_allowed_dl()
3074 if (!cpumask_intersects(src_rd->span, ctx->new_mask)) { in set_cpus_allowed_dl()
3083 raw_spin_lock(&src_dl_b->lock); in set_cpus_allowed_dl()
3084 __dl_sub(src_dl_b, p->dl.dl_bw, dl_bw_cpus(task_cpu(p))); in set_cpus_allowed_dl()
3085 raw_spin_unlock(&src_dl_b->lock); in set_cpus_allowed_dl()
3091 /* Assumes rq->lock is held */
3094 if (rq->dl.overloaded) in rq_online_dl()
3097 if (rq->dl.dl_nr_running > 0) in rq_online_dl()
3098 cpudl_set(&rq->rd->cpudl, rq->cpu, rq->dl.earliest_dl.curr); in rq_online_dl()
3100 cpudl_clear(&rq->rd->cpudl, rq->cpu, true); in rq_online_dl()
3103 /* Assumes rq->lock is held */
3106 if (rq->dl.overloaded) in rq_offline_dl()
3109 cpudl_clear(&rq->rd->cpudl, rq->cpu, false); in rq_offline_dl()
3122 * This function always returns a non-empty bitmap in @cpus. This is because
3132 if (!cpumask_intersects(p->cpus_ptr, hk_msk)) { in dl_get_task_effective_cpus()
3159 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); in dl_add_task_root_domain()
3160 if (!dl_task(p) || dl_entity_is_special(&p->dl)) { in dl_add_task_root_domain()
3161 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); in dl_add_task_root_domain()
3166 * Get an active rq, whose rq->rd traces the correct root in dl_add_task_root_domain()
3168 * Ideally this would be under cpuset reader lock until rq->rd is in dl_add_task_root_domain()
3177 dl_b = &rq->rd->dl_bw; in dl_add_task_root_domain()
3180 raw_spin_lock(&dl_b->lock); in dl_add_task_root_domain()
3181 __dl_add(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span)); in dl_add_task_root_domain()
3182 raw_spin_unlock(&dl_b->lock); in dl_add_task_root_domain()
3183 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); in dl_add_task_root_domain()
3190 guard(raw_spinlock_irqsave)(&rd->dl_bw.lock); in dl_clear_root_domain()
3194 * loop will add dl-servers contributions back properly, in dl_clear_root_domain()
3196 rd->dl_bw.total_bw = 0; in dl_clear_root_domain()
3197 for_each_cpu(i, rd->span) in dl_clear_root_domain()
3198 cpu_rq(i)->dl.extra_bw = cpu_rq(i)->dl.max_bw; in dl_clear_root_domain()
3204 for_each_cpu(i, rd->span) { in dl_clear_root_domain()
3205 struct sched_dl_entity *dl_se = &cpu_rq(i)->fair_server; in dl_clear_root_domain()
3208 __dl_add(&rd->dl_bw, dl_se->dl_bw, dl_bw_cpus(i)); in dl_clear_root_domain()
3214 dl_clear_root_domain(cpu_rq(cpu)->rd); in dl_clear_root_domain_cpu()
3220 * task_non_contending() can start the "inactive timer" (if the 0-lag in switched_from_dl()
3224 * SCHED_DEADLINE until the 0-lag time passes, inactive_task_timer() in switched_from_dl()
3227 if (task_on_rq_queued(p) && p->dl.dl_runtime) in switched_from_dl()
3228 task_non_contending(&p->dl, false); in switched_from_dl()
3243 if (p->dl.dl_non_contending) in switched_from_dl()
3244 sub_running_bw(&p->dl, &rq->dl); in switched_from_dl()
3245 sub_rq_bw(&p->dl, &rq->dl); in switched_from_dl()
3250 * at the 0-lag time, because the task could have been migrated in switched_from_dl()
3253 if (p->dl.dl_non_contending) in switched_from_dl()
3254 p->dl.dl_non_contending = 0; in switched_from_dl()
3257 * Since this might be the only -deadline task on the rq, in switched_from_dl()
3261 if (!task_on_rq_queued(p) || rq->dl.dl_nr_running) in switched_from_dl()
3268 * When switching to -deadline, we may overload the rq, then
3273 cancel_inactive_timer(&p->dl); in switched_to_dl()
3283 add_rq_bw(&p->dl, &rq->dl); in switched_to_dl()
3288 if (rq->donor != p) { in switched_to_dl()
3289 if (p->nr_cpus_allowed > 1 && rq->dl.overloaded) in switched_to_dl()
3291 if (dl_task(rq->donor)) in switched_to_dl()
3302 return p->dl.deadline; in get_prio_dl()
3306 * If the scheduling parameters of a -deadline task changed,
3314 if (p->dl.deadline == old_deadline) in prio_changed_dl()
3317 if (dl_time_before(old_deadline, p->dl.deadline)) in prio_changed_dl()
3326 if (dl_time_before(rq->dl.earliest_dl.curr, p->dl.deadline)) in prio_changed_dl()
3335 if (!dl_task(rq->curr) || in prio_changed_dl()
3336 dl_time_before(p->dl.deadline, rq->curr->dl.deadline)) in prio_changed_dl()
3344 return p->dl.dl_throttled; in task_is_throttled_dl()
3415 raw_spin_lock_irqsave(&dl_b->lock, flags); in sched_dl_global_validate()
3416 if (new_bw * cpus < dl_b->total_bw) in sched_dl_global_validate()
3417 ret = -EBUSY; in sched_dl_global_validate()
3418 raw_spin_unlock_irqrestore(&dl_b->lock, flags); in sched_dl_global_validate()
3433 dl_rq->bw_ratio = 1 << RATIO_SHIFT; in init_dl_rq_bw_ratio()
3434 dl_rq->max_bw = dl_rq->extra_bw = 1 << BW_SHIFT; in init_dl_rq_bw_ratio()
3436 dl_rq->bw_ratio = to_ratio(global_rt_runtime(), in init_dl_rq_bw_ratio()
3437 global_rt_period()) >> (BW_SHIFT - RATIO_SHIFT); in init_dl_rq_bw_ratio()
3438 dl_rq->max_bw = dl_rq->extra_bw = in init_dl_rq_bw_ratio()
3445 u64 new_bw = -1; in sched_dl_do_global()
3455 init_dl_rq_bw_ratio(&cpu_rq(cpu)->dl); in sched_dl_do_global()
3467 raw_spin_lock_irqsave(&dl_b->lock, flags); in sched_dl_do_global()
3468 dl_b->bw = new_bw; in sched_dl_do_global()
3469 raw_spin_unlock_irqrestore(&dl_b->lock, flags); in sched_dl_do_global()
3481 * This function is called while holding p's rq->lock.
3486 u64 period = attr->sched_period ?: attr->sched_deadline; in sched_dl_overflow()
3487 u64 runtime = attr->sched_runtime; in sched_dl_overflow()
3489 int cpus, err = -1, cpu = task_cpu(p); in sched_dl_overflow()
3493 if (attr->sched_flags & SCHED_FLAG_SUGOV) in sched_dl_overflow()
3497 if (new_bw == p->dl.dl_bw && task_has_dl_policy(p)) in sched_dl_overflow()
3501 * Either if a task, enters, leave, or stays -deadline but changes in sched_dl_overflow()
3503 * allocated bandwidth of the container. in sched_dl_overflow()
3505 raw_spin_lock(&dl_b->lock); in sched_dl_overflow()
3511 if (hrtimer_active(&p->dl.inactive_timer)) in sched_dl_overflow()
3512 __dl_sub(dl_b, p->dl.dl_bw, cpus); in sched_dl_overflow()
3516 !__dl_overflow(dl_b, cap, p->dl.dl_bw, new_bw)) { in sched_dl_overflow()
3520 * utilization change until the task's 0-lag point. in sched_dl_overflow()
3524 __dl_sub(dl_b, p->dl.dl_bw, cpus); in sched_dl_overflow()
3532 * (0-lag) time. in sched_dl_overflow()
3536 raw_spin_unlock(&dl_b->lock); in sched_dl_overflow()
3551 struct sched_dl_entity *dl_se = &p->dl; in __setparam_dl()
3553 dl_se->dl_runtime = attr->sched_runtime; in __setparam_dl()
3554 dl_se->dl_deadline = attr->sched_deadline; in __setparam_dl()
3555 dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline; in __setparam_dl()
3556 dl_se->flags = attr->sched_flags & SCHED_DL_FLAGS; in __setparam_dl()
3557 dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); in __setparam_dl()
3558 dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime); in __setparam_dl()
3563 struct sched_dl_entity *dl_se = &p->dl; in __getparam_dl()
3565 attr->sched_priority = p->rt_priority; in __getparam_dl()
3566 attr->sched_runtime = dl_se->dl_runtime; in __getparam_dl()
3567 attr->sched_deadline = dl_se->dl_deadline; in __getparam_dl()
3568 attr->sched_period = dl_se->dl_period; in __getparam_dl()
3569 attr->sched_flags &= ~SCHED_DL_FLAGS; in __getparam_dl()
3570 attr->sched_flags |= dl_se->flags; in __getparam_dl()
3574 * This function validates the new parameters of a -deadline task.
3588 if (attr->sched_flags & SCHED_FLAG_SUGOV) in __checkparam_dl()
3592 if (attr->sched_deadline == 0) in __checkparam_dl()
3599 if (attr->sched_runtime < (1ULL << DL_SCALE)) in __checkparam_dl()
3603 * Since we use the MSB for wrap-around and sign issues, make in __checkparam_dl()
3606 if (attr->sched_deadline & (1ULL << 63) || in __checkparam_dl()
3607 attr->sched_period & (1ULL << 63)) in __checkparam_dl()
3610 period = attr->sched_period; in __checkparam_dl()
3612 period = attr->sched_deadline; in __checkparam_dl()
3615 if (period < attr->sched_deadline || in __checkparam_dl()
3616 attr->sched_deadline < attr->sched_runtime) in __checkparam_dl()
3633 dl_se->dl_runtime = 0; in __dl_clear_params()
3634 dl_se->dl_deadline = 0; in __dl_clear_params()
3635 dl_se->dl_period = 0; in __dl_clear_params()
3636 dl_se->flags = 0; in __dl_clear_params()
3637 dl_se->dl_bw = 0; in __dl_clear_params()
3638 dl_se->dl_density = 0; in __dl_clear_params()
3640 dl_se->dl_throttled = 0; in __dl_clear_params()
3641 dl_se->dl_yielded = 0; in __dl_clear_params()
3642 dl_se->dl_non_contending = 0; in __dl_clear_params()
3643 dl_se->dl_overrun = 0; in __dl_clear_params()
3644 dl_se->dl_server = 0; in __dl_clear_params()
3647 dl_se->pi_se = dl_se; in __dl_clear_params()
3653 RB_CLEAR_NODE(&dl_se->rb_node); in init_dl_entity()
3661 struct sched_dl_entity *dl_se = &p->dl; in dl_param_changed()
3663 if (dl_se->dl_runtime != attr->sched_runtime || in dl_param_changed()
3664 dl_se->dl_deadline != attr->sched_deadline || in dl_param_changed()
3665 dl_se->dl_period != attr->sched_period || in dl_param_changed()
3666 dl_se->flags != (attr->sched_flags & SCHED_DL_FLAGS)) in dl_param_changed()
3682 raw_spin_lock_irqsave(&cur_dl_b->lock, flags); in dl_cpuset_cpumask_can_shrink()
3685 raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags); in dl_cpuset_cpumask_can_shrink()
3706 raw_spin_lock_irqsave(&dl_b->lock, flags); in dl_bw_manage()
3732 cap -= arch_scale_cpu_capacity(cpu); in dl_bw_manage()
3740 if (cpu_rq(cpu)->fair_server.dl_server) in dl_bw_manage()
3741 fair_server_bw = cpu_rq(cpu)->fair_server.dl_bw; in dl_bw_manage()
3748 if (dl_b->total_bw - fair_server_bw > 0) { in dl_bw_manage()
3754 if (dl_bw_cpus(cpu) - 1) in dl_bw_manage()
3763 raw_spin_unlock_irqrestore(&dl_b->lock, flags); in dl_bw_manage()
3766 return overflow ? -EBUSY : 0; in dl_bw_manage()
3786 print_dl_rq(m, cpu, &cpu_rq(cpu)->dl); in print_dl_stats()