Lines Matching +full:timer +full:- +full:cannot +full:- +full:wake +full:- +full:cpu
1 // SPDX-License-Identifier: GPL-2.0
24 * guard against timer DoS.
58 return dl_se->dl_server; in dl_server()
74 struct rq *rq = dl_se->rq; in rq_of_dl_se()
84 return &rq_of_dl_se(dl_se)->dl; in dl_rq_of_se()
89 return !RB_EMPTY_NODE(&dl_se->rb_node); in on_dl_rq()
95 return dl_se->pi_se; in pi_of()
119 return &cpu_rq(i)->rd->dl_bw; in dl_bw_of()
124 struct root_domain *rd = cpu_rq(i)->rd; in dl_bw_cpus()
130 if (cpumask_subset(rd->span, cpu_active_mask)) in dl_bw_cpus()
131 return cpumask_weight(rd->span); in dl_bw_cpus()
135 for_each_cpu_and(i, rd->span, cpu_active_mask) in dl_bw_cpus()
153 * XXX Fix: If 'rq->rd == def_root_domain' perform AC against capacity
154 * of the CPU the task is running on rather rd's \Sum CPU capacity.
165 return __dl_bw_capacity(cpu_rq(i)->rd->span); in dl_bw_capacity()
169 bool dl_bw_visited(int cpu, u64 cookie) in dl_bw_visited() argument
171 struct root_domain *rd = cpu_rq(cpu)->rd; in dl_bw_visited()
173 if (rd->visit_cookie == cookie) in dl_bw_visited()
176 rd->visit_cookie = cookie; in dl_bw_visited()
188 for_each_cpu_and(i, rd->span, cpu_active_mask) { in __dl_update()
191 rq->dl.extra_bw += bw; in __dl_update()
197 return &cpu_rq(i)->dl.dl_bw; in dl_bw_of()
210 bool dl_bw_visited(int cpu, u64 cookie) in dl_bw_visited() argument
220 dl->extra_bw += bw; in __dl_update()
227 dl_b->total_bw -= tsk_bw; in __dl_sub()
234 dl_b->total_bw += tsk_bw; in __dl_add()
235 __dl_update(dl_b, -((s32)tsk_bw / cpus)); in __dl_add()
241 return dl_b->bw != -1 && in __dl_overflow()
242 cap_scale(dl_b->bw, cap) < dl_b->total_bw - old_bw + new_bw; in __dl_overflow()
248 u64 old = dl_rq->running_bw; in __add_running_bw()
251 dl_rq->running_bw += dl_bw; in __add_running_bw()
252 WARN_ON_ONCE(dl_rq->running_bw < old); /* overflow */ in __add_running_bw()
253 WARN_ON_ONCE(dl_rq->running_bw > dl_rq->this_bw); in __add_running_bw()
261 u64 old = dl_rq->running_bw; in __sub_running_bw()
264 dl_rq->running_bw -= dl_bw; in __sub_running_bw()
265 WARN_ON_ONCE(dl_rq->running_bw > old); /* underflow */ in __sub_running_bw()
266 if (dl_rq->running_bw > old) in __sub_running_bw()
267 dl_rq->running_bw = 0; in __sub_running_bw()
275 u64 old = dl_rq->this_bw; in __add_rq_bw()
278 dl_rq->this_bw += dl_bw; in __add_rq_bw()
279 WARN_ON_ONCE(dl_rq->this_bw < old); /* overflow */ in __add_rq_bw()
285 u64 old = dl_rq->this_bw; in __sub_rq_bw()
288 dl_rq->this_bw -= dl_bw; in __sub_rq_bw()
289 WARN_ON_ONCE(dl_rq->this_bw > old); /* underflow */ in __sub_rq_bw()
290 if (dl_rq->this_bw > old) in __sub_rq_bw()
291 dl_rq->this_bw = 0; in __sub_rq_bw()
292 WARN_ON_ONCE(dl_rq->running_bw > dl_rq->this_bw); in __sub_rq_bw()
299 __add_rq_bw(dl_se->dl_bw, dl_rq); in add_rq_bw()
306 __sub_rq_bw(dl_se->dl_bw, dl_rq); in sub_rq_bw()
313 __add_running_bw(dl_se->dl_bw, dl_rq); in add_running_bw()
320 __sub_running_bw(dl_se->dl_bw, dl_rq); in sub_running_bw()
325 if (dl_se->dl_non_contending) { in dl_rq_change_utilization()
326 sub_running_bw(dl_se, &rq->dl); in dl_rq_change_utilization()
327 dl_se->dl_non_contending = 0; in dl_rq_change_utilization()
330 * If the timer handler is currently running and the in dl_rq_change_utilization()
331 * timer cannot be canceled, inactive_task_timer() in dl_rq_change_utilization()
336 if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1) { in dl_rq_change_utilization()
341 __sub_rq_bw(dl_se->dl_bw, &rq->dl); in dl_rq_change_utilization()
342 __add_rq_bw(new_bw, &rq->dl); in dl_rq_change_utilization()
346 void cancel_dl_timer(struct sched_dl_entity *dl_se, struct hrtimer *timer) in cancel_dl_timer() argument
349 * If the timer callback was running (hrtimer_try_to_cancel == -1), in cancel_dl_timer()
352 if (hrtimer_try_to_cancel(timer) == 1 && !dl_server(dl_se)) in cancel_dl_timer()
359 cancel_dl_timer(dl_se, &dl_se->dl_timer); in cancel_replenish_timer()
365 cancel_dl_timer(dl_se, &dl_se->inactive_timer); in cancel_inactive_timer()
370 WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV); in dl_change_utilization()
375 dl_rq_change_utilization(task_rq(p), &p->dl, new_bw); in dl_change_utilization()
381 * The utilization of a task cannot be immediately removed from
383 * Instead, we have to wait for the so called "0-lag time".
385 * If a task blocks before the "0-lag time", a timer (the inactive
386 * timer) is armed, and running_bw is decreased when the timer
389 * If the task wakes up again before the inactive timer fires,
390 * the timer is canceled, whereas if the task wakes up after the
391 * inactive timer fired (and running_bw has been decreased) the
394 * is used to avoid race conditions between the inactive timer handler
400 * "ACTIVE non contending" task is a blocked task for which the "0-lag time"
401 * has not passed yet. An "INACTIVE" task is a task for which the "0-lag"
403 * +------------------+
405 * +------------------>+ contending |
407 * | +----+------+------+
410 * +--------+-------+ | |
411 * | | t >= 0-lag | | wakeup
412 * | INACTIVE |<---------------+ |
414 * +--------+-------+ | |
416 * | t < 0-lag | |
419 * | +----+------+------+
421 * +-------------------+ |
422 * inactive timer | non contending |
423 * fired +------------------+
426 * blocks, and checks if the 0-lag time already passed or
428 * in the second case, it arms the inactive timer).
436 struct hrtimer *timer = &dl_se->inactive_timer; in task_non_contending() local
438 struct dl_rq *dl_rq = &rq->dl; in task_non_contending()
442 * If this is a non-deadline task that has been boosted, in task_non_contending()
445 if (dl_se->dl_runtime == 0) in task_non_contending()
451 WARN_ON(dl_se->dl_non_contending); in task_non_contending()
453 zerolag_time = dl_se->deadline - in task_non_contending()
454 div64_long((dl_se->runtime * dl_se->dl_period), in task_non_contending()
455 dl_se->dl_runtime); in task_non_contending()
458 * Using relative times instead of the absolute "0-lag time" in task_non_contending()
461 zerolag_time -= rq_clock(rq); in task_non_contending()
464 * If the "0-lag time" already passed, decrease the active in task_non_contending()
465 * utilization now, instead of starting a timer in task_non_contending()
467 if ((zerolag_time < 0) || hrtimer_active(&dl_se->inactive_timer)) { in task_non_contending()
476 if (!dl_task(p) || READ_ONCE(p->__state) == TASK_DEAD) { in task_non_contending()
479 if (READ_ONCE(p->__state) == TASK_DEAD) in task_non_contending()
480 sub_rq_bw(dl_se, &rq->dl); in task_non_contending()
481 raw_spin_lock(&dl_b->lock); in task_non_contending()
482 __dl_sub(dl_b, dl_se->dl_bw, dl_bw_cpus(task_cpu(p))); in task_non_contending()
483 raw_spin_unlock(&dl_b->lock); in task_non_contending()
491 dl_se->dl_non_contending = 1; in task_non_contending()
495 hrtimer_start(timer, ns_to_ktime(zerolag_time), HRTIMER_MODE_REL_HARD); in task_non_contending()
503 * If this is a non-deadline task that has been boosted, in task_contending()
506 if (dl_se->dl_runtime == 0) in task_contending()
512 if (dl_se->dl_non_contending) { in task_contending()
513 dl_se->dl_non_contending = 0; in task_contending()
515 * If the timer handler is currently running and the in task_contending()
516 * timer cannot be canceled, inactive_task_timer() in task_contending()
527 * when the "inactive timer" fired). in task_contending()
536 return rb_first_cached(&dl_rq->root) == &dl_se->rb_node; in is_leftmost()
543 raw_spin_lock_init(&dl_b->lock); in init_dl_bw()
545 dl_b->bw = -1; in init_dl_bw()
547 dl_b->bw = to_ratio(global_rt_period(), global_rt_runtime()); in init_dl_bw()
548 dl_b->total_bw = 0; in init_dl_bw()
553 dl_rq->root = RB_ROOT_CACHED; in init_dl_rq()
556 /* zero means no -deadline tasks */ in init_dl_rq()
557 dl_rq->earliest_dl.curr = dl_rq->earliest_dl.next = 0; in init_dl_rq()
559 dl_rq->overloaded = 0; in init_dl_rq()
560 dl_rq->pushable_dl_tasks_root = RB_ROOT_CACHED; in init_dl_rq()
562 init_dl_bw(&dl_rq->dl_bw); in init_dl_rq()
565 dl_rq->running_bw = 0; in init_dl_rq()
566 dl_rq->this_bw = 0; in init_dl_rq()
574 return atomic_read(&rq->rd->dlo_count); in dl_overloaded()
579 if (!rq->online) in dl_set_overload()
582 cpumask_set_cpu(rq->cpu, rq->rd->dlo_mask); in dl_set_overload()
590 atomic_inc(&rq->rd->dlo_count); in dl_set_overload()
595 if (!rq->online) in dl_clear_overload()
598 atomic_dec(&rq->rd->dlo_count); in dl_clear_overload()
599 cpumask_clear_cpu(rq->cpu, rq->rd->dlo_mask); in dl_clear_overload()
607 return dl_entity_preempt(&__node_2_pdl(a)->dl, &__node_2_pdl(b)->dl); in __pushable_less()
612 return !RB_EMPTY_ROOT(&rq->dl.pushable_dl_tasks_root.rb_root); in has_pushable_dl_tasks()
616 * The list of pushable -deadline task is not a plist, like in
617 * sched_rt.c, it is an rb-tree with tasks ordered by deadline.
623 WARN_ON_ONCE(!RB_EMPTY_NODE(&p->pushable_dl_tasks)); in enqueue_pushable_dl_task()
625 leftmost = rb_add_cached(&p->pushable_dl_tasks, in enqueue_pushable_dl_task()
626 &rq->dl.pushable_dl_tasks_root, in enqueue_pushable_dl_task()
629 rq->dl.earliest_dl.next = p->dl.deadline; in enqueue_pushable_dl_task()
631 if (!rq->dl.overloaded) { in enqueue_pushable_dl_task()
633 rq->dl.overloaded = 1; in enqueue_pushable_dl_task()
639 struct dl_rq *dl_rq = &rq->dl; in dequeue_pushable_dl_task()
640 struct rb_root_cached *root = &dl_rq->pushable_dl_tasks_root; in dequeue_pushable_dl_task()
643 if (RB_EMPTY_NODE(&p->pushable_dl_tasks)) in dequeue_pushable_dl_task()
646 leftmost = rb_erase_cached(&p->pushable_dl_tasks, root); in dequeue_pushable_dl_task()
648 dl_rq->earliest_dl.next = __node_2_pdl(leftmost)->dl.deadline; in dequeue_pushable_dl_task()
650 RB_CLEAR_NODE(&p->pushable_dl_tasks); in dequeue_pushable_dl_task()
652 if (!has_pushable_dl_tasks(rq) && rq->dl.overloaded) { in dequeue_pushable_dl_task()
654 rq->dl.overloaded = 0; in dequeue_pushable_dl_task()
662 return rq->online && dl_task(prev); in need_pull_dl_task()
676 queue_balance_callback(rq, &per_cpu(dl_push_head, rq->cpu), push_dl_tasks); in deadline_queue_push_tasks()
681 queue_balance_callback(rq, &per_cpu(dl_pull_head, rq->cpu), pull_dl_task); in deadline_queue_pull_task()
693 int cpu; in dl_task_offline_migration() local
696 * If we cannot preempt any rq, fall back to pick any in dl_task_offline_migration()
697 * online CPU: in dl_task_offline_migration()
699 cpu = cpumask_any_and(cpu_active_mask, p->cpus_ptr); in dl_task_offline_migration()
700 if (cpu >= nr_cpu_ids) { in dl_task_offline_migration()
702 * Failed to find any suitable CPU. in dl_task_offline_migration()
712 cpu = cpumask_any(cpu_active_mask); in dl_task_offline_migration()
714 later_rq = cpu_rq(cpu); in dl_task_offline_migration()
718 if (p->dl.dl_non_contending || p->dl.dl_throttled) { in dl_task_offline_migration()
720 * Inactive timer is armed (or callback is running, but in dl_task_offline_migration()
725 sub_running_bw(&p->dl, &rq->dl); in dl_task_offline_migration()
726 sub_rq_bw(&p->dl, &rq->dl); in dl_task_offline_migration()
728 add_rq_bw(&p->dl, &later_rq->dl); in dl_task_offline_migration()
729 add_running_bw(&p->dl, &later_rq->dl); in dl_task_offline_migration()
731 sub_rq_bw(&p->dl, &rq->dl); in dl_task_offline_migration()
732 add_rq_bw(&p->dl, &later_rq->dl); in dl_task_offline_migration()
740 dl_b = &rq->rd->dl_bw; in dl_task_offline_migration()
741 raw_spin_lock(&dl_b->lock); in dl_task_offline_migration()
742 __dl_sub(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span)); in dl_task_offline_migration()
743 raw_spin_unlock(&dl_b->lock); in dl_task_offline_migration()
745 dl_b = &later_rq->rd->dl_bw; in dl_task_offline_migration()
746 raw_spin_lock(&dl_b->lock); in dl_task_offline_migration()
747 __dl_add(dl_b, p->dl.dl_bw, cpumask_weight(later_rq->rd->span)); in dl_task_offline_migration()
748 raw_spin_unlock(&dl_b->lock); in dl_task_offline_migration()
750 set_task_cpu(p, later_rq->cpu); in dl_task_offline_migration()
796 /* for non-boosted task, pi_of(dl_se) == dl_se */ in replenish_dl_new_period()
797 dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; in replenish_dl_new_period()
798 dl_se->runtime = pi_of(dl_se)->dl_runtime; in replenish_dl_new_period()
804 if (dl_se->dl_defer && !dl_se->dl_defer_running) { in replenish_dl_new_period()
805 dl_se->dl_throttled = 1; in replenish_dl_new_period()
806 dl_se->dl_defer_armed = 1; in replenish_dl_new_period()
813 * - the absolute deadline of the entity has to be placed at
815 * - the runtime of the entity has to be set to the maximum value.
817 * The capability of specifying such event is useful whenever a -deadline
828 WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline)); in setup_new_dl_entity()
831 * We are racing with the deadline timer. So, do nothing because in setup_new_dl_entity()
832 * the deadline timer handler will take care of properly recharging in setup_new_dl_entity()
835 if (dl_se->dl_throttled) in setup_new_dl_entity()
872 WARN_ON_ONCE(pi_of(dl_se)->dl_runtime <= 0); in replenish_dl_entity()
875 * This could be the case for a !-dl task that is boosted. in replenish_dl_entity()
884 if (dl_se->dl_deadline == 0 || in replenish_dl_entity()
885 (dl_se->dl_defer_armed && dl_entity_overflow(dl_se, rq_clock(rq)))) { in replenish_dl_entity()
886 dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; in replenish_dl_entity()
887 dl_se->runtime = pi_of(dl_se)->dl_runtime; in replenish_dl_entity()
890 if (dl_se->dl_yielded && dl_se->runtime > 0) in replenish_dl_entity()
891 dl_se->runtime = 0; in replenish_dl_entity()
899 while (dl_se->runtime <= 0) { in replenish_dl_entity()
900 dl_se->deadline += pi_of(dl_se)->dl_period; in replenish_dl_entity()
901 dl_se->runtime += pi_of(dl_se)->dl_runtime; in replenish_dl_entity()
906 * the future" with respect to rq->clock. If it's in replenish_dl_entity()
913 if (dl_time_before(dl_se->deadline, rq_clock(rq))) { in replenish_dl_entity()
918 if (dl_se->dl_yielded) in replenish_dl_entity()
919 dl_se->dl_yielded = 0; in replenish_dl_entity()
920 if (dl_se->dl_throttled) in replenish_dl_entity()
921 dl_se->dl_throttled = 0; in replenish_dl_entity()
927 if (dl_se->dl_defer_armed) { in replenish_dl_entity()
928 dl_se->dl_defer_armed = 0; in replenish_dl_entity()
935 * and arm the defer timer. in replenish_dl_entity()
937 if (dl_se->dl_defer && !dl_se->dl_defer_running && in replenish_dl_entity()
938 dl_time_before(rq_clock(dl_se->rq), dl_se->deadline - dl_se->runtime)) { in replenish_dl_entity()
939 if (!is_dl_boosted(dl_se) && dl_se->server_has_tasks(dl_se)) { in replenish_dl_entity()
942 * Set dl_se->dl_defer_armed and dl_throttled variables to in replenish_dl_entity()
946 dl_se->dl_defer_armed = 1; in replenish_dl_entity()
947 dl_se->dl_throttled = 1; in replenish_dl_entity()
950 * If for whatever reason (delays), a previous timer was in replenish_dl_entity()
954 hrtimer_try_to_cancel(&dl_se->dl_timer); in replenish_dl_entity()
955 dl_se->dl_defer_armed = 0; in replenish_dl_entity()
956 dl_se->dl_throttled = 0; in replenish_dl_entity()
963 * Here we check if --at time t-- an entity (which is probably being
974 * Documentation/scheduler/sched-deadline.rst for more information).
978 * runtime / (deadline - t) > dl_runtime / dl_deadline ,
999 * and (deadline - t), since t is rq->clock, is the time left in dl_entity_overflow()
1008 left = (pi_of(dl_se)->dl_deadline >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); in dl_entity_overflow()
1009 right = ((dl_se->deadline - t) >> DL_SCALE) * in dl_entity_overflow()
1010 (pi_of(dl_se)->dl_runtime >> DL_SCALE); in dl_entity_overflow()
1016 * Revised wakeup rule [1]: For self-suspending tasks, rather then
1017 * re-initializing task's runtime and deadline, the revised wakeup
1022 * runtime / (deadline - t) > dl_runtime / dl_deadline
1025 * runtime = (dl_runtime / dl_deadline) * (deadline - t)
1031 * bandwidth server revisited. SIGBED Rev. 11, 4 (January 2015), 19-24.
1036 u64 laxity = dl_se->deadline - rq_clock(rq); in update_dl_revised_wakeup()
1044 WARN_ON(dl_time_before(dl_se->deadline, rq_clock(rq))); in update_dl_revised_wakeup()
1046 dl_se->runtime = (dl_se->dl_density * laxity) >> BW_SHIFT; in update_dl_revised_wakeup()
1062 return dl_se->dl_deadline == dl_se->dl_period; in dl_is_implicit()
1067 * might need to be updated. This is done by a CBS wake up rule. There are two
1099 if (dl_time_before(dl_se->deadline, rq_clock(rq)) || in update_dl_entity()
1103 !dl_time_before(dl_se->deadline, rq_clock(rq)) && in update_dl_entity()
1110 } else if (dl_server(dl_se) && dl_se->dl_defer) { in update_dl_entity()
1115 if (!dl_se->dl_defer_running) { in update_dl_entity()
1116 dl_se->dl_defer_armed = 1; in update_dl_entity()
1117 dl_se->dl_throttled = 1; in update_dl_entity()
1124 return dl_se->deadline - dl_se->dl_deadline + dl_se->dl_period; in dl_next_period()
1130 * set the bandwidth replenishment timer to the replenishment instant
1133 * Notice that it is important for the caller to know if the timer
1139 struct hrtimer *timer = &dl_se->dl_timer; in start_dl_timer() local
1148 * We want the timer to fire at the deadline, but considering in start_dl_timer()
1149 * that it is actually coming from rq->clock and not from in start_dl_timer()
1152 * The deferred reservation will have its timer set to in start_dl_timer()
1153 * (deadline - runtime). At that point, the CBS rule will decide in start_dl_timer()
1158 if (dl_se->dl_defer_armed) { in start_dl_timer()
1159 WARN_ON_ONCE(!dl_se->dl_throttled); in start_dl_timer()
1160 act = ns_to_ktime(dl_se->deadline - dl_se->runtime); in start_dl_timer()
1162 /* act = deadline - rel-deadline + period */ in start_dl_timer()
1166 now = hrtimer_cb_get_time(timer); in start_dl_timer()
1167 delta = ktime_to_ns(now) - rq_clock(rq); in start_dl_timer()
1173 * start the timer in the past! in start_dl_timer()
1183 * harmless because we're holding task_rq()->lock, therefore the timer in start_dl_timer()
1187 if (!hrtimer_is_queued(timer)) { in start_dl_timer()
1190 hrtimer_start(timer, act, HRTIMER_MODE_ABS_HARD); in start_dl_timer()
1205 * Nothing relies on rq->lock after this, so its safe to drop in __push_dl_task()
1206 * rq->lock. in __push_dl_task()
1215 /* a defer timer will not be reset if the runtime consumed was < dl_server_min_res */
1218 static enum hrtimer_restart dl_server_timer(struct hrtimer *timer, struct sched_dl_entity *dl_se) in dl_server_timer() argument
1226 if (!dl_se->dl_throttled || !dl_se->dl_runtime) in dl_server_timer()
1232 if (!dl_se->dl_runtime) in dl_server_timer()
1235 if (!dl_se->server_has_tasks(dl_se)) { in dl_server_timer()
1240 if (dl_se->dl_defer_armed) { in dl_server_timer()
1243 * If so, it is possible to push the defer timer for this amount in dl_server_timer()
1245 * forwarding the timer for a too small amount of time. in dl_server_timer()
1247 if (dl_time_before(rq_clock(dl_se->rq), in dl_server_timer()
1248 (dl_se->deadline - dl_se->runtime - dl_server_min_res))) { in dl_server_timer()
1250 /* reset the defer timer */ in dl_server_timer()
1251 fw = dl_se->deadline - rq_clock(dl_se->rq) - dl_se->runtime; in dl_server_timer()
1253 hrtimer_forward_now(timer, ns_to_ktime(fw)); in dl_server_timer()
1257 dl_se->dl_defer_running = 1; in dl_server_timer()
1262 if (!dl_task(dl_se->rq->curr) || dl_entity_preempt(dl_se, &dl_se->rq->curr->dl)) in dl_server_timer()
1272 * This is the bandwidth enforcement timer callback. If here, we know
1273 * a task is not on its dl_rq, since the fact that the timer was running
1284 static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) in dl_task_timer() argument
1286 struct sched_dl_entity *dl_se = container_of(timer, in dl_task_timer()
1294 return dl_server_timer(timer, dl_se); in dl_task_timer()
1314 * Spurious timer due to start_dl_timer() race; or we already received in dl_task_timer()
1317 if (!dl_se->dl_throttled) in dl_task_timer()
1324 * If the throttle happened during sched-out; like: in dl_task_timer()
1332 * prev->on_rq = 0; in dl_task_timer()
1335 * but do not enqueue -- wait for our wakeup to do that. in dl_task_timer()
1343 if (unlikely(!rq->online)) { in dl_task_timer()
1362 if (dl_task(rq->donor)) in dl_task_timer()
1383 struct hrtimer *timer = &dl_se->dl_timer; in init_dl_task_timer() local
1385 hrtimer_setup(timer, dl_task_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); in init_dl_task_timer()
1391 * cannot use the runtime, and so it replenishes the task. This rule
1403 * task and set the replenishing timer to the begin of the next period,
1410 if (dl_time_before(dl_se->deadline, rq_clock(rq)) && in dl_check_constrained_dl()
1414 dl_se->dl_throttled = 1; in dl_check_constrained_dl()
1415 if (dl_se->runtime > 0) in dl_check_constrained_dl()
1416 dl_se->runtime = 0; in dl_check_constrained_dl()
1423 return (dl_se->runtime <= 0); in dl_runtime_exceeded()
1428 * GRUB reclaiming algorithm, the runtime is not decreased as "dq = -dt",
1429 * but as "dq = -(max{u, (Umax - Uinact - Uextra)} / Umax) dt",
1431 * utilization, Uinact is the (per-runqueue) inactive utilization, computed
1435 * Since rq->dl.running_bw and rq->dl.this_bw contain utilizations multiplied
1437 * Since rq->dl.bw_ratio contains 1 / Umax multiplied by 2^RATIO_SHIFT, dl_bw
1438 * is multiplied by rq->dl.bw_ratio and shifted right by RATIO_SHIFT.
1440 * larger than 2^(64 - 20 - 8), which is more than 64 seconds. So, overflow is
1446 u64 u_inact = rq->dl.this_bw - rq->dl.running_bw; /* Utot - Uact */ in grub_reclaim()
1449 * Instead of computing max{u, (u_max - u_inact - u_extra)}, we in grub_reclaim()
1450 * compare u_inact + u_extra with u_max - u, because u_inact + u_extra in grub_reclaim()
1451 * can be larger than u_max. So, u_max - u_inact - u_extra would be in grub_reclaim()
1454 if (u_inact + rq->dl.extra_bw > rq->dl.max_bw - dl_se->dl_bw) in grub_reclaim()
1455 u_act = dl_se->dl_bw; in grub_reclaim()
1457 u_act = rq->dl.max_bw - u_inact - rq->dl.extra_bw; in grub_reclaim()
1459 u_act = (u_act * rq->dl.bw_ratio) >> RATIO_SHIFT; in grub_reclaim()
1468 * For tasks that participate in GRUB, we implement GRUB-PA: the in dl_scaled_delta_exec()
1472 * according to current frequency and CPU maximum capacity. in dl_scaled_delta_exec()
1474 if (unlikely(dl_se->flags & SCHED_FLAG_RECLAIM)) { in dl_scaled_delta_exec()
1477 int cpu = cpu_of(rq); in dl_scaled_delta_exec() local
1478 unsigned long scale_freq = arch_scale_freq_capacity(cpu); in dl_scaled_delta_exec()
1479 unsigned long scale_cpu = arch_scale_cpu_capacity(cpu); in dl_scaled_delta_exec()
1496 if (unlikely(dl_se->dl_yielded)) in update_curr_dl_se()
1501 if (dl_server(dl_se) && dl_se->dl_throttled && !dl_se->dl_defer) in update_curr_dl_se()
1511 dl_se->runtime -= scaled_delta_exec; in update_curr_dl_se()
1521 if (dl_se->dl_defer && dl_se->dl_throttled && dl_runtime_exceeded(dl_se)) { in update_curr_dl_se()
1523 * If the server was previously activated - the starving condition in update_curr_dl_se()
1528 dl_se->dl_defer_running = 0; in update_curr_dl_se()
1530 hrtimer_try_to_cancel(&dl_se->dl_timer); in update_curr_dl_se()
1532 replenish_dl_new_period(dl_se, dl_se->rq); in update_curr_dl_se()
1535 * Not being able to start the timer seems problematic. If it could not in update_curr_dl_se()
1546 if (dl_runtime_exceeded(dl_se) || dl_se->dl_yielded) { in update_curr_dl_se()
1547 dl_se->dl_throttled = 1; in update_curr_dl_se()
1551 (dl_se->flags & SCHED_FLAG_DL_OVERRUN)) in update_curr_dl_se()
1552 dl_se->dl_overrun = 1; in update_curr_dl_se()
1556 update_stats_dequeue_dl(&rq->dl, dl_se, 0); in update_curr_dl_se()
1567 if (!is_leftmost(dl_se, &rq->dl)) in update_curr_dl_se()
1572 * The fair server (sole dl_server) does not account for real-time in update_curr_dl_se()
1575 if (dl_se == &rq->fair_server) in update_curr_dl_se()
1580 * Because -- for now -- we share the rt bandwidth, we need to in update_curr_dl_se()
1587 * using deadline servers -- however there's a few nasties to figure in update_curr_dl_se()
1591 struct rt_rq *rt_rq = &rq->rt; in update_curr_dl_se()
1593 raw_spin_lock(&rt_rq->rt_runtime_lock); in update_curr_dl_se()
1600 rt_rq->rt_time += delta_exec; in update_curr_dl_se()
1601 raw_spin_unlock(&rt_rq->rt_runtime_lock); in update_curr_dl_se()
1607 * In the non-defer mode, the idle time is not accounted, as the
1618 if (!rq->fair_server.dl_defer) in dl_server_update_idle_time()
1622 if (rq->fair_server.runtime < 0) in dl_server_update_idle_time()
1625 delta_exec = rq_clock_task(rq) - p->se.exec_start; in dl_server_update_idle_time()
1629 rq->fair_server.runtime -= delta_exec; in dl_server_update_idle_time()
1631 if (rq->fair_server.runtime < 0) { in dl_server_update_idle_time()
1632 rq->fair_server.dl_defer_running = 0; in dl_server_update_idle_time()
1633 rq->fair_server.runtime = 0; in dl_server_update_idle_time()
1636 p->se.exec_start = rq_clock_task(rq); in dl_server_update_idle_time()
1642 if (dl_se->dl_runtime) in dl_server_update()
1643 update_curr_dl_se(dl_se->rq, dl_se, delta_exec); in dl_server_update()
1648 struct rq *rq = dl_se->rq; in dl_server_start()
1661 dl_se->dl_server = 1; in dl_server_start()
1662 dl_se->dl_defer = 1; in dl_server_start()
1666 if (!dl_se->dl_runtime) in dl_server_start()
1669 dl_se->dl_server_active = 1; in dl_server_start()
1671 if (!dl_task(dl_se->rq->curr) || dl_entity_preempt(dl_se, &rq->curr->dl)) in dl_server_start()
1672 resched_curr(dl_se->rq); in dl_server_start()
1677 if (!dl_se->dl_runtime) in dl_server_stop()
1681 hrtimer_try_to_cancel(&dl_se->dl_timer); in dl_server_stop()
1682 dl_se->dl_defer_armed = 0; in dl_server_stop()
1683 dl_se->dl_throttled = 0; in dl_server_stop()
1684 dl_se->dl_server_active = 0; in dl_server_stop()
1691 dl_se->rq = rq; in dl_server_init()
1692 dl_se->server_has_tasks = has_tasks; in dl_server_init()
1693 dl_se->server_pick_task = pick_task; in dl_server_init()
1698 u64 new_bw = dl_se->dl_bw; in __dl_server_attach_root()
1699 int cpu = cpu_of(rq); in __dl_server_attach_root() local
1703 guard(raw_spinlock)(&dl_b->lock); in __dl_server_attach_root()
1705 if (!dl_bw_cpus(cpu)) in __dl_server_attach_root()
1708 __dl_add(dl_b, new_bw, dl_bw_cpus(cpu)); in __dl_server_attach_root()
1713 u64 old_bw = init ? 0 : to_ratio(dl_se->dl_period, dl_se->dl_runtime); in dl_server_apply_params()
1715 struct rq *rq = dl_se->rq; in dl_server_apply_params()
1716 int cpu = cpu_of(rq); in dl_server_apply_params() local
1722 dl_b = dl_bw_of(cpu); in dl_server_apply_params()
1723 guard(raw_spinlock)(&dl_b->lock); in dl_server_apply_params()
1725 cpus = dl_bw_cpus(cpu); in dl_server_apply_params()
1726 cap = dl_bw_capacity(cpu); in dl_server_apply_params()
1729 return -EBUSY; in dl_server_apply_params()
1732 __add_rq_bw(new_bw, &rq->dl); in dl_server_apply_params()
1735 __dl_sub(dl_b, dl_se->dl_bw, cpus); in dl_server_apply_params()
1741 dl_se->dl_runtime = runtime; in dl_server_apply_params()
1742 dl_se->dl_deadline = period; in dl_server_apply_params()
1743 dl_se->dl_period = period; in dl_server_apply_params()
1745 dl_se->runtime = 0; in dl_server_apply_params()
1746 dl_se->deadline = 0; in dl_server_apply_params()
1748 dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); in dl_server_apply_params()
1749 dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime); in dl_server_apply_params()
1756 * a -deadline task and has not been removed from the dl_rq).
1760 struct task_struct *donor = rq->donor; in update_curr_dl()
1761 struct sched_dl_entity *dl_se = &donor->dl; in update_curr_dl()
1779 static enum hrtimer_restart inactive_task_timer(struct hrtimer *timer) in inactive_task_timer() argument
1781 struct sched_dl_entity *dl_se = container_of(timer, in inactive_task_timer()
1792 rq = dl_se->rq; in inactive_task_timer()
1802 if (!dl_task(p) || READ_ONCE(p->__state) == TASK_DEAD) { in inactive_task_timer()
1805 if (READ_ONCE(p->__state) == TASK_DEAD && dl_se->dl_non_contending) { in inactive_task_timer()
1806 sub_running_bw(&p->dl, dl_rq_of_se(&p->dl)); in inactive_task_timer()
1807 sub_rq_bw(&p->dl, dl_rq_of_se(&p->dl)); in inactive_task_timer()
1808 dl_se->dl_non_contending = 0; in inactive_task_timer()
1811 raw_spin_lock(&dl_b->lock); in inactive_task_timer()
1812 __dl_sub(dl_b, p->dl.dl_bw, dl_bw_cpus(task_cpu(p))); in inactive_task_timer()
1813 raw_spin_unlock(&dl_b->lock); in inactive_task_timer()
1820 if (dl_se->dl_non_contending == 0) in inactive_task_timer()
1823 sub_running_bw(dl_se, &rq->dl); in inactive_task_timer()
1824 dl_se->dl_non_contending = 0; in inactive_task_timer()
1839 struct hrtimer *timer = &dl_se->inactive_timer; in init_dl_inactive_task_timer() local
1841 hrtimer_setup(timer, inactive_task_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); in init_dl_inactive_task_timer()
1853 if (dl_rq->earliest_dl.curr == 0 || in inc_dl_deadline()
1854 dl_time_before(deadline, dl_rq->earliest_dl.curr)) { in inc_dl_deadline()
1855 if (dl_rq->earliest_dl.curr == 0) in inc_dl_deadline()
1856 cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_HIGHER); in inc_dl_deadline()
1857 dl_rq->earliest_dl.curr = deadline; in inc_dl_deadline()
1858 cpudl_set(&rq->rd->cpudl, rq->cpu, deadline); in inc_dl_deadline()
1870 if (!dl_rq->dl_nr_running) { in dec_dl_deadline()
1871 dl_rq->earliest_dl.curr = 0; in dec_dl_deadline()
1872 dl_rq->earliest_dl.next = 0; in dec_dl_deadline()
1873 cpudl_clear(&rq->rd->cpudl, rq->cpu); in dec_dl_deadline()
1874 cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr); in dec_dl_deadline()
1876 struct rb_node *leftmost = rb_first_cached(&dl_rq->root); in dec_dl_deadline()
1879 dl_rq->earliest_dl.curr = entry->deadline; in dec_dl_deadline()
1880 cpudl_set(&rq->rd->cpudl, rq->cpu, entry->deadline); in dec_dl_deadline()
1894 u64 deadline = dl_se->deadline; in inc_dl_tasks()
1896 dl_rq->dl_nr_running++; in inc_dl_tasks()
1905 WARN_ON(!dl_rq->dl_nr_running); in dec_dl_tasks()
1906 dl_rq->dl_nr_running--; in dec_dl_tasks()
1909 dec_dl_deadline(dl_rq, dl_se->deadline); in dec_dl_tasks()
1914 return dl_time_before(__node_2_dle(a)->deadline, __node_2_dle(b)->deadline); in __dl_less()
1926 return &dl_task_of(dl_se)->stats; in __schedstats_from_dl_se()
1976 state = READ_ONCE(p->__state); in update_stats_dequeue_dl()
1978 __schedstat_set(p->stats.sleep_start, in update_stats_dequeue_dl()
1982 __schedstat_set(p->stats.block_start, in update_stats_dequeue_dl()
1991 WARN_ON_ONCE(!RB_EMPTY_NODE(&dl_se->rb_node)); in __enqueue_dl_entity()
1993 rb_add_cached(&dl_se->rb_node, &dl_rq->root, __dl_less); in __enqueue_dl_entity()
2002 if (RB_EMPTY_NODE(&dl_se->rb_node)) in __dequeue_dl_entity()
2005 rb_erase_cached(&dl_se->rb_node, &dl_rq->root); in __dequeue_dl_entity()
2007 RB_CLEAR_NODE(&dl_se->rb_node); in __dequeue_dl_entity()
2023 * the replenishment timer will be set to the next period. in enqueue_dl_entity()
2025 if (!dl_se->dl_throttled && !dl_is_implicit(dl_se)) in enqueue_dl_entity()
2038 * its rq, the bandwidth timer callback (which clearly has not in enqueue_dl_entity()
2042 * task's state - in GRUB parlance, "inactive" vs "active contending"). in enqueue_dl_entity()
2047 if (!dl_se->dl_defer && dl_se->dl_throttled && !(flags & ENQUEUE_REPLENISH)) { in enqueue_dl_entity()
2066 dl_time_before(dl_se->deadline, rq_clock(rq_of_dl_se(dl_se)))) { in enqueue_dl_entity()
2074 if (dl_se->dl_throttled && start_dl_timer(dl_se)) in enqueue_dl_entity()
2078 * We're about to enqueue, make sure we're not ->dl_throttled! in enqueue_dl_entity()
2079 * In case the timer was not started, say because the defer time in enqueue_dl_entity()
2083 if (dl_se->dl_throttled) { in enqueue_dl_entity()
2084 hrtimer_try_to_cancel(&dl_se->dl_timer); in enqueue_dl_entity()
2085 dl_se->dl_defer_armed = 0; in enqueue_dl_entity()
2086 dl_se->dl_throttled = 0; in enqueue_dl_entity()
2104 * This check allows to start the inactive timer (or to immediately in dequeue_dl_entity()
2107 * (p->state == TASK_DEAD). We can handle the two cases in the same in dequeue_dl_entity()
2118 if (is_dl_boosted(&p->dl)) { in enqueue_task_dl()
2131 if (p->dl.dl_throttled) { in enqueue_task_dl()
2133 * The replenish timer needs to be canceled. No in enqueue_task_dl()
2137 cancel_replenish_timer(&p->dl); in enqueue_task_dl()
2138 p->dl.dl_throttled = 0; in enqueue_task_dl()
2140 } else if (!dl_prio(p->normal_prio)) { in enqueue_task_dl()
2146 * clear the flag, otherwise the task may wake up as throttled after in enqueue_task_dl()
2150 p->dl.dl_throttled = 0; in enqueue_task_dl()
2152 printk_deferred_once("sched: DL de-boosted task PID %d: REPLENISH flag missing\n", in enqueue_task_dl()
2159 update_stats_wait_start_dl(dl_rq_of_se(&p->dl), &p->dl); in enqueue_task_dl()
2161 if (p->on_rq == TASK_ON_RQ_MIGRATING) in enqueue_task_dl()
2164 enqueue_dl_entity(&p->dl, flags); in enqueue_task_dl()
2166 if (dl_server(&p->dl)) in enqueue_task_dl()
2169 if (!task_current(rq, p) && !p->dl.dl_throttled && p->nr_cpus_allowed > 1) in enqueue_task_dl()
2177 if (p->on_rq == TASK_ON_RQ_MIGRATING) in dequeue_task_dl()
2180 dequeue_dl_entity(&p->dl, flags); in dequeue_task_dl()
2181 if (!p->dl.dl_throttled && !dl_server(&p->dl)) in dequeue_task_dl()
2188 * Yield task semantic for -deadline tasks is:
2190 * get off from the CPU until our next instance, with
2202 * it and the bandwidth timer will wake it up and will give it in yield_task_dl()
2205 rq->curr->dl.dl_yielded = 1; in yield_task_dl()
2222 return (!rq->dl.dl_nr_running || in dl_task_is_earliest_deadline()
2223 dl_time_before(p->dl.deadline, in dl_task_is_earliest_deadline()
2224 rq->dl.earliest_dl.curr)); in dl_task_is_earliest_deadline()
2230 select_task_rq_dl(struct task_struct *p, int cpu, int flags) in select_task_rq_dl() argument
2239 rq = cpu_rq(cpu); in select_task_rq_dl()
2242 curr = READ_ONCE(rq->curr); /* unlocked access */ in select_task_rq_dl()
2243 donor = READ_ONCE(rq->donor); in select_task_rq_dl()
2246 * If we are dealing with a -deadline task, we must in select_task_rq_dl()
2247 * decide where to wake it up. in select_task_rq_dl()
2255 (curr->nr_cpus_allowed < 2 || in select_task_rq_dl()
2256 !dl_entity_preempt(&p->dl, &donor->dl)) && in select_task_rq_dl()
2257 p->nr_cpus_allowed > 1; in select_task_rq_dl()
2260 * Take the capacity of the CPU into account to in select_task_rq_dl()
2264 select_rq |= !dl_task_fits_capacity(p, cpu); in select_task_rq_dl()
2269 if (target != -1 && in select_task_rq_dl()
2271 cpu = target; in select_task_rq_dl()
2276 return cpu; in select_task_rq_dl()
2284 if (READ_ONCE(p->__state) != TASK_WAKING) in migrate_task_rq_dl()
2289 * Since p->state == TASK_WAKING, set_task_cpu() has been called in migrate_task_rq_dl()
2290 * from try_to_wake_up(). Hence, p->pi_lock is locked, but in migrate_task_rq_dl()
2291 * rq->lock is not... So, lock it in migrate_task_rq_dl()
2294 if (p->dl.dl_non_contending) { in migrate_task_rq_dl()
2296 sub_running_bw(&p->dl, &rq->dl); in migrate_task_rq_dl()
2297 p->dl.dl_non_contending = 0; in migrate_task_rq_dl()
2299 * If the timer handler is currently running and the in migrate_task_rq_dl()
2300 * timer cannot be canceled, inactive_task_timer() in migrate_task_rq_dl()
2305 cancel_inactive_timer(&p->dl); in migrate_task_rq_dl()
2307 sub_rq_bw(&p->dl, &rq->dl); in migrate_task_rq_dl()
2317 if (rq->curr->nr_cpus_allowed == 1 || in check_preempt_equal_dl()
2318 !cpudl_find(&rq->rd->cpudl, rq->donor, NULL)) in check_preempt_equal_dl()
2325 if (p->nr_cpus_allowed != 1 && in check_preempt_equal_dl()
2326 cpudl_find(&rq->rd->cpudl, p, NULL)) in check_preempt_equal_dl()
2334 if (!on_dl_rq(&p->dl) && need_pull_dl_task(rq, p)) { in balance_dl()
2337 * picked for load-balance and preemption/IRQs are still in balance_dl()
2351 * Only called when both the current and waking task are -deadline
2357 if (dl_entity_preempt(&p->dl, &rq->donor->dl)) { in wakeup_preempt_dl()
2367 if ((p->dl.deadline == rq->donor->dl.deadline) && in wakeup_preempt_dl()
2368 !test_tsk_need_resched(rq->curr)) in wakeup_preempt_dl()
2376 hrtick_start(rq, dl_se->runtime); in start_hrtick_dl()
2386 struct sched_dl_entity *dl_se = &p->dl; in set_next_task_dl()
2387 struct dl_rq *dl_rq = &rq->dl; in set_next_task_dl()
2389 p->se.exec_start = rq_clock_task(rq); in set_next_task_dl()
2390 if (on_dl_rq(&p->dl)) in set_next_task_dl()
2399 if (rq->donor->sched_class != &dl_sched_class) in set_next_task_dl()
2405 start_hrtick_dl(rq, &p->dl); in set_next_task_dl()
2410 struct rb_node *left = rb_first_cached(&dl_rq->root); in pick_next_dl_entity()
2419 * __pick_next_task_dl - Helper to pick the next -deadline task to run.
2425 struct dl_rq *dl_rq = &rq->dl; in __pick_task_dl()
2436 p = dl_se->server_pick_task(dl_se); in __pick_task_dl()
2439 dl_se->dl_yielded = 1; in __pick_task_dl()
2444 rq->dl_server = dl_se; in __pick_task_dl()
2459 struct sched_dl_entity *dl_se = &p->dl; in put_prev_task_dl()
2460 struct dl_rq *dl_rq = &rq->dl; in put_prev_task_dl()
2462 if (on_dl_rq(&p->dl)) in put_prev_task_dl()
2468 if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1) in put_prev_task_dl()
2490 if (hrtick_enabled_dl(rq) && queued && p->dl.runtime > 0 && in task_tick_dl()
2491 is_leftmost(&p->dl, &rq->dl)) in task_tick_dl()
2492 start_hrtick_dl(rq, &p->dl); in task_tick_dl()
2498 * SCHED_DEADLINE tasks cannot fork and this is achieved through in task_fork_dl()
2510 * on the CPU, NULL otherwise:
2512 static struct task_struct *pick_earliest_pushable_dl_task(struct rq *rq, int cpu) in pick_earliest_pushable_dl_task() argument
2520 next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root); in pick_earliest_pushable_dl_task()
2524 if (task_is_pushable(rq, p, cpu)) in pick_earliest_pushable_dl_task()
2540 int cpu = task_cpu(task); in find_later_rq() local
2544 return -1; in find_later_rq()
2546 if (task->nr_cpus_allowed == 1) in find_later_rq()
2547 return -1; in find_later_rq()
2551 * first, then we can look for a suitable CPU. in find_later_rq()
2553 if (!cpudl_find(&task_rq(task)->rd->cpudl, task, later_mask)) in find_later_rq()
2554 return -1; in find_later_rq()
2565 * The last CPU where the task run is our first in find_later_rq()
2566 * guess, since it is most likely cache-hot there. in find_later_rq()
2568 if (cpumask_test_cpu(cpu, later_mask)) in find_later_rq()
2569 return cpu; in find_later_rq()
2575 this_cpu = -1; in find_later_rq()
2578 for_each_domain(cpu, sd) { in find_later_rq()
2579 if (sd->flags & SD_WAKE_AFFINE) { in find_later_rq()
2586 if (this_cpu != -1 && in find_later_rq()
2595 * Last chance: if a CPU being in both later_mask in find_later_rq()
2597 * choice. Of course, the latest possible CPU is in find_later_rq()
2612 if (this_cpu != -1) in find_later_rq()
2615 cpu = cpumask_any_distribute(later_mask); in find_later_rq()
2616 if (cpu < nr_cpu_ids) in find_later_rq()
2617 return cpu; in find_later_rq()
2619 return -1; in find_later_rq()
2627 int cpu; in find_lock_later_rq() local
2630 cpu = find_later_rq(task); in find_lock_later_rq()
2632 if ((cpu == -1) || (cpu == rq->cpu)) in find_lock_later_rq()
2635 later_rq = cpu_rq(cpu); in find_lock_later_rq()
2650 !cpumask_test_cpu(later_rq->cpu, &task->cpus_mask) || in find_lock_later_rq()
2662 * If the rq we found has no -deadline task, or in find_lock_later_rq()
2684 p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root)); in pick_next_pushable_dl_task()
2686 WARN_ON_ONCE(rq->cpu != task_cpu(p)); in pick_next_pushable_dl_task()
2688 WARN_ON_ONCE(p->nr_cpus_allowed <= 1); in pick_next_pushable_dl_task()
2697 * See if the non running -deadline tasks on this rq
2698 * can be sent to some other CPU where they can preempt
2713 * If next_task preempts rq->curr, and rq->curr in push_dl_task()
2717 if (dl_task(rq->donor) && in push_dl_task()
2718 dl_time_before(next_task->dl.deadline, rq->donor->dl.deadline) && in push_dl_task()
2719 rq->curr->nr_cpus_allowed > 1) { in push_dl_task()
2727 if (WARN_ON(next_task == rq->curr)) in push_dl_task()
2740 * find_lock_later_rq releases rq->lock and it is in push_dl_task()
2747 * again, some other CPU will pull it when ready. in push_dl_task()
2776 /* push_dl_task() will return true if it moved a -deadline task */ in push_dl_tasks()
2783 int this_cpu = this_rq->cpu, cpu; in pull_dl_task() local
2798 for_each_cpu(cpu, this_rq->rd->dlo_mask) { in pull_dl_task()
2799 if (this_cpu == cpu) in pull_dl_task()
2802 src_rq = cpu_rq(cpu); in pull_dl_task()
2808 if (this_rq->dl.dl_nr_running && in pull_dl_task()
2809 dl_time_before(this_rq->dl.earliest_dl.curr, in pull_dl_task()
2810 src_rq->dl.earliest_dl.next)) in pull_dl_task()
2813 /* Might drop this_rq->lock */ in pull_dl_task()
2821 if (src_rq->dl.dl_nr_running <= 1) in pull_dl_task()
2828 * - it preempts our current (if there's one), in pull_dl_task()
2829 * - it will preempt the last one we pulled (if any). in pull_dl_task()
2831 if (p && dl_time_before(p->dl.deadline, dmin) && in pull_dl_task()
2833 WARN_ON(p == src_rq->curr); in pull_dl_task()
2840 if (dl_time_before(p->dl.deadline, in pull_dl_task()
2841 src_rq->donor->dl.deadline)) in pull_dl_task()
2848 dmin = p->dl.deadline; in pull_dl_task()
2860 stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop, in pull_dl_task()
2861 push_task, &src_rq->push_work); in pull_dl_task()
2878 !test_tsk_need_resched(rq->curr) && in task_woken_dl()
2879 p->nr_cpus_allowed > 1 && in task_woken_dl()
2880 dl_task(rq->donor) && in task_woken_dl()
2881 (rq->curr->nr_cpus_allowed < 2 || in task_woken_dl()
2882 !dl_entity_preempt(&p->dl, &rq->donor->dl))) { in task_woken_dl()
2896 src_rd = rq->rd; in set_cpus_allowed_dl()
2903 if (!cpumask_intersects(src_rd->span, ctx->new_mask)) { in set_cpus_allowed_dl()
2912 raw_spin_lock(&src_dl_b->lock); in set_cpus_allowed_dl()
2913 __dl_sub(src_dl_b, p->dl.dl_bw, dl_bw_cpus(task_cpu(p))); in set_cpus_allowed_dl()
2914 raw_spin_unlock(&src_dl_b->lock); in set_cpus_allowed_dl()
2920 /* Assumes rq->lock is held */
2923 if (rq->dl.overloaded) in rq_online_dl()
2926 cpudl_set_freecpu(&rq->rd->cpudl, rq->cpu); in rq_online_dl()
2927 if (rq->dl.dl_nr_running > 0) in rq_online_dl()
2928 cpudl_set(&rq->rd->cpudl, rq->cpu, rq->dl.earliest_dl.curr); in rq_online_dl()
2931 /* Assumes rq->lock is held */
2934 if (rq->dl.overloaded) in rq_offline_dl()
2937 cpudl_clear(&rq->rd->cpudl, rq->cpu); in rq_offline_dl()
2938 cpudl_clear_freecpu(&rq->rd->cpudl, rq->cpu); in rq_offline_dl()
2956 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); in dl_add_task_root_domain()
2957 if (!dl_task(p) || dl_entity_is_special(&p->dl)) { in dl_add_task_root_domain()
2958 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); in dl_add_task_root_domain()
2964 dl_b = &rq->rd->dl_bw; in dl_add_task_root_domain()
2965 raw_spin_lock(&dl_b->lock); in dl_add_task_root_domain()
2967 __dl_add(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span)); in dl_add_task_root_domain()
2969 raw_spin_unlock(&dl_b->lock); in dl_add_task_root_domain()
2978 guard(raw_spinlock_irqsave)(&rd->dl_bw.lock); in dl_clear_root_domain()
2979 rd->dl_bw.total_bw = 0; in dl_clear_root_domain()
2985 for_each_cpu(i, rd->span) { in dl_clear_root_domain()
2986 struct sched_dl_entity *dl_se = &cpu_rq(i)->fair_server; in dl_clear_root_domain()
2989 __dl_add(&rd->dl_bw, dl_se->dl_bw, dl_bw_cpus(i)); in dl_clear_root_domain()
2993 void dl_clear_root_domain_cpu(int cpu) in dl_clear_root_domain_cpu() argument
2995 dl_clear_root_domain(cpu_rq(cpu)->rd); in dl_clear_root_domain_cpu()
3003 * task_non_contending() can start the "inactive timer" (if the 0-lag in switched_from_dl()
3005 * the "inactive timer" fires, it can continue to consume its current in switched_from_dl()
3007 * SCHED_DEADLINE until the 0-lag time passes, inactive_task_timer() in switched_from_dl()
3010 if (task_on_rq_queued(p) && p->dl.dl_runtime) in switched_from_dl()
3011 task_non_contending(&p->dl); in switched_from_dl()
3021 * Inactive timer is armed. However, p is leaving DEADLINE and in switched_from_dl()
3026 if (p->dl.dl_non_contending) in switched_from_dl()
3027 sub_running_bw(&p->dl, &rq->dl); in switched_from_dl()
3028 sub_rq_bw(&p->dl, &rq->dl); in switched_from_dl()
3032 * We cannot use inactive_task_timer() to invoke sub_running_bw() in switched_from_dl()
3033 * at the 0-lag time, because the task could have been migrated in switched_from_dl()
3036 if (p->dl.dl_non_contending) in switched_from_dl()
3037 p->dl.dl_non_contending = 0; in switched_from_dl()
3040 * Since this might be the only -deadline task on the rq, in switched_from_dl()
3042 * from an overloaded CPU, if any. in switched_from_dl()
3044 if (!task_on_rq_queued(p) || rq->dl.dl_nr_running) in switched_from_dl()
3051 * When switching to -deadline, we may overload the rq, then
3056 cancel_inactive_timer(&p->dl); in switched_to_dl()
3066 add_rq_bw(&p->dl, &rq->dl); in switched_to_dl()
3071 if (rq->donor != p) { in switched_to_dl()
3073 if (p->nr_cpus_allowed > 1 && rq->dl.overloaded) in switched_to_dl()
3076 if (dl_task(rq->donor)) in switched_to_dl()
3086 * If the scheduling parameters of a -deadline task changed,
3102 if (!rq->dl.overloaded) in prio_changed_dl()
3111 if (dl_time_before(rq->dl.earliest_dl.curr, p->dl.deadline)) in prio_changed_dl()
3120 if (!dl_task(rq->curr) || in prio_changed_dl()
3121 dl_time_before(p->dl.deadline, rq->curr->dl.deadline)) in prio_changed_dl()
3134 static int task_is_throttled_dl(struct task_struct *p, int cpu) in task_is_throttled_dl() argument
3136 return p->dl.dl_throttled; in task_is_throttled_dl()
3189 int cpu, cpus, ret = 0; in sched_dl_global_validate() local
3197 for_each_online_cpu(cpu) { in sched_dl_global_validate()
3200 if (dl_bw_visited(cpu, cookie)) in sched_dl_global_validate()
3203 dl_b = dl_bw_of(cpu); in sched_dl_global_validate()
3204 cpus = dl_bw_cpus(cpu); in sched_dl_global_validate()
3206 raw_spin_lock_irqsave(&dl_b->lock, flags); in sched_dl_global_validate()
3207 if (new_bw * cpus < dl_b->total_bw) in sched_dl_global_validate()
3208 ret = -EBUSY; in sched_dl_global_validate()
3209 raw_spin_unlock_irqrestore(&dl_b->lock, flags); in sched_dl_global_validate()
3224 dl_rq->bw_ratio = 1 << RATIO_SHIFT; in init_dl_rq_bw_ratio()
3225 dl_rq->max_bw = dl_rq->extra_bw = 1 << BW_SHIFT; in init_dl_rq_bw_ratio()
3227 dl_rq->bw_ratio = to_ratio(global_rt_runtime(), in init_dl_rq_bw_ratio()
3228 global_rt_period()) >> (BW_SHIFT - RATIO_SHIFT); in init_dl_rq_bw_ratio()
3229 dl_rq->max_bw = dl_rq->extra_bw = in init_dl_rq_bw_ratio()
3236 u64 new_bw = -1; in sched_dl_do_global()
3239 int cpu; in sched_dl_do_global() local
3245 for_each_possible_cpu(cpu) { in sched_dl_do_global()
3248 if (dl_bw_visited(cpu, cookie)) { in sched_dl_do_global()
3253 dl_b = dl_bw_of(cpu); in sched_dl_do_global()
3255 raw_spin_lock_irqsave(&dl_b->lock, flags); in sched_dl_do_global()
3256 dl_b->bw = new_bw; in sched_dl_do_global()
3257 raw_spin_unlock_irqrestore(&dl_b->lock, flags); in sched_dl_do_global()
3260 init_dl_rq_bw_ratio(&cpu_rq(cpu)->dl); in sched_dl_do_global()
3270 * This function is called while holding p's rq->lock.
3275 u64 period = attr->sched_period ?: attr->sched_deadline; in sched_dl_overflow()
3276 u64 runtime = attr->sched_runtime; in sched_dl_overflow()
3278 int cpus, err = -1, cpu = task_cpu(p); in sched_dl_overflow() local
3279 struct dl_bw *dl_b = dl_bw_of(cpu); in sched_dl_overflow()
3282 if (attr->sched_flags & SCHED_FLAG_SUGOV) in sched_dl_overflow()
3286 if (new_bw == p->dl.dl_bw && task_has_dl_policy(p)) in sched_dl_overflow()
3290 * Either if a task, enters, leave, or stays -deadline but changes in sched_dl_overflow()
3294 raw_spin_lock(&dl_b->lock); in sched_dl_overflow()
3295 cpus = dl_bw_cpus(cpu); in sched_dl_overflow()
3296 cap = dl_bw_capacity(cpu); in sched_dl_overflow()
3300 if (hrtimer_active(&p->dl.inactive_timer)) in sched_dl_overflow()
3301 __dl_sub(dl_b, p->dl.dl_bw, cpus); in sched_dl_overflow()
3305 !__dl_overflow(dl_b, cap, p->dl.dl_bw, new_bw)) { in sched_dl_overflow()
3309 * utilization change until the task's 0-lag point. in sched_dl_overflow()
3311 * timer" when the task is not inactive. in sched_dl_overflow()
3313 __dl_sub(dl_b, p->dl.dl_bw, cpus); in sched_dl_overflow()
3321 * (0-lag) time. in sched_dl_overflow()
3325 raw_spin_unlock(&dl_b->lock); in sched_dl_overflow()
3340 struct sched_dl_entity *dl_se = &p->dl; in __setparam_dl()
3342 dl_se->dl_runtime = attr->sched_runtime; in __setparam_dl()
3343 dl_se->dl_deadline = attr->sched_deadline; in __setparam_dl()
3344 dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline; in __setparam_dl()
3345 dl_se->flags = attr->sched_flags & SCHED_DL_FLAGS; in __setparam_dl()
3346 dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); in __setparam_dl()
3347 dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime); in __setparam_dl()
3352 struct sched_dl_entity *dl_se = &p->dl; in __getparam_dl()
3354 attr->sched_priority = p->rt_priority; in __getparam_dl()
3355 attr->sched_runtime = dl_se->dl_runtime; in __getparam_dl()
3356 attr->sched_deadline = dl_se->dl_deadline; in __getparam_dl()
3357 attr->sched_period = dl_se->dl_period; in __getparam_dl()
3358 attr->sched_flags &= ~SCHED_DL_FLAGS; in __getparam_dl()
3359 attr->sched_flags |= dl_se->flags; in __getparam_dl()
3363 * This function validates the new parameters of a -deadline task.
3377 if (attr->sched_flags & SCHED_FLAG_SUGOV) in __checkparam_dl()
3381 if (attr->sched_deadline == 0) in __checkparam_dl()
3388 if (attr->sched_runtime < (1ULL << DL_SCALE)) in __checkparam_dl()
3392 * Since we use the MSB for wrap-around and sign issues, make in __checkparam_dl()
3395 if (attr->sched_deadline & (1ULL << 63) || in __checkparam_dl()
3396 attr->sched_period & (1ULL << 63)) in __checkparam_dl()
3399 period = attr->sched_period; in __checkparam_dl()
3401 period = attr->sched_deadline; in __checkparam_dl()
3404 if (period < attr->sched_deadline || in __checkparam_dl()
3405 attr->sched_deadline < attr->sched_runtime) in __checkparam_dl()
3422 dl_se->dl_runtime = 0; in __dl_clear_params()
3423 dl_se->dl_deadline = 0; in __dl_clear_params()
3424 dl_se->dl_period = 0; in __dl_clear_params()
3425 dl_se->flags = 0; in __dl_clear_params()
3426 dl_se->dl_bw = 0; in __dl_clear_params()
3427 dl_se->dl_density = 0; in __dl_clear_params()
3429 dl_se->dl_throttled = 0; in __dl_clear_params()
3430 dl_se->dl_yielded = 0; in __dl_clear_params()
3431 dl_se->dl_non_contending = 0; in __dl_clear_params()
3432 dl_se->dl_overrun = 0; in __dl_clear_params()
3433 dl_se->dl_server = 0; in __dl_clear_params()
3436 dl_se->pi_se = dl_se; in __dl_clear_params()
3442 RB_CLEAR_NODE(&dl_se->rb_node); in init_dl_entity()
3450 struct sched_dl_entity *dl_se = &p->dl; in dl_param_changed()
3452 if (dl_se->dl_runtime != attr->sched_runtime || in dl_param_changed()
3453 dl_se->dl_deadline != attr->sched_deadline || in dl_param_changed()
3454 dl_se->dl_period != attr->sched_period || in dl_param_changed()
3455 dl_se->flags != (attr->sched_flags & SCHED_DL_FLAGS)) in dl_param_changed()
3472 raw_spin_lock_irqsave(&cur_dl_b->lock, flags); in dl_cpuset_cpumask_can_shrink()
3475 raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags); in dl_cpuset_cpumask_can_shrink()
3487 static int dl_bw_manage(enum dl_bw_request req, int cpu, u64 dl_bw) in dl_bw_manage() argument
3495 dl_b = dl_bw_of(cpu); in dl_bw_manage()
3496 raw_spin_lock_irqsave(&dl_b->lock, flags); in dl_bw_manage()
3498 cap = dl_bw_capacity(cpu); in dl_bw_manage()
3501 __dl_sub(dl_b, dl_bw, dl_bw_cpus(cpu)); in dl_bw_manage()
3513 __dl_add(dl_b, dl_bw, dl_bw_cpus(cpu)); in dl_bw_manage()
3518 * cpu is not off yet, but we need to do the math by in dl_bw_manage()
3520 * turn cpu off?). in dl_bw_manage()
3522 cap -= arch_scale_cpu_capacity(cpu); in dl_bw_manage()
3525 * cpu is going offline and NORMAL tasks will be moved away in dl_bw_manage()
3528 * the cpu is off. in dl_bw_manage()
3530 if (cpu_rq(cpu)->fair_server.dl_server) in dl_bw_manage()
3531 fair_server_bw = cpu_rq(cpu)->fair_server.dl_bw; in dl_bw_manage()
3538 if (dl_b->total_bw - fair_server_bw > 0) { in dl_bw_manage()
3540 * Leaving at least one CPU for DEADLINE tasks seems a in dl_bw_manage()
3541 * wise thing to do. As said above, cpu is not offline in dl_bw_manage()
3544 if (dl_bw_cpus(cpu) - 1) in dl_bw_manage()
3553 raw_spin_unlock_irqrestore(&dl_b->lock, flags); in dl_bw_manage()
3556 return overflow ? -EBUSY : 0; in dl_bw_manage()
3559 int dl_bw_deactivate(int cpu) in dl_bw_deactivate() argument
3561 return dl_bw_manage(dl_bw_req_deactivate, cpu, 0); in dl_bw_deactivate()
3564 int dl_bw_alloc(int cpu, u64 dl_bw) in dl_bw_alloc() argument
3566 return dl_bw_manage(dl_bw_req_alloc, cpu, dl_bw); in dl_bw_alloc()
3569 void dl_bw_free(int cpu, u64 dl_bw) in dl_bw_free() argument
3571 dl_bw_manage(dl_bw_req_free, cpu, dl_bw); in dl_bw_free()
3575 void print_dl_stats(struct seq_file *m, int cpu) in print_dl_stats() argument
3577 print_dl_rq(m, cpu, &cpu_rq(cpu)->dl); in print_dl_stats()