Lines Matching defs:rq
239 /* unserved remainder of @rq's rescuee's admitted slice, 0 once fully served */
240 static s64 scx_rescue_slice_remaining(struct rq *rq)
242 s64 served = rq->scx.rescue.curr->se.sum_exec_runtime - rq->scx.rescue.exec_snap;
244 return max(rq->scx.rescue.slice - served, 0);
266 * @rq: rq the rescuee is running on
273 void scx_rescue_charge(struct rq *rq, s64 delta_exec)
277 lockdep_assert_rq_held(rq);
285 rq->scx.rescue.budget -= delta_exec;
288 pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq));
291 if (!scx_rescue_slice_remaining(rq))
292 scx_task_slice_ended(rq, rq->scx.rescue.curr);
296 * scx_rescue_end - End the rescue execution on @rq
297 * @rq: rq of interest
304 void scx_rescue_end(struct rq *rq)
306 lockdep_assert_rq_held(rq);
308 rq->scx.rescue.curr = NULL;
309 if (list_empty(&rq->scx.rescue.dsq.list))
310 rq->scx.rescue.budget = min(rq->scx.rescue.budget, scx_rescue_quantum_ns);
315 * @rq: rq @p is running on
326 * @p is leaving the rq or bypass is dismantling rescues.
328 bool scx_rescue_keep(struct rq *rq, struct task_struct *p)
330 s64 remaining = scx_rescue_slice_remaining(rq);
332 lockdep_assert_rq_held(rq);
335 scx_bypassing(scx_task_sched(p), cpu_of(rq)))
344 * @rq: rq of interest
352 static void scx_rescue_accrue(struct rq *rq)
354 bool in_session = rq->scx.rescue.curr || !list_empty(&rq->scx.rescue.dsq.list);
359 lockdep_assert_rq_held(rq);
361 /* not every path here holds an updated rq clock, use __scx_bpf_now() */
362 now = __scx_bpf_now(rq);
363 delta = now - rq->scx.rescue.clock;
364 rq->scx.rescue.clock = now;
371 rq->scx.rescue.budget = cap;
373 rq->scx.rescue.budget =
374 min(cap, rq->scx.rescue.budget +
382 static s64 scx_rescue_next_slice(struct rq *rq)
385 u32 depth = rq->scx.rescue.dsq.nr ?: 1;
390 static void scx_rescue_timer_arm(struct rq *rq)
392 struct timer_list *timer = &rq->scx.rescue.timer;
402 if (!rq->scx.rescue.curr && rq->scx.rescue.budget < scx_rescue_quantum_ns) {
403 s64 deficit = scx_rescue_quantum_ns - rq->scx.rescue.budget;
411 add_timer_on(timer, cpu_of(rq));
415 * scx_rescue_admit - Start rescuing @p on @rq
416 * @rq: rq @p is being admitted on
426 static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice)
428 lockdep_assert_rq_held(rq);
429 WARN_ON_ONCE(rq->scx.rescue.curr);
431 rq->scx.rescue.curr = p;
432 rq->scx.rescue.slice = slice;
433 rq->scx.rescue.exec_snap = p->se.sum_exec_runtime;
435 scx_rescue_timer_arm(rq);
440 * @rq: rq @p is being inserted on
445 * and should be inserted at the tail of @rq's local DSQ, %false if it has to
448 static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p)
450 scx_rescue_accrue(rq);
452 if (!rq->scx.rescue.curr && list_empty(&rq->scx.rescue.dsq.list) &&
453 rq->scx.rescue.budget >= scx_rescue_quantum_ns) {
454 scx_rescue_admit(rq, p, scx_rescue_quantum_ns);
458 scx_rescue_timer_arm(rq);
464 * @rq: rq whose rescue timer fired
466 * If the oldest waiter on @rq's rescue DSQ has been queued for too long, rescue
471 static void scx_rescue_check_overload(struct rq *rq)
475 int cpu = cpu_of(rq);
479 lockdep_assert_rq_held(rq);
481 p = list_first_entry_or_null(&rq->scx.rescue.dsq.list, struct task_struct,
495 if (time_before64(get_jiffies_64(), rq->scx.rescue.kill_at +
512 rq->scx.rescue.kill_at = get_jiffies_64();
522 * @timer: rq->scx.rescue.timer
534 struct rq *rq = timer_container_of(rq, timer, scx.rescue.timer);
537 guard(rq_lock_irqsave)(rq);
539 p = rq->scx.rescue.curr;
540 if (!p && list_empty(&rq->scx.rescue.dsq.list))
543 scx_rescue_accrue(rq);
544 scx_rescue_check_overload(rq);
547 s64 slice = scx_rescue_next_slice(rq);
550 if (rq->scx.rescue.budget < scx_rescue_quantum_ns)
554 p = list_first_entry(&rq->scx.rescue.dsq.list, struct task_struct,
556 scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);
557 scx_rescue_admit(rq, p, slice);
559 &rq->scx.rescue.dsq, rq);
560 if (sched_class_above(&ext_sched_class, rq->curr->sched_class))
561 resched_curr(rq);
562 } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) {
570 scx_set_task_slice(p, scx_rescue_slice_remaining(rq));
572 scx_task_unlink_from_dsq(p, &rq->scx.local_dsq);
575 &rq->scx.local_dsq, rq);
578 scx_rescue_timer_arm(rq);
582 void scx_rescue_flush(struct rq *rq)
586 lockdep_assert_rq_held(rq);
589 if (cpu_active(cpu_of(rq)))
593 if (rq->scx.rescue.curr)
594 scx_task_slice_ended(rq, rq->scx.rescue.curr);
597 list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) {
598 scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);
600 &rq->scx.rescue.dsq, rq);
603 timer_delete(&rq->scx.rescue.timer);
606 void scx_rescue_dump(struct seq_buf *s, struct rq *rq)
608 struct task_struct *p = rq->scx.rescue.curr;
611 rq->scx.rescue.dsq.nr,
612 div_s64(rq->scx.rescue.budget, NSEC_PER_USEC),
670 void scx_rescue_init(struct rq *rq)
672 BUG_ON(scx_init_dsq(&rq->scx.rescue.dsq, SCX_DSQ_RESCUE, NULL));
673 timer_setup(&rq->scx.rescue.timer, scx_rescue_timerfn, TIMER_PINNED);
674 rq->scx.rescue.kill_at = get_jiffies_64();
680 * @rq: rq whose local DSQ @p targets
684 * Return @rq's local DSQ if @sch holds the required caps on @rq's cid.
685 * Otherwise, return @rq's rescue DSQ if the insert carries %SCX_ENQ_RESCUE and
686 * rescue is enabled, or @rq's reject DSQ after recording the reenq reason on
697 struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
701 return &rq->scx.local_dsq;
703 s32 cid = __scx_cpu_to_cid(cpu_of(rq));
704 struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;
713 needed |= scx_caps_for_preempt(asch, rq, *enq_flags);
714 missing = scx_missing_caps(asch, cpu_of(rq), needed);
718 return &rq->scx.local_dsq;
721 * The task must run on this CPU regardless of caps: the rq is draining
726 if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) ||
730 return &rq->scx.local_dsq;
745 if (scx_rescue_try_admit(rq, p))
746 return &rq->scx.local_dsq;
750 return &rq->scx.rescue.dsq;
756 return &rq->scx.reject_dsq;
759 /* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */
760 bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p)
765 if (is_migration_disabled(p) || p == scx_rescuee(rq))
768 missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p));
773 p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq));
778 * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides
785 void scx_reenq_reject(struct rq *rq)
790 lockdep_assert_rq_held(rq);
792 if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))
799 list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) {
804 scx_dispatch_dequeue(rq, p);
816 scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
940 /* discard @rq's queued ecaps syncs */
941 static void discard_queued_syncs(struct rq *rq)
945 lockdep_assert_rq_held(rq);
947 llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync))
953 * @rq: the cid's cpu rq
954 * @prev: @rq's previous task from the in-progress dispatch
957 * transposed copy owned by the cid's cpu and written only here under @rq's
961 * learns the cid's idle state. Such a gain arms the per-rq
964 void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
966 s32 cpu = cpu_of(rq);
971 lockdep_assert_rq_held(rq);
973 if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync)))
985 discard_queued_syncs(rq);
993 batch = llist_del_all(&rq->scx.ecaps_to_sync);
1016 * and re-acquire the rq lock temporarily while the rest of
1019 * cpu, which under core scheduling can differ from @rq's cpu,
1020 * so the context is set up there. The rq recorded in it keeps
1021 * the dispatches targeting @rq.
1028 dspc->rq = rq;
1030 rq->scx.sub_dispatch_prev = prev;
1031 SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu),
1033 rq->scx.sub_dispatch_prev = NULL;
1034 scx_flush_dispatch_buf(pcpu->sch, rq);
1040 * learns the cpu's idle state. Arm the per-rq gate so the next
1045 rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY;
1057 scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE);
1062 * @rq: rq of the cpu leaving bypass
1063 * @sch: scheduler that just left bypass on @rq's cpu
1072 void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch)
1074 s32 cpu = cpu_of(rq);
1079 lockdep_assert_rq_held(rq);
1100 void scx_online_ecaps(struct rq *rq)
1114 guard(rq_lock_irqsave)(rq);
1117 cid = __scx_cpu_to_cid(cpu_of(rq));
1141 void scx_offline_ecaps(struct rq *rq)
1143 s32 cpu = cpu_of(rq);
1146 guard(rq_lock_irqsave)(rq);
1161 * sync node. Remove the node from @rq's pending list so the pcpu can be freed.
1165 struct rq *rq = cpu_rq(cpu);
1175 scoped_guard (rq_lock_irqsave, rq) {
1176 llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) {
1187 llist_add_batch(head, tail, &rq->scx.ecaps_to_sync);
1193 * privately across dispatch-induced rq unlocks, reading as on-list.
1196 * @pcpu accesses share one contiguous lock hold, off-list under the rq
1200 scoped_guard (rq_lock_irqsave, rq) {
1223 struct rq *rq = cpu_rq(cpu);
1225 guard(rq_lock_irqsave)(rq);
1226 discard_queued_syncs(rq);
1553 struct rq *rq;
1593 rq = task_rq_lock(p, &rf);
1598 * and this rq lock and ran exit_task() on @sch (the
1604 task_rq_unlock(rq, p, &rf);
1611 task_rq_unlock(rq, p, &rf);
1876 struct rq *rq;
1910 rq = task_rq_lock(p, &rf);
1915 * and this rq lock and ran exit_task() on $parent (the
1921 task_rq_unlock(rq, p, &rf);
1927 task_rq_unlock(rq, p, &rf);
2081 struct rq *rq;
2091 rq = task_rq_lock(p, &rf);
2093 task_rq_unlock(rq, p, &rf);
2108 struct rq *rq;
2118 rq = task_rq_lock(p, &rf);
2120 task_rq_unlock(rq, p, &rf);
2231 struct rq *rq = scx_locked_rq();
2255 if (scx_missing_caps(child, cpu_of(rq), SCX_CAP_BASE))
2258 return scx_dispatch_sched(child, rq, rq->scx.sub_dispatch_prev, true) !=