Lines Matching refs:rq
239 /* unserved remainder of @rq's rescuee's admitted slice, 0 once fully served */
240 static s64 scx_rescue_slice_remaining(struct rq *rq)
242 s64 served = rq->scx.rescue.curr->se.sum_exec_runtime - rq->scx.rescue.exec_snap;
244 return max(rq->scx.rescue.slice - served, 0);
266 * @rq: rq the rescuee is running on
273 void scx_rescue_charge(struct rq *rq, s64 delta_exec)
277 lockdep_assert_rq_held(rq);
285 rq->scx.rescue.budget -= delta_exec;
288 pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq));
291 if (!scx_rescue_slice_remaining(rq))
292 scx_task_slice_ended(rq, rq->scx.rescue.curr);
296 * scx_rescue_end - End the rescue execution on @rq
297 * @rq: rq of interest
304 void scx_rescue_end(struct rq *rq)
306 lockdep_assert_rq_held(rq);
308 rq->scx.rescue.curr = NULL;
309 if (list_empty(&rq->scx.rescue.dsq.list))
310 rq->scx.rescue.budget = min(rq->scx.rescue.budget, scx_rescue_quantum_ns);
315 * @rq: rq @p is running on
326 * @p is leaving the rq or bypass is dismantling rescues.
328 bool scx_rescue_keep(struct rq *rq, struct task_struct *p)
330 s64 remaining = scx_rescue_slice_remaining(rq);
332 lockdep_assert_rq_held(rq);
335 scx_bypassing(scx_task_sched(p), cpu_of(rq)))
344 * @rq: rq of interest
352 static void scx_rescue_accrue(struct rq *rq)
354 bool in_session = rq->scx.rescue.curr || !list_empty(&rq->scx.rescue.dsq.list);
359 lockdep_assert_rq_held(rq);
361 /* not every path here holds an updated rq clock, use __scx_bpf_now() */
362 now = __scx_bpf_now(rq);
363 delta = now - rq->scx.rescue.clock;
364 rq->scx.rescue.clock = now;
371 rq->scx.rescue.budget = cap;
373 rq->scx.rescue.budget =
374 min(cap, rq->scx.rescue.budget +
382 static s64 scx_rescue_next_slice(struct rq *rq)
385 u32 depth = rq->scx.rescue.dsq.nr ?: 1;
390 static void scx_rescue_timer_arm(struct rq *rq)
392 struct timer_list *timer = &rq->scx.rescue.timer;
402 if (!rq->scx.rescue.curr && rq->scx.rescue.budget < scx_rescue_quantum_ns) {
403 s64 deficit = scx_rescue_quantum_ns - rq->scx.rescue.budget;
411 add_timer_on(timer, cpu_of(rq));
415 * scx_rescue_admit - Start rescuing @p on @rq
416 * @rq: rq @p is being admitted on
426 static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice)
428 lockdep_assert_rq_held(rq);
429 WARN_ON_ONCE(rq->scx.rescue.curr);
431 rq->scx.rescue.curr = p;
432 rq->scx.rescue.slice = slice;
433 rq->scx.rescue.exec_snap = p->se.sum_exec_runtime;
435 scx_rescue_timer_arm(rq);
440 * @rq: rq @p is being inserted on
445 * and should be inserted at the tail of @rq's local DSQ, %false if it has to
448 static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p)
450 scx_rescue_accrue(rq);
452 if (!rq->scx.rescue.curr && list_empty(&rq->scx.rescue.dsq.list) &&
453 rq->scx.rescue.budget >= scx_rescue_quantum_ns) {
454 scx_rescue_admit(rq, p, scx_rescue_quantum_ns);
458 scx_rescue_timer_arm(rq);
464 * @rq: rq whose rescue timer fired
466 * If the oldest waiter on @rq's rescue DSQ has been queued for too long, rescue
471 static void scx_rescue_check_overload(struct rq *rq)
475 int cpu = cpu_of(rq);
479 lockdep_assert_rq_held(rq);
481 p = list_first_entry_or_null(&rq->scx.rescue.dsq.list, struct task_struct,
495 if (time_before64(get_jiffies_64(), rq->scx.rescue.kill_at +
512 rq->scx.rescue.kill_at = get_jiffies_64();
522 * @timer: rq->scx.rescue.timer
534 struct rq *rq = timer_container_of(rq, timer, scx.rescue.timer);
537 guard(rq_lock_irqsave)(rq);
539 p = rq->scx.rescue.curr;
540 if (!p && list_empty(&rq->scx.rescue.dsq.list))
543 scx_rescue_accrue(rq);
544 scx_rescue_check_overload(rq);
547 s64 slice = scx_rescue_next_slice(rq);
550 if (rq->scx.rescue.budget < scx_rescue_quantum_ns)
554 p = list_first_entry(&rq->scx.rescue.dsq.list, struct task_struct,
556 scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);
557 scx_rescue_admit(rq, p, slice);
559 SCX_ENQ_IGNORE_CAPS, rq);
560 if (sched_class_above(&ext_sched_class, rq->curr->sched_class))
561 resched_curr(rq);
562 } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) {
570 scx_set_task_slice(p, scx_rescue_slice_remaining(rq));
572 scx_task_unlink_from_dsq(p, &rq->scx.local_dsq);
575 rq);
578 scx_rescue_timer_arm(rq);
582 void scx_rescue_flush(struct rq *rq)
586 lockdep_assert_rq_held(rq);
589 if (rq->scx.rescue.curr)
590 scx_task_slice_ended(rq, rq->scx.rescue.curr);
593 list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) {
594 scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);
596 SCX_ENQ_IGNORE_CAPS, rq);
599 timer_delete(&rq->scx.rescue.timer);
602 void scx_rescue_dump(struct seq_buf *s, struct rq *rq)
604 struct task_struct *p = rq->scx.rescue.curr;
607 rq->scx.rescue.dsq.nr,
608 div_s64(rq->scx.rescue.budget, NSEC_PER_USEC),
666 void scx_rescue_init(struct rq *rq)
668 BUG_ON(scx_init_dsq(&rq->scx.rescue.dsq, SCX_DSQ_RESCUE, NULL));
669 timer_setup(&rq->scx.rescue.timer, scx_rescue_timerfn, TIMER_PINNED);
670 rq->scx.rescue.kill_at = get_jiffies_64();
676 * @rq: rq whose local DSQ @p targets
680 * Return @rq's local DSQ if @sch holds the required caps on @rq's cid.
681 * Otherwise, return @rq's rescue DSQ if the insert carries %SCX_ENQ_RESCUE and
682 * rescue is enabled, or @rq's reject DSQ after recording the reenq reason on
693 struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
697 return &rq->scx.local_dsq;
699 s32 cid = __scx_cpu_to_cid(cpu_of(rq));
700 struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;
709 needed |= scx_caps_for_preempt(asch, rq, *enq_flags);
710 missing = scx_missing_caps(asch, cpu_of(rq), needed);
714 return &rq->scx.local_dsq;
717 * The task must run on this CPU regardless of caps: the rq is draining
722 if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) ||
726 return &rq->scx.local_dsq;
741 if (scx_rescue_try_admit(rq, p))
742 return &rq->scx.local_dsq;
746 return &rq->scx.rescue.dsq;
752 return &rq->scx.reject_dsq;
755 /* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */
756 bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p)
761 if (is_migration_disabled(p) || p == scx_rescuee(rq))
764 missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p));
769 p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq));
774 * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides
781 void scx_reenq_reject(struct rq *rq)
786 lockdep_assert_rq_held(rq);
788 if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))
795 list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) {
801 scx_dispatch_dequeue(rq, p);
813 scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
937 /* discard @rq's queued ecaps syncs */
938 static void discard_queued_syncs(struct rq *rq)
942 lockdep_assert_rq_held(rq);
944 llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync))
950 * @rq: the cid's cpu rq
951 * @prev: @rq's previous task from the in-progress dispatch
954 * transposed copy owned by the cid's cpu and written only here under @rq's
958 * learns the cid's idle state. Such a gain arms the per-rq
961 void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
963 s32 cpu = cpu_of(rq);
968 lockdep_assert_rq_held(rq);
970 if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync)))
982 discard_queued_syncs(rq);
990 batch = llist_del_all(&rq->scx.ecaps_to_sync);
1013 * and re-acquire the rq lock temporarily while the rest of
1016 * cpu, which under core scheduling can differ from @rq's cpu,
1017 * so the context is set up there. The rq recorded in it keeps
1018 * the dispatches targeting @rq.
1025 dspc->rq = rq;
1027 rq->scx.sub_dispatch_prev = prev;
1028 SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu),
1030 rq->scx.sub_dispatch_prev = NULL;
1031 scx_flush_dispatch_buf(pcpu->sch, rq);
1037 * learns the cpu's idle state. Arm the per-rq gate so the next
1042 rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY;
1054 scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE);
1059 * @rq: rq of the cpu leaving bypass
1060 * @sch: scheduler that just left bypass on @rq's cpu
1069 void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch)
1071 s32 cpu = cpu_of(rq);
1076 lockdep_assert_rq_held(rq);
1097 void scx_online_ecaps(struct rq *rq)
1111 guard(rq_lock_irqsave)(rq);
1114 cid = __scx_cpu_to_cid(cpu_of(rq));
1138 void scx_offline_ecaps(struct rq *rq)
1140 s32 cpu = cpu_of(rq);
1143 guard(rq_lock_irqsave)(rq);
1158 * sync node. Remove the node from @rq's pending list so the pcpu can be freed.
1162 struct rq *rq = cpu_rq(cpu);
1172 scoped_guard (rq_lock_irqsave, rq) {
1173 llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) {
1184 llist_add_batch(head, tail, &rq->scx.ecaps_to_sync);
1190 * privately across dispatch-induced rq unlocks, reading as on-list.
1193 * @pcpu accesses share one contiguous lock hold, off-list under the rq
1197 scoped_guard (rq_lock_irqsave, rq) {
1220 struct rq *rq = cpu_rq(cpu);
1222 guard(rq_lock_irqsave)(rq);
1223 discard_queued_syncs(rq);
1550 struct rq *rq;
1590 rq = task_rq_lock(p, &rf);
1595 * and this rq lock and ran exit_task() on @sch (the
1601 task_rq_unlock(rq, p, &rf);
1608 task_rq_unlock(rq, p, &rf);
1873 struct rq *rq;
1907 rq = task_rq_lock(p, &rf);
1912 * and this rq lock and ran exit_task() on $parent (the
1918 task_rq_unlock(rq, p, &rf);
1924 task_rq_unlock(rq, p, &rf);
2078 struct rq *rq;
2088 rq = task_rq_lock(p, &rf);
2090 task_rq_unlock(rq, p, &rf);
2105 struct rq *rq;
2115 rq = task_rq_lock(p, &rf);
2117 task_rq_unlock(rq, p, &rf);
2228 struct rq *rq = scx_locked_rq();
2252 if (scx_missing_caps(child, cpu_of(rq), SCX_CAP_BASE))
2255 return scx_dispatch_sched(child, rq, rq->scx.sub_dispatch_prev, true) !=