Lines Matching defs:rq
108 /* is @dsq synchronized by the containing rq lock instead of dsq->lock? */
260 static void run_deferred(struct rq *rq);
345 * rq_is_open - Is the rq available for immediate execution of an SCX task?
346 * @rq: rq to test
349 * Returns %true if @rq is currently open for executing an SCX task. After a
350 * %false return, @rq is guaranteed to invoke SCX dispatch path at least once
351 * before going to idle and not inserting a task into @rq's local DSQ after a
352 * %false return doesn't cause @rq to stall.
354 static bool rq_is_open(struct rq *rq, u64 enq_flags)
356 lockdep_assert_rq_held(rq);
360 * waking up on @rq.
362 if (sched_class_above(rq->next_class, &ext_sched_class))
366 * @rq is either in transition to or in idle and there is no
369 if (sched_class_above(&ext_sched_class, rq->next_class))
373 * @rq is either picking, in transition to, or running an SCX task.
377 * If we're in the dispatch path holding rq lock, $curr may or may not
382 if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
391 struct task_struct *curr = rq->curr;
402 * @rq is either in transition to or running an SCX task and can't go
409 * Track the rq currently locked.
411 * This allows kfuncs to safely operate on rq from any scx ops callback,
412 * knowing which rq is already locked.
414 DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);
417 * Under core scheduling, a pick that releases the rq lock invalidates the
421 static void scx_rq_lock_drop(struct rq *rq)
423 lockdep_assert_rq_held(rq);
425 if (sched_core_enabled(rq))
426 rq->scx.lock_drop_seq++;
430 static void switch_rq_lock(struct rq *from, struct rq *to)
452 * @rq: rq to update as the currently-locked rq, or NULL
458 * must hold @rq's rq lock so this_cpu_ptr is stable across the call.
460 static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq,
472 * trampoline rebases it into BPF's arena pointer form. The rq
477 SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, kern_va);
479 SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask);
585 * @rq: rq @p was on
596 struct rq *rq, struct scx_dispatch_q *dsq,
599 lockdep_assert_rq_held(rq);
611 /* if @p has stayed on @dsq, its rq couldn't have changed */
612 if (WARN_ON_ONCE(rq != task_rq(p)))
700 * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock()
733 __balance_callbacks(iter->rq, &iter->rf);
734 task_rq_unlock(iter->rq, iter->locked_task, &iter->rf);
740 * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator
743 * If @iter is in the middle of a locked iteration, it may be locking the rq of
766 * scx_task_iter_relock - Re-acquire scx_tasks_lock and, optionally, @p's rq
768 * @p: task whose rq to lock, or %NULL for scx_tasks_lock only
770 * Counterpart to scx_task_iter_unlock(). Locking @p's rq is optional. Once
778 iter->rq = task_rq_lock(p, &iter->rf);
788 * which is released on return. If the iterator holds a task's rq lock, that rq
859 * scx_task_iter_next_locked - Next non-idle task with its rq locked
862 * Visit the non-idle task with its rq lock held. Allows callers to specify
901 iter->rq = task_rq_lock(p, &iter->rf);
908 * set by sched_ext_dead() under @p's rq lock. Test it to
995 static void deferred_bal_cb_workfn(struct rq *rq)
997 run_deferred(rq);
1002 struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work);
1004 raw_spin_rq_lock(rq);
1005 run_deferred(rq);
1006 scx_rq_lock_drop(rq);
1007 raw_spin_rq_unlock(rq);
1011 * schedule_deferred - Schedule execution of deferred actions on an rq
1012 * @rq: target rq
1014 * Schedule execution of deferred actions on @rq. Deferred actions are executed
1015 * with @rq locked but unpinned, and thus can unlock @rq to e.g. migrate tasks
1018 static void schedule_deferred(struct rq *rq)
1032 irq_work_queue_on(&rq->scx.deferred_irq_work, cpu_of(rq));
1036 * schedule_deferred_locked - Schedule execution of deferred actions on an rq
1037 * @rq: target rq
1039 * Schedule execution of deferred actions on @rq. Equivalent to
1040 * schedule_deferred() but requires @rq to be locked and can be more efficient.
1042 static void schedule_deferred_locked(struct rq *rq)
1044 lockdep_assert_rq_held(rq);
1051 if (rq->scx.flags & SCX_RQ_IN_WAKEUP)
1055 if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)
1059 * If in dispatch, the balance callbacks will be called before rq lock
1064 * rq's list: The call can drop its lock and make the pending balance
1069 if (rq->scx.flags & SCX_RQ_IN_DISPATCH) {
1070 rq->scx.flags |= SCX_RQ_BAL_CB_PENDING;
1079 schedule_deferred(rq);
1083 u64 reenq_flags, struct rq *locked_rq)
1085 struct rq *rq;
1095 rq = container_of(dsq, struct rq, scx.local_dsq);
1104 if (unlikely(scx_missing_caps(sch, cpu_of(rq), SCX_CAP_BASE))) {
1109 struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq));
1121 guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
1124 list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals);
1128 rq = this_rq();
1130 struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq));
1142 guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
1145 list_move_tail(&dru->node, &rq->scx.deferred_reenq_users);
1153 if (rq == locked_rq)
1154 schedule_deferred_locked(rq);
1156 schedule_deferred(rq);
1188 * While @p is running, sleeping or queued on an rq-owned DSQ, both fields are
1189 * protected by the rq lock. While running, the rq lock is required because
1191 * only reliable under the rq lock.
1195 * scheduler's responsibility. An rq-locked scx_bpf_task_set_slice() write and a
1203 * only if @p's rq lock is already held, otherwise it bounces through
1204 * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration.
1235 * Only rq-owned DSQs can hold protected tasks and the associated rq
1291 * @rq: rq @p is on
1303 void scx_task_slice_ended(struct rq *rq, struct task_struct *p)
1305 lockdep_assert_rq_held(rq);
1308 if (unlikely(p == scx_rescuee(rq)))
1309 scx_rescue_end(rq);
1331 * Apply a pending out-of-band slice request under @rq's lock. A request whose
1337 static void apply_task_slice_oob(struct rq *rq, struct task_struct *p)
1341 lockdep_assert_rq_held(rq);
1359 unlikely(scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) {
1391 static void update_curr_scx(struct rq *rq)
1393 struct task_struct *curr = rq->curr;
1397 apply_task_slice_oob(rq, curr);
1399 delta_exec = update_curr_common(rq);
1406 if (unlikely(curr == scx_rescuee(rq)))
1407 scx_rescue_charge(rq, delta_exec);
1409 dl_server_update(&rq->ext_server, delta_exec);
1445 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
1450 rq->scx.nr_immed++;
1453 * If @rq already had other tasks or the current task is not
1456 if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags)))
1457 scx_schedule_reenq_local(rq, 0);
1467 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
1470 WARN_ON_ONCE(rq->scx.nr_immed <= 0))
1473 rq->scx.nr_immed--;
1506 static void call_task_dequeue(struct scx_sched *sch, struct rq *rq,
1513 SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags);
1518 static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,
1522 call_task_dequeue(sch, rq, p, 0);
1530 schedule_deferred_locked(rq);
1535 * Note that @rq's lock may be dropped between this enqueue and @p
1537 * an opportunity to wake up on @rq and prevent @p from running.
1545 * We must drop the local rq lock in order to finish the second
1546 * dispatch. In that time, an RT task can wake up on the local rq.
1551 * We must drop the remote rq lock before the dispatched task can run,
1552 * which gives an RT task an opportunity to wake up on the remote rq.
1558 * from @rq's local DSQ. scx_wakeup_preempt() serves exactly this
1560 * @rq->next_class to &ext_sched_class if it's currently idle.
1563 * @rq->next_class is below &ext_sched_class, it will also
1564 * resched_curr(rq).
1566 if (sched_class_above(p->sched_class, rq->next_class))
1567 wakeup_preempt(rq, p, 0);
1570 * If @rq is in dispatch, the CPU is already vacant and looking for the
1574 * a resched if @rq->next_class was idle. It's harmless, since
1577 if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
1580 if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
1581 rq->curr->sched_class == &ext_sched_class) {
1582 if (likely(scx_set_task_slice(rq->curr, 0)))
1583 resched_curr(rq);
1589 static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
1596 dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags);
1706 rq_owned_post_enq(sch, rq, dsq, p, enq_flags);
1711 * without @p's rq lock (finish_dispatch() passes the dispatch
1712 * rq); that's safe because dequeue_task_scx() waits on
1717 call_task_dequeue(sch, rq, p, 0);
1753 void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p)
1758 lockdep_assert_rq_held(rq);
1762 * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals.
1808 * @p's rq and dsq are locked.
1821 struct rq *rq, u64 dsq_id,
1827 return &rq->scx.local_dsq;
1912 struct rq *rq = task_rq(p);
1914 find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p));
1920 * We are in the enqueue path with @rq locked and pinned, and thus can't
1921 * double lock a remote rq and enqueue to its local DSQ. For
1923 * the enqueue so that it's executed when @rq can be unlocked.
1925 if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) {
1949 &rq->scx.ddsp_deferred_locals);
1950 schedule_deferred_locked(rq);
1959 scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime,
1963 bool scx_rq_online(struct rq *rq)
1970 * we aren't locking @rq.
1972 return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq)));
1975 void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
1985 /* internal movements - rq migration / RESTORE */
1986 if (sticky_cpu == cpu_of(rq))
2008 __scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq),
2020 if (!scx_rq_online(rq))
2023 if (scx_bypassing(sch, cpu_of(rq))) {
2050 qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT;
2059 SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags);
2082 scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags);
2085 dsq = &rq->scx.local_dsq;
2097 scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
2105 static void set_task_runnable(struct rq *rq, struct task_struct *p)
2107 lockdep_assert_rq_held(rq);
2118 list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list);
2121 * Record the rq @p is runnable on, maintained under the rq lock so it
2125 WRITE_ONCE(p->scx.runnable_cpu, cpu_of(rq));
2138 static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags)
2142 u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags;
2145 rq->scx.flags |= SCX_RQ_IN_WAKEUP;
2154 if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) {
2155 sticky_cpu = cpu_of(rq);
2164 set_task_runnable(rq, p);
2166 rq->scx.nr_running++;
2167 add_nr_running(rq, 1);
2170 SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags);
2173 if (rq->scx.nr_running == 1)
2174 dl_server_start(&rq->ext_server);
2176 scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu);
2181 rq->scx.flags &= ~SCX_RQ_IN_WAKEUP;
2184 unlikely(cpu_of(rq) != p->scx.selected_cpu))
2188 static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags)
2205 * QUEUEING is started and finished while holding @p's rq lock.
2206 * As we're holding the rq lock now, we shouldn't see QUEUEING.
2215 * store does not require this rq's lock, so retrying with
2234 * As we're waiting on DISPATCHING with the rq locked, the
2235 * dispatching side shouldn't try to lock the rq while
2261 call_task_dequeue(sch, rq, p, deq_flags);
2264 static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags)
2281 ops_dequeue(rq, p, deq_flags);
2284 * A currently running task which is going off @rq first gets dequeued
2291 * dispatch_one() leaves @rq unlocked. However, they don't contain any
2295 if (task_current(rq, p) &&
2296 (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) {
2297 update_curr_scx(rq);
2299 SCX_CALL_OP_TASK(sch, stopping, rq, p, false);
2303 SCX_CALL_OP_TASK(sch, quiescent, rq, p, deq_flags);
2311 rq->scx.nr_running--;
2312 sub_nr_running(rq, 1);
2314 scx_dispatch_dequeue(rq, p);
2317 if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p)))
2318 scx_task_slice_ended(rq, p);
2324 static void yield_task_scx(struct rq *rq)
2326 struct task_struct *p = rq->donor;
2330 scx_task_slice_ended(rq, p);
2333 SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL);
2338 static bool yield_to_task_scx(struct rq *rq, struct task_struct *to)
2340 struct task_struct *from = rq->donor;
2344 scx_task_slice_ended(rq, from);
2347 return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to);
2352 static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags)
2368 * - @rq is waking from idle due to a SCX task waking to it.
2372 if (rq->scx.nr_immed)
2373 scx_schedule_reenq_local(rq, 0);
2378 struct rq *dst_rq)
2382 /* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */
2401 * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ
2405 * @src_rq: rq to move the task from, locked on entry, released on return
2406 * @dst_rq: rq to move the task into, locked on return
2412 struct rq *src_rq, struct rq *dst_rq)
2428 * @p's owner, dropping our scx flags and the placing @sch. We own @rq,
2456 * - The BPF scheduler is bypassed while the rq is offline and we can always say
2459 * The caller must ensure that @p and @rq are on different CPUs.
2460 * If enforce == true, caller must hold @p's rq lock.
2463 struct task_struct *p, struct rq *rq,
2466 s32 cpu = cpu_of(rq);
2470 * out, make sure we're holding @p's rq lock so as not to risk
2510 if (!scx_rq_online(rq)) {
2520 * unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock
2523 * @locked_rq: currently locked rq
2524 * @src_rq: rq @p is currently on, stable with @dsq locked
2535 * this may race with dequeue, which can't drop the rq lock or fail, do a little
2551 struct rq *locked_rq,
2552 struct rq *src_rq)
2571 static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq,
2573 struct scx_dispatch_q *dsq, struct rq *src_rq)
2600 static struct rq *move_task_between_dsqs(struct scx_sched *sch,
2605 struct rq *src_rq = task_rq(p), *dst_rq;
2612 dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
2652 bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
2668 struct rq *task_rq = task_rq(p);
2681 if (rq == task_rq) {
2683 scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq);
2688 if (task_can_run_on_remote_rq(sch, p, rq, false)) {
2689 if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq)))
2699 bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq)
2701 int node = cpu_to_node(cpu_of(rq));
2703 return scx_consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0);
2709 * @rq: current rq which is locked
2716 * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local
2718 * local DSQs are protected with rq locks.
2723 static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
2727 struct rq *src_rq = task_rq(p);
2728 struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
2729 struct rq *locked_rq = rq;
2735 * If dispatching to @rq that @p is already on, no lock dancing needed.
2737 if (rq == src_rq && rq == dst_rq) {
2738 scx_dispatch_enqueue(sch, rq, dst_dsq, p, slice, vtime,
2770 * If @p is staying on the same rq, there's no need to go
2797 /* switch back to @rq lock */
2798 if (locked_rq != rq)
2799 switch_rq_lock(locked_rq, rq);
2805 * @rq: current rq which is locked
2814 * require rq lock dancing. As we don't wanna do either while inside
2824 static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_struct *p,
2872 * task to the BPF scheduler while holding @p's rq lock. As we
2882 dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));
2885 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
2887 scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime,
2891 void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq)
2899 finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id,
2907 static inline void maybe_queue_balance_callback(struct rq *rq)
2909 lockdep_assert_rq_held(rq);
2911 if (!(rq->scx.flags & SCX_RQ_BAL_CB_PENDING))
2914 queue_balance_callback(rq, &rq->scx.deferred_bal_cb,
2917 rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;
2920 static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev)
2924 s32 cpu = cpu_of(rq);
2926 lockdep_assert_rq_held(rq);
2927 rq->scx.flags |= SCX_RQ_IN_DISPATCH;
2929 scx_process_sync_ecaps(rq, prev);
2932 unlikely(rq->scx.cpu_released)) {
2940 SCX_CALL_OP(root_sch, cpu_acquire, rq, cpu, NULL);
2941 rq->scx.cpu_released = false;
2945 update_curr_scx(rq);
2965 if (rq->scx.local_dsq.nr) {
2970 verdict = scx_dispatch_sched(root_sch, rq, prev, false);
2978 * may have dropped the rq lock while @prev changed class or scheduler.
2984 scx_bypassing(prev_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) {
2990 rq->scx.flags &= ~SCX_RQ_IN_DISPATCH;
2995 * @rq may have extra IMMED tasks without reenq scheduled:
3004 if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed))
3005 scx_schedule_reenq_local(rq, 0);
3007 rq->scx.flags &= ~SCX_RQ_IN_DISPATCH;
3011 static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
3020 ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC);
3021 scx_dispatch_dequeue(rq, p);
3024 p->se.exec_start = rq_clock_task(rq);
3028 SCX_CALL_OP_TASK(sch, running, rq, p);
3033 apply_task_slice_oob(rq, p);
3045 if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
3052 rq->scx.flags |= SCX_RQ_CAN_STOP_TICK;
3053 sched_update_tick_dependency(rq);
3055 update_other_load_avgs(rq);
3058 if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) {
3059 rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
3060 update_other_load_avgs(rq);
3064 * @rq still references the outgoing scheduling context. A finite
3067 if (tick_nohz_full_cpu(cpu_of(rq)))
3068 tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
3084 static void switch_class(struct rq *rq, struct task_struct *next)
3110 if (!rq->scx.cpu_released) {
3117 SCX_CALL_OP(sch, cpu_release, rq, cpu_of(rq), &args);
3119 rq->scx.cpu_released = true;
3123 static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
3130 smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
3132 update_curr_scx(rq);
3139 if (unlikely(p == scx_rescuee(rq)))
3140 rescue_keep = scx_rescue_keep(rq, p);
3142 scx_task_slice_ended(rq, p);
3147 SCX_CALL_OP_TASK(sch, stopping, rq, p, true);
3150 set_task_runnable(rq, p);
3162 if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) &&
3163 !scx_bypassing(sch, cpu_of(rq))) {
3166 scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
3176 if (unlikely(p == scx_rescuee(rq))) {
3184 scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0,
3202 scx_task_can_stay_on_cpu(rq, p)) {
3203 WARN_ON_ONCE(!sched_core_enabled(rq) &&
3205 scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
3207 scx_do_enqueue_task(rq, p, 0, -1);
3213 switch_class(rq, next);
3216 static void kick_sync_wait_bal_cb(struct rq *rq)
3224 * This callback is queued and normally flushed within @rq's own
3225 * scheduling pass. However, dispatch can drop the rq lock while it sits
3232 if (unlikely(cpu_of(rq) != smp_processor_id()))
3239 * Drop rq lock and enable IRQs while waiting. IRQs must be enabled
3249 for_each_cpu(cpu, rq->scx.cpus_to_sync) {
3254 if (cpu == cpu_of(rq) ||
3256 cpumask_clear_cpu(cpu, rq->scx.cpus_to_sync);
3260 scx_rq_lock_drop(rq);
3261 raw_spin_rq_unlock_irq(rq);
3263 smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
3266 raw_spin_rq_lock_irq(rq);
3274 static struct task_struct *first_local_task(struct rq *rq)
3276 return list_first_entry_or_null(&rq->scx.local_dsq.list,
3283 static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf,
3288 rq_unpin_lock(rq, rf);
3289 verdict = dispatch_one(rq, prev);
3290 rq_repin_lock(rq, rf);
3291 maybe_queue_balance_callback(rq);
3294 * Defer to a balance callback which can drop rq lock and enable IRQs.
3298 if (unlikely(rq->scx.kick_sync_pending)) {
3299 rq->scx.kick_sync_pending = false;
3300 queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb,
3311 * throughout. If the dispatch released the rq lock, anything can have happened
3315 static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf,
3319 u32 seq = rq->scx.lock_drop_seq;
3321 /* another dispatch is in flight on @rq, let that handle it */
3322 if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
3325 rq_unpin_lock(rq, rf);
3327 verdict = dispatch_one(rq, prev);
3329 if (cpu_of(rq) == smp_processor_id()) {
3330 maybe_queue_balance_callback(rq);
3333 if (unlikely(rq->scx.kick_sync_pending)) {
3334 rq->scx.kick_sync_pending = false;
3335 queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb,
3338 } else if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) {
3341 * so they can't be queued on another CPU's rq. Run the deferred
3344 rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;
3345 run_deferred(rq);
3348 rq_repin_lock(rq, rf);
3350 /* if dispatch_one() released the rq lock, restart the selection */
3351 if (rq->scx.lock_drop_seq != seq)
3357 static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf,
3365 do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
3367 struct task_struct *prev = rq->curr;
3372 smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
3374 rq_modified_begin(rq, &ext_sched_class);
3376 if (sched_core_enabled(rq))
3377 verdict = dispatch_core_pick(rq, rf, prev);
3379 verdict = dispatch_pick(rq, rf, prev);
3386 * rq during dispatch_one(), abort and return RETRY_TASK, so that the
3392 if (!force_scx && rq_modified_above(rq, &ext_sched_class))
3403 scx_task_slice_ended(rq, p);
3407 p = first_local_task(rq);
3411 if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) {
3414 if (!scx_bypassing(sch, cpu_of(rq)) &&
3427 static struct task_struct *pick_task_scx(struct rq *rq, struct rq_flags *rf)
3429 return do_pick_task_scx(rq, rf, false);
3444 return do_pick_task_scx(dl_se->rq, rf, true);
3450 void ext_server_init(struct rq *rq)
3452 struct sched_dl_entity *dl_se = &rq->ext_server;
3456 dl_server_init(dl_se, rq, ext_server_pick_task);
3546 * immediate rq migration. However, for SCX, as the current rq
3595 * it under rq lock. See the write rules.
3610 static void task_woken_scx(struct rq *rq, struct task_struct *p)
3612 run_deferred(rq);
3637 static void handle_hotplug(struct rq *rq, bool online)
3640 s32 cpu = cpu_of(rq);
3657 scx_online_ecaps(rq);
3659 scx_offline_ecaps(rq);
3700 void scx_rq_activate(struct rq *rq)
3702 handle_hotplug(rq, true);
3705 void scx_rq_deactivate(struct rq *rq)
3707 handle_hotplug(rq, false);
3710 static void rq_online_scx(struct rq *rq)
3712 rq->scx.flags |= SCX_RQ_ONLINE;
3715 static void rq_offline_scx(struct rq *rq)
3717 rq->scx.flags &= ~SCX_RQ_ONLINE;
3718 scx_rescue_flush(rq);
3721 static bool check_rq_for_timeouts(struct rq *rq)
3728 rq_lock_irqsave(rq, &rf);
3733 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) {
3752 __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq),
3761 rq_unlock_irqrestore(rq, &rf);
3784 void scx_tick(struct rq *rq)
3806 update_other_load_avgs(rq);
3809 static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
3813 update_curr_scx(rq);
3819 if (scx_bypassing(sch, cpu_of(rq)))
3822 SCX_CALL_OP_TASK(sch, tick, rq, curr);
3825 resched_curr(rq);
3893 struct rq *rq;
3896 rq = task_rq_lock(p, &rf);
3910 task_rq_unlock(rq, p, &rf);
3919 struct rq *rq = task_rq(p);
3922 lockdep_assert_rq_held(rq);
3943 SCX_CALL_OP_TASK(sch, enable, rq, p);
3946 SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
3957 struct rq *rq = task_rq(p);
3959 lockdep_assert_rq_held(rq);
3965 SCX_CALL_OP_TASK(sch, disable, rq, p);
3973 scx_task_slice_ended(rq, p);
4143 struct rq *rq;
4145 rq = task_rq_lock(p, &rf);
4147 task_rq_unlock(rq, p, &rf);
4163 struct rq *rq;
4166 rq = task_rq_lock(p, &rf);
4169 task_rq_unlock(rq, p, &rf);
4191 struct rq *rq = task_rq(p);
4193 lockdep_assert_rq_held(rq);
4201 !task_on_cpu(rq, p);
4208 * for the last time and then dropped the rq lock - task_dead_and_done()
4235 struct rq *rq;
4237 rq = task_rq_lock(p, &rf);
4241 task_rq_unlock(rq, p, &rf);
4245 static void reweight_task_scx(struct rq *rq, struct task_struct *p,
4268 SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
4271 static void prio_changed_scx(struct rq *rq, struct task_struct *p, u64 oldprio)
4275 static void switching_to_scx(struct rq *rq, struct task_struct *p)
4289 scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr);
4292 static void switched_from_scx(struct rq *rq, struct task_struct *p)
4310 static void switched_to_scx(struct rq *rq, struct task_struct *p) {}
4324 static void process_ddsp_deferred_locals(struct rq *rq)
4328 lockdep_assert_rq_held(rq);
4331 * Now that @rq can be unlocked, execute the deferred enqueueing of
4334 * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq
4337 while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals,
4349 dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));
4351 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
4378 static bool local_task_should_reenq(struct rq *rq, struct task_struct *p,
4386 if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || p == scx_rescuee(rq)))
4399 scx_task_reenq_on_cap_revoke(rq, p)) {
4407 static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
4413 lockdep_assert_rq_held(rq);
4417 if (rq_is_open(rq, 0))
4422 * @rq->scx.local_dsq. Move all candidate tasks off to a private list
4425 list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list,
4447 if (!local_task_should_reenq(rq, p, &reenq_flags, &reason))
4450 scx_dispatch_dequeue(rq, p);
4462 scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
4476 rq->curr->sched_class == &ext_sched_class &&
4477 scx_task_reenq_on_cap_revoke(rq, rq->curr)) {
4478 scx_set_task_slice(rq->curr, 0);
4479 resched_curr(rq);
4485 static void process_deferred_reenq_locals(struct rq *rq)
4487 lockdep_assert_rq_held(rq);
4498 scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) {
4500 list_first_entry_or_null(&rq->scx.deferred_reenq_locals,
4520 reenq_local(sch, rq, reenq_flags);
4530 static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags)
4532 struct rq *locked_rq = rq;
4538 lockdep_assert_rq_held(rq);
4543 struct rq *task_rq;
4572 /* @p is on @dsq, its rq and @dsq are locked */
4597 if (locked_rq != rq) {
4602 raw_spin_rq_lock(rq);
4606 static void process_deferred_reenq_users(struct rq *rq)
4608 lockdep_assert_rq_held(rq);
4614 scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) {
4616 list_first_entry_or_null(&rq->scx.deferred_reenq_users,
4641 reenq_user(rq, dsq, reenq_flags);
4645 static void run_deferred(struct rq *rq)
4647 process_ddsp_deferred_locals(rq);
4649 if (!list_empty(&rq->scx.deferred_reenq_locals))
4650 process_deferred_reenq_locals(rq);
4652 if (!list_empty(&rq->scx.deferred_reenq_users))
4653 process_deferred_reenq_users(rq);
4655 scx_reenq_reject(rq);
4659 bool scx_can_stop_tick(struct rq *rq)
4661 struct task_struct *p = rq->curr;
4668 * @rq->curr may still reference an outgoing EXT task after it has been
4669 * dequeued. If no EXT tasks are accounted on @rq, ignore its stale
4673 if (!rq->scx.nr_running)
4676 if (scx_bypassing(sch, cpu_of(rq)))
4683 if (unlikely(p == scx_rescuee(rq)))
4687 * @rq can dispatch from different DSQs, so we can't tell whether it
4691 return rq->scx.flags & SCX_RQ_CAN_STOP_TICK;
5093 struct rq *rq = cpu_rq(cpu);
5100 guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
5752 struct rq *donor_rq = cpu_rq(donor);
5792 * without its rq locked. Skip.
5804 * $p's rq is not locked but $p's DSQ lock protects its
5811 * Moving $p from one non-local DSQ to another. The source rq
6043 * @rq: rq of the cpu leaving bypass
6044 * @pos: scheduler that just left bypass on @rq's cpu
6045 * @pcpu: @pos's per-cpu state for @rq's cpu
6057 * However, the root uses a separate per-rq flag so its re-notify keeps working
6060 static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos,
6064 rq->scx.flags |= SCX_RQ_ROOT_IDLE_RENOTIFY;
6069 rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY;
6141 * state. As an optimization, walk each rq's runnable_list instead of
6148 struct rq *rq = cpu_rq(cpu);
6151 raw_spin_rq_lock(rq);
6163 unbypass_renotify_idle(rq, pos, pcpu);
6164 scx_unbypass_replay_ecaps(rq, pos);
6177 scx_rq_lock_drop(rq);
6178 raw_spin_rq_unlock(rq);
6189 list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list,
6198 if (bypass && task_current(rq, p))
6199 scx_task_slice_ended(rq, p);
6209 resched_curr(rq);
6211 scx_rq_lock_drop(rq);
6212 raw_spin_rq_unlock(rq);
6514 struct rq *rq = cpu_rq(cpu);
6516 scoped_guard(rq_lock_irqsave, rq) {
6517 update_rq_clock(rq);
6519 if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server,
6520 &rq->fair_server)))
6523 dl_server_detach_bw(&rq->ext_server);
6817 struct rq *rq, struct task_struct *p, char marker)
6856 SCX_CALL_OP(sch, dump_task, rq, dctx, p);
6873 struct rq *rq = cpu_rq(cpu);
6882 rq_lock_irqsave(rq, &rf);
6884 idle = list_empty(&rq->scx.runnable_list) &&
6885 rq->curr->sched_class == &idle_sched_class;
6901 cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released,
6902 rq->scx.ops_qseq, rq->scx.kick_sync);
6903 scx_rescue_dump(&ns, rq);
6905 rq->curr->comm, rq->curr->pid, rq->curr->sched_class);
6918 if (!cpumask_empty(rq->scx.cpus_to_sync))
6920 cpumask_pr_args(rq->scx.cpus_to_sync));
6925 SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle);
6946 if (rq->curr->sched_class == &ext_sched_class &&
6947 (dump_all_tasks || scx_task_on_sched(sch, rq->curr)))
6948 scx_dump_task(sch, s, dctx, rq, rq->curr, '*');
6950 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
6952 scx_dump_task(sch, s, dctx, rq, p, ' ');
6954 rq_unlock_irqrestore(rq, &rf);
7570 struct rq *rq = cpu_rq(cpu);
7572 rq->scx.local_dsq.sched = sch;
7573 rq->scx.cpuperf_target = SCX_CPUPERF_ONE;
7695 struct rq *rq = cpu_rq(cpu);
7697 scoped_guard(rq_lock_irqsave, rq) {
7698 update_rq_clock(rq);
7699 ret = dl_server_attach_bw(&rq->ext_server);
7770 * Set %INIT_BEGIN under the iter's rq lock so that a concurrent
7877 struct rq *rq = cpu_rq(cpu);
7879 guard(rq_lock_irqsave)(rq);
7880 update_rq_clock(rq);
7881 dl_server_detach_bw(&rq->fair_server);
8471 static bool can_skip_idle_kick(struct rq *rq)
8473 lockdep_assert_rq_held(rq);
8476 * We can skip idle kicking if @rq is going to go through at least one
8479 * dispatch_one() trying to pull the next task from a remote rq, which
8480 * may fail, and @rq may become idle afterwards.
8482 * The race window is small and we don't and can't guarantee that @rq is
8485 return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_DISPATCH);
8488 static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_rq,
8491 struct rq *rq = cpu_rq(cpu);
8498 raw_spin_rq_lock_irqsave(rq, flags);
8499 cur_class = rq->curr->sched_class;
8507 * cap check: ecaps is read here under @rq's lock.
8515 u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0);
8519 else if (unlikely(!scx_set_task_slice(rq->curr, 0)))
8528 ksyncs[cpu] = rq->scx.kick_sync;
8534 resched_curr(rq);
8543 scx_rq_lock_drop(rq);
8544 raw_spin_rq_unlock_irqrestore(rq, flags);
8550 struct rq *this_rq)
8552 struct rq *rq = cpu_rq(cpu);
8555 raw_spin_rq_lock_irqsave(rq, flags);
8558 if (!can_skip_idle_kick(rq) &&
8561 resched_curr(rq);
8566 scx_rq_lock_drop(rq);
8567 raw_spin_rq_unlock_irqrestore(rq, flags);
8572 struct rq *this_rq = this_rq();
8720 struct rq *rq = cpu_rq(cpu);
8724 BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
8726 BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL));
8727 scx_rescue_init(rq);
8730 INIT_LIST_HEAD(&rq->scx.runnable_list);
8731 INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals);
8733 BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n));
8734 INIT_LIST_HEAD(&rq->scx.sched_pcpus_to_kick);
8735 raw_spin_lock_init(&rq->scx.deferred_reenq_lock);
8736 INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals);
8737 INIT_LIST_HEAD(&rq->scx.deferred_reenq_users);
8738 rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn);
8739 rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn);
9028 struct rq *p_rq, *src_rq, *locked_rq;
9057 * Can be called from either ops.dispatch() holding the dispatched rq's
9058 * lock or any context where no rq lock is held. If latter, lock @p's
9066 * Under core scheduling, dispatch can run for a sibling rq, so the
9067 * locked rq is not necessarily this CPU's.
9194 * before trying to move from the specified DSQ. It may also grab rq locks and
9218 scx_flush_dispatch_buf(sch, dspc->rq);
9226 if (scx_consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) {
9303 * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq
9379 struct rq *rq;
9386 rq = cpu_rq(smp_processor_id());
9387 lockdep_assert_rq_held(rq);
9389 return reenq_local(sch, rq, SCX_REENQ_ANY);
9491 * applied whether or not the caller holds @p's rq lock - see the slice write
9506 struct rq *locked_rq;
9514 * Directly write only when we hold the lock of the rq @p is queued or
9534 /* under the rq lock: apply now, extensions gated on baseline access */
9573 struct rq *this_rq;
9600 * rq locks. We can probably be smarter and avoid bouncing if called
9601 * from ops which don't hold a rq lock.
9607 struct rq *target_rq = cpu_rq(cpu);
9694 * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler
9695 * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() -
9696 * or the calling CPU's when no rq is locked.
9893 * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler
9894 * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() -
9895 * or the calling CPU's when no rq is locked.
9900 struct rq *locked_rq = scx_locked_rq();
10224 struct rq *rq, *locked_rq;
10236 rq = cpu_rq(cpu);
10240 * When called with an rq lock held, restrict the operation to the
10243 if (locked_rq && rq != locked_rq) {
10249 * If no rq lock is held, allow to operate on any CPU by acquiring
10250 * the corresponding rq lock.
10253 rq_lock_irqsave(rq, &rf);
10254 update_rq_clock(rq);
10258 * ecaps updates are folded under the rq lock, making this test
10263 rq->scx.cpuperf_target = perf;
10264 cpufreq_update_util(rq, 0);
10272 rq_unlock_irqrestore(rq, &rf);
10499 * scx_bpf_locked_rq - Return the rq currently locked by SCX
10502 * Returns the rq if a rq lock is currently held by SCX.
10505 __bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux)
10508 struct rq *rq;
10516 rq = scx_locked_rq();
10517 if (!rq) {
10518 scx_error(sch, "accessing rq without holding rq lock");
10522 return rq;
10600 u64 __scx_bpf_now(struct rq *rq)
10602 /* the caller must be on @rq's cpu or hold its lock */
10603 lockdep_assert((rq == this_rq() && !preemptible()) ||
10604 lockdep_is_held(__rq_lockp(rq)));
10606 if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) {
10607 /* if the rq clock is valid, use the cached rq clock */
10608 return READ_ONCE(rq->scx.clock);
10611 * Otherwise, return a fresh rq clock.
10613 * The rq clock is updated outside of the rq lock.
10614 * In this case, keep the updated rq clock invalid so the next
10615 * read outside the rq lock gets a fresh rq clock.
10617 return sched_clock_cpu(cpu_of(rq));
10632 * using the rq clock in the scheduler core whenever possible.
10637 * uses the rq clock in the scheduler core whenever it is valid. It considers
10638 * that the rq clock is valid from the time the rq clock is updated
10639 * (update_rq_clock) until the rq is unlocked (rq_unpin_lock).
10647 * during the same period of when the rq clock is valid.