1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst 4 * 5 * Inline definitions layered on top of internal.h and cid.h. 6 * 7 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. 8 * Copyright (c) 2026 Tejun Heo <tj@kernel.org> 9 */ 10 #ifndef _KERNEL_SCHED_EXT_INLINES_H 11 #define _KERNEL_SCHED_EXT_INLINES_H 12 13 #include "internal.h" 14 #include "cid.h" 15 16 /* what dispatch concluded, consumed by the pick that follows */ 17 enum scx_dsp_verdict { 18 SCX_DSP_NONE, /* nothing to run */ 19 SCX_DSP_LOCAL, /* local DSQ has tasks */ 20 SCX_DSP_PREV, /* keep running @prev */ 21 SCX_DSP_RETRY, /* pick helpers only: restart the pick */ 22 }; 23 24 /* 25 * One user of this function is scx_bpf_sub_dispatch() which can be called 26 * recursively as sub-sched dispatches nest. Always inline to reduce stack usage 27 * from the call frame. 28 */ 29 static __always_inline enum scx_dsp_verdict 30 scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, 31 struct task_struct *prev, bool nested) 32 { 33 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; 34 int nr_loops = SCX_DSP_MAX_LOOPS; 35 s32 cpu = cpu_of(rq); 36 bool prev_on_sch = (prev->sched_class == &ext_sched_class) && 37 scx_task_on_sched(sch, prev); 38 39 if (scx_consume_global_dsq(sch, rq)) 40 return SCX_DSP_LOCAL; 41 42 if (scx_bypass_dsp_enabled(sch)) { 43 /* if @sch is bypassing, only the bypass DSQs are active */ 44 if (scx_bypassing(sch, cpu)) { 45 if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) 46 return SCX_DSP_LOCAL; 47 return SCX_DSP_NONE; 48 } 49 50 #ifdef CONFIG_EXT_SUB_SCHED 51 /* 52 * If @sch isn't bypassing but its children are, @sch is 53 * responsible for making forward progress for both its own 54 * tasks that aren't bypassing and the bypassing descendants' 55 * tasks. The following implements a simple built-in behavior - 56 * let each CPU try to run the bypass DSQ every Nth time. 57 * 58 * Later, if necessary, we can add an ops flag to suppress the 59 * auto-consumption and a kfunc to consume the bypass DSQ and, 60 * so that the BPF scheduler can fully control scheduling of 61 * bypassed tasks. 62 */ 63 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); 64 65 if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && 66 scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { 67 __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); 68 return SCX_DSP_LOCAL; 69 } 70 #endif /* CONFIG_EXT_SUB_SCHED */ 71 } 72 73 /* 74 * scx_rq_online() can't be used. Its cpu_active() test goes false 75 * before CPU hotplug waits for an RCU grace period, and 76 * rq_offline_scx() moves this CPU's tasks to the local DSQ only after 77 * the wait. The grace period can depend on those tasks running. 78 */ 79 if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !(rq->scx.flags & SCX_RQ_ONLINE)) 80 return SCX_DSP_NONE; 81 82 dspc->rq = rq; 83 84 /* 85 * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq 86 * lock, the local DSQ might still end up empty after a successful 87 * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() 88 * produced some tasks, retry. The BPF scheduler may depend on this 89 * looping behavior to simplify its implementation. 90 */ 91 do { 92 dspc->nr_tasks = 0; 93 94 #ifdef CONFIG_EXT_SUB_SCHED 95 /* stash @prev so that nested invocations can access it */ 96 if (!nested) 97 rq->scx.sub_dispatch_prev = prev; 98 #endif 99 100 SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), 101 prev_on_sch ? prev : NULL); 102 103 #ifdef CONFIG_EXT_SUB_SCHED 104 if (!nested) 105 rq->scx.sub_dispatch_prev = NULL; 106 #endif 107 108 scx_flush_dispatch_buf(sch, rq); 109 110 if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) 111 return SCX_DSP_PREV; 112 if (rq->scx.local_dsq.nr) 113 return SCX_DSP_LOCAL; 114 if (scx_consume_global_dsq(sch, rq)) 115 return SCX_DSP_LOCAL; 116 117 /* 118 * ops.dispatch() can trap us in this loop by repeatedly 119 * dispatching ineligible tasks. Break out once in a while to 120 * allow the watchdog to run. As IRQ can't be enabled in 121 * dispatch, we want to complete this scheduling cycle and then 122 * start a new one. IOW, we want to call resched_curr() on the 123 * next, most likely idle, task, not the current one. Use 124 * __scx_bpf_kick_cpu() for deferred kicking. 125 */ 126 if (unlikely(!--nr_loops)) { 127 scx_kick_cpu(sch, cpu, 0); 128 break; 129 } 130 } while (dspc->nr_tasks); 131 132 /* 133 * Prevent the CPU from going idle while bypassed descendants have tasks 134 * queued. Without this fallback, bypassed tasks could stall if the host 135 * scheduler's ops.dispatch() doesn't yield any tasks. 136 */ 137 if (scx_bypass_dsp_enabled(sch) && 138 scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { 139 __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); 140 return SCX_DSP_LOCAL; 141 } 142 143 return SCX_DSP_NONE; 144 } 145 146 #endif /* _KERNEL_SCHED_EXT_INLINES_H */ 147