1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
4 *
5 * Inline definitions layered on top of internal.h and cid.h.
6 *
7 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
8 * Copyright (c) 2026 Tejun Heo <tj@kernel.org>
9 */
10 #ifndef _KERNEL_SCHED_EXT_INLINES_H
11 #define _KERNEL_SCHED_EXT_INLINES_H
12
13 #include "internal.h"
14 #include "cid.h"
15
16 /* what dispatch concluded, consumed by the pick that follows */
17 enum scx_dsp_verdict {
18 SCX_DSP_NONE, /* nothing to run */
19 SCX_DSP_LOCAL, /* local DSQ has tasks */
20 SCX_DSP_PREV, /* keep running @prev */
21 SCX_DSP_RETRY, /* pick helpers only: restart the pick */
22 };
23
24 /*
25 * One user of this function is scx_bpf_sub_dispatch() which can be called
26 * recursively as sub-sched dispatches nest. Always inline to reduce stack usage
27 * from the call frame.
28 */
29 static __always_inline enum scx_dsp_verdict
scx_dispatch_sched(struct scx_sched * sch,struct rq * rq,struct task_struct * prev,bool nested)30 scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
31 struct task_struct *prev, bool nested)
32 {
33 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
34 int nr_loops = SCX_DSP_MAX_LOOPS;
35 s32 cpu = cpu_of(rq);
36 bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
37 scx_task_on_sched(sch, prev);
38
39 if (scx_consume_global_dsq(sch, rq))
40 return SCX_DSP_LOCAL;
41
42 if (scx_bypass_dsp_enabled(sch)) {
43 /* if @sch is bypassing, only the bypass DSQs are active */
44 if (scx_bypassing(sch, cpu)) {
45 if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0))
46 return SCX_DSP_LOCAL;
47 return SCX_DSP_NONE;
48 }
49
50 #ifdef CONFIG_EXT_SUB_SCHED
51 /*
52 * If @sch isn't bypassing but its children are, @sch is
53 * responsible for making forward progress for both its own
54 * tasks that aren't bypassing and the bypassing descendants'
55 * tasks. The following implements a simple built-in behavior -
56 * let each CPU try to run the bypass DSQ every Nth time.
57 *
58 * Later, if necessary, we can add an ops flag to suppress the
59 * auto-consumption and a kfunc to consume the bypass DSQ and,
60 * so that the BPF scheduler can fully control scheduling of
61 * bypassed tasks.
62 */
63 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
64
65 if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) &&
66 scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
67 __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
68 return SCX_DSP_LOCAL;
69 }
70 #endif /* CONFIG_EXT_SUB_SCHED */
71 }
72
73 if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq))
74 return SCX_DSP_NONE;
75
76 dspc->rq = rq;
77
78 /*
79 * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq
80 * lock, the local DSQ might still end up empty after a successful
81 * ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
82 * produced some tasks, retry. The BPF scheduler may depend on this
83 * looping behavior to simplify its implementation.
84 */
85 do {
86 dspc->nr_tasks = 0;
87
88 #ifdef CONFIG_EXT_SUB_SCHED
89 /* stash @prev so that nested invocations can access it */
90 if (!nested)
91 rq->scx.sub_dispatch_prev = prev;
92 #endif
93
94 SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
95 prev_on_sch ? prev : NULL);
96
97 #ifdef CONFIG_EXT_SUB_SCHED
98 if (!nested)
99 rq->scx.sub_dispatch_prev = NULL;
100 #endif
101
102 scx_flush_dispatch_buf(sch, rq);
103
104 if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice)
105 return SCX_DSP_PREV;
106 if (rq->scx.local_dsq.nr)
107 return SCX_DSP_LOCAL;
108 if (scx_consume_global_dsq(sch, rq))
109 return SCX_DSP_LOCAL;
110
111 /*
112 * ops.dispatch() can trap us in this loop by repeatedly
113 * dispatching ineligible tasks. Break out once in a while to
114 * allow the watchdog to run. As IRQ can't be enabled in
115 * dispatch, we want to complete this scheduling cycle and then
116 * start a new one. IOW, we want to call resched_curr() on the
117 * next, most likely idle, task, not the current one. Use
118 * __scx_bpf_kick_cpu() for deferred kicking.
119 */
120 if (unlikely(!--nr_loops)) {
121 scx_kick_cpu(sch, cpu, 0);
122 break;
123 }
124 } while (dspc->nr_tasks);
125
126 /*
127 * Prevent the CPU from going idle while bypassed descendants have tasks
128 * queued. Without this fallback, bypassed tasks could stall if the host
129 * scheduler's ops.dispatch() doesn't yield any tasks.
130 */
131 if (scx_bypass_dsp_enabled(sch) &&
132 scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0))
133 return SCX_DSP_LOCAL;
134
135 return SCX_DSP_NONE;
136 }
137
138 #endif /* _KERNEL_SCHED_EXT_INLINES_H */
139