1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
4 *
5 * Sub-scheduler hierarchy support.
6 *
7 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
8 * Copyright (c) 2026 Tejun Heo <tj@kernel.org>
9 */
10 #ifndef _KERNEL_SCHED_EXT_SUB_H
11 #define _KERNEL_SCHED_EXT_SUB_H
12
13 #include "internal.h"
14
15 #ifdef CONFIG_EXT_SUB_SCHED
16
17 struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root);
18 struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root);
19 void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch);
20 struct cgroup *sch_cgroup(struct scx_sched *sch);
21 void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch);
22 void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog);
23 void scx_pstack_recursion_on_caps_updated(struct bpf_prog *prog);
24 void drain_descendants(struct scx_sched *sch);
25 void scx_sub_disable(struct scx_sched *sch);
26 void scx_sub_enable_workfn(struct kthread_work *work);
27 bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux);
28 void scx_free_pshards(struct scx_sched *sch);
29 s32 scx_alloc_pshards(struct scx_sched *sch);
30 void scx_init_root_caps(struct scx_sched *sch);
31 void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
32 void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch);
33 void scx_online_ecaps(struct rq *rq);
34 void scx_offline_ecaps(struct rq *rq);
35 void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu);
36 void scx_discard_stale_ecaps_syncs(void);
37 struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
38 struct task_struct *p, u64 *enq_flags);
39 bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p);
40 void scx_reenq_reject(struct rq *rq);
41 void scx_rescue_charge(struct rq *rq, s64 delta_exec);
42 void scx_rescue_end(struct rq *rq);
43 bool scx_rescue_keep(struct rq *rq, struct task_struct *p);
44 void scx_rescue_flush(struct rq *rq);
45 void scx_rescue_dump(struct seq_buf *s, struct rq *rq);
46 void scx_rescue_set_knobs(struct scx_sched *sch);
47 void scx_rescue_init(struct rq *rq);
48
49 /*
50 * cgrp->scx_sched is written by root/sub enable/disable under all of
51 * scx_enable_mutex, scx_fork_rwsem and cgroup_mutex. A new cgroup inherits the
52 * parent's sched under just cgroup_mutex but is not yet reachable by the other
53 * two lock holders. Any one of the three locks stabilizes the association.
54 */
scx_cgroup_sched(struct cgroup * cgrp)55 static inline struct scx_sched *scx_cgroup_sched(struct cgroup *cgrp)
56 {
57 return rcu_dereference_check(cgrp->scx_sched,
58 lockdep_is_held(&cgroup_mutex) ||
59 percpu_rwsem_is_held(&scx_fork_rwsem) ||
60 lockdep_is_held(&scx_enable_mutex));
61 }
62
sch_cgrp_path(struct scx_sched * sch)63 static inline const char *sch_cgrp_path(struct scx_sched *sch)
64 {
65 return sch->cgrp_path;
66 }
67
68 /* a dying sub's hot-path influence ends in scx_sched_free_rcu_work() */
scx_dec_has_subs(struct scx_sched * sch)69 static inline void scx_dec_has_subs(struct scx_sched *sch)
70 {
71 if (sch->level)
72 static_branch_dec(&__scx_has_subs);
73 }
74
75 #else /* CONFIG_EXT_SUB_SCHED */
76
scx_next_descendant_pre(struct scx_sched * pos,struct scx_sched * root)77 static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
scx_skip_subtree_pre(struct scx_sched * pos,struct scx_sched * root)78 static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; }
scx_set_task_sched(struct task_struct * p,struct scx_sched * sch)79 static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}
sch_cgroup(struct scx_sched * sch)80 static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }
sch_cgrp_path(struct scx_sched * sch)81 static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; }
set_cgroup_sched(struct cgroup * cgrp,struct scx_sched * sch)82 static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}
drain_descendants(struct scx_sched * sch)83 static inline void drain_descendants(struct scx_sched *sch) { }
scx_sub_disable(struct scx_sched * sch)84 static inline void scx_sub_disable(struct scx_sched *sch) { }
scx_free_pshards(struct scx_sched * sch)85 static inline void scx_free_pshards(struct scx_sched *sch) {}
scx_alloc_pshards(struct scx_sched * sch)86 static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; }
scx_init_root_caps(struct scx_sched * sch)87 static inline void scx_init_root_caps(struct scx_sched *sch) {}
scx_process_sync_ecaps(struct rq * rq,struct task_struct * prev)88 static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {}
scx_unbypass_replay_ecaps(struct rq * rq,struct scx_sched * sch)89 static inline void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) {}
scx_online_ecaps(struct rq * rq)90 static inline void scx_online_ecaps(struct rq *rq) {}
scx_offline_ecaps(struct rq * rq)91 static inline void scx_offline_ecaps(struct rq *rq) {}
scx_discard_ecaps_to_sync(s32 cpu,struct scx_sched_pcpu * pcpu)92 static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {}
scx_discard_stale_ecaps_syncs(void)93 static inline void scx_discard_stale_ecaps_syncs(void) {}
scx_resolve_local_dsq(struct scx_sched * sch,struct rq * rq,struct task_struct * p,u64 * enq_flags)94 static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
scx_task_reenq_on_cap_revoke(struct rq * rq,struct task_struct * p)95 static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; }
scx_reenq_reject(struct rq * rq)96 static inline void scx_reenq_reject(struct rq *rq) {}
scx_rescue_charge(struct rq * rq,s64 delta_exec)97 static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {}
scx_rescue_end(struct rq * rq)98 static inline void scx_rescue_end(struct rq *rq) {}
scx_rescue_keep(struct rq * rq,struct task_struct * p)99 static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; }
scx_rescue_flush(struct rq * rq)100 static inline void scx_rescue_flush(struct rq *rq) {}
scx_rescue_dump(struct seq_buf * s,struct rq * rq)101 static inline void scx_rescue_dump(struct seq_buf *s, struct rq *rq) {}
scx_rescue_set_knobs(struct scx_sched * sch)102 static inline void scx_rescue_set_knobs(struct scx_sched *sch) {}
scx_rescue_init(struct rq * rq)103 static inline void scx_rescue_init(struct rq *rq) {}
scx_dec_has_subs(struct scx_sched * sch)104 static inline void scx_dec_has_subs(struct scx_sched *sch) {}
105
106 #endif /* CONFIG_EXT_SUB_SCHED */
107
108 /**
109 * scx_for_each_descendant_pre - pre-order walk of a sched's descendants
110 * @pos: iteration cursor
111 * @root: sched to walk the descendants of
112 *
113 * Walk @root's descendants. @root is included in the iteration and the first
114 * node to be visited. Must be called with scx_enable_mutex, scx_sched_lock, or
115 * RCU read lock.
116 */
117 #define scx_for_each_descendant_pre(pos, root) \
118 for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \
119 (pos) = scx_next_descendant_pre((pos), (root)))
120
121 #ifdef CONFIG_EXT_SUB_SCHED
122
123 /**
124 * scx_missing_caps - The caps in @needed that @sch lacks on @cpu
125 * @sch: sched to test
126 * @cpu: cpu to test on
127 * @needed: bitmask of SCX_CAP_* values
128 *
129 * Return the caps in @needed that @sch lacks for @cpu, 0 if it holds them all.
130 */
scx_missing_caps(struct scx_sched * sch,s32 cpu,u64 needed)131 static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed)
132 {
133 u64 ecaps;
134
135 /* no sub-scheds, no missing caps */
136 if (!scx_has_subs())
137 return 0;
138
139 /* root holds every cap on every cpu */
140 if (!sch->level)
141 return 0;
142
143 ecaps = READ_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps);
144
145 return needed & ~ecaps;
146 }
147
148 /*
149 * Cap semantics: which caps an action requires, and which caps a cap implies.
150 * Keep all such mappings collected here.
151 */
152
153 /* map @enq_flags to the SCX_CAP_* bit required for the local-DSQ insert */
scx_caps_for_enq(u64 enq_flags)154 static inline u64 scx_caps_for_enq(u64 enq_flags)
155 {
156 /* a restored task must be put into the local DSQ regardless of caps */
157 if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS))
158 return 0;
159 if (enq_flags & SCX_ENQ_IMMED)
160 return SCX_CAP_ENQ_IMMED;
161 return SCX_CAP_ENQ;
162 }
163
164 /* map queued @p to the SCX_CAP_* bit required to stay on its local DSQ */
scx_caps_for_task(struct task_struct * p)165 static inline u64 scx_caps_for_task(struct task_struct *p)
166 {
167 if (p->scx.flags & SCX_TASK_IMMED)
168 return SCX_CAP_ENQ_IMMED;
169 return SCX_CAP_ENQ;
170 }
171
172 /* the cap @sch needs to preempt @rq's current task, 0 if none */
scx_caps_for_preempt(struct scx_sched * sch,struct rq * rq,u64 enq_flags)173 static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags)
174 {
175 struct task_struct *curr = rq->curr;
176
177 /* a kernel-forced placement preempts regardless of caps */
178 if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS))
179 return 0;
180 /* a non-ext task can't be preempted by ext, own-subtree needs no cap */
181 if (curr->sched_class != &ext_sched_class ||
182 scx_is_descendant(scx_task_sched(curr), sch))
183 return 0;
184 return SCX_CAP_PREEMPT;
185 }
186
187 /* caps implied by holding @cap */
scx_caps_implied(u64 cap)188 static inline u64 scx_caps_implied(u64 cap)
189 {
190 switch (cap) {
191 case SCX_CAP_PREEMPT:
192 return SCX_CAP_ENQ | SCX_CAP_ENQ_IMMED;
193 case SCX_CAP_ENQ:
194 return SCX_CAP_ENQ_IMMED;
195 }
196 return 0;
197 }
198
199 /* may @p keep running on @rq's cpu? requires baseline cpu access */
scx_task_can_stay_on_cpu(struct rq * rq,struct task_struct * p)200 static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p)
201 {
202 if (!scx_has_subs())
203 return true;
204
205 /* a migration-disabled task is let in without caps, keep it likewise */
206 if (unlikely(is_migration_disabled(p)))
207 return true;
208
209 return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE));
210 }
211
212 /* the task admitted for rescue on @rq, NULL if none */
scx_rescuee(struct rq * rq)213 static inline struct task_struct *scx_rescuee(struct rq *rq)
214 {
215 lockdep_assert_rq_held(rq);
216
217 if (!scx_has_subs())
218 return NULL;
219
220 return rq->scx.rescue.curr;
221 }
222
223 #else /* CONFIG_EXT_SUB_SCHED */
224
scx_missing_caps(struct scx_sched * sch,s32 cpu,u64 needed)225 static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; }
scx_caps_for_preempt(struct scx_sched * sch,struct rq * rq,u64 enq_flags)226 static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; }
scx_task_can_stay_on_cpu(struct rq * rq,struct task_struct * p)227 static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; }
scx_rescuee(struct rq * rq)228 static inline struct task_struct *scx_rescuee(struct rq *rq) { return NULL; }
229
230 #endif /* CONFIG_EXT_SUB_SCHED */
231
232 #endif /* _KERNEL_SCHED_EXT_SUB_H */
233