1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
4 * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
5 * Copyright (c) 2024 David Vernet <dvernet@meta.com>
6 */
7 #ifndef __SCX_COMPAT_BPF_H
8 #define __SCX_COMPAT_BPF_H
9
10 #define __COMPAT_ENUM_OR_ZERO(__type, __ent) \
11 ({ \
12 __type __ret = 0; \
13 if (bpf_core_enum_value_exists(__type, __ent)) \
14 __ret = __ent; \
15 __ret; \
16 })
17
18 /* v6.12: 819513666966 ("sched_ext: Add cgroup support") */
19 struct cgroup *scx_bpf_task_cgroup___new(struct task_struct *p) __ksym __weak;
20
21 #define scx_bpf_task_cgroup(p) \
22 (bpf_ksym_exists(scx_bpf_task_cgroup___new) ? \
23 scx_bpf_task_cgroup___new((p)) : NULL)
24
25 /*
26 * v6.13: The verb `dispatch` was too overloaded and confusing. kfuncs are
27 * renamed to unload the verb.
28 *
29 * scx_bpf_dispatch_from_dsq() and friends were added during v6.12 by
30 * 4c30f5ce4f7a ("sched_ext: Implement scx_bpf_dispatch[_vtime]_from_dsq()").
31 */
32 bool scx_bpf_dsq_move_to_local___new(u64 dsq_id) __ksym __weak;
33 void scx_bpf_dsq_move_set_slice___new(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak;
34 void scx_bpf_dsq_move_set_vtime___new(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak;
35 bool scx_bpf_dsq_move___new(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
36 bool scx_bpf_dsq_move_vtime___new(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
37
38 bool scx_bpf_consume___old(u64 dsq_id) __ksym __weak;
39 void scx_bpf_dispatch_from_dsq_set_slice___old(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak;
40 void scx_bpf_dispatch_from_dsq_set_vtime___old(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak;
41 bool scx_bpf_dispatch_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
42 bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
43
44 #define scx_bpf_dsq_move_to_local(dsq_id) \
45 (bpf_ksym_exists(scx_bpf_dsq_move_to_local___new) ? \
46 scx_bpf_dsq_move_to_local___new((dsq_id)) : \
47 scx_bpf_consume___old((dsq_id)))
48
49 #define scx_bpf_dsq_move_set_slice(it__iter, slice) \
50 (bpf_ksym_exists(scx_bpf_dsq_move_set_slice___new) ? \
51 scx_bpf_dsq_move_set_slice___new((it__iter), (slice)) : \
52 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_slice___old) ? \
53 scx_bpf_dispatch_from_dsq_set_slice___old((it__iter), (slice)) : \
54 (void)0))
55
56 #define scx_bpf_dsq_move_set_vtime(it__iter, vtime) \
57 (bpf_ksym_exists(scx_bpf_dsq_move_set_vtime___new) ? \
58 scx_bpf_dsq_move_set_vtime___new((it__iter), (vtime)) : \
59 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_vtime___old) ? \
60 scx_bpf_dispatch_from_dsq_set_vtime___old((it__iter), (vtime)) : \
61 (void)0))
62
63 #define scx_bpf_dsq_move(it__iter, p, dsq_id, enq_flags) \
64 (bpf_ksym_exists(scx_bpf_dsq_move___new) ? \
65 scx_bpf_dsq_move___new((it__iter), (p), (dsq_id), (enq_flags)) : \
66 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq___old) ? \
67 scx_bpf_dispatch_from_dsq___old((it__iter), (p), (dsq_id), (enq_flags)) : \
68 false))
69
70 #define scx_bpf_dsq_move_vtime(it__iter, p, dsq_id, enq_flags) \
71 (bpf_ksym_exists(scx_bpf_dsq_move_vtime___new) ? \
72 scx_bpf_dsq_move_vtime___new((it__iter), (p), (dsq_id), (enq_flags)) : \
73 (bpf_ksym_exists(scx_bpf_dispatch_vtime_from_dsq___old) ? \
74 scx_bpf_dispatch_vtime_from_dsq___old((it__iter), (p), (dsq_id), (enq_flags)) : \
75 false))
76
77 /*
78 * v6.15: 950ad93df2fc ("bpf: add kfunc for populating cpumask bits")
79 *
80 * Compat macro will be dropped on v6.19 release.
81 */
82 int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym __weak;
83
84 #define __COMPAT_bpf_cpumask_populate(cpumask, src, size__sz) \
85 (bpf_ksym_exists(bpf_cpumask_populate) ? \
86 (bpf_cpumask_populate(cpumask, src, size__sz)) : -EOPNOTSUPP)
87
88 /*
89 * v6.19: Introduce lockless peek API for user DSQs.
90 *
91 * Preserve the following macro until v6.21.
92 */
__COMPAT_scx_bpf_dsq_peek(u64 dsq_id)93 static inline struct task_struct *__COMPAT_scx_bpf_dsq_peek(u64 dsq_id)
94 {
95 struct task_struct *p = NULL;
96 struct bpf_iter_scx_dsq it;
97
98 if (bpf_ksym_exists(scx_bpf_dsq_peek))
99 return scx_bpf_dsq_peek(dsq_id);
100 if (!bpf_iter_scx_dsq_new(&it, dsq_id, 0))
101 p = bpf_iter_scx_dsq_next(&it);
102 bpf_iter_scx_dsq_destroy(&it);
103 return p;
104 }
105
106 /**
107 * __COMPAT_is_enq_cpu_selected - Test if SCX_ENQ_CPU_SELECTED is on
108 * in a compatible way. We will preserve this __COMPAT helper until v6.16.
109 *
110 * @enq_flags: enqueue flags from ops.enqueue()
111 *
112 * Return: True if SCX_ENQ_CPU_SELECTED is turned on in @enq_flags
113 */
__COMPAT_is_enq_cpu_selected(u64 enq_flags)114 static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags)
115 {
116 #ifdef HAVE_SCX_ENQ_CPU_SELECTED
117 /*
118 * This is the case that a BPF code compiled against vmlinux.h
119 * where the enum SCX_ENQ_CPU_SELECTED exists.
120 */
121
122 /*
123 * We should temporarily suspend the macro expansion of
124 * 'SCX_ENQ_CPU_SELECTED'. This avoids 'SCX_ENQ_CPU_SELECTED' being
125 * rewritten to '__SCX_ENQ_CPU_SELECTED' when 'SCX_ENQ_CPU_SELECTED'
126 * is defined in 'scripts/gen_enums.py'.
127 */
128 #pragma push_macro("SCX_ENQ_CPU_SELECTED")
129 #undef SCX_ENQ_CPU_SELECTED
130 u64 flag;
131
132 /*
133 * When the kernel did not have SCX_ENQ_CPU_SELECTED,
134 * select_task_rq_scx() has never been skipped. Thus, this case
135 * should be considered that the CPU has already been selected.
136 */
137 if (!bpf_core_enum_value_exists(enum scx_enq_flags,
138 SCX_ENQ_CPU_SELECTED))
139 return true;
140
141 flag = bpf_core_enum_value(enum scx_enq_flags, SCX_ENQ_CPU_SELECTED);
142 return enq_flags & flag;
143
144 /*
145 * Once done, resume the macro expansion of 'SCX_ENQ_CPU_SELECTED'.
146 */
147 #pragma pop_macro("SCX_ENQ_CPU_SELECTED")
148 #else
149 /*
150 * This is the case that a BPF code compiled against vmlinux.h
151 * where the enum SCX_ENQ_CPU_SELECTED does NOT exist.
152 */
153 return true;
154 #endif /* HAVE_SCX_ENQ_CPU_SELECTED */
155 }
156
157
158 #define scx_bpf_now() \
159 (bpf_ksym_exists(scx_bpf_now) ? \
160 scx_bpf_now() : \
161 bpf_ktime_get_ns())
162
163 /*
164 * v6.15: Introduce event counters.
165 *
166 * Preserve the following macro until v6.17.
167 */
168 #define __COMPAT_scx_bpf_events(events, size) \
169 (bpf_ksym_exists(scx_bpf_events) ? \
170 scx_bpf_events(events, size) : ({}))
171
172 /*
173 * v6.15: Introduce NUMA-aware kfuncs to operate with per-node idle
174 * cpumasks.
175 *
176 * Preserve the following __COMPAT_scx_*_node macros until v6.17.
177 */
178 #define __COMPAT_scx_bpf_nr_node_ids() \
179 (bpf_ksym_exists(scx_bpf_nr_node_ids) ? \
180 scx_bpf_nr_node_ids() : 1U)
181
182 #define __COMPAT_scx_bpf_cpu_node(cpu) \
183 (bpf_ksym_exists(scx_bpf_cpu_node) ? \
184 scx_bpf_cpu_node(cpu) : 0)
185
186 #define __COMPAT_scx_bpf_get_idle_cpumask_node(node) \
187 (bpf_ksym_exists(scx_bpf_get_idle_cpumask_node) ? \
188 scx_bpf_get_idle_cpumask_node(node) : \
189 scx_bpf_get_idle_cpumask()) \
190
191 #define __COMPAT_scx_bpf_get_idle_smtmask_node(node) \
192 (bpf_ksym_exists(scx_bpf_get_idle_smtmask_node) ? \
193 scx_bpf_get_idle_smtmask_node(node) : \
194 scx_bpf_get_idle_smtmask())
195
196 #define __COMPAT_scx_bpf_pick_idle_cpu_node(cpus_allowed, node, flags) \
197 (bpf_ksym_exists(scx_bpf_pick_idle_cpu_node) ? \
198 scx_bpf_pick_idle_cpu_node(cpus_allowed, node, flags) : \
199 scx_bpf_pick_idle_cpu(cpus_allowed, flags))
200
201 #define __COMPAT_scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) \
202 (bpf_ksym_exists(scx_bpf_pick_any_cpu_node) ? \
203 scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \
204 scx_bpf_pick_any_cpu(cpus_allowed, flags))
205
206 /*
207 * v6.18: Add a helper to retrieve the current task running on a CPU.
208 *
209 * Keep this helper available until v6.20 for compatibility.
210 */
__COMPAT_scx_bpf_cpu_curr(int cpu)211 static inline struct task_struct *__COMPAT_scx_bpf_cpu_curr(int cpu)
212 {
213 struct rq *rq;
214
215 if (bpf_ksym_exists(scx_bpf_cpu_curr))
216 return scx_bpf_cpu_curr(cpu);
217
218 rq = scx_bpf_cpu_rq(cpu);
219
220 return rq ? rq->curr : NULL;
221 }
222
223 /*
224 * v6.19: To work around BPF maximum parameter limit, the following kfuncs are
225 * replaced with variants that pack scalar arguments in a struct. Wrappers are
226 * provided to maintain source compatibility.
227 *
228 * v6.13: scx_bpf_dsq_insert_vtime() renaming is also handled here. See the
229 * block on dispatch renaming above for more details.
230 *
231 * The kernel will carry the compat variants until v6.23 to maintain binary
232 * compatibility. After v6.23 release, remove the compat handling and move the
233 * wrappers to common.bpf.h.
234 */
235 s32 scx_bpf_select_cpu_and___compat(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
236 const struct cpumask *cpus_allowed, u64 flags) __ksym __weak;
237 void scx_bpf_dispatch_vtime___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak;
238 void scx_bpf_dsq_insert_vtime___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak;
239
240 /**
241 * scx_bpf_select_cpu_and - Pick an idle CPU usable by task @p
242 * @p: task_struct to select a CPU for
243 * @prev_cpu: CPU @p was on previously
244 * @wake_flags: %SCX_WAKE_* flags
245 * @cpus_allowed: cpumask of allowed CPUs
246 * @flags: %SCX_PICK_IDLE* flags
247 *
248 * Inline wrapper that packs scalar arguments into a struct and calls
249 * __scx_bpf_select_cpu_and(). See __scx_bpf_select_cpu_and() for details.
250 */
251 static inline s32
scx_bpf_select_cpu_and(struct task_struct * p,s32 prev_cpu,u64 wake_flags,const struct cpumask * cpus_allowed,u64 flags)252 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
253 const struct cpumask *cpus_allowed, u64 flags)
254 {
255 if (bpf_core_type_exists(struct scx_bpf_select_cpu_and_args)) {
256 struct scx_bpf_select_cpu_and_args args = {
257 .prev_cpu = prev_cpu,
258 .wake_flags = wake_flags,
259 .flags = flags,
260 };
261
262 return __scx_bpf_select_cpu_and(p, cpus_allowed, &args);
263 } else {
264 return scx_bpf_select_cpu_and___compat(p, prev_cpu, wake_flags,
265 cpus_allowed, flags);
266 }
267 }
268
269 /**
270 * scx_bpf_dsq_insert_vtime - Insert a task into the vtime priority queue of a DSQ
271 * @p: task_struct to insert
272 * @dsq_id: DSQ to insert into
273 * @slice: duration @p can run for in nsecs, 0 to keep the current value
274 * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ
275 * @enq_flags: SCX_ENQ_*
276 *
277 * Inline wrapper that packs scalar arguments into a struct and calls
278 * __scx_bpf_dsq_insert_vtime(). See __scx_bpf_dsq_insert_vtime() for details.
279 */
280 static inline bool
scx_bpf_dsq_insert_vtime(struct task_struct * p,u64 dsq_id,u64 slice,u64 vtime,u64 enq_flags)281 scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime,
282 u64 enq_flags)
283 {
284 if (bpf_core_type_exists(struct scx_bpf_dsq_insert_vtime_args)) {
285 struct scx_bpf_dsq_insert_vtime_args args = {
286 .dsq_id = dsq_id,
287 .slice = slice,
288 .vtime = vtime,
289 .enq_flags = enq_flags,
290 };
291
292 return __scx_bpf_dsq_insert_vtime(p, &args);
293 } else if (bpf_ksym_exists(scx_bpf_dsq_insert_vtime___compat)) {
294 scx_bpf_dsq_insert_vtime___compat(p, dsq_id, slice, vtime,
295 enq_flags);
296 return true;
297 } else {
298 scx_bpf_dispatch_vtime___compat(p, dsq_id, slice, vtime,
299 enq_flags);
300 return true;
301 }
302 }
303
304 /*
305 * v6.19: scx_bpf_dsq_insert() now returns bool instead of void. Move
306 * scx_bpf_dsq_insert() decl to common.bpf.h and drop compat helper after v6.22.
307 * The extra ___compat suffix is to work around libbpf not ignoring __SUFFIX on
308 * kernel side. The entire suffix can be dropped later.
309 *
310 * v6.13: scx_bpf_dsq_insert() renaming is also handled here. See the block on
311 * dispatch renaming above for more details.
312 */
313 bool scx_bpf_dsq_insert___v2___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;
314 void scx_bpf_dsq_insert___v1(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;
315 void scx_bpf_dispatch___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;
316
317 static inline bool
scx_bpf_dsq_insert(struct task_struct * p,u64 dsq_id,u64 slice,u64 enq_flags)318 scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags)
319 {
320 if (bpf_ksym_exists(scx_bpf_dsq_insert___v2___compat)) {
321 return scx_bpf_dsq_insert___v2___compat(p, dsq_id, slice, enq_flags);
322 } else if (bpf_ksym_exists(scx_bpf_dsq_insert___v1)) {
323 scx_bpf_dsq_insert___v1(p, dsq_id, slice, enq_flags);
324 return true;
325 } else {
326 scx_bpf_dispatch___compat(p, dsq_id, slice, enq_flags);
327 return true;
328 }
329 }
330
331 /*
332 * v6.19: scx_bpf_task_set_slice() and scx_bpf_task_set_dsq_vtime() added to for
333 * sub-sched authority checks. Drop the wrappers and move the decls to
334 * common.bpf.h after v6.22.
335 */
336 bool scx_bpf_task_set_slice___new(struct task_struct *p, u64 slice) __ksym __weak;
337 bool scx_bpf_task_set_dsq_vtime___new(struct task_struct *p, u64 vtime) __ksym __weak;
338
scx_bpf_task_set_slice(struct task_struct * p,u64 slice)339 static inline void scx_bpf_task_set_slice(struct task_struct *p, u64 slice)
340 {
341 if (bpf_ksym_exists(scx_bpf_task_set_slice___new))
342 scx_bpf_task_set_slice___new(p, slice);
343 else
344 p->scx.slice = slice;
345 }
346
scx_bpf_task_set_dsq_vtime(struct task_struct * p,u64 vtime)347 static inline void scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime)
348 {
349 if (bpf_ksym_exists(scx_bpf_task_set_dsq_vtime___new))
350 scx_bpf_task_set_dsq_vtime___new(p, vtime);
351 else
352 p->scx.dsq_vtime = vtime;
353 }
354
355 /*
356 * v6.19: The new void variant can be called from anywhere while the older v1
357 * variant can only be called from ops.cpu_release(). The double ___ prefixes on
358 * the v2 variant need to be removed once libbpf is updated to ignore ___ prefix
359 * on kernel side. Drop the wrapper and move the decl to common.bpf.h after
360 * v6.22.
361 */
362 u32 scx_bpf_reenqueue_local___v1(void) __ksym __weak;
363 void scx_bpf_reenqueue_local___v2___compat(void) __ksym __weak;
364
__COMPAT_scx_bpf_reenqueue_local_from_anywhere(void)365 static inline bool __COMPAT_scx_bpf_reenqueue_local_from_anywhere(void)
366 {
367 return bpf_ksym_exists(scx_bpf_reenqueue_local___v2___compat);
368 }
369
scx_bpf_reenqueue_local(void)370 static inline void scx_bpf_reenqueue_local(void)
371 {
372 if (__COMPAT_scx_bpf_reenqueue_local_from_anywhere())
373 scx_bpf_reenqueue_local___v2___compat();
374 else
375 scx_bpf_reenqueue_local___v1();
376 }
377
378 /*
379 * Define sched_ext_ops. This may be expanded to define multiple variants for
380 * backward compatibility. See compat.h::SCX_OPS_LOAD/ATTACH().
381 */
382 #define SCX_OPS_DEFINE(__name, ...) \
383 SEC(".struct_ops.link") \
384 struct sched_ext_ops __name = { \
385 __VA_ARGS__, \
386 };
387
388 #endif /* __SCX_COMPAT_BPF_H */
389