1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
4 * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
5 * Copyright (c) 2024 David Vernet <dvernet@meta.com>
6 */
7 #ifndef __SCX_COMPAT_BPF_H
8 #define __SCX_COMPAT_BPF_H
9
10 #define __COMPAT_ENUM_OR_ZERO(__type, __ent) \
11 ({ \
12 __type __ret = 0; \
13 if (bpf_core_enum_value_exists(__type, __ent)) \
14 __ret = __ent; \
15 __ret; \
16 })
17
18 /* v6.12: 819513666966 ("sched_ext: Add cgroup support") */
19 struct cgroup *scx_bpf_task_cgroup___new(struct task_struct *p) __ksym __weak;
20
21 #define scx_bpf_task_cgroup(p) \
22 (bpf_ksym_exists(scx_bpf_task_cgroup___new) ? \
23 scx_bpf_task_cgroup___new((p)) : NULL)
24
25 /*
26 * v6.13: The verb `dispatch` was too overloaded and confusing. kfuncs are
27 * renamed to unload the verb.
28 *
29 * scx_bpf_dispatch_from_dsq() and friends were added during v6.12 by
30 * 4c30f5ce4f7a ("sched_ext: Implement scx_bpf_dispatch[_vtime]_from_dsq()").
31 *
32 * v7.1: scx_bpf_dsq_move_to_local___v2() to add @enq_flags.
33 */
34 bool scx_bpf_dsq_move_to_local___v2___compat(u64 dsq_id, u64 enq_flags) __ksym __weak;
35 bool scx_bpf_dsq_move_to_local___v1(u64 dsq_id) __ksym __weak;
36 void scx_bpf_dsq_move_set_slice___new(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak;
37 void scx_bpf_dsq_move_set_vtime___new(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak;
38 bool scx_bpf_dsq_move___new(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
39 bool scx_bpf_dsq_move_vtime___new(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
40
41 bool scx_bpf_consume___old(u64 dsq_id) __ksym __weak;
42 void scx_bpf_dispatch_from_dsq_set_slice___old(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak;
43 void scx_bpf_dispatch_from_dsq_set_vtime___old(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak;
44 bool scx_bpf_dispatch_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
45 bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
46
47 #define scx_bpf_dsq_move_to_local(dsq_id, enq_flags) \
48 (bpf_ksym_exists(scx_bpf_dsq_move_to_local___v2___compat) ? \
49 scx_bpf_dsq_move_to_local___v2___compat((dsq_id), (enq_flags)) : \
50 (bpf_ksym_exists(scx_bpf_dsq_move_to_local___v1) ? \
51 scx_bpf_dsq_move_to_local___v1((dsq_id)) : \
52 scx_bpf_consume___old((dsq_id))))
53
54 #define scx_bpf_dsq_move_set_slice(it__iter, slice) \
55 (bpf_ksym_exists(scx_bpf_dsq_move_set_slice___new) ? \
56 scx_bpf_dsq_move_set_slice___new((it__iter), (slice)) : \
57 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_slice___old) ? \
58 scx_bpf_dispatch_from_dsq_set_slice___old((it__iter), (slice)) : \
59 (void)0))
60
61 #define scx_bpf_dsq_move_set_vtime(it__iter, vtime) \
62 (bpf_ksym_exists(scx_bpf_dsq_move_set_vtime___new) ? \
63 scx_bpf_dsq_move_set_vtime___new((it__iter), (vtime)) : \
64 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_vtime___old) ? \
65 scx_bpf_dispatch_from_dsq_set_vtime___old((it__iter), (vtime)) : \
66 (void)0))
67
68 #define scx_bpf_dsq_move(it__iter, p, dsq_id, enq_flags) \
69 (bpf_ksym_exists(scx_bpf_dsq_move___new) ? \
70 scx_bpf_dsq_move___new((it__iter), (p), (dsq_id), (enq_flags)) : \
71 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq___old) ? \
72 scx_bpf_dispatch_from_dsq___old((it__iter), (p), (dsq_id), (enq_flags)) : \
73 false))
74
75 #define scx_bpf_dsq_move_vtime(it__iter, p, dsq_id, enq_flags) \
76 (bpf_ksym_exists(scx_bpf_dsq_move_vtime___new) ? \
77 scx_bpf_dsq_move_vtime___new((it__iter), (p), (dsq_id), (enq_flags)) : \
78 (bpf_ksym_exists(scx_bpf_dispatch_vtime_from_dsq___old) ? \
79 scx_bpf_dispatch_vtime_from_dsq___old((it__iter), (p), (dsq_id), (enq_flags)) : \
80 false))
81
82 /*
83 * v6.15: 950ad93df2fc ("bpf: add kfunc for populating cpumask bits")
84 *
85 * Compat macro will be dropped on v6.19 release.
86 */
87 int bpf_cpumask_populate(struct bpf_cpumask *dst, void *src, size_t src__sz) __ksym __weak;
88
89 #define __COMPAT_bpf_cpumask_populate(cpumask, src, size__sz) \
90 (bpf_ksym_exists(bpf_cpumask_populate) ? \
91 (bpf_cpumask_populate(cpumask, src, size__sz)) : -EOPNOTSUPP)
92
93 /*
94 * v6.19: Introduce lockless peek API for user DSQs.
95 *
96 * Preserve the following macro until v6.21.
97 */
__COMPAT_scx_bpf_dsq_peek(u64 dsq_id)98 static inline struct task_struct *__COMPAT_scx_bpf_dsq_peek(u64 dsq_id)
99 {
100 struct task_struct *p = NULL;
101 struct bpf_iter_scx_dsq it;
102
103 if (bpf_ksym_exists(scx_bpf_dsq_peek))
104 return scx_bpf_dsq_peek(dsq_id);
105 if (!bpf_iter_scx_dsq_new(&it, dsq_id, 0))
106 p = bpf_iter_scx_dsq_next(&it);
107 bpf_iter_scx_dsq_destroy(&it);
108 return p;
109 }
110
111 /*
112 * v7.1: scx_bpf_sub_dispatch() for sub-sched dispatch. Preserve until
113 * we drop the compat layer for older kernels that lack the kfunc.
114 */
115 bool scx_bpf_sub_dispatch___compat(u64 cgroup_id) __ksym __weak;
116
scx_bpf_sub_dispatch(u64 cgroup_id)117 static inline bool scx_bpf_sub_dispatch(u64 cgroup_id)
118 {
119 if (bpf_ksym_exists(scx_bpf_sub_dispatch___compat))
120 return scx_bpf_sub_dispatch___compat(cgroup_id);
121 return false;
122 }
123
124 /*
125 * v7.3: scx_bpf_cid_override() for explicit cid and shard mapping. Ignore if
126 * missing.
127 */
128 void scx_bpf_cid_override___compat(const s32 __arena *cpu_to_cid__arena,
129 u32 cpu_to_cid_cnt,
130 const s32 __arena *shard_start__arena,
131 u32 shard_start_cnt) __ksym __weak;
132
scx_bpf_cid_override(const s32 __arena * cpu_to_cid,u32 cpu_to_cid_cnt,const s32 __arena * shard_start,u32 shard_start_cnt)133 static inline void scx_bpf_cid_override(const s32 __arena *cpu_to_cid, u32 cpu_to_cid_cnt,
134 const s32 __arena *shard_start, u32 shard_start_cnt)
135 {
136 if (bpf_ksym_exists(scx_bpf_cid_override___compat))
137 scx_bpf_cid_override___compat(cpu_to_cid, cpu_to_cid_cnt,
138 shard_start, shard_start_cnt);
139 }
140
141 /**
142 * __COMPAT_is_enq_cpu_selected - Test if SCX_ENQ_CPU_SELECTED is on
143 * in a compatible way. We will preserve this __COMPAT helper until v6.16.
144 *
145 * @enq_flags: enqueue flags from ops.enqueue()
146 *
147 * Return: True if SCX_ENQ_CPU_SELECTED is turned on in @enq_flags
148 */
__COMPAT_is_enq_cpu_selected(u64 enq_flags)149 static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags)
150 {
151 #ifdef HAVE_SCX_ENQ_CPU_SELECTED
152 /*
153 * This is the case that a BPF code compiled against vmlinux.h
154 * where the enum SCX_ENQ_CPU_SELECTED exists.
155 */
156
157 /*
158 * We should temporarily suspend the macro expansion of
159 * 'SCX_ENQ_CPU_SELECTED'. This avoids 'SCX_ENQ_CPU_SELECTED' being
160 * rewritten to '__SCX_ENQ_CPU_SELECTED' when 'SCX_ENQ_CPU_SELECTED'
161 * is defined in 'scripts/gen_enums.py'.
162 */
163 #pragma push_macro("SCX_ENQ_CPU_SELECTED")
164 #undef SCX_ENQ_CPU_SELECTED
165 u64 flag;
166
167 /*
168 * When the kernel did not have SCX_ENQ_CPU_SELECTED,
169 * select_task_rq_scx() has never been skipped. Thus, this case
170 * should be considered that the CPU has already been selected.
171 */
172 if (!bpf_core_enum_value_exists(enum scx_enq_flags,
173 SCX_ENQ_CPU_SELECTED))
174 return true;
175
176 flag = bpf_core_enum_value(enum scx_enq_flags, SCX_ENQ_CPU_SELECTED);
177 return enq_flags & flag;
178
179 /*
180 * Once done, resume the macro expansion of 'SCX_ENQ_CPU_SELECTED'.
181 */
182 #pragma pop_macro("SCX_ENQ_CPU_SELECTED")
183 #else
184 /*
185 * This is the case that a BPF code compiled against vmlinux.h
186 * where the enum SCX_ENQ_CPU_SELECTED does NOT exist.
187 */
188 return true;
189 #endif /* HAVE_SCX_ENQ_CPU_SELECTED */
190 }
191
192
193 #define scx_bpf_now() \
194 (bpf_ksym_exists(scx_bpf_now) ? \
195 scx_bpf_now() : \
196 bpf_ktime_get_ns())
197
198 /*
199 * v6.15: Introduce event counters.
200 *
201 * Preserve the following macro until v6.17.
202 */
203 #define __COMPAT_scx_bpf_events(events, size) \
204 (bpf_ksym_exists(scx_bpf_events) ? \
205 scx_bpf_events(events, size) : ({}))
206
207 /*
208 * v6.15: Introduce NUMA-aware kfuncs to operate with per-node idle
209 * cpumasks.
210 *
211 * Preserve the following __COMPAT_scx_*_node macros until v6.17.
212 */
213 #define __COMPAT_scx_bpf_nr_node_ids() \
214 (bpf_ksym_exists(scx_bpf_nr_node_ids) ? \
215 scx_bpf_nr_node_ids() : 1U)
216
217 #define __COMPAT_scx_bpf_cpu_node(cpu) \
218 (bpf_ksym_exists(scx_bpf_cpu_node) ? \
219 scx_bpf_cpu_node(cpu) : 0)
220
221 #define __COMPAT_scx_bpf_get_idle_cpumask_node(node) \
222 (bpf_ksym_exists(scx_bpf_get_idle_cpumask_node) ? \
223 scx_bpf_get_idle_cpumask_node(node) : \
224 scx_bpf_get_idle_cpumask()) \
225
226 #define __COMPAT_scx_bpf_get_idle_smtmask_node(node) \
227 (bpf_ksym_exists(scx_bpf_get_idle_smtmask_node) ? \
228 scx_bpf_get_idle_smtmask_node(node) : \
229 scx_bpf_get_idle_smtmask())
230
231 #define __COMPAT_scx_bpf_pick_idle_cpu_node(cpus_allowed, node, flags) \
232 (bpf_ksym_exists(scx_bpf_pick_idle_cpu_node) ? \
233 scx_bpf_pick_idle_cpu_node(cpus_allowed, node, flags) : \
234 scx_bpf_pick_idle_cpu(cpus_allowed, flags))
235
236 #define __COMPAT_scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) \
237 (bpf_ksym_exists(scx_bpf_pick_any_cpu_node) ? \
238 scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \
239 scx_bpf_pick_any_cpu(cpus_allowed, flags))
240
241 /*
242 * v6.19: To work around BPF maximum parameter limit, the following kfuncs are
243 * replaced with variants that pack scalar arguments in a struct. Wrappers are
244 * provided to maintain source compatibility.
245 *
246 * v6.13: scx_bpf_dsq_insert_vtime() renaming is also handled here. See the
247 * block on dispatch renaming above for more details.
248 *
249 * The kernel will carry the compat variants until v6.23 to maintain binary
250 * compatibility. After v6.23 release, remove the compat handling and move the
251 * wrappers to common.bpf.h.
252 */
253 s32 scx_bpf_select_cpu_and___compat(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
254 const struct cpumask *cpus_allowed, u64 flags) __ksym __weak;
255 void scx_bpf_dispatch_vtime___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak;
256 void scx_bpf_dsq_insert_vtime___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak;
257
258 /**
259 * scx_bpf_select_cpu_and - Pick an idle CPU usable by task @p
260 * @p: task_struct to select a CPU for
261 * @prev_cpu: CPU @p was on previously
262 * @wake_flags: %SCX_WAKE_* flags
263 * @cpus_allowed: cpumask of allowed CPUs
264 * @flags: %SCX_PICK_IDLE* flags
265 *
266 * Inline wrapper that packs scalar arguments into a struct and calls
267 * __scx_bpf_select_cpu_and(). See __scx_bpf_select_cpu_and() for details.
268 */
269 static inline s32
scx_bpf_select_cpu_and(struct task_struct * p,s32 prev_cpu,u64 wake_flags,const struct cpumask * cpus_allowed,u64 flags)270 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
271 const struct cpumask *cpus_allowed, u64 flags)
272 {
273 if (bpf_core_type_exists(struct scx_bpf_select_cpu_and_args)) {
274 struct scx_bpf_select_cpu_and_args args = {
275 .prev_cpu = prev_cpu,
276 .wake_flags = wake_flags,
277 .flags = flags,
278 };
279
280 return __scx_bpf_select_cpu_and(p, cpus_allowed, &args);
281 } else {
282 return scx_bpf_select_cpu_and___compat(p, prev_cpu, wake_flags,
283 cpus_allowed, flags);
284 }
285 }
286
287 /*
288 * scx_bpf_select_cpu_and() is now an inline wrapper. Use this instead of
289 * bpf_ksym_exists(scx_bpf_select_cpu_and) to test availability.
290 */
291 #define __COMPAT_HAS_scx_bpf_select_cpu_and \
292 (bpf_core_type_exists(struct scx_bpf_select_cpu_and_args) || \
293 bpf_ksym_exists(scx_bpf_select_cpu_and___compat))
294
295 /**
296 * scx_bpf_dsq_insert_vtime - Insert a task into the vtime priority queue of a DSQ
297 * @p: task_struct to insert
298 * @dsq_id: DSQ to insert into
299 * @slice: duration @p can run for in nsecs, 0 to keep the current value
300 * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ
301 * @enq_flags: SCX_ENQ_*
302 *
303 * Inline wrapper that packs scalar arguments into a struct and calls
304 * __scx_bpf_dsq_insert_vtime(). See __scx_bpf_dsq_insert_vtime() for details.
305 */
306 static inline bool
scx_bpf_dsq_insert_vtime(struct task_struct * p,u64 dsq_id,u64 slice,u64 vtime,u64 enq_flags)307 scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime,
308 u64 enq_flags)
309 {
310 if (bpf_core_type_exists(struct scx_bpf_dsq_insert_vtime_args)) {
311 struct scx_bpf_dsq_insert_vtime_args args = {
312 .dsq_id = dsq_id,
313 .slice = slice,
314 .vtime = vtime,
315 .enq_flags = enq_flags,
316 };
317
318 return __scx_bpf_dsq_insert_vtime(p, &args);
319 } else if (bpf_ksym_exists(scx_bpf_dsq_insert_vtime___compat)) {
320 scx_bpf_dsq_insert_vtime___compat(p, dsq_id, slice, vtime,
321 enq_flags);
322 return true;
323 } else {
324 scx_bpf_dispatch_vtime___compat(p, dsq_id, slice, vtime,
325 enq_flags);
326 return true;
327 }
328 }
329
330 /*
331 * v6.19: scx_bpf_dsq_insert() now returns bool instead of void. Move
332 * scx_bpf_dsq_insert() decl to common.bpf.h and drop compat helper after v6.22.
333 * The extra ___compat suffix is to work around libbpf not ignoring __SUFFIX on
334 * kernel side. The entire suffix can be dropped later.
335 *
336 * v6.13: scx_bpf_dsq_insert() renaming is also handled here. See the block on
337 * dispatch renaming above for more details.
338 */
339 bool scx_bpf_dsq_insert___v2___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;
340 void scx_bpf_dsq_insert___v1(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;
341 void scx_bpf_dispatch___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;
342
343 static inline bool
scx_bpf_dsq_insert(struct task_struct * p,u64 dsq_id,u64 slice,u64 enq_flags)344 scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags)
345 {
346 if (bpf_ksym_exists(scx_bpf_dsq_insert___v2___compat)) {
347 return scx_bpf_dsq_insert___v2___compat(p, dsq_id, slice, enq_flags);
348 } else if (bpf_ksym_exists(scx_bpf_dsq_insert___v1)) {
349 scx_bpf_dsq_insert___v1(p, dsq_id, slice, enq_flags);
350 return true;
351 } else {
352 scx_bpf_dispatch___compat(p, dsq_id, slice, enq_flags);
353 return true;
354 }
355 }
356
357 /*
358 * v6.19: scx_bpf_task_set_slice() and scx_bpf_task_set_dsq_vtime() added to for
359 * sub-sched authority checks. Drop the wrappers and move the decls to
360 * common.bpf.h after v6.22.
361 */
362 bool scx_bpf_task_set_slice___new(struct task_struct *p, u64 slice) __ksym __weak;
363 bool scx_bpf_task_set_dsq_vtime___new(struct task_struct *p, u64 vtime) __ksym __weak;
364
scx_bpf_task_set_slice(struct task_struct * p,u64 slice)365 static inline void scx_bpf_task_set_slice(struct task_struct *p, u64 slice)
366 {
367 if (bpf_ksym_exists(scx_bpf_task_set_slice___new))
368 scx_bpf_task_set_slice___new(p, slice);
369 else
370 p->scx.slice = slice;
371 }
372
scx_bpf_task_set_dsq_vtime(struct task_struct * p,u64 vtime)373 static inline void scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime)
374 {
375 if (bpf_ksym_exists(scx_bpf_task_set_dsq_vtime___new))
376 scx_bpf_task_set_dsq_vtime___new(p, vtime);
377 else
378 p->scx.dsq_vtime = vtime;
379 }
380
381 /*
382 * v6.19: The new void variant can be called from anywhere while the older v1
383 * variant can only be called from ops.cpu_release(). The double ___ prefixes on
384 * the v2 variant need to be removed once libbpf is updated to ignore ___ prefix
385 * on kernel side. Drop the wrapper and move the decl to common.bpf.h after
386 * v6.22.
387 */
388 u32 scx_bpf_reenqueue_local___v1(void) __ksym __weak;
389 void scx_bpf_reenqueue_local___v2___compat(void) __ksym __weak;
390
__COMPAT_scx_bpf_reenqueue_local_from_anywhere(void)391 static inline bool __COMPAT_scx_bpf_reenqueue_local_from_anywhere(void)
392 {
393 return bpf_ksym_exists(scx_bpf_reenqueue_local___v2___compat);
394 }
395
scx_bpf_reenqueue_local(void)396 static inline void scx_bpf_reenqueue_local(void)
397 {
398 if (__COMPAT_scx_bpf_reenqueue_local_from_anywhere())
399 scx_bpf_reenqueue_local___v2___compat();
400 else
401 scx_bpf_reenqueue_local___v1();
402 }
403
404 /*
405 * v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This
406 * will eventually deprecate scx_bpf_reenqueue_local().
407 */
408 void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags) __ksym __weak;
409
__COMPAT_has_generic_reenq(void)410 static inline bool __COMPAT_has_generic_reenq(void)
411 {
412 return bpf_ksym_exists(scx_bpf_dsq_reenq___compat);
413 }
414
scx_bpf_dsq_reenq(u64 dsq_id,u64 reenq_flags)415 static inline void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags)
416 {
417 if (bpf_ksym_exists(scx_bpf_dsq_reenq___compat))
418 scx_bpf_dsq_reenq___compat(dsq_id, reenq_flags);
419 else if (dsq_id == SCX_DSQ_LOCAL && reenq_flags == 0)
420 scx_bpf_reenqueue_local();
421 else
422 scx_bpf_error("kernel too old to reenqueue foreign local or user DSQs");
423 }
424
425 /*
426 * Define sched_ext_ops. See compat.h::SCX_OPS_OPEN() for how backward
427 * compatibility is handled (this macro can be expanded to emit multiple
428 * variants for incompatible op changes; SCX_OPS_OPEN() handles purely
429 * additive changes at load time).
430 */
431 #define SCX_OPS_DEFINE(__name, ...) \
432 SEC(".struct_ops.link") \
433 struct sched_ext_ops __name = { \
434 __VA_ARGS__, \
435 };
436
437 /*
438 * Define a cid-form sched_ext_ops. Programs targeting this struct_ops type
439 * use cid-form callback signatures (select_cid, set_cmask, cid_online/offline,
440 * dispatch with cid arg, etc.) and may only call the cid-form scx_bpf_*
441 * kfuncs (kick_cid, task_cid, this_cid, ...).
442 */
443 #define SCX_OPS_CID_DEFINE(__name, ...) \
444 SEC(".struct_ops.link") \
445 struct sched_ext_ops_cid __name = { \
446 __VA_ARGS__, \
447 };
448
449 #endif /* __SCX_COMPAT_BPF_H */
450