1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
4 * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
5 * Copyright (c) 2024 David Vernet <dvernet@meta.com>
6 */
7 #ifndef __SCX_COMPAT_H
8 #define __SCX_COMPAT_H
9
10 #include <bpf/btf.h>
11 #include <bpf/libbpf.h>
12 #include <fcntl.h>
13 #include <stdint.h>
14 #include <stdio.h>
15 #include <stdlib.h>
16 #include <string.h>
17 #include <unistd.h>
18
19 #include "enums_abi.autogen.h"
20
21 struct btf *__COMPAT_vmlinux_btf __attribute__((weak));
22
__COMPAT_load_vmlinux_btf(void)23 static inline void __COMPAT_load_vmlinux_btf(void)
24 {
25 if (!__COMPAT_vmlinux_btf) {
26 __COMPAT_vmlinux_btf = btf__load_vmlinux_btf();
27 SCX_BUG_ON(!__COMPAT_vmlinux_btf, "btf__load_vmlinux_btf()");
28 }
29 }
30
31 /*
32 * Recover the true value of a 64-bit enum enumerator whose kernel BTF entry
33 * was truncated to its low 32 bits.
34 *
35 * Kernels whose BTF was generated without BTF_KIND_ENUM64 support encode
36 * 64-bit enums as 8-byte BTF_KIND_ENUM entries whose enumerator values only
37 * carry the low 32 bits. This happens with pahole < 1.24, which predates
38 * ENUM64, and with pahole passing --skip_encoding_btf_enum64 (e.g. Google's
39 * Container-Optimized OS / GKE kernels deliberately pass it for backward
40 * compatibility with older BTF consumers). The high bits
41 * can't be recovered from kernel BTF, so substitute the value from the
42 * vmlinux.h this tree was built against, cross-checked against the low 32
43 * bits the kernel did provide.
44 *
45 * Note that this is a best-effort recovery, not a ground truth. The
46 * substitution assumes the running kernel agrees with this tree's vmlinux.h
47 * on the high 32 bits, but only the low 32 bits can actually be verified.
48 * The cross-check is vacuous for enumerators whose value has no low bits
49 * set (e.g. SCX_DSQ_FLAG_BUILTIN, __SCX_ENQ_INTERNAL_MASK,
50 * SCX_ENQ_CLEAR_OPSS, SCX_ECODE_*): their lo32 is 0 and matches anything,
51 * so those substitutions rest entirely on the high bits never moving. An
52 * enumerator missing from the table (a kernel newer than this tree's
53 * vmlinux.h, or a stale autogen table) can't be recovered at all. If a
54 * substitution is ever wrong, the scheduler operates on bogus values (e.g.
55 * dispatching to nonexistent DSQ ids or silently dropping flags) and can
56 * wildly malfunction, which is why the mismatch and table-miss paths refuse
57 * instead of guessing.
58 */
__COMPAT_recover_truncated_enum64(const char * type,const char * name,u32 lo32,u64 * v)59 static inline bool __COMPAT_recover_truncated_enum64(const char *type,
60 const char *name,
61 u32 lo32, u64 *v)
62 {
63 static bool warned;
64 size_t i;
65
66 for (i = 0; i < sizeof(__scx_enum_abi_vals) / sizeof(__scx_enum_abi_vals[0]); i++) {
67 const struct __scx_enum_abi_val *e = &__scx_enum_abi_vals[i];
68
69 if (strcmp(e->type, type) || strcmp(e->name, name))
70 continue;
71
72 if (e->val <= (u64)UINT32_MAX) {
73 *v = lo32;
74 return true;
75 }
76
77 if ((u32)e->val != lo32) {
78 fprintf(stderr, "ERROR: kernel BTF value of %s::%s (0x%x) doesn't match the low 32 bits of the vmlinux.h value (0x%llx); refusing to substitute\n",
79 type, name, lo32, (unsigned long long)e->val);
80 return false;
81 }
82
83 if (!warned) {
84 fprintf(stderr,
85 "WARNING: kernel BTF lacks BTF_KIND_ENUM64 encoding (generated by\n"
86 "WARNING: pahole < 1.24 or with --skip_encoding_btf_enum64), so 64-bit\n"
87 "WARNING: scx enum values are truncated to their low 32 bits in kernel\n"
88 "WARNING: BTF. Substituting the full 64-bit values from the vmlinux.h\n"
89 "WARNING: this binary was built against, cross-checked against the low\n"
90 "WARNING: 32 bits the kernel does provide. The high 32 bits cannot be\n"
91 "WARNING: verified: if the running kernel's actual values differ from\n"
92 "WARNING: the build-time vmlinux.h (e.g. an enum that moved in a newer\n"
93 "WARNING: kernel), the scheduler will operate on bogus values, such as\n"
94 "WARNING: dispatching to nonexistent DSQ ids, and can wildly malfunction.\n");
95 warned = true;
96 }
97 *v = e->val;
98 return true;
99 }
100
101 /*
102 * Unknown enumerator (likely a stale autogen table). Fail
103 * pessimistically to avoid returning an invalid value.
104 */
105 fprintf(stderr, "ERROR: kernel BTF truncates 64-bit enum %s::%s to 0x%x; 64-bit variant not found in vmlinux.h\n",
106 type, name, lo32);
107 return false;
108 }
109
__COMPAT_read_enum(const char * type,const char * name,u64 * v)110 static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v)
111 {
112 const struct btf_type *t;
113 const char *n;
114 s32 tid;
115 __u32 i;
116
117 __COMPAT_load_vmlinux_btf();
118
119 tid = btf__find_by_name(__COMPAT_vmlinux_btf, type);
120 if (tid < 0)
121 return false;
122
123 t = btf__type_by_id(__COMPAT_vmlinux_btf, tid);
124 SCX_BUG_ON(!t, "btf__type_by_id(%d)", tid);
125
126 if (btf_is_enum(t)) {
127 struct btf_enum *e = btf_enum(t);
128
129 for (i = 0; i < btf_vlen(t); i++) {
130 n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off);
131 SCX_BUG_ON(!n, "btf__name_by_offset()");
132 if (!strcmp(n, name)) {
133 /*
134 * Try to recover a 64-bit enum from an 8-byte
135 * BTF_KIND_ENUM that was encoded without ENUM64
136 * support (old pahole or
137 * --skip_encoding_btf_enum64). Only scx_*
138 * types are covered by the substitution table;
139 * non-scx types fall through to the raw value
140 * so this generic utility keeps working for
141 * them.
142 */
143 if (t->size == 8 && !strncmp(type, "scx_", 4))
144 return __COMPAT_recover_truncated_enum64(type, name,
145 (u32)e[i].val, v);
146 *v = e[i].val;
147 return true;
148 }
149 }
150 } else if (btf_is_enum64(t)) {
151 struct btf_enum64 *e = btf_enum64(t);
152
153 for (i = 0; i < btf_vlen(t); i++) {
154 n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off);
155 SCX_BUG_ON(!n, "btf__name_by_offset()");
156 if (!strcmp(n, name)) {
157 *v = btf_enum64_value(&e[i]);
158 return true;
159 }
160 }
161 }
162
163 return false;
164 }
165
166 #define __COMPAT_ENUM_OR_ZERO(__type, __ent) \
167 ({ \
168 u64 __val = 0; \
169 __COMPAT_read_enum(__type, __ent, &__val); \
170 __val; \
171 })
172
__COMPAT_has_ksym(const char * ksym)173 static inline bool __COMPAT_has_ksym(const char *ksym)
174 {
175 __COMPAT_load_vmlinux_btf();
176 return btf__find_by_name(__COMPAT_vmlinux_btf, ksym) >= 0;
177 }
178
__COMPAT_struct_has_field(const char * type,const char * field)179 static inline bool __COMPAT_struct_has_field(const char *type, const char *field)
180 {
181 const struct btf_type *t;
182 const struct btf_member *m;
183 const char *n;
184 s32 tid;
185 __u32 i;
186
187 __COMPAT_load_vmlinux_btf();
188 tid = btf__find_by_name_kind(__COMPAT_vmlinux_btf, type, BTF_KIND_STRUCT);
189 if (tid < 0)
190 return false;
191
192 t = btf__type_by_id(__COMPAT_vmlinux_btf, tid);
193 SCX_BUG_ON(!t, "btf__type_by_id(%d)", tid);
194
195 m = btf_members(t);
196
197 for (i = 0; i < btf_vlen(t); i++) {
198 n = btf__name_by_offset(__COMPAT_vmlinux_btf, m[i].name_off);
199 SCX_BUG_ON(!n, "btf__name_by_offset()");
200 if (!strcmp(n, field))
201 return true;
202 }
203
204 return false;
205 }
206
207 #define SCX_OPS_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_ops_flags", #name)
208
209 #define SCX_OPS_KEEP_BUILTIN_IDLE SCX_OPS_FLAG(SCX_OPS_KEEP_BUILTIN_IDLE)
210 #define SCX_OPS_ENQ_LAST SCX_OPS_FLAG(SCX_OPS_ENQ_LAST)
211 #define SCX_OPS_ENQ_EXITING SCX_OPS_FLAG(SCX_OPS_ENQ_EXITING)
212 #define SCX_OPS_SWITCH_PARTIAL SCX_OPS_FLAG(SCX_OPS_SWITCH_PARTIAL)
213 #define SCX_OPS_ENQ_MIGRATION_DISABLED SCX_OPS_FLAG(SCX_OPS_ENQ_MIGRATION_DISABLED)
214 #define SCX_OPS_ALLOW_QUEUED_WAKEUP SCX_OPS_FLAG(SCX_OPS_ALLOW_QUEUED_WAKEUP)
215 #define SCX_OPS_BUILTIN_IDLE_PER_NODE SCX_OPS_FLAG(SCX_OPS_BUILTIN_IDLE_PER_NODE)
216 #define SCX_OPS_ALWAYS_ENQ_IMMED SCX_OPS_FLAG(SCX_OPS_ALWAYS_ENQ_IMMED)
217
218 #define SCX_PICK_IDLE_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_pick_idle_cpu_flags", #name)
219
220 #define SCX_PICK_IDLE_CORE SCX_PICK_IDLE_FLAG(SCX_PICK_IDLE_CORE)
221 #define SCX_PICK_IDLE_IN_NODE SCX_PICK_IDLE_FLAG(SCX_PICK_IDLE_IN_NODE)
222
scx_hotplug_seq(void)223 static inline long scx_hotplug_seq(void)
224 {
225 int fd;
226 char buf[32];
227 char *endptr;
228 ssize_t len;
229 long val;
230
231 fd = open("/sys/kernel/sched_ext/hotplug_seq", O_RDONLY);
232 if (fd < 0)
233 return -ENOENT;
234
235 len = read(fd, buf, sizeof(buf) - 1);
236 SCX_BUG_ON(len <= 0, "read failed (%ld)", len);
237 buf[len] = 0;
238 close(fd);
239
240 errno = 0;
241 val = strtoul(buf, &endptr, 10);
242 SCX_BUG_ON(errno == ERANGE || endptr == buf ||
243 (*endptr != '\n' && *endptr != '\0'), "invalid num hotplug events: %ld", val);
244
245 return val;
246 }
247
248 /*
249 * Open the sched_ext_ops skeleton.
250 *
251 * struct sched_ext_ops can change over time. Two complementary mechanisms
252 * keep BPF schedulers built against newer headers running on older kernels:
253 *
254 * 1. Load-time fix-up (SCX_OPS_OPEN()). For each optional ops callback or field
255 * added to struct sched_ext_ops, an explicit stanza below probes the
256 * running kernel's BTF via __COMPAT_struct_has_field() and, if the field
257 * is missing, clears it in the in-memory struct_ops (with a warning to
258 * stderr) before load. Handles additive changes - a new stanza must be
259 * added here for each new optional field.
260 *
261 * 2. Multi-variant struct_ops via compat.bpf.h::SCX_OPS_DEFINE(). That
262 * macro can be expanded to emit several variants of struct sched_ext_ops,
263 * and SCX_OPS_LOAD()/ATTACH() can pick the right one based on what the
264 * kernel supports. Needed when an existing operation has to change
265 * incompatibly (e.g. a callback signature changes); the load-time
266 * fix-up above only handles purely additive changes.
267 *
268 * ec7e3b0463e1 ("implement-ops") in https://github.com/sched-ext/sched_ext is
269 * the current minimum required kernel version.
270 *
271 * COMPAT:
272 * - v6.17: ops.cgroup_set_bandwidth()
273 * - v6.19: ops.cgroup_set_idle()
274 * - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id
275 * - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us
276 */
277 #define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \
278 struct __scx_name *__oskel; \
279 \
280 SCX_BUG_ON(!__COMPAT_struct_has_field(__ops_struct, "dump"), \
281 __ops_struct ".dump() missing, kernel too old?"); \
282 \
283 __oskel = __scx_name##__open(); \
284 SCX_BUG_ON(!__oskel, "Could not open " #__scx_name); \
285 __oskel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \
286 SCX_ENUM_INIT(__oskel); \
287 __oskel; \
288 })
289
290 #define SCX_OPS_OPEN(__ops_name, __scx_name) ({ \
291 struct __scx_name *__skel; \
292 \
293 __skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops"); \
294 if (__skel->struct_ops.__ops_name->cgroup_set_bandwidth && \
295 !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_bandwidth")) { \
296 fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_bandwidth()\n"); \
297 __skel->struct_ops.__ops_name->cgroup_set_bandwidth = NULL; \
298 } \
299 if (__skel->struct_ops.__ops_name->cgroup_set_idle && \
300 !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_idle")) { \
301 fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_idle()\n"); \
302 __skel->struct_ops.__ops_name->cgroup_set_idle = NULL; \
303 } \
304 if (__skel->struct_ops.__ops_name->sub_attach && \
305 !__COMPAT_struct_has_field("sched_ext_ops", "sub_attach")) { \
306 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_attach()\n"); \
307 __skel->struct_ops.__ops_name->sub_attach = NULL; \
308 } \
309 if (__skel->struct_ops.__ops_name->sub_detach && \
310 !__COMPAT_struct_has_field("sched_ext_ops", "sub_detach")) { \
311 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_detach()\n"); \
312 __skel->struct_ops.__ops_name->sub_detach = NULL; \
313 } \
314 if (__skel->struct_ops.__ops_name->sub_cgroup_id > 0 && \
315 !__COMPAT_struct_has_field("sched_ext_ops", "sub_cgroup_id")) { \
316 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_cgroup_id\n"); \
317 __skel->struct_ops.__ops_name->sub_cgroup_id = 0; \
318 } \
319 if (__skel->struct_ops.__ops_name->rescue_bandwidth_ppt > 0 && \
320 !__COMPAT_struct_has_field("sched_ext_ops", "rescue_bandwidth_ppt")) { \
321 fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_bandwidth_ppt\n"); \
322 __skel->struct_ops.__ops_name->rescue_bandwidth_ppt = 0; \
323 } \
324 if (__skel->struct_ops.__ops_name->rescue_quantum_us > 0 && \
325 !__COMPAT_struct_has_field("sched_ext_ops", "rescue_quantum_us")) { \
326 fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_quantum_us\n"); \
327 __skel->struct_ops.__ops_name->rescue_quantum_us = 0; \
328 } \
329 __skel; \
330 })
331
332 /*
333 * Open a cid-form (struct sched_ext_ops_cid) skeleton. The cid form postdates
334 * every op the load-time fix-ups above handle, so none of them apply.
335 */
336 #define SCX_OPS_CID_OPEN(__ops_name, __scx_name) \
337 __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid")
338
339 /*
340 * Associate non-struct_ops BPF programs with the scheduler's struct_ops map so
341 * that scx_prog_sched() can determine which scheduler a BPF program belongs
342 * to. Requires libbpf >= 1.7.
343 */
344 #if LIBBPF_MAJOR_VERSION > 1 || \
345 (LIBBPF_MAJOR_VERSION == 1 && LIBBPF_MINOR_VERSION >= 7)
__scx_ops_assoc_prog(struct bpf_program * prog,struct bpf_map * map,const char * ops_name)346 static inline void __scx_ops_assoc_prog(struct bpf_program *prog,
347 struct bpf_map *map,
348 const char *ops_name)
349 {
350 s32 err = bpf_program__assoc_struct_ops(prog, map, NULL);
351 if (err)
352 fprintf(stderr,
353 "ERROR: Failed to associate %s with %s: %d\n",
354 bpf_program__name(prog), ops_name, err);
355 }
356 #else
__scx_ops_assoc_prog(struct bpf_program * prog,struct bpf_map * map,const char * ops_name)357 static inline void __scx_ops_assoc_prog(struct bpf_program *prog,
358 struct bpf_map *map,
359 const char *ops_name)
360 {
361 }
362 #endif
363
364 /* See SCX_OPS_OPEN() above for backward-compatibility handling. */
365 #define SCX_OPS_LOAD(__skel, __ops_name, __scx_name, __uei_name) ({ \
366 struct bpf_program *__prog; \
367 UEI_SET_SIZE(__skel, __ops_name, __uei_name); \
368 SCX_BUG_ON(__scx_name##__load((__skel)), "Failed to load skel"); \
369 bpf_object__for_each_program(__prog, (__skel)->obj) { \
370 if (bpf_program__type(__prog) == BPF_PROG_TYPE_STRUCT_OPS) \
371 continue; \
372 __scx_ops_assoc_prog(__prog, (__skel)->maps.__ops_name, \
373 #__ops_name); \
374 } \
375 })
376
377 /*
378 * New versions of bpftool now emit additional link placeholders for BPF maps,
379 * and set up BPF skeleton in such a way that libbpf will auto-attach BPF maps
380 * automatically, assuming libbpf is recent enough (v1.5+). Old libbpf will do
381 * nothing with those links and won't attempt to auto-attach maps.
382 *
383 * To maintain compatibility with older libbpf while avoiding trying to attach
384 * twice, disable the autoattach feature on newer libbpf.
385 */
386 #if LIBBPF_MAJOR_VERSION > 1 || \
387 (LIBBPF_MAJOR_VERSION == 1 && LIBBPF_MINOR_VERSION >= 5)
388 #define __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) \
389 bpf_map__set_autoattach((__skel)->maps.__ops_name, false)
390 #else
391 #define __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) do {} while (0)
392 #endif
393
394 #define SCX_OPS_ATTACH(__skel, __ops_name, __scx_name) ({ \
395 struct bpf_link *__link; \
396 __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name); \
397 SCX_BUG_ON(__scx_name##__attach((__skel)), "Failed to attach skel"); \
398 __link = bpf_map__attach_struct_ops((__skel)->maps.__ops_name); \
399 SCX_BUG_ON(!__link, "Failed to attach struct_ops"); \
400 __link; \
401 })
402
403 #endif /* __SCX_COMPAT_H */
404