1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. 4 * Copyright (c) 2024 Tejun Heo <tj@kernel.org> 5 * Copyright (c) 2024 David Vernet <dvernet@meta.com> 6 */ 7 #ifndef __SCX_COMPAT_H 8 #define __SCX_COMPAT_H 9 10 #include <bpf/btf.h> 11 #include <bpf/libbpf.h> 12 #include <fcntl.h> 13 #include <stdint.h> 14 #include <stdio.h> 15 #include <stdlib.h> 16 #include <string.h> 17 #include <unistd.h> 18 19 #include "enums_abi.autogen.h" 20 21 struct btf *__COMPAT_vmlinux_btf __attribute__((weak)); 22 23 static inline void __COMPAT_load_vmlinux_btf(void) 24 { 25 if (!__COMPAT_vmlinux_btf) { 26 __COMPAT_vmlinux_btf = btf__load_vmlinux_btf(); 27 SCX_BUG_ON(!__COMPAT_vmlinux_btf, "btf__load_vmlinux_btf()"); 28 } 29 } 30 31 /* 32 * Recover the true value of a 64-bit enum enumerator whose kernel BTF entry 33 * was truncated to its low 32 bits. 34 * 35 * Kernels whose BTF was generated without BTF_KIND_ENUM64 support encode 36 * 64-bit enums as 8-byte BTF_KIND_ENUM entries whose enumerator values only 37 * carry the low 32 bits. This happens with pahole < 1.24, which predates 38 * ENUM64, and with pahole passing --skip_encoding_btf_enum64 (e.g. Google's 39 * Container-Optimized OS / GKE kernels deliberately pass it for backward 40 * compatibility with older BTF consumers). The high bits 41 * can't be recovered from kernel BTF, so substitute the value from the 42 * vmlinux.h this tree was built against, cross-checked against the low 32 43 * bits the kernel did provide. 44 * 45 * Note that this is a best-effort recovery, not a ground truth. The 46 * substitution assumes the running kernel agrees with this tree's vmlinux.h 47 * on the high 32 bits, but only the low 32 bits can actually be verified. 48 * The cross-check is vacuous for enumerators whose value has no low bits 49 * set (e.g. SCX_DSQ_FLAG_BUILTIN, __SCX_ENQ_INTERNAL_MASK, 50 * SCX_ENQ_CLEAR_OPSS, SCX_ECODE_*): their lo32 is 0 and matches anything, 51 * so those substitutions rest entirely on the high bits never moving. An 52 * enumerator missing from the table (a kernel newer than this tree's 53 * vmlinux.h, or a stale autogen table) can't be recovered at all. If a 54 * substitution is ever wrong, the scheduler operates on bogus values (e.g. 55 * dispatching to nonexistent DSQ ids or silently dropping flags) and can 56 * wildly malfunction, which is why the mismatch and table-miss paths refuse 57 * instead of guessing. 58 */ 59 static inline bool __COMPAT_recover_truncated_enum64(const char *type, 60 const char *name, 61 u32 lo32, u64 *v) 62 { 63 static bool warned; 64 size_t i; 65 66 for (i = 0; i < sizeof(__scx_enum_abi_vals) / sizeof(__scx_enum_abi_vals[0]); i++) { 67 const struct __scx_enum_abi_val *e = &__scx_enum_abi_vals[i]; 68 69 if (strcmp(e->type, type) || strcmp(e->name, name)) 70 continue; 71 72 if (e->val <= (u64)UINT32_MAX) { 73 *v = lo32; 74 return true; 75 } 76 77 if ((u32)e->val != lo32) { 78 fprintf(stderr, "ERROR: kernel BTF value of %s::%s (0x%x) doesn't match the low 32 bits of the vmlinux.h value (0x%llx); refusing to substitute\n", 79 type, name, lo32, (unsigned long long)e->val); 80 return false; 81 } 82 83 if (!warned) { 84 fprintf(stderr, 85 "WARNING: kernel BTF lacks BTF_KIND_ENUM64 encoding (generated by\n" 86 "WARNING: pahole < 1.24 or with --skip_encoding_btf_enum64), so 64-bit\n" 87 "WARNING: scx enum values are truncated to their low 32 bits in kernel\n" 88 "WARNING: BTF. Substituting the full 64-bit values from the vmlinux.h\n" 89 "WARNING: this binary was built against, cross-checked against the low\n" 90 "WARNING: 32 bits the kernel does provide. The high 32 bits cannot be\n" 91 "WARNING: verified: if the running kernel's actual values differ from\n" 92 "WARNING: the build-time vmlinux.h (e.g. an enum that moved in a newer\n" 93 "WARNING: kernel), the scheduler will operate on bogus values, such as\n" 94 "WARNING: dispatching to nonexistent DSQ ids, and can wildly malfunction.\n"); 95 warned = true; 96 } 97 *v = e->val; 98 return true; 99 } 100 101 /* 102 * Unknown enumerator (likely a stale autogen table). Fail 103 * pessimistically to avoid returning an invalid value. 104 */ 105 fprintf(stderr, "ERROR: kernel BTF truncates 64-bit enum %s::%s to 0x%x; 64-bit variant not found in vmlinux.h\n", 106 type, name, lo32); 107 return false; 108 } 109 110 static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v) 111 { 112 const struct btf_type *t; 113 const char *n; 114 s32 tid; 115 __u32 i; 116 117 __COMPAT_load_vmlinux_btf(); 118 119 tid = btf__find_by_name(__COMPAT_vmlinux_btf, type); 120 if (tid < 0) 121 return false; 122 123 t = btf__type_by_id(__COMPAT_vmlinux_btf, tid); 124 SCX_BUG_ON(!t, "btf__type_by_id(%d)", tid); 125 126 if (btf_is_enum(t)) { 127 struct btf_enum *e = btf_enum(t); 128 129 for (i = 0; i < btf_vlen(t); i++) { 130 n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); 131 SCX_BUG_ON(!n, "btf__name_by_offset()"); 132 if (!strcmp(n, name)) { 133 /* 134 * Try to recover a 64-bit enum from an 8-byte 135 * BTF_KIND_ENUM that was encoded without ENUM64 136 * support (old pahole or 137 * --skip_encoding_btf_enum64). Only scx_* 138 * types are covered by the substitution table; 139 * non-scx types fall through to the raw value 140 * so this generic utility keeps working for 141 * them. 142 */ 143 if (t->size == 8 && !strncmp(type, "scx_", 4)) 144 return __COMPAT_recover_truncated_enum64(type, name, 145 (u32)e[i].val, v); 146 *v = e[i].val; 147 return true; 148 } 149 } 150 } else if (btf_is_enum64(t)) { 151 struct btf_enum64 *e = btf_enum64(t); 152 153 for (i = 0; i < btf_vlen(t); i++) { 154 n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); 155 SCX_BUG_ON(!n, "btf__name_by_offset()"); 156 if (!strcmp(n, name)) { 157 *v = btf_enum64_value(&e[i]); 158 return true; 159 } 160 } 161 } 162 163 return false; 164 } 165 166 #define __COMPAT_ENUM_OR_ZERO(__type, __ent) \ 167 ({ \ 168 u64 __val = 0; \ 169 __COMPAT_read_enum(__type, __ent, &__val); \ 170 __val; \ 171 }) 172 173 static inline bool __COMPAT_has_ksym(const char *ksym) 174 { 175 __COMPAT_load_vmlinux_btf(); 176 return btf__find_by_name(__COMPAT_vmlinux_btf, ksym) >= 0; 177 } 178 179 static inline bool __COMPAT_struct_has_field(const char *type, const char *field) 180 { 181 const struct btf_type *t; 182 const struct btf_member *m; 183 const char *n; 184 s32 tid; 185 __u32 i; 186 187 __COMPAT_load_vmlinux_btf(); 188 tid = btf__find_by_name_kind(__COMPAT_vmlinux_btf, type, BTF_KIND_STRUCT); 189 if (tid < 0) 190 return false; 191 192 t = btf__type_by_id(__COMPAT_vmlinux_btf, tid); 193 SCX_BUG_ON(!t, "btf__type_by_id(%d)", tid); 194 195 m = btf_members(t); 196 197 for (i = 0; i < btf_vlen(t); i++) { 198 n = btf__name_by_offset(__COMPAT_vmlinux_btf, m[i].name_off); 199 SCX_BUG_ON(!n, "btf__name_by_offset()"); 200 if (!strcmp(n, field)) 201 return true; 202 } 203 204 return false; 205 } 206 207 #define SCX_OPS_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_ops_flags", #name) 208 209 #define SCX_OPS_KEEP_BUILTIN_IDLE SCX_OPS_FLAG(SCX_OPS_KEEP_BUILTIN_IDLE) 210 #define SCX_OPS_ENQ_LAST SCX_OPS_FLAG(SCX_OPS_ENQ_LAST) 211 #define SCX_OPS_ENQ_EXITING SCX_OPS_FLAG(SCX_OPS_ENQ_EXITING) 212 #define SCX_OPS_SWITCH_PARTIAL SCX_OPS_FLAG(SCX_OPS_SWITCH_PARTIAL) 213 #define SCX_OPS_ENQ_MIGRATION_DISABLED SCX_OPS_FLAG(SCX_OPS_ENQ_MIGRATION_DISABLED) 214 #define SCX_OPS_ALLOW_QUEUED_WAKEUP SCX_OPS_FLAG(SCX_OPS_ALLOW_QUEUED_WAKEUP) 215 #define SCX_OPS_BUILTIN_IDLE_PER_NODE SCX_OPS_FLAG(SCX_OPS_BUILTIN_IDLE_PER_NODE) 216 #define SCX_OPS_ALWAYS_ENQ_IMMED SCX_OPS_FLAG(SCX_OPS_ALWAYS_ENQ_IMMED) 217 #define SCX_OPS_ENQ_BLOCKED SCX_OPS_FLAG(SCX_OPS_ENQ_BLOCKED) 218 219 #define SCX_PICK_IDLE_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_pick_idle_cpu_flags", #name) 220 221 #define SCX_PICK_IDLE_CORE SCX_PICK_IDLE_FLAG(SCX_PICK_IDLE_CORE) 222 #define SCX_PICK_IDLE_IN_NODE SCX_PICK_IDLE_FLAG(SCX_PICK_IDLE_IN_NODE) 223 224 static inline long scx_hotplug_seq(void) 225 { 226 int fd; 227 char buf[32]; 228 char *endptr; 229 ssize_t len; 230 long val; 231 232 fd = open("/sys/kernel/sched_ext/hotplug_seq", O_RDONLY); 233 if (fd < 0) 234 return -ENOENT; 235 236 len = read(fd, buf, sizeof(buf) - 1); 237 SCX_BUG_ON(len <= 0, "read failed (%ld)", len); 238 buf[len] = 0; 239 close(fd); 240 241 errno = 0; 242 val = strtoul(buf, &endptr, 10); 243 SCX_BUG_ON(errno == ERANGE || endptr == buf || 244 (*endptr != '\n' && *endptr != '\0'), "invalid num hotplug events: %ld", val); 245 246 return val; 247 } 248 249 /* 250 * Open the sched_ext_ops skeleton. 251 * 252 * struct sched_ext_ops can change over time. Two complementary mechanisms 253 * keep BPF schedulers built against newer headers running on older kernels: 254 * 255 * 1. Load-time fix-up (SCX_OPS_OPEN()). For each optional ops callback or field 256 * added to struct sched_ext_ops, an explicit stanza below probes the 257 * running kernel's BTF via __COMPAT_struct_has_field() and, if the field 258 * is missing, clears it in the in-memory struct_ops (with a warning to 259 * stderr) before load. Handles additive changes - a new stanza must be 260 * added here for each new optional field. 261 * 262 * 2. Multi-variant struct_ops via compat.bpf.h::SCX_OPS_DEFINE(). That 263 * macro can be expanded to emit several variants of struct sched_ext_ops, 264 * and SCX_OPS_LOAD()/ATTACH() can pick the right one based on what the 265 * kernel supports. Needed when an existing operation has to change 266 * incompatibly (e.g. a callback signature changes); the load-time 267 * fix-up above only handles purely additive changes. 268 * 269 * ec7e3b0463e1 ("implement-ops") in https://github.com/sched-ext/sched_ext is 270 * the current minimum required kernel version. 271 * 272 * COMPAT: 273 * - v6.17: ops.cgroup_set_bandwidth() 274 * - v6.19: ops.cgroup_set_idle() 275 * - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id 276 * - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us 277 */ 278 #define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct, __opts) ({ \ 279 struct __scx_name *__oskel; \ 280 \ 281 SCX_BUG_ON(!__COMPAT_struct_has_field(__ops_struct, "dump"), \ 282 __ops_struct ".dump() missing, kernel too old?"); \ 283 \ 284 __oskel = __scx_name##__open_opts(__opts); \ 285 SCX_BUG_ON(!__oskel, "Could not open " #__scx_name); \ 286 __oskel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \ 287 SCX_ENUM_INIT(__oskel); \ 288 __oskel; \ 289 }) 290 291 #define SCX_OPS_OPEN_OPTS(__ops_name, __scx_name, __opts) ({ \ 292 struct __scx_name *__skel; \ 293 \ 294 __skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops", __opts); \ 295 if (__skel->struct_ops.__ops_name->cgroup_set_bandwidth && \ 296 !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_bandwidth")) { \ 297 fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_bandwidth()\n"); \ 298 __skel->struct_ops.__ops_name->cgroup_set_bandwidth = NULL; \ 299 } \ 300 if (__skel->struct_ops.__ops_name->cgroup_set_idle && \ 301 !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_idle")) { \ 302 fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_idle()\n"); \ 303 __skel->struct_ops.__ops_name->cgroup_set_idle = NULL; \ 304 } \ 305 if (__skel->struct_ops.__ops_name->sub_attach && \ 306 !__COMPAT_struct_has_field("sched_ext_ops", "sub_attach")) { \ 307 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_attach()\n"); \ 308 __skel->struct_ops.__ops_name->sub_attach = NULL; \ 309 } \ 310 if (__skel->struct_ops.__ops_name->sub_detach && \ 311 !__COMPAT_struct_has_field("sched_ext_ops", "sub_detach")) { \ 312 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_detach()\n"); \ 313 __skel->struct_ops.__ops_name->sub_detach = NULL; \ 314 } \ 315 if (__skel->struct_ops.__ops_name->sub_cgroup_id > 0 && \ 316 !__COMPAT_struct_has_field("sched_ext_ops", "sub_cgroup_id")) { \ 317 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_cgroup_id\n"); \ 318 __skel->struct_ops.__ops_name->sub_cgroup_id = 0; \ 319 } \ 320 if (__skel->struct_ops.__ops_name->rescue_bandwidth_ppt > 0 && \ 321 !__COMPAT_struct_has_field("sched_ext_ops", "rescue_bandwidth_ppt")) { \ 322 fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_bandwidth_ppt\n"); \ 323 __skel->struct_ops.__ops_name->rescue_bandwidth_ppt = 0; \ 324 } \ 325 if (__skel->struct_ops.__ops_name->rescue_quantum_us > 0 && \ 326 !__COMPAT_struct_has_field("sched_ext_ops", "rescue_quantum_us")) { \ 327 fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_quantum_us\n"); \ 328 __skel->struct_ops.__ops_name->rescue_quantum_us = 0; \ 329 } \ 330 __skel; \ 331 }) 332 333 #define SCX_OPS_OPEN(__ops_name, __scx_name) \ 334 SCX_OPS_OPEN_OPTS(__ops_name, __scx_name, 0) 335 336 /* 337 * Open a cid-form (struct sched_ext_ops_cid) skeleton. The cid form postdates 338 * every op the load-time fix-ups above handle, so none of them apply. 339 * 340 * A cid-form scheduler registers through bpf_scx_reg_cid() rather than 341 * bpf_scx_reg(), so repoint common.bpf.h's prolog probe at it. Both the cid 342 * form and bpf_scx_reg_cid() appeared in v7.2, so a kernel that accepts this 343 * skeleton always has the symbol. 344 */ 345 #define SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, __opts) ({ \ 346 struct __scx_name *__cskel; \ 347 \ 348 __cskel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid", __opts); \ 349 bpf_program__set_attach_target(__cskel->progs.scx_lib_init_probe, 0, \ 350 "bpf_scx_reg_cid"); \ 351 __cskel; \ 352 }) 353 354 #define SCX_OPS_CID_OPEN(__ops_name, __scx_name) \ 355 SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, 0) 356 357 /* 358 * Associate non-struct_ops BPF programs with the scheduler's struct_ops map so 359 * that scx_prog_sched() can determine which scheduler a BPF program belongs 360 * to. Requires libbpf >= 1.7. 361 */ 362 #if LIBBPF_MAJOR_VERSION > 1 || \ 363 (LIBBPF_MAJOR_VERSION == 1 && LIBBPF_MINOR_VERSION >= 7) 364 static inline void __scx_ops_assoc_prog(struct bpf_program *prog, 365 struct bpf_map *map, 366 const char *ops_name) 367 { 368 s32 err = bpf_program__assoc_struct_ops(prog, map, NULL); 369 if (err) 370 fprintf(stderr, 371 "ERROR: Failed to associate %s with %s: %d\n", 372 bpf_program__name(prog), ops_name, err); 373 } 374 #else 375 static inline void __scx_ops_assoc_prog(struct bpf_program *prog, 376 struct bpf_map *map, 377 const char *ops_name) 378 { 379 } 380 #endif 381 382 /* See SCX_OPS_OPEN() above for backward-compatibility handling. */ 383 #define SCX_OPS_LOAD(__skel, __ops_name, __scx_name, __uei_name) ({ \ 384 struct bpf_program *__prog; \ 385 UEI_SET_SIZE(__skel, __ops_name, __uei_name); \ 386 SCX_BUG_ON(__scx_name##__load((__skel)), "Failed to load skel"); \ 387 bpf_object__for_each_program(__prog, (__skel)->obj) { \ 388 if (bpf_program__type(__prog) == BPF_PROG_TYPE_STRUCT_OPS) \ 389 continue; \ 390 __scx_ops_assoc_prog(__prog, (__skel)->maps.__ops_name, \ 391 #__ops_name); \ 392 } \ 393 }) 394 395 /* 396 * New versions of bpftool now emit additional link placeholders for BPF maps, 397 * and set up BPF skeleton in such a way that libbpf will auto-attach BPF maps 398 * automatically, assuming libbpf is recent enough (v1.5+). Old libbpf will do 399 * nothing with those links and won't attempt to auto-attach maps. 400 * 401 * To maintain compatibility with older libbpf while avoiding trying to attach 402 * twice, disable the autoattach feature on newer libbpf. 403 */ 404 #if LIBBPF_MAJOR_VERSION > 1 || \ 405 (LIBBPF_MAJOR_VERSION == 1 && LIBBPF_MINOR_VERSION >= 5) 406 #define __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) \ 407 bpf_map__set_autoattach((__skel)->maps.__ops_name, false) 408 #else 409 #define __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) do {} while (0) 410 #endif 411 412 #define SCX_OPS_ATTACH(__skel, __ops_name, __scx_name) ({ \ 413 struct bpf_link *__link; \ 414 __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name); \ 415 SCX_BUG_ON(__scx_name##__attach((__skel)), "Failed to attach skel"); \ 416 __link = bpf_map__attach_struct_ops((__skel)->maps.__ops_name); \ 417 SCX_BUG_ON(!__link, "Failed to attach struct_ops"); \ 418 __link; \ 419 }) 420 421 #endif /* __SCX_COMPAT_H */ 422