1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. 4 * Copyright (c) 2024 Tejun Heo <tj@kernel.org> 5 * Copyright (c) 2024 David Vernet <dvernet@meta.com> 6 */ 7 #ifndef __SCX_COMPAT_H 8 #define __SCX_COMPAT_H 9 10 #include <bpf/btf.h> 11 #include <bpf/libbpf.h> 12 #include <fcntl.h> 13 #include <stdint.h> 14 #include <stdio.h> 15 #include <stdlib.h> 16 #include <string.h> 17 #include <unistd.h> 18 19 #include "enums_abi.autogen.h" 20 21 struct btf *__COMPAT_vmlinux_btf __attribute__((weak)); 22 23 static inline void __COMPAT_load_vmlinux_btf(void) 24 { 25 if (!__COMPAT_vmlinux_btf) { 26 __COMPAT_vmlinux_btf = btf__load_vmlinux_btf(); 27 SCX_BUG_ON(!__COMPAT_vmlinux_btf, "btf__load_vmlinux_btf()"); 28 } 29 } 30 31 /* 32 * Recover the true value of a 64-bit enum enumerator whose kernel BTF entry 33 * was truncated to its low 32 bits. 34 * 35 * Kernels whose BTF was generated without BTF_KIND_ENUM64 support encode 36 * 64-bit enums as 8-byte BTF_KIND_ENUM entries whose enumerator values only 37 * carry the low 32 bits. This happens with pahole < 1.24, which predates 38 * ENUM64, and with pahole passing --skip_encoding_btf_enum64 (e.g. Google's 39 * Container-Optimized OS / GKE kernels deliberately pass it for backward 40 * compatibility with older BTF consumers). The high bits 41 * can't be recovered from kernel BTF, so substitute the value from the 42 * vmlinux.h this tree was built against, cross-checked against the low 32 43 * bits the kernel did provide. 44 * 45 * Note that this is a best-effort recovery, not a ground truth. The 46 * substitution assumes the running kernel agrees with this tree's vmlinux.h 47 * on the high 32 bits, but only the low 32 bits can actually be verified. 48 * The cross-check is vacuous for enumerators whose value has no low bits 49 * set (e.g. SCX_DSQ_FLAG_BUILTIN, __SCX_ENQ_INTERNAL_MASK, 50 * SCX_ENQ_CLEAR_OPSS, SCX_ECODE_*): their lo32 is 0 and matches anything, 51 * so those substitutions rest entirely on the high bits never moving. An 52 * enumerator missing from the table (a kernel newer than this tree's 53 * vmlinux.h, or a stale autogen table) can't be recovered at all. If a 54 * substitution is ever wrong, the scheduler operates on bogus values (e.g. 55 * dispatching to nonexistent DSQ ids or silently dropping flags) and can 56 * wildly malfunction, which is why the mismatch and table-miss paths refuse 57 * instead of guessing. 58 */ 59 static inline bool __COMPAT_recover_truncated_enum64(const char *type, 60 const char *name, 61 u32 lo32, u64 *v) 62 { 63 static bool warned; 64 size_t i; 65 66 for (i = 0; i < sizeof(__scx_enum_abi_vals) / sizeof(__scx_enum_abi_vals[0]); i++) { 67 const struct __scx_enum_abi_val *e = &__scx_enum_abi_vals[i]; 68 69 if (strcmp(e->type, type) || strcmp(e->name, name)) 70 continue; 71 72 if (e->val <= (u64)UINT32_MAX) { 73 *v = lo32; 74 return true; 75 } 76 77 if ((u32)e->val != lo32) { 78 fprintf(stderr, "ERROR: kernel BTF value of %s::%s (0x%x) doesn't match the low 32 bits of the vmlinux.h value (0x%llx); refusing to substitute\n", 79 type, name, lo32, (unsigned long long)e->val); 80 return false; 81 } 82 83 if (!warned) { 84 fprintf(stderr, 85 "WARNING: kernel BTF lacks BTF_KIND_ENUM64 encoding (generated by\n" 86 "WARNING: pahole < 1.24 or with --skip_encoding_btf_enum64), so 64-bit\n" 87 "WARNING: scx enum values are truncated to their low 32 bits in kernel\n" 88 "WARNING: BTF. Substituting the full 64-bit values from the vmlinux.h\n" 89 "WARNING: this binary was built against, cross-checked against the low\n" 90 "WARNING: 32 bits the kernel does provide. The high 32 bits cannot be\n" 91 "WARNING: verified: if the running kernel's actual values differ from\n" 92 "WARNING: the build-time vmlinux.h (e.g. an enum that moved in a newer\n" 93 "WARNING: kernel), the scheduler will operate on bogus values, such as\n" 94 "WARNING: dispatching to nonexistent DSQ ids, and can wildly malfunction.\n"); 95 warned = true; 96 } 97 *v = e->val; 98 return true; 99 } 100 101 /* 102 * Unknown enumerator (likely a stale autogen table). Fail 103 * pessimistically to avoid returning an invalid value. 104 */ 105 fprintf(stderr, "ERROR: kernel BTF truncates 64-bit enum %s::%s to 0x%x; 64-bit variant not found in vmlinux.h\n", 106 type, name, lo32); 107 return false; 108 } 109 110 static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v) 111 { 112 const struct btf_type *t; 113 const char *n; 114 s32 tid; 115 __u32 i; 116 117 __COMPAT_load_vmlinux_btf(); 118 119 tid = btf__find_by_name(__COMPAT_vmlinux_btf, type); 120 if (tid < 0) 121 return false; 122 123 t = btf__type_by_id(__COMPAT_vmlinux_btf, tid); 124 SCX_BUG_ON(!t, "btf__type_by_id(%d)", tid); 125 126 if (btf_is_enum(t)) { 127 struct btf_enum *e = btf_enum(t); 128 129 for (i = 0; i < btf_vlen(t); i++) { 130 n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); 131 SCX_BUG_ON(!n, "btf__name_by_offset()"); 132 if (!strcmp(n, name)) { 133 /* 134 * Try to recover a 64-bit enum from an 8-byte 135 * BTF_KIND_ENUM that was encoded without ENUM64 136 * support (old pahole or 137 * --skip_encoding_btf_enum64). Only scx_* 138 * types are covered by the substitution table; 139 * non-scx types fall through to the raw value 140 * so this generic utility keeps working for 141 * them. 142 */ 143 if (t->size == 8 && !strncmp(type, "scx_", 4)) 144 return __COMPAT_recover_truncated_enum64(type, name, 145 (u32)e[i].val, v); 146 *v = e[i].val; 147 return true; 148 } 149 } 150 } else if (btf_is_enum64(t)) { 151 struct btf_enum64 *e = btf_enum64(t); 152 153 for (i = 0; i < btf_vlen(t); i++) { 154 n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); 155 SCX_BUG_ON(!n, "btf__name_by_offset()"); 156 if (!strcmp(n, name)) { 157 *v = btf_enum64_value(&e[i]); 158 return true; 159 } 160 } 161 } 162 163 return false; 164 } 165 166 #define __COMPAT_ENUM_OR_ZERO(__type, __ent) \ 167 ({ \ 168 u64 __val = 0; \ 169 __COMPAT_read_enum(__type, __ent, &__val); \ 170 __val; \ 171 }) 172 173 static inline bool __COMPAT_has_ksym(const char *ksym) 174 { 175 __COMPAT_load_vmlinux_btf(); 176 return btf__find_by_name(__COMPAT_vmlinux_btf, ksym) >= 0; 177 } 178 179 static inline bool __COMPAT_struct_has_field(const char *type, const char *field) 180 { 181 const struct btf_type *t; 182 const struct btf_member *m; 183 const char *n; 184 s32 tid; 185 __u32 i; 186 187 __COMPAT_load_vmlinux_btf(); 188 tid = btf__find_by_name_kind(__COMPAT_vmlinux_btf, type, BTF_KIND_STRUCT); 189 if (tid < 0) 190 return false; 191 192 t = btf__type_by_id(__COMPAT_vmlinux_btf, tid); 193 SCX_BUG_ON(!t, "btf__type_by_id(%d)", tid); 194 195 m = btf_members(t); 196 197 for (i = 0; i < btf_vlen(t); i++) { 198 n = btf__name_by_offset(__COMPAT_vmlinux_btf, m[i].name_off); 199 SCX_BUG_ON(!n, "btf__name_by_offset()"); 200 if (!strcmp(n, field)) 201 return true; 202 } 203 204 return false; 205 } 206 207 #define SCX_OPS_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_ops_flags", #name) 208 209 #define SCX_OPS_KEEP_BUILTIN_IDLE SCX_OPS_FLAG(SCX_OPS_KEEP_BUILTIN_IDLE) 210 #define SCX_OPS_ENQ_LAST SCX_OPS_FLAG(SCX_OPS_ENQ_LAST) 211 #define SCX_OPS_ENQ_EXITING SCX_OPS_FLAG(SCX_OPS_ENQ_EXITING) 212 #define SCX_OPS_SWITCH_PARTIAL SCX_OPS_FLAG(SCX_OPS_SWITCH_PARTIAL) 213 #define SCX_OPS_ENQ_MIGRATION_DISABLED SCX_OPS_FLAG(SCX_OPS_ENQ_MIGRATION_DISABLED) 214 #define SCX_OPS_ALLOW_QUEUED_WAKEUP SCX_OPS_FLAG(SCX_OPS_ALLOW_QUEUED_WAKEUP) 215 #define SCX_OPS_BUILTIN_IDLE_PER_NODE SCX_OPS_FLAG(SCX_OPS_BUILTIN_IDLE_PER_NODE) 216 #define SCX_OPS_ALWAYS_ENQ_IMMED SCX_OPS_FLAG(SCX_OPS_ALWAYS_ENQ_IMMED) 217 218 #define SCX_PICK_IDLE_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_pick_idle_cpu_flags", #name) 219 220 #define SCX_PICK_IDLE_CORE SCX_PICK_IDLE_FLAG(SCX_PICK_IDLE_CORE) 221 #define SCX_PICK_IDLE_IN_NODE SCX_PICK_IDLE_FLAG(SCX_PICK_IDLE_IN_NODE) 222 223 static inline long scx_hotplug_seq(void) 224 { 225 int fd; 226 char buf[32]; 227 char *endptr; 228 ssize_t len; 229 long val; 230 231 fd = open("/sys/kernel/sched_ext/hotplug_seq", O_RDONLY); 232 if (fd < 0) 233 return -ENOENT; 234 235 len = read(fd, buf, sizeof(buf) - 1); 236 SCX_BUG_ON(len <= 0, "read failed (%ld)", len); 237 buf[len] = 0; 238 close(fd); 239 240 errno = 0; 241 val = strtoul(buf, &endptr, 10); 242 SCX_BUG_ON(errno == ERANGE || endptr == buf || 243 (*endptr != '\n' && *endptr != '\0'), "invalid num hotplug events: %ld", val); 244 245 return val; 246 } 247 248 /* 249 * Open the sched_ext_ops skeleton. 250 * 251 * struct sched_ext_ops can change over time. Two complementary mechanisms 252 * keep BPF schedulers built against newer headers running on older kernels: 253 * 254 * 1. Load-time fix-up (SCX_OPS_OPEN()). For each optional ops callback or field 255 * added to struct sched_ext_ops, an explicit stanza below probes the 256 * running kernel's BTF via __COMPAT_struct_has_field() and, if the field 257 * is missing, clears it in the in-memory struct_ops (with a warning to 258 * stderr) before load. Handles additive changes - a new stanza must be 259 * added here for each new optional field. 260 * 261 * 2. Multi-variant struct_ops via compat.bpf.h::SCX_OPS_DEFINE(). That 262 * macro can be expanded to emit several variants of struct sched_ext_ops, 263 * and SCX_OPS_LOAD()/ATTACH() can pick the right one based on what the 264 * kernel supports. Needed when an existing operation has to change 265 * incompatibly (e.g. a callback signature changes); the load-time 266 * fix-up above only handles purely additive changes. 267 * 268 * ec7e3b0463e1 ("implement-ops") in https://github.com/sched-ext/sched_ext is 269 * the current minimum required kernel version. 270 * 271 * COMPAT: 272 * - v6.17: ops.cgroup_set_bandwidth() 273 * - v6.19: ops.cgroup_set_idle() 274 * - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id 275 * - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us 276 */ 277 #define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \ 278 struct __scx_name *__oskel; \ 279 \ 280 SCX_BUG_ON(!__COMPAT_struct_has_field(__ops_struct, "dump"), \ 281 __ops_struct ".dump() missing, kernel too old?"); \ 282 \ 283 __oskel = __scx_name##__open(); \ 284 SCX_BUG_ON(!__oskel, "Could not open " #__scx_name); \ 285 __oskel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \ 286 SCX_ENUM_INIT(__oskel); \ 287 __oskel; \ 288 }) 289 290 #define SCX_OPS_OPEN(__ops_name, __scx_name) ({ \ 291 struct __scx_name *__skel; \ 292 \ 293 __skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops"); \ 294 if (__skel->struct_ops.__ops_name->cgroup_set_bandwidth && \ 295 !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_bandwidth")) { \ 296 fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_bandwidth()\n"); \ 297 __skel->struct_ops.__ops_name->cgroup_set_bandwidth = NULL; \ 298 } \ 299 if (__skel->struct_ops.__ops_name->cgroup_set_idle && \ 300 !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_idle")) { \ 301 fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_idle()\n"); \ 302 __skel->struct_ops.__ops_name->cgroup_set_idle = NULL; \ 303 } \ 304 if (__skel->struct_ops.__ops_name->sub_attach && \ 305 !__COMPAT_struct_has_field("sched_ext_ops", "sub_attach")) { \ 306 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_attach()\n"); \ 307 __skel->struct_ops.__ops_name->sub_attach = NULL; \ 308 } \ 309 if (__skel->struct_ops.__ops_name->sub_detach && \ 310 !__COMPAT_struct_has_field("sched_ext_ops", "sub_detach")) { \ 311 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_detach()\n"); \ 312 __skel->struct_ops.__ops_name->sub_detach = NULL; \ 313 } \ 314 if (__skel->struct_ops.__ops_name->sub_cgroup_id > 0 && \ 315 !__COMPAT_struct_has_field("sched_ext_ops", "sub_cgroup_id")) { \ 316 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_cgroup_id\n"); \ 317 __skel->struct_ops.__ops_name->sub_cgroup_id = 0; \ 318 } \ 319 if (__skel->struct_ops.__ops_name->rescue_bandwidth_ppt > 0 && \ 320 !__COMPAT_struct_has_field("sched_ext_ops", "rescue_bandwidth_ppt")) { \ 321 fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_bandwidth_ppt\n"); \ 322 __skel->struct_ops.__ops_name->rescue_bandwidth_ppt = 0; \ 323 } \ 324 if (__skel->struct_ops.__ops_name->rescue_quantum_us > 0 && \ 325 !__COMPAT_struct_has_field("sched_ext_ops", "rescue_quantum_us")) { \ 326 fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_quantum_us\n"); \ 327 __skel->struct_ops.__ops_name->rescue_quantum_us = 0; \ 328 } \ 329 __skel; \ 330 }) 331 332 /* 333 * Open a cid-form (struct sched_ext_ops_cid) skeleton. The cid form postdates 334 * every op the load-time fix-ups above handle, so none of them apply. 335 */ 336 #define SCX_OPS_CID_OPEN(__ops_name, __scx_name) \ 337 __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid") 338 339 /* 340 * Associate non-struct_ops BPF programs with the scheduler's struct_ops map so 341 * that scx_prog_sched() can determine which scheduler a BPF program belongs 342 * to. Requires libbpf >= 1.7. 343 */ 344 #if LIBBPF_MAJOR_VERSION > 1 || \ 345 (LIBBPF_MAJOR_VERSION == 1 && LIBBPF_MINOR_VERSION >= 7) 346 static inline void __scx_ops_assoc_prog(struct bpf_program *prog, 347 struct bpf_map *map, 348 const char *ops_name) 349 { 350 s32 err = bpf_program__assoc_struct_ops(prog, map, NULL); 351 if (err) 352 fprintf(stderr, 353 "ERROR: Failed to associate %s with %s: %d\n", 354 bpf_program__name(prog), ops_name, err); 355 } 356 #else 357 static inline void __scx_ops_assoc_prog(struct bpf_program *prog, 358 struct bpf_map *map, 359 const char *ops_name) 360 { 361 } 362 #endif 363 364 /* See SCX_OPS_OPEN() above for backward-compatibility handling. */ 365 #define SCX_OPS_LOAD(__skel, __ops_name, __scx_name, __uei_name) ({ \ 366 struct bpf_program *__prog; \ 367 UEI_SET_SIZE(__skel, __ops_name, __uei_name); \ 368 SCX_BUG_ON(__scx_name##__load((__skel)), "Failed to load skel"); \ 369 bpf_object__for_each_program(__prog, (__skel)->obj) { \ 370 if (bpf_program__type(__prog) == BPF_PROG_TYPE_STRUCT_OPS) \ 371 continue; \ 372 __scx_ops_assoc_prog(__prog, (__skel)->maps.__ops_name, \ 373 #__ops_name); \ 374 } \ 375 }) 376 377 /* 378 * New versions of bpftool now emit additional link placeholders for BPF maps, 379 * and set up BPF skeleton in such a way that libbpf will auto-attach BPF maps 380 * automatically, assuming libbpf is recent enough (v1.5+). Old libbpf will do 381 * nothing with those links and won't attempt to auto-attach maps. 382 * 383 * To maintain compatibility with older libbpf while avoiding trying to attach 384 * twice, disable the autoattach feature on newer libbpf. 385 */ 386 #if LIBBPF_MAJOR_VERSION > 1 || \ 387 (LIBBPF_MAJOR_VERSION == 1 && LIBBPF_MINOR_VERSION >= 5) 388 #define __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) \ 389 bpf_map__set_autoattach((__skel)->maps.__ops_name, false) 390 #else 391 #define __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) do {} while (0) 392 #endif 393 394 #define SCX_OPS_ATTACH(__skel, __ops_name, __scx_name) ({ \ 395 struct bpf_link *__link; \ 396 __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name); \ 397 SCX_BUG_ON(__scx_name##__attach((__skel)), "Failed to attach skel"); \ 398 __link = bpf_map__attach_struct_ops((__skel)->maps.__ops_name); \ 399 SCX_BUG_ON(!__link, "Failed to attach struct_ops"); \ 400 __link; \ 401 }) 402 403 #endif /* __SCX_COMPAT_H */ 404