1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. 4 * Copyright (c) 2024 Tejun Heo <tj@kernel.org> 5 * Copyright (c) 2024 David Vernet <dvernet@meta.com> 6 */ 7 #ifndef __SCX_COMPAT_H 8 #define __SCX_COMPAT_H 9 10 #include <bpf/btf.h> 11 #include <bpf/libbpf.h> 12 #include <fcntl.h> 13 #include <stdlib.h> 14 #include <unistd.h> 15 16 struct btf *__COMPAT_vmlinux_btf __attribute__((weak)); 17 18 static inline void __COMPAT_load_vmlinux_btf(void) 19 { 20 if (!__COMPAT_vmlinux_btf) { 21 __COMPAT_vmlinux_btf = btf__load_vmlinux_btf(); 22 SCX_BUG_ON(!__COMPAT_vmlinux_btf, "btf__load_vmlinux_btf()"); 23 } 24 } 25 26 static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v) 27 { 28 const struct btf_type *t; 29 const char *n; 30 s32 tid; 31 __u32 i; 32 33 __COMPAT_load_vmlinux_btf(); 34 35 tid = btf__find_by_name(__COMPAT_vmlinux_btf, type); 36 if (tid < 0) 37 return false; 38 39 t = btf__type_by_id(__COMPAT_vmlinux_btf, tid); 40 SCX_BUG_ON(!t, "btf__type_by_id(%d)", tid); 41 42 if (btf_is_enum(t)) { 43 struct btf_enum *e = btf_enum(t); 44 45 for (i = 0; i < btf_vlen(t); i++) { 46 n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); 47 SCX_BUG_ON(!n, "btf__name_by_offset()"); 48 if (!strcmp(n, name)) { 49 *v = e[i].val; 50 return true; 51 } 52 } 53 } else if (btf_is_enum64(t)) { 54 struct btf_enum64 *e = btf_enum64(t); 55 56 for (i = 0; i < btf_vlen(t); i++) { 57 n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); 58 SCX_BUG_ON(!n, "btf__name_by_offset()"); 59 if (!strcmp(n, name)) { 60 *v = btf_enum64_value(&e[i]); 61 return true; 62 } 63 } 64 } 65 66 return false; 67 } 68 69 #define __COMPAT_ENUM_OR_ZERO(__type, __ent) \ 70 ({ \ 71 u64 __val = 0; \ 72 __COMPAT_read_enum(__type, __ent, &__val); \ 73 __val; \ 74 }) 75 76 static inline bool __COMPAT_has_ksym(const char *ksym) 77 { 78 __COMPAT_load_vmlinux_btf(); 79 return btf__find_by_name(__COMPAT_vmlinux_btf, ksym) >= 0; 80 } 81 82 static inline bool __COMPAT_struct_has_field(const char *type, const char *field) 83 { 84 const struct btf_type *t; 85 const struct btf_member *m; 86 const char *n; 87 s32 tid; 88 __u32 i; 89 90 __COMPAT_load_vmlinux_btf(); 91 tid = btf__find_by_name_kind(__COMPAT_vmlinux_btf, type, BTF_KIND_STRUCT); 92 if (tid < 0) 93 return false; 94 95 t = btf__type_by_id(__COMPAT_vmlinux_btf, tid); 96 SCX_BUG_ON(!t, "btf__type_by_id(%d)", tid); 97 98 m = btf_members(t); 99 100 for (i = 0; i < btf_vlen(t); i++) { 101 n = btf__name_by_offset(__COMPAT_vmlinux_btf, m[i].name_off); 102 SCX_BUG_ON(!n, "btf__name_by_offset()"); 103 if (!strcmp(n, field)) 104 return true; 105 } 106 107 return false; 108 } 109 110 #define SCX_OPS_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_ops_flags", #name) 111 112 #define SCX_OPS_KEEP_BUILTIN_IDLE SCX_OPS_FLAG(SCX_OPS_KEEP_BUILTIN_IDLE) 113 #define SCX_OPS_ENQ_LAST SCX_OPS_FLAG(SCX_OPS_ENQ_LAST) 114 #define SCX_OPS_ENQ_EXITING SCX_OPS_FLAG(SCX_OPS_ENQ_EXITING) 115 #define SCX_OPS_SWITCH_PARTIAL SCX_OPS_FLAG(SCX_OPS_SWITCH_PARTIAL) 116 #define SCX_OPS_ENQ_MIGRATION_DISABLED SCX_OPS_FLAG(SCX_OPS_ENQ_MIGRATION_DISABLED) 117 #define SCX_OPS_ALLOW_QUEUED_WAKEUP SCX_OPS_FLAG(SCX_OPS_ALLOW_QUEUED_WAKEUP) 118 #define SCX_OPS_BUILTIN_IDLE_PER_NODE SCX_OPS_FLAG(SCX_OPS_BUILTIN_IDLE_PER_NODE) 119 #define SCX_OPS_ALWAYS_ENQ_IMMED SCX_OPS_FLAG(SCX_OPS_ALWAYS_ENQ_IMMED) 120 121 #define SCX_PICK_IDLE_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_pick_idle_cpu_flags", #name) 122 123 #define SCX_PICK_IDLE_CORE SCX_PICK_IDLE_FLAG(SCX_PICK_IDLE_CORE) 124 #define SCX_PICK_IDLE_IN_NODE SCX_PICK_IDLE_FLAG(SCX_PICK_IDLE_IN_NODE) 125 126 static inline long scx_hotplug_seq(void) 127 { 128 int fd; 129 char buf[32]; 130 char *endptr; 131 ssize_t len; 132 long val; 133 134 fd = open("/sys/kernel/sched_ext/hotplug_seq", O_RDONLY); 135 if (fd < 0) 136 return -ENOENT; 137 138 len = read(fd, buf, sizeof(buf) - 1); 139 SCX_BUG_ON(len <= 0, "read failed (%ld)", len); 140 buf[len] = 0; 141 close(fd); 142 143 errno = 0; 144 val = strtoul(buf, &endptr, 10); 145 SCX_BUG_ON(errno == ERANGE || endptr == buf || 146 (*endptr != '\n' && *endptr != '\0'), "invalid num hotplug events: %ld", val); 147 148 return val; 149 } 150 151 /* 152 * Open the sched_ext_ops skeleton. 153 * 154 * struct sched_ext_ops can change over time. Two complementary mechanisms 155 * keep BPF schedulers built against newer headers running on older kernels: 156 * 157 * 1. Load-time fix-up (SCX_OPS_OPEN()). For each optional ops callback or field 158 * added to struct sched_ext_ops, an explicit stanza below probes the 159 * running kernel's BTF via __COMPAT_struct_has_field() and, if the field 160 * is missing, clears it in the in-memory struct_ops (with a warning to 161 * stderr) before load. Handles additive changes - a new stanza must be 162 * added here for each new optional field. 163 * 164 * 2. Multi-variant struct_ops via compat.bpf.h::SCX_OPS_DEFINE(). That 165 * macro can be expanded to emit several variants of struct sched_ext_ops, 166 * and SCX_OPS_LOAD()/ATTACH() can pick the right one based on what the 167 * kernel supports. Needed when an existing operation has to change 168 * incompatibly (e.g. a callback signature changes); the load-time 169 * fix-up above only handles purely additive changes. 170 * 171 * ec7e3b0463e1 ("implement-ops") in https://github.com/sched-ext/sched_ext is 172 * the current minimum required kernel version. 173 * 174 * COMPAT: 175 * - v6.17: ops.cgroup_set_bandwidth() 176 * - v6.19: ops.cgroup_set_idle() 177 * - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id 178 * - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us 179 */ 180 #define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \ 181 struct __scx_name *__oskel; \ 182 \ 183 SCX_BUG_ON(!__COMPAT_struct_has_field(__ops_struct, "dump"), \ 184 __ops_struct ".dump() missing, kernel too old?"); \ 185 \ 186 __oskel = __scx_name##__open(); \ 187 SCX_BUG_ON(!__oskel, "Could not open " #__scx_name); \ 188 __oskel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \ 189 SCX_ENUM_INIT(__oskel); \ 190 __oskel; \ 191 }) 192 193 #define SCX_OPS_OPEN(__ops_name, __scx_name) ({ \ 194 struct __scx_name *__skel; \ 195 \ 196 __skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops"); \ 197 if (__skel->struct_ops.__ops_name->cgroup_set_bandwidth && \ 198 !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_bandwidth")) { \ 199 fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_bandwidth()\n"); \ 200 __skel->struct_ops.__ops_name->cgroup_set_bandwidth = NULL; \ 201 } \ 202 if (__skel->struct_ops.__ops_name->cgroup_set_idle && \ 203 !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_idle")) { \ 204 fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_idle()\n"); \ 205 __skel->struct_ops.__ops_name->cgroup_set_idle = NULL; \ 206 } \ 207 if (__skel->struct_ops.__ops_name->sub_attach && \ 208 !__COMPAT_struct_has_field("sched_ext_ops", "sub_attach")) { \ 209 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_attach()\n"); \ 210 __skel->struct_ops.__ops_name->sub_attach = NULL; \ 211 } \ 212 if (__skel->struct_ops.__ops_name->sub_detach && \ 213 !__COMPAT_struct_has_field("sched_ext_ops", "sub_detach")) { \ 214 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_detach()\n"); \ 215 __skel->struct_ops.__ops_name->sub_detach = NULL; \ 216 } \ 217 if (__skel->struct_ops.__ops_name->sub_cgroup_id > 0 && \ 218 !__COMPAT_struct_has_field("sched_ext_ops", "sub_cgroup_id")) { \ 219 fprintf(stderr, "WARNING: kernel doesn't support ops.sub_cgroup_id\n"); \ 220 __skel->struct_ops.__ops_name->sub_cgroup_id = 0; \ 221 } \ 222 if (__skel->struct_ops.__ops_name->rescue_bandwidth_ppt > 0 && \ 223 !__COMPAT_struct_has_field("sched_ext_ops", "rescue_bandwidth_ppt")) { \ 224 fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_bandwidth_ppt\n"); \ 225 __skel->struct_ops.__ops_name->rescue_bandwidth_ppt = 0; \ 226 } \ 227 if (__skel->struct_ops.__ops_name->rescue_quantum_us > 0 && \ 228 !__COMPAT_struct_has_field("sched_ext_ops", "rescue_quantum_us")) { \ 229 fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_quantum_us\n"); \ 230 __skel->struct_ops.__ops_name->rescue_quantum_us = 0; \ 231 } \ 232 __skel; \ 233 }) 234 235 /* 236 * Open a cid-form (struct sched_ext_ops_cid) skeleton. The cid form postdates 237 * every op the load-time fix-ups above handle, so none of them apply. 238 */ 239 #define SCX_OPS_CID_OPEN(__ops_name, __scx_name) \ 240 __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid") 241 242 /* 243 * Associate non-struct_ops BPF programs with the scheduler's struct_ops map so 244 * that scx_prog_sched() can determine which scheduler a BPF program belongs 245 * to. Requires libbpf >= 1.7. 246 */ 247 #if LIBBPF_MAJOR_VERSION > 1 || \ 248 (LIBBPF_MAJOR_VERSION == 1 && LIBBPF_MINOR_VERSION >= 7) 249 static inline void __scx_ops_assoc_prog(struct bpf_program *prog, 250 struct bpf_map *map, 251 const char *ops_name) 252 { 253 s32 err = bpf_program__assoc_struct_ops(prog, map, NULL); 254 if (err) 255 fprintf(stderr, 256 "ERROR: Failed to associate %s with %s: %d\n", 257 bpf_program__name(prog), ops_name, err); 258 } 259 #else 260 static inline void __scx_ops_assoc_prog(struct bpf_program *prog, 261 struct bpf_map *map, 262 const char *ops_name) 263 { 264 } 265 #endif 266 267 /* See SCX_OPS_OPEN() above for backward-compatibility handling. */ 268 #define SCX_OPS_LOAD(__skel, __ops_name, __scx_name, __uei_name) ({ \ 269 struct bpf_program *__prog; \ 270 UEI_SET_SIZE(__skel, __ops_name, __uei_name); \ 271 SCX_BUG_ON(__scx_name##__load((__skel)), "Failed to load skel"); \ 272 bpf_object__for_each_program(__prog, (__skel)->obj) { \ 273 if (bpf_program__type(__prog) == BPF_PROG_TYPE_STRUCT_OPS) \ 274 continue; \ 275 __scx_ops_assoc_prog(__prog, (__skel)->maps.__ops_name, \ 276 #__ops_name); \ 277 } \ 278 }) 279 280 /* 281 * New versions of bpftool now emit additional link placeholders for BPF maps, 282 * and set up BPF skeleton in such a way that libbpf will auto-attach BPF maps 283 * automatically, assuming libbpf is recent enough (v1.5+). Old libbpf will do 284 * nothing with those links and won't attempt to auto-attach maps. 285 * 286 * To maintain compatibility with older libbpf while avoiding trying to attach 287 * twice, disable the autoattach feature on newer libbpf. 288 */ 289 #if LIBBPF_MAJOR_VERSION > 1 || \ 290 (LIBBPF_MAJOR_VERSION == 1 && LIBBPF_MINOR_VERSION >= 5) 291 #define __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) \ 292 bpf_map__set_autoattach((__skel)->maps.__ops_name, false) 293 #else 294 #define __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) do {} while (0) 295 #endif 296 297 #define SCX_OPS_ATTACH(__skel, __ops_name, __scx_name) ({ \ 298 struct bpf_link *__link; \ 299 __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name); \ 300 SCX_BUG_ON(__scx_name##__attach((__skel)), "Failed to attach skel"); \ 301 __link = bpf_map__attach_struct_ops((__skel)->maps.__ops_name); \ 302 SCX_BUG_ON(!__link, "Failed to attach struct_ops"); \ 303 __link; \ 304 }) 305 306 #endif /* __SCX_COMPAT_H */ 307