1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. 4 * Copyright (c) 2022 Tejun Heo <tj@kernel.org> 5 * Copyright (c) 2022 David Vernet <dvernet@meta.com> 6 */ 7 #ifndef __SCX_COMMON_BPF_H 8 #define __SCX_COMMON_BPF_H 9 10 /* 11 * The generated kfunc prototypes in vmlinux.h are missing address space 12 * attributes which cause build failures. For now, suppress the generated 13 * prototypes. See https://github.com/sched-ext/scx/issues/1111. 14 */ 15 #define BPF_NO_KFUNC_PROTOTYPES 16 17 #ifdef LSP 18 #define __bpf__ 19 #include "../vmlinux.h" 20 #else 21 #include "vmlinux.h" 22 #endif 23 24 #include <bpf/bpf_helpers.h> 25 #include <bpf/bpf_tracing.h> 26 #include <asm-generic/errno.h> 27 #include "user_exit_info.bpf.h" 28 #include "enum_defs.autogen.h" 29 #include "bpf_arena_common.bpf.h" 30 #include "const-defs.h" 31 32 #define PF_IDLE 0x00000002 /* I am an IDLE thread */ 33 #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ 34 #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ 35 #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ 36 #define PF_EXITING 0x00000004 37 #define CLOCK_MONOTONIC 1 38 39 #ifndef NR_CPUS 40 #define NR_CPUS 1024 41 #endif 42 43 #ifndef NUMA_NO_NODE 44 #define NUMA_NO_NODE (-1) 45 #endif 46 47 extern int LINUX_KERNEL_VERSION __kconfig; 48 extern const char CONFIG_CC_VERSION_TEXT[64] __kconfig __weak; 49 extern const char CONFIG_LOCALVERSION[64] __kconfig __weak; 50 extern bool CONFIG_PREEMPT_RCU __kconfig __weak; 51 52 /* 53 * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can 54 * lead to really confusing misbehaviors. Let's trigger a build failure. 55 */ 56 static inline void ___vmlinux_h_sanity_check___(void) 57 { 58 _Static_assert(SCX_DSQ_FLAG_BUILTIN, 59 "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); 60 } 61 62 s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; 63 s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym; 64 s32 __scx_bpf_select_cpu_and(struct task_struct *p, const struct cpumask *cpus_allowed, 65 struct scx_bpf_select_cpu_and_args *args) __ksym __weak; 66 bool __scx_bpf_dsq_insert_vtime(struct task_struct *p, struct scx_bpf_dsq_insert_vtime_args *args) __ksym __weak; 67 u32 scx_bpf_dispatch_nr_slots(void) __ksym; 68 void scx_bpf_dispatch_cancel(void) __ksym; 69 void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; 70 s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; 71 void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; 72 struct task_struct *scx_bpf_dsq_peek(u64 dsq_id) __ksym __weak; 73 int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak; 74 struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak; 75 void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak; 76 bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it, struct task_struct *p) __ksym __weak; 77 void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak; 78 void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; 79 void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak; 80 u32 scx_bpf_cpuperf_cap(s32 cpu) __ksym __weak; 81 u32 scx_bpf_cpuperf_cur(s32 cpu) __ksym __weak; 82 void scx_bpf_cpuperf_set(s32 cpu, u32 perf) __ksym __weak; 83 u32 scx_bpf_nr_node_ids(void) __ksym __weak; 84 u32 scx_bpf_nr_cpu_ids(void) __ksym __weak; 85 int scx_bpf_cpu_node(s32 cpu) __ksym __weak; 86 const struct cpumask *scx_bpf_get_possible_cpumask(void) __ksym __weak; 87 const struct cpumask *scx_bpf_get_online_cpumask(void) __ksym __weak; 88 void scx_bpf_put_cpumask(const struct cpumask *cpumask) __ksym __weak; 89 const struct cpumask *scx_bpf_get_idle_cpumask_node(int node) __ksym __weak; 90 const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; 91 const struct cpumask *scx_bpf_get_idle_smtmask_node(int node) __ksym __weak; 92 const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; 93 void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; 94 bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; 95 s32 scx_bpf_pick_idle_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags) __ksym __weak; 96 s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; 97 s32 scx_bpf_pick_any_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags) __ksym __weak; 98 s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; 99 bool scx_bpf_task_running(const struct task_struct *p) __ksym; 100 s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; 101 struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym __weak; 102 struct rq *scx_bpf_locked_rq(void) __ksym; 103 struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak; 104 struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak; 105 u64 scx_bpf_now(void) __ksym __weak; 106 void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak; 107 u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp) __ksym __weak; 108 s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; 109 s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; 110 s32 scx_bpf_cid_node(s32 cid) __ksym __weak; 111 /* scx_bpf_cid_topo() and scx_bpf_kick_cid() are declared in compat.bpf.h */ 112 s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; 113 s32 scx_bpf_this_cid(void) __ksym __weak; 114 struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak; 115 u32 scx_bpf_nr_cids(void) __ksym __weak; 116 u32 scx_bpf_nr_online_cids(void) __ksym __weak; 117 const void __arena *scx_bpf_online_cmask(void) __ksym __weak; 118 u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak; 119 u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; 120 /* scx_bpf_cidperf_set() is declared in compat.bpf.h */ 121 122 /* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */ 123 s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena, struct scx_cmask __arena *denied_out__arena__nullable) __ksym __weak; 124 void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena) __ksym __weak; 125 s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask __arena *out__arena) __ksym __weak; 126 s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak; 127 128 /* 129 * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from 130 * within bpf_for_each() loops. 131 */ 132 #define BPF_FOR_EACH_ITER (&___it) 133 134 #define scx_read_event(e, name) \ 135 (bpf_core_field_exists((e)->name) ? (e)->name : 0) 136 137 static inline __attribute__((format(printf, 1, 2))) 138 void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {} 139 140 #define SCX_STRINGIFY(x) #x 141 #define SCX_TOSTRING(x) SCX_STRINGIFY(x) 142 143 /* 144 * Helper macro for initializing the fmt and variadic argument inputs to both 145 * bstr exit kfuncs. Callers to this function should use ___fmt and ___param to 146 * refer to the initialized list of inputs to the bstr kfunc. 147 */ 148 #define scx_bpf_bstr_preamble(fmt, args...) \ 149 static char ___fmt[] = fmt; \ 150 /* \ 151 * Note that __param[] must have at least one \ 152 * element to keep the verifier happy. \ 153 */ \ 154 unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ 155 \ 156 _Pragma("GCC diagnostic push") \ 157 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ 158 ___bpf_fill(___param, args); \ 159 _Pragma("GCC diagnostic pop") 160 161 /* 162 * scx_bpf_exit() wraps the scx_bpf_exit_bstr() kfunc with variadic arguments 163 * instead of an array of u64. Using this macro will cause the scheduler to 164 * exit cleanly with the specified exit code being passed to user space. 165 */ 166 #define scx_bpf_exit(code, fmt, args...) \ 167 ({ \ 168 scx_bpf_bstr_preamble(fmt, args) \ 169 scx_bpf_exit_bstr(code, ___fmt, ___param, sizeof(___param)); \ 170 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 171 }) 172 173 /* 174 * scx_bpf_sub_kill() wraps the scx_bpf_sub_kill_bstr() kfunc with variadic 175 * arguments instead of an array of u64. It kills the direct child sub-scheduler 176 * @cgid, passing the formatted reason to its user space, and evaluates to the 177 * kfunc's return value. On a kernel without sub-scheduler support the kfunc is 178 * absent and it returns -EOPNOTSUPP. 179 */ 180 #define scx_bpf_sub_kill(cgid, fmt, args...) \ 181 ({ \ 182 scx_bpf_bstr_preamble(fmt, args) \ 183 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 184 bpf_ksym_exists(scx_bpf_sub_kill_bstr) ? \ 185 scx_bpf_sub_kill_bstr((cgid), ___fmt, ___param, \ 186 sizeof(___param)) : -EOPNOTSUPP; \ 187 }) 188 189 /* 190 * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments 191 * instead of an array of u64. Invoking this macro will cause the scheduler to 192 * exit in an erroneous state, with diagnostic information being passed to the 193 * user. It appends the file and line number to aid debugging. 194 */ 195 #define scx_bpf_error(fmt, args...) \ 196 ({ \ 197 scx_bpf_bstr_preamble( \ 198 __FILE__ ":" SCX_TOSTRING(__LINE__) ": " fmt, ##args) \ 199 scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ 200 ___scx_bpf_bstr_format_checker( \ 201 __FILE__ ":" SCX_TOSTRING(__LINE__) ": " fmt, ##args); \ 202 }) 203 204 /* 205 * scx_bpf_dump() wraps the scx_bpf_dump_bstr() kfunc with variadic arguments 206 * instead of an array of u64. To be used from ops.dump() and friends. 207 */ 208 #define scx_bpf_dump(fmt, args...) \ 209 ({ \ 210 scx_bpf_bstr_preamble(fmt, args) \ 211 scx_bpf_dump_bstr(___fmt, ___param, sizeof(___param)); \ 212 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 213 }) 214 215 /* 216 * scx_bpf_dump_header() is a wrapper around scx_bpf_dump that adds a header 217 * of system information for debugging. 218 */ 219 #define scx_bpf_dump_header() \ 220 ({ \ 221 scx_bpf_dump("kernel: %d.%d.%d %s\ncc: %s\n", \ 222 LINUX_KERNEL_VERSION >> 16, \ 223 LINUX_KERNEL_VERSION >> 8 & 0xFF, \ 224 LINUX_KERNEL_VERSION & 0xFF, \ 225 CONFIG_LOCALVERSION, \ 226 CONFIG_CC_VERSION_TEXT); \ 227 }) 228 229 #define BPF_STRUCT_OPS(name, args...) \ 230 SEC("struct_ops/"#name) \ 231 BPF_PROG(name, ##args) 232 233 #define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ 234 SEC("struct_ops.s/"#name) \ 235 BPF_PROG(name, ##args) 236 237 /** 238 * RESIZABLE_ARRAY - Generates annotations for an array that may be resized 239 * @elfsec: the data section of the BPF program in which to place the array 240 * @arr: the name of the array 241 * 242 * libbpf has an API for setting map value sizes. Since data sections (i.e. 243 * bss, data, rodata) themselves are maps, a data section can be resized. If 244 * a data section has an array as its last element, the BTF info for that 245 * array will be adjusted so that length of the array is extended to meet the 246 * new length of the data section. This macro annotates an array to have an 247 * element count of one with the assumption that this array can be resized 248 * within the userspace program. It also annotates the section specifier so 249 * this array exists in a custom sub data section which can be resized 250 * independently. 251 * 252 * See RESIZE_ARRAY() for the userspace convenience macro for resizing an 253 * array declared with RESIZABLE_ARRAY(). 254 */ 255 #define RESIZABLE_ARRAY(elfsec, arr) arr[1] SEC("."#elfsec"."#arr) 256 257 /** 258 * MEMBER_VPTR - Obtain the verified pointer to a struct or array member 259 * @base: struct or array to index 260 * @member: dereferenced member (e.g. .field, [idx0][idx1], .field[idx0] ...) 261 * 262 * The verifier often gets confused by the instruction sequence the compiler 263 * generates for indexing struct fields or arrays. This macro forces the 264 * compiler to generate a code sequence which first calculates the byte offset, 265 * checks it against the struct or array size and add that byte offset to 266 * generate the pointer to the member to help the verifier. 267 * 268 * Ideally, we want to abort if the calculated offset is out-of-bounds. However, 269 * BPF currently doesn't support abort, so evaluate to %NULL instead. The caller 270 * must check for %NULL and take appropriate action to appease the verifier. To 271 * avoid confusing the verifier, it's best to check for %NULL and dereference 272 * immediately. 273 * 274 * vptr = MEMBER_VPTR(my_array, [i][j]); 275 * if (!vptr) 276 * return error; 277 * *vptr = new_value; 278 * 279 * sizeof(@base) should encompass the memory area to be accessed and thus can't 280 * be a pointer to the area. Use `MEMBER_VPTR(*ptr, .member)` instead of 281 * `MEMBER_VPTR(ptr, ->member)`. 282 */ 283 #ifndef MEMBER_VPTR 284 #define MEMBER_VPTR(base, member) (typeof((base) member) *) \ 285 ({ \ 286 u64 __base = (u64)&(base); \ 287 u64 __addr = (u64)&((base) member) - __base; \ 288 _Static_assert(sizeof(base) >= sizeof((base) member), \ 289 "@base is smaller than @member, is @base a pointer?"); \ 290 asm volatile ( \ 291 "if %0 <= %[max] goto +2\n" \ 292 "%0 = 0\n" \ 293 "goto +1\n" \ 294 "%0 += %1\n" \ 295 : "+r"(__addr) \ 296 : "r"(__base), \ 297 [max]"i"(sizeof(base) - sizeof((base) member))); \ 298 __addr; \ 299 }) 300 #endif /* MEMBER_VPTR */ 301 302 /** 303 * ARRAY_ELEM_PTR - Obtain the verified pointer to an array element 304 * @arr: array to index into 305 * @i: array index 306 * @n: number of elements in array 307 * 308 * Similar to MEMBER_VPTR() but is intended for use with arrays where the 309 * element count needs to be explicit. 310 * It can be used in cases where a global array is defined with an initial 311 * size but is intended to be resized before loading the BPF program. 312 * Without this version of the macro, MEMBER_VPTR() will use the compile time 313 * size of the array to compute the max, which will result in rejection by 314 * the verifier. 315 */ 316 #ifndef ARRAY_ELEM_PTR 317 #define ARRAY_ELEM_PTR(arr, i, n) (typeof(arr[i]) *) \ 318 ({ \ 319 u64 __base = (u64)arr; \ 320 u64 __addr = (u64)&(arr[i]) - __base; \ 321 asm volatile ( \ 322 "if %0 <= %[max] goto +2\n" \ 323 "%0 = 0\n" \ 324 "goto +1\n" \ 325 "%0 += %1\n" \ 326 : "+r"(__addr) \ 327 : "r"(__base), \ 328 [max]"r"(sizeof(arr[0]) * ((n) - 1))); \ 329 __addr; \ 330 }) 331 #endif /* ARRAY_ELEM_PTR */ 332 333 /** 334 * __sink - Hide @expr's value from the compiler and BPF verifier 335 * @expr: The expression whose value should be opacified 336 * 337 * No-op at runtime. The empty inline assembly with a read-write constraint 338 * ("+g") has two effects at compile/verify time: 339 * 340 * 1. Compiler: treats @expr as both read and written, preventing dead-code 341 * elimination and keeping @expr (and any side effects that produced it) 342 * alive. 343 * 344 * 2. BPF verifier: forgets the precise value/range of @expr ("makes it 345 * imprecise"). The verifier normally tracks exact ranges for every register 346 * and stack slot. While useful, precision means each distinct value creates a 347 * separate verifier state. Inside loops this leads to state explosion - each 348 * iteration carries different precise values so states never merge and the 349 * verifier explores every iteration individually. 350 * 351 * Example - preventing loop state explosion:: 352 * 353 * u32 nr_intersects = 0, nr_covered = 0; 354 * __sink(nr_intersects); 355 * __sink(nr_covered); 356 * bpf_for(i, 0, nr_nodes) { 357 * if (intersects(cpumask, node_mask[i])) 358 * nr_intersects++; 359 * if (covers(cpumask, node_mask[i])) 360 * nr_covered++; 361 * } 362 * 363 * Without __sink(), the verifier tracks every possible (nr_intersects, 364 * nr_covered) pair across iterations, causing "BPF program is too large". With 365 * __sink(), the values become unknown scalars so all iterations collapse into 366 * one reusable state. 367 * 368 * Example - keeping a reference alive:: 369 * 370 * struct task_struct *t = bpf_task_acquire(task); 371 * __sink(t); 372 * 373 * Follows the convention from BPF selftests (bpf_misc.h). 374 */ 375 #define __sink(expr) asm volatile ("" : "+g"(expr)) 376 377 /* 378 * BPF declarations and helpers 379 */ 380 381 /* list and rbtree */ 382 #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) 383 #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) 384 385 void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; 386 void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; 387 388 #define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) 389 #define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) 390 391 int bpf_list_push_front_impl(struct bpf_list_head *head, 392 struct bpf_list_node *node, 393 void *meta, __u64 off) __ksym; 394 #define bpf_list_push_front(head, node) bpf_list_push_front_impl(head, node, NULL, 0) 395 396 int bpf_list_push_back_impl(struct bpf_list_head *head, 397 struct bpf_list_node *node, 398 void *meta, __u64 off) __ksym; 399 #define bpf_list_push_back(head, node) bpf_list_push_back_impl(head, node, NULL, 0) 400 401 struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; 402 struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; 403 struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, 404 struct bpf_rb_node *node) __ksym; 405 int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, 406 bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b), 407 void *meta, __u64 off) __ksym; 408 #define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0) 409 410 struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; 411 412 void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym; 413 #define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL) 414 415 /* task */ 416 struct task_struct *bpf_task_from_pid(s32 pid) __ksym; 417 struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; 418 void bpf_task_release(struct task_struct *p) __ksym; 419 420 /* cgroup */ 421 struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; 422 struct cgroup *bpf_cgroup_acquire(struct cgroup *cgrp) __ksym; 423 void bpf_cgroup_release(struct cgroup *cgrp) __ksym; 424 struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; 425 426 /* css iteration */ 427 struct bpf_iter_css; 428 struct cgroup_subsys_state; 429 extern int bpf_iter_css_new(struct bpf_iter_css *it, 430 struct cgroup_subsys_state *start, 431 unsigned int flags) __weak __ksym; 432 extern struct cgroup_subsys_state * 433 bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym; 434 extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym; 435 436 /* cpumask */ 437 struct bpf_cpumask *bpf_cpumask_create(void) __ksym; 438 struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; 439 void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; 440 u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; 441 u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; 442 void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 443 void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 444 bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; 445 bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 446 bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 447 void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; 448 void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; 449 bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, 450 const struct cpumask *src2) __ksym; 451 void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, 452 const struct cpumask *src2) __ksym; 453 void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, 454 const struct cpumask *src2) __ksym; 455 bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; 456 bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; 457 bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; 458 bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; 459 bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; 460 void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; 461 u32 bpf_cpumask_any_distribute(const struct cpumask *cpumask) __ksym; 462 u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, 463 const struct cpumask *src2) __ksym; 464 u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym; 465 466 int bpf_iter_bits_new(struct bpf_iter_bits *it, const u64 *unsafe_ptr__ign, u32 nr_words) __ksym; 467 int *bpf_iter_bits_next(struct bpf_iter_bits *it) __ksym; 468 void bpf_iter_bits_destroy(struct bpf_iter_bits *it) __ksym; 469 470 #define def_iter_struct(name) \ 471 struct bpf_iter_##name { \ 472 struct bpf_iter_bits it; \ 473 const struct cpumask *bitmap; \ 474 }; 475 476 #define def_iter_new(name) \ 477 static inline int bpf_iter_##name##_new( \ 478 struct bpf_iter_##name *it, const u64 *unsafe_ptr__ign, u32 nr_words) \ 479 { \ 480 it->bitmap = scx_bpf_get_##name##_cpumask(); \ 481 return bpf_iter_bits_new(&it->it, (const u64 *)it->bitmap, \ 482 sizeof(struct cpumask) / 8); \ 483 } 484 485 #define def_iter_next(name) \ 486 static inline int *bpf_iter_##name##_next(struct bpf_iter_##name *it) { \ 487 return bpf_iter_bits_next(&it->it); \ 488 } 489 490 #define def_iter_destroy(name) \ 491 static inline void bpf_iter_##name##_destroy(struct bpf_iter_##name *it) { \ 492 scx_bpf_put_cpumask(it->bitmap); \ 493 bpf_iter_bits_destroy(&it->it); \ 494 } 495 #define def_for_each_cpu(cpu, name) for_each_##name##_cpu(cpu) 496 497 /// Provides iterator for possible and online cpus. 498 /// 499 /// # Example 500 /// 501 /// ``` 502 /// static inline void example_use() { 503 /// int *cpu; 504 /// 505 /// for_each_possible_cpu(cpu){ 506 /// bpf_printk("CPU %d is possible", *cpu); 507 /// } 508 /// 509 /// for_each_online_cpu(cpu){ 510 /// bpf_printk("CPU %d is online", *cpu); 511 /// } 512 /// } 513 /// ``` 514 def_iter_struct(possible); 515 def_iter_new(possible); 516 def_iter_next(possible); 517 def_iter_destroy(possible); 518 #define for_each_possible_cpu(cpu) bpf_for_each(possible, cpu, NULL, 0) 519 520 def_iter_struct(online); 521 def_iter_new(online); 522 def_iter_next(online); 523 def_iter_destroy(online); 524 #define for_each_online_cpu(cpu) bpf_for_each(online, cpu, NULL, 0) 525 526 /* 527 * Access a cpumask in read-only mode (typically to check bits). 528 */ 529 static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask) 530 { 531 return (const struct cpumask *)mask; 532 } 533 534 /* 535 * True if the non-sleepable BPF trampoline prolog (__bpf_prog_enter) calls 536 * migrate_disable() for the current task. Recorded once by 537 * scx_lib_init_probe, an fentry program that fires during the natural 538 * scheduler-attach call chain (auto-attached by scx_ops_attach!). 539 * 540 * Defaults to false (conservative). Over-reporting in is_migration_disabled() 541 * causes local-only dispatch, which is safe. Under-reporting can crash the 542 * scheduler, so we err high if the probe somehow fails to run. 543 */ 544 bool __scx_prolog_disables_migration __weak = false; 545 546 /* 547 * scx_lib_init_probe - non-sleepable prolog probe. 548 * 549 * Attached to bpf_scx_reg(), the .reg callback in bpf_sched_ext_ops 550 * (kernel/sched/ext.c). The kernel's struct_ops machinery invokes 551 * bpf_scx_reg when userspace creates the scheduler link, before 552 * ops.init() fires. Its address is taken in the vtable, so the symbol 553 * is non-inlinable and has been stable since introduction. 554 * 555 * A cid-form scheduler registers through bpf_scx_reg_cid(), the .reg 556 * callback of bpf_sched_ext_ops_cid, and so never enters bpf_scx_reg(). 557 * The cid-form open paths (SCX_OPS_CID_OPEN(), scx_ops_cid_open!()) 558 * therefore repoint this program at bpf_scx_reg_cid(). Repointing rather 559 * than adding a second program keeps the object loadable on kernels 560 * predating the cid form, where bpf_scx_reg_cid() has no BTF entry. 561 * 562 * Entering via fentry runs us through __bpf_prog_enter -- the 563 * non-sleepable prolog that consumers of is_migration_disabled() live 564 * under. 565 * 566 * Loud warning: the prolog adds at most 1 to migration_disabled. 567 * Reading > 1 means something upstream in the 568 * bpf_struct_ops_link_create -> .reg path disabled migration 569 * before the prolog ran, invalidating the probe; audit and adjust. 570 */ 571 SEC("fentry/bpf_scx_reg") __weak 572 int scx_lib_init_probe(void *ctx) 573 { 574 if (bpf_core_field_exists(((struct task_struct *)0)->migration_disabled)) { 575 const struct task_struct *p = bpf_get_current_task_btf(); 576 unsigned int md = p->migration_disabled; 577 578 if (md > 1) 579 bpf_printk("scx_lib_init_probe: unexpected migration_disabled=%u " 580 "upstream of BPF prolog; probe result unreliable", 581 md); 582 583 __scx_prolog_disables_migration = md > 0; 584 } 585 return 0; 586 } 587 588 /* 589 * Return true if task @p cannot migrate to a different CPU, false 590 * otherwise. 591 * 592 * IMPORTANT: designed for NON-SLEEPABLE BPF contexts only. Sleepable 593 * contexts (BPF_STRUCT_OPS_SLEEPABLE, SEC("syscall"), 594 * SEC("fentry.s/...")) enter via __bpf_prog_enter_sleepable() or 595 * __bpf_prog_enter_sleepable_recur(), both of which unconditionally 596 * call migrate_disable(); this helper can yield a false negative for 597 * p == current there, which can crash the scheduler. 598 */ 599 static inline bool is_migration_disabled(const struct task_struct *p) 600 { 601 /* 602 * Testing p->migration_disabled in BPF is tricky because the BPF prolog 603 * (__bpf_prog_enter) may call migrate_disable() for the current task, 604 * making migration_disabled == 1 even for tasks that are not truly 605 * migration-disabled. 606 * 607 * Since commit 8e4f0b1ebcf2 ("bpf: use rcu_read_lock_dont_migrate() for 608 * trampoline.c"), the BPF prolog calls migrate_disable() only when 609 * CONFIG_PREEMPT_RCU is enabled. Two fast paths cover the common cases: 610 * 611 * 1) CONFIG_PREEMPT_RCU: prolog always calls migrate_disable(), so 612 * migration_disabled == 1 for the current task is ambiguous. 613 * Disambiguate by checking p == current. 614 * 615 * 2) v6.18+ without CONFIG_PREEMPT_RCU: prolog never calls 616 * migrate_disable(), so migration_disabled == 1 is unambiguously 617 * a real migrate_disable() call. 618 * 619 * A slow path handles pre-v6.18 kernels without CONFIG_PREEMPT_RCU, 620 * where the prolog historically called migrate_disable() unconditionally 621 * but a cherry-picked downstream kernel may not. The runtime-probed flag 622 * __scx_prolog_disables_migration (set by scx_lib_init_probe) distinguishes 623 * the two cases without relying on the kernel version alone. 624 */ 625 if (bpf_core_field_exists(p->migration_disabled)) { 626 if (p->migration_disabled == 1) { 627 /* Fast path: prolog always disables migration */ 628 if (CONFIG_PREEMPT_RCU) 629 return bpf_get_current_task_btf() != p; 630 /* Fast path: prolog never disables migration */ 631 if (LINUX_KERNEL_VERSION >= KERNEL_VERSION(6, 18, 0)) 632 return true; 633 /* Slow path: pre-v6.18, !PREEMPT_RCU - use runtime flag */ 634 return __scx_prolog_disables_migration ? 635 bpf_get_current_task_btf() != p : true; 636 } 637 return p->migration_disabled; 638 } 639 return false; 640 } 641 642 /* rcu */ 643 void bpf_rcu_read_lock(void) __ksym; 644 void bpf_rcu_read_unlock(void) __ksym; 645 646 /* resilient qspinlock */ 647 int bpf_res_spin_lock(struct bpf_res_spin_lock *lock) __ksym __weak; 648 void bpf_res_spin_unlock(struct bpf_res_spin_lock *lock) __ksym __weak; 649 650 /* 651 * Time helpers, most of which are from jiffies.h. 652 */ 653 654 /** 655 * time_delta - Calculate the delta between new and old time stamp 656 * @after: first comparable as u64 657 * @before: second comparable as u64 658 * 659 * Return: the time difference, which is >= 0 660 */ 661 static inline s64 time_delta(u64 after, u64 before) 662 { 663 return (s64)(after - before) > 0 ? (s64)(after - before) : 0; 664 } 665 666 /** 667 * time_after - returns true if the time a is after time b. 668 * @a: first comparable as u64 669 * @b: second comparable as u64 670 * 671 * Do this with "<0" and ">=0" to only test the sign of the result. A 672 * good compiler would generate better code (and a really good compiler 673 * wouldn't care). Gcc is currently neither. 674 * 675 * Return: %true is time a is after time b, otherwise %false. 676 */ 677 static inline bool time_after(u64 a, u64 b) 678 { 679 return (s64)(b - a) < 0; 680 } 681 682 /** 683 * time_before - returns true if the time a is before time b. 684 * @a: first comparable as u64 685 * @b: second comparable as u64 686 * 687 * Return: %true is time a is before time b, otherwise %false. 688 */ 689 static inline bool time_before(u64 a, u64 b) 690 { 691 return time_after(b, a); 692 } 693 694 /** 695 * time_after_eq - returns true if the time a is after or the same as time b. 696 * @a: first comparable as u64 697 * @b: second comparable as u64 698 * 699 * Return: %true is time a is after or the same as time b, otherwise %false. 700 */ 701 static inline bool time_after_eq(u64 a, u64 b) 702 { 703 return (s64)(a - b) >= 0; 704 } 705 706 /** 707 * time_before_eq - returns true if the time a is before or the same as time b. 708 * @a: first comparable as u64 709 * @b: second comparable as u64 710 * 711 * Return: %true is time a is before or the same as time b, otherwise %false. 712 */ 713 static inline bool time_before_eq(u64 a, u64 b) 714 { 715 return time_after_eq(b, a); 716 } 717 718 /** 719 * time_in_range - Calculate whether a is in the range of [b, c]. 720 * @a: time to test 721 * @b: beginning of the range 722 * @c: end of the range 723 * 724 * Return: %true is time a is in the range [b, c], otherwise %false. 725 */ 726 static inline bool time_in_range(u64 a, u64 b, u64 c) 727 { 728 return time_after_eq(a, b) && time_before_eq(a, c); 729 } 730 731 /** 732 * time_in_range_open - Calculate whether a is in the range of [b, c). 733 * @a: time to test 734 * @b: beginning of the range 735 * @c: end of the range 736 * 737 * Return: %true is time a is in the range [b, c), otherwise %false. 738 */ 739 static inline bool time_in_range_open(u64 a, u64 b, u64 c) 740 { 741 return time_after_eq(a, b) && time_before(a, c); 742 } 743 744 745 /* 746 * Other helpers 747 */ 748 749 /* useful compiler attributes */ 750 #ifndef likely 751 #define likely(x) __builtin_expect(!!(x), 1) 752 #endif 753 #ifndef unlikely 754 #define unlikely(x) __builtin_expect(!!(x), 0) 755 #endif 756 #ifndef __maybe_unused 757 #define __maybe_unused __attribute__((__unused__)) 758 #endif 759 760 /* 761 * READ/WRITE_ONCE() are from kernel (include/asm-generic/rwonce.h). They 762 * prevent compiler from caching, redoing or reordering reads or writes. 763 */ 764 typedef __u8 __attribute__((__may_alias__)) __u8_alias_t; 765 typedef __u16 __attribute__((__may_alias__)) __u16_alias_t; 766 typedef __u32 __attribute__((__may_alias__)) __u32_alias_t; 767 typedef __u64 __attribute__((__may_alias__)) __u64_alias_t; 768 769 static __always_inline void __read_once_size(const volatile void *p, void *res, int size) 770 { 771 switch (size) { 772 case 1: *(__u8_alias_t *) res = *(volatile __u8_alias_t *) p; break; 773 case 2: *(__u16_alias_t *) res = *(volatile __u16_alias_t *) p; break; 774 case 4: *(__u32_alias_t *) res = *(volatile __u32_alias_t *) p; break; 775 case 8: *(__u64_alias_t *) res = *(volatile __u64_alias_t *) p; break; 776 default: 777 barrier(); 778 __builtin_memcpy((void *)res, (const void *)p, size); 779 barrier(); 780 } 781 } 782 783 static __always_inline void __write_once_size(volatile void *p, void *res, int size) 784 { 785 switch (size) { 786 case 1: *(volatile __u8_alias_t *) p = *(__u8_alias_t *) res; break; 787 case 2: *(volatile __u16_alias_t *) p = *(__u16_alias_t *) res; break; 788 case 4: *(volatile __u32_alias_t *) p = *(__u32_alias_t *) res; break; 789 case 8: *(volatile __u64_alias_t *) p = *(__u64_alias_t *) res; break; 790 default: 791 barrier(); 792 __builtin_memcpy((void *)p, (const void *)res, size); 793 barrier(); 794 } 795 } 796 797 /* 798 * __unqual_typeof(x) - Declare an unqualified scalar type, leaving 799 * non-scalar types unchanged, 800 * 801 * Prefer C11 _Generic for better compile-times and simpler code. Note: 'char' 802 * is not type-compatible with 'signed char', and we define a separate case. 803 * 804 * This is copied verbatim from kernel's include/linux/compiler_types.h, but 805 * with default expression (for pointers) changed from (x) to (typeof(x)0). 806 * 807 * This is because LLVM has a bug where for lvalue (x), it does not get rid of 808 * an extra address_space qualifier, but does in case of rvalue (typeof(x)0). 809 * Hence, for pointers, we need to create an rvalue expression to get the 810 * desired type. See https://github.com/llvm/llvm-project/issues/53400. 811 */ 812 #define __scalar_type_to_expr_cases(type) \ 813 unsigned type : (unsigned type)0, signed type : (signed type)0 814 815 #define __unqual_typeof(x) \ 816 typeof(_Generic((x), \ 817 char: (char)0, \ 818 __scalar_type_to_expr_cases(char), \ 819 __scalar_type_to_expr_cases(short), \ 820 __scalar_type_to_expr_cases(int), \ 821 __scalar_type_to_expr_cases(long), \ 822 __scalar_type_to_expr_cases(long long), \ 823 default: (typeof(x))0)) 824 825 #define READ_ONCE(x) \ 826 ({ \ 827 union { __unqual_typeof(x) __val; char __c[1]; } __u = \ 828 { .__c = { 0 } }; \ 829 __read_once_size((__unqual_typeof(x) *)&(x), __u.__c, sizeof(x)); \ 830 __u.__val; \ 831 }) 832 833 #define WRITE_ONCE(x, val) \ 834 ({ \ 835 union { __unqual_typeof(x) __val; char __c[1]; } __u = \ 836 { .__val = (val) }; \ 837 __write_once_size((__unqual_typeof(x) *)&(x), __u.__c, sizeof(x)); \ 838 __u.__val; \ 839 }) 840 841 /* 842 * __calc_avg - Calculate exponential weighted moving average (EWMA) with 843 * @old and @new values. @decay represents how large the @old value remains. 844 * With a larger @decay value, the moving average changes slowly, exhibiting 845 * fewer fluctuations. 846 */ 847 #define __calc_avg(old, new, decay) ({ \ 848 typeof(decay) thr = 1 << (decay); \ 849 typeof(old) ret; \ 850 if (((old) < thr) || ((new) < thr)) { \ 851 if (((old) == 1) && ((new) == 0)) \ 852 ret = 0; \ 853 else \ 854 ret = ((old) - ((old) >> 1)) + ((new) >> 1); \ 855 } else { \ 856 ret = ((old) - ((old) >> (decay))) + ((new) >> (decay)); \ 857 } \ 858 ret; \ 859 }) 860 861 /* 862 * log2_u32 - Compute the base 2 logarithm of a 32-bit exponential value. 863 * @v: The value for which we're computing the base 2 logarithm. 864 */ 865 static inline u32 log2_u32(u32 v) 866 { 867 u32 r; 868 u32 shift; 869 870 r = (v > 0xFFFF) << 4; v >>= r; 871 shift = (v > 0xFF) << 3; v >>= shift; r |= shift; 872 shift = (v > 0xF) << 2; v >>= shift; r |= shift; 873 shift = (v > 0x3) << 1; v >>= shift; r |= shift; 874 r |= (v >> 1); 875 return r; 876 } 877 878 /* 879 * log2_u64 - Compute the base 2 logarithm of a 64-bit exponential value. 880 * @v: The value for which we're computing the base 2 logarithm. 881 */ 882 static inline u32 log2_u64(u64 v) 883 { 884 u32 hi = v >> 32; 885 if (hi) 886 return log2_u32(hi) + 32 + 1; 887 else 888 return log2_u32(v) + 1; 889 } 890 891 /* 892 * sqrt_u64 - Calculate the square root of value @x using Newton's method. 893 */ 894 static inline u64 __sqrt_u64(u64 x) 895 { 896 if (x == 0 || x == 1) 897 return x; 898 899 u64 r = ((1ULL << 32) > x) ? x : (1ULL << 32); 900 901 for (int i = 0; i < 8; ++i) { 902 u64 q = x / r; 903 if (r <= q) 904 break; 905 r = (r + q) >> 1; 906 } 907 return r; 908 } 909 910 /* 911 * ctzll -- Counts trailing zeros in an unsigned long long. If the input value 912 * is zero, the return value is undefined. 913 */ 914 static inline int ctzll(u64 v) 915 { 916 #if (!defined(__BPF__) && defined(__SCX_TARGET_ARCH_x86)) || \ 917 (defined(__BPF__) && defined(__clang_major__) && __clang_major__ >= 19) 918 /* 919 * Use the ctz builtin when: (1) building for native x86, or 920 * (2) building for BPF with clang >= 19 (BPF backend supports 921 * the intrinsic from clang 19 onward; earlier versions hit 922 * "unimplemented opcode" in the backend). 923 */ 924 return __builtin_ctzll(v); 925 #else 926 /* 927 * If neither the target architecture nor the toolchains support ctzll, 928 * use software-based emulation. Let's use the De Bruijn sequence-based 929 * approach to find LSB fastly. See the details of De Bruijn sequence: 930 * 931 * https://en.wikipedia.org/wiki/De_Bruijn_sequence 932 * https://www.chessprogramming.org/BitScan#De_Bruijn_Multiplication 933 */ 934 const int lookup_table[64] = { 935 0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4, 936 62, 55, 59, 36, 53, 51, 43, 22, 45, 39, 33, 30, 24, 18, 12, 5, 937 63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 44, 32, 23, 11, 938 46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, 939 }; 940 const u64 DEBRUIJN_CONSTANT = 0x03f79d71b4cb0a89ULL; 941 unsigned int index; 942 u64 lowest_bit; 943 const int *lt; 944 945 if (v == 0) 946 return -1; 947 948 /* 949 * Isolate the least significant bit (LSB). 950 * For example, if v = 0b...10100, then v & -v = 0b...00100 951 */ 952 lowest_bit = v & -v; 953 954 /* 955 * Each isolated bit produces a unique 6-bit value, guaranteed by the 956 * De Bruijn property. Calculate a unique index into the lookup table 957 * using the magic constant and a right shift. 958 * 959 * Multiplying by the 64-bit constant "spreads out" that 1-bit into a 960 * unique pattern in the top 6 bits. This uniqueness property is 961 * exactly what a De Bruijn sequence guarantees: Every possible 6-bit 962 * pattern (in top bits) occurs exactly once for each LSB position. So, 963 * the constant 0x03f79d71b4cb0a89ULL is carefully chosen to be a 964 * De Bruijn sequence, ensuring no collisions in the table index. 965 */ 966 index = (lowest_bit * DEBRUIJN_CONSTANT) >> 58; 967 968 /* 969 * Lookup in a precomputed table. No collision is guaranteed by the 970 * De Bruijn property. 971 */ 972 lt = MEMBER_VPTR(lookup_table, [index]); 973 return (lt)? *lt : -1; 974 #endif 975 } 976 977 /* 978 * Return a value proportionally scaled to the task's weight. 979 */ 980 static inline u64 scale_by_task_weight(const struct task_struct *p, u64 value) 981 { 982 return (value * p->scx.weight) / 100; 983 } 984 985 /* 986 * Return a value inversely proportional to the task's weight. 987 */ 988 static inline u64 scale_by_task_weight_inverse(const struct task_struct *p, u64 value) 989 { 990 return value * 100 / p->scx.weight; 991 } 992 993 994 /* 995 * Get a random u64 from the kernel's pseudo-random generator. 996 */ 997 static inline u64 get_prandom_u64() 998 { 999 return ((u64)bpf_get_prandom_u32() << 32) | bpf_get_prandom_u32(); 1000 } 1001 1002 /* 1003 * Define the shadow structure to avoid a compilation error when 1004 * vmlinux.h does not enable necessary kernel configs. The ___local 1005 * suffix is a CO-RE convention that tells the loader to match this 1006 * against the base struct rq in the kernel. The attribute 1007 * preserve_access_index tells the compiler to generate a CO-RE 1008 * relocation for these fields. 1009 */ 1010 struct rq___local { 1011 /* 1012 * A monotonically increasing clock per CPU. It is rq->clock minus 1013 * cumulative IRQ time and hypervisor steal time. Unlike rq->clock, 1014 * it does not advance during IRQ processing or hypervisor preemption. 1015 * It does advance during idle (the idle task counts as a running task 1016 * for this purpose). 1017 */ 1018 u64 clock_task; 1019 /* 1020 * Invariant version of clock_task scaled by CPU capacity and 1021 * frequency. For example, clock_pelt advances 2x slower on a CPU 1022 * with half the capacity. 1023 * 1024 * At idle exit, rq->clock_pelt jumps forward to resync with 1025 * clock_task. The kernel's rq_clock_pelt() corrects for this jump 1026 * by subtracting lost_idle_time, yielding a clock that appears 1027 * continuous across idle transitions. scx_clock_pelt() mirrors 1028 * rq_clock_pelt() by performing the same subtraction. 1029 */ 1030 u64 clock_pelt; 1031 /* 1032 * Accumulates the magnitude of each clock_pelt jump at idle exit. 1033 * Subtracting this from clock_pelt gives rq_clock_pelt(): a 1034 * continuous, capacity-invariant clock suitable for both task 1035 * execution time stamping and cross-idle measurements. 1036 */ 1037 unsigned long lost_idle_time; 1038 /* 1039 * Shadow of paravirt_steal_clock() (the hypervisor's cumulative 1040 * stolen time counter). Stays frozen while the hypervisor preempts 1041 * the vCPU; catches up the next time update_rq_clock_task() is 1042 * called. The delta is the stolen time not yet subtracted from 1043 * clock_task. 1044 * 1045 * Unlike irqtime->total (a plain kernel-side field), the live stolen 1046 * time counter lives in hypervisor-specific shared memory and has no 1047 * kernel-side equivalent readable from BPF in a hypervisor-agnostic 1048 * way. This field is therefore the only portable BPF-accessible 1049 * approximation of cumulative steal time. 1050 * 1051 * Available only when CONFIG_PARAVIRT_TIME_ACCOUNTING is on. 1052 */ 1053 u64 prev_steal_time_rq; 1054 } __attribute__((preserve_access_index)); 1055 1056 extern struct rq runqueues __ksym; 1057 1058 /* 1059 * Define the shadow structure to avoid a compilation error when 1060 * vmlinux.h does not enable necessary kernel configs. 1061 */ 1062 struct irqtime___local { 1063 /* 1064 * Cumulative IRQ time counter for this CPU, in nanoseconds. Advances 1065 * immediately at the exit of every hardirq and non-ksoftirqd softirq 1066 * via irqtime_account_irq(). ksoftirqd time is counted as normal 1067 * task time and is NOT included. NMI time is also NOT included. 1068 * 1069 * The companion field irqtime->sync (struct u64_stats_sync) protects 1070 * against 64-bit tearing on 32-bit architectures. On 64-bit kernels, 1071 * u64_stats_sync is an empty struct and all seqcount operations are 1072 * no-ops, so a plain BPF_CORE_READ of this field is safe. 1073 * 1074 * Available only when CONFIG_IRQ_TIME_ACCOUNTING is on. 1075 */ 1076 u64 total; 1077 } __attribute__((preserve_access_index)); 1078 1079 /* 1080 * cpu_irqtime is a per-CPU variable defined only when 1081 * CONFIG_IRQ_TIME_ACCOUNTING is on. Declare it as __weak so the BPF 1082 * loader sets its address to 0 (rather than failing) when the symbol 1083 * is absent from the running kernel. 1084 */ 1085 extern struct irqtime___local cpu_irqtime __ksym __weak; 1086 1087 static inline struct rq___local *get_current_rq(u32 cpu) 1088 { 1089 /* 1090 * This is a workaround to get an rq pointer now that 1091 * scx_bpf_cpu_rq() has been removed. 1092 * 1093 * WARNING: The caller must hold the rq lock for @cpu. This is 1094 * guaranteed when called from scheduling callbacks (ops.running, 1095 * ops.stopping, ops.enqueue, ops.dequeue, ops.dispatch, etc.). 1096 * There is no runtime check available in BPF for kernel spinlock 1097 * state — correctness is enforced by calling context only. 1098 */ 1099 return (void *)bpf_per_cpu_ptr(&runqueues, cpu); 1100 } 1101 1102 static inline u64 scx_clock_task(u32 cpu) 1103 { 1104 struct rq___local *rq = get_current_rq(cpu); 1105 1106 /* 1107 * Equivalent to the kernel's rq_clock_task(): wall-clock time minus 1108 * cumulative IRQ time (CONFIG_IRQ_TIME_ACCOUNTING) and hypervisor 1109 * steal time (CONFIG_PARAVIRT_TIME_ACCOUNTING). Without those configs, 1110 * it equals rq->clock. 1111 * 1112 * Conceptually this clock advances during idle (the idle task counts 1113 * as a running task), but rq->clock_task is only updated on scheduling 1114 * events. With NO_HZ_IDLE (the default), the periodic tick is stopped 1115 * on idle CPUs, so rq->clock_task is not refreshed while a CPU is 1116 * idle. Reading this clock for a remote idle CPU from a BPF timer 1117 * callback returns the value from when the CPU last went idle, making 1118 * the delta over an idle interval effectively zero. 1119 */ 1120 return rq ? rq->clock_task : 0; 1121 } 1122 1123 static inline u64 scx_clock_pelt(u32 cpu) 1124 { 1125 struct rq___local *rq = get_current_rq(cpu); 1126 1127 /* 1128 * Equivalent to the kernel's rq_clock_pelt(): subtracts 1129 * lost_idle_time from clock_pelt to absorb the jump that occurs 1130 * when clock_pelt resyncs with clock_task at idle exit. The intent 1131 * is a continuous, capacity- and frequency-invariant clock that is 1132 * frozen during idle, IRQ, and hypervisor steal. 1133 * 1134 * However, like scx_clock_task(), this clock has a stale-read issue 1135 * for remote idle CPUs with NO_HZ_IDLE (the default). clock_pelt 1136 * itself advances at wall-clock rate (hardware-clock based), but 1137 * lost_idle_time is only updated via update_rq_clock_pelt(), which 1138 * requires update_rq_clock() to be called. With NO_HZ_IDLE, the 1139 * periodic tick is stopped on idle CPUs, so lost_idle_time is not 1140 * refreshed during idle. Reading this clock for a remote idle CPU 1141 * from a BPF timer callback therefore returns a value that drifts 1142 * at wall-clock rate -- the same stale behaviour as scx_clock_task(). 1143 * 1144 * Without NO_HZ_IDLE, periodic ticks keep lost_idle_time nearly in 1145 * sync (stale by at most one tick period, ~1 ms), so the result is 1146 * accurate. 1147 */ 1148 return rq ? (rq->clock_pelt - rq->lost_idle_time) : 0; 1149 } 1150 1151 static inline u64 scx_clock_virt(u32 cpu) 1152 { 1153 struct rq___local *rq; 1154 1155 /* 1156 * Check field existence before calling get_current_rq() so we avoid 1157 * the per_cpu lookup entirely on kernels built without 1158 * CONFIG_PARAVIRT_TIME_ACCOUNTING. 1159 */ 1160 if (!bpf_core_field_exists(((struct rq___local *)0)->prev_steal_time_rq)) 1161 return 0; 1162 1163 /* Lagging shadow of the kernel's paravirt_steal_clock(). */ 1164 rq = get_current_rq(cpu); 1165 return rq ? BPF_CORE_READ(rq, prev_steal_time_rq) : 0; 1166 } 1167 1168 static inline u64 scx_clock_irq(u32 cpu) 1169 { 1170 struct irqtime___local *irqt; 1171 1172 /* 1173 * bpf_core_type_exists() resolves at load time: if struct irqtime is 1174 * absent from kernel BTF (CONFIG_IRQ_TIME_ACCOUNTING off), the loader 1175 * patches this into an unconditional return 0, making the 1176 * bpf_per_cpu_ptr() call below dead code that the verifier never sees. 1177 */ 1178 if (!bpf_core_type_exists(struct irqtime___local)) 1179 return 0; 1180 1181 /* Equivalent to the kernel's irq_time_read(). */ 1182 irqt = bpf_per_cpu_ptr(&cpu_irqtime, cpu); 1183 return irqt ? BPF_CORE_READ(irqt, total) : 0; 1184 } 1185 1186 /* Abbreviated forms of <linux/overflow.h>'s struct_size() family. */ 1187 #define flex_array_size(p, member, count) \ 1188 ((count) * sizeof(*(p)->member)) 1189 1190 #define struct_size(p, member, count) \ 1191 (offsetof(typeof(*(p)), member) + flex_array_size(p, member, count)) 1192 1193 #define struct_size_t(type, member, count) \ 1194 struct_size((type *)NULL, member, count) 1195 1196 /* 1197 * <linux/stddef.h>'s TRAILING_OVERLAP(): embed a struct that ends in a flexible 1198 * array member together with its storage. @NAME is a complete @TYPE and the 1199 * @MEMBERS declared after the array reserve the space the array grows into. 1200 * Unlike the kernel's, the padding is named after @NAME so that one struct can 1201 * embed several, and it is sized with __builtin_offsetof() because 1202 * bpf_helpers.h redefines offsetof() as a pointer cast, which is not a constant 1203 * expression. 1204 */ 1205 #define __TRAILING_OVERLAP(TYPE, NAME, FAM, ATTRS, MEMBERS) \ 1206 union { \ 1207 TYPE NAME; \ 1208 struct { \ 1209 unsigned char __offset_to_##NAME[__builtin_offsetof(TYPE, FAM)]; \ 1210 MEMBERS \ 1211 } ATTRS; \ 1212 } 1213 1214 #define TRAILING_OVERLAP(TYPE, NAME, FAM, MEMBERS) \ 1215 __TRAILING_OVERLAP(TYPE, NAME, FAM, /* no attrs */, MEMBERS) 1216 1217 /* 1218 * Loop counters the verifier cannot see through. 1219 * 1220 * A may_goto loop converges when the state at its head is within the state 1221 * of a previous iteration. A counter tracked as a precise constant prevents 1222 * that convergence: every iteration creates a new state, and the verifier 1223 * unrolls the loop until it runs out of budget whenever the counter feeds an 1224 * operation that requires precision. 1225 * 1226 * Loading a writable global gives the verifier an unknown scalar while its 1227 * runtime value remains one. Using it as the step makes the counter unknown 1228 * after the first iteration, and the load on every increment keeps the loop 1229 * body from making it precise again. volatile is required to keep the compiler 1230 * from hoisting or eliminating the loads, and the value must remain non-const 1231 * so the verifier cannot resolve it. The variable is weak so the objects linked 1232 * into a scheduler share one copy. Its runtime value must never be changed from 1233 * one. 1234 * 1235 * bpf_for() avoids this verifier behavior too, but calls bpf_iter_num_next() 1236 * on every iteration. bpf_arena_for() is intended for hot scheduler walks 1237 * where that cost matters. @var must be no wider than u32, and @start and 1238 * @end must be representable as u32. 1239 */ 1240 volatile u32 bpf_arena_loop_one __weak = 1; 1241 1242 /* 1243 * A loop-carried accumulator that a caller later compares is kept precise, and 1244 * a precise scalar whose range grows every iteration keeps a may_goto loop from 1245 * converging. Initializing it from this zero makes it unknown from the first 1246 * iteration, so every pass through the loop head looks the same. 1247 */ 1248 volatile u32 bpf_arena_loop_zero __weak = 0; 1249 1250 #define __bpf_arena_loop_start(var, start) \ 1251 ({ \ 1252 _Static_assert(sizeof(var) <= sizeof(u32), \ 1253 "bpf_arena_for() index must fit in u32"); \ 1254 (start); \ 1255 }) 1256 1257 #define bpf_arena_for(var, start, end) \ 1258 for (var = __bpf_arena_loop_start(var, start); \ 1259 var < (end) && can_loop; \ 1260 var += bpf_arena_loop_one) 1261 1262 #include "compat.bpf.h" 1263 #include "enums.bpf.h" 1264 #include "features.bpf.h" 1265 #include "cid.bpf.h" 1266 1267 #endif /* __SCX_COMMON_BPF_H */ 1268