1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. 4 * Copyright (c) 2022 Tejun Heo <tj@kernel.org> 5 * Copyright (c) 2022 David Vernet <dvernet@meta.com> 6 */ 7 #ifndef __SCX_COMMON_BPF_H 8 #define __SCX_COMMON_BPF_H 9 10 /* 11 * The generated kfunc prototypes in vmlinux.h are missing address space 12 * attributes which cause build failures. For now, suppress the generated 13 * prototypes. See https://github.com/sched-ext/scx/issues/1111. 14 */ 15 #define BPF_NO_KFUNC_PROTOTYPES 16 17 #ifdef LSP 18 #define __bpf__ 19 #include "../vmlinux.h" 20 #else 21 #include "vmlinux.h" 22 #endif 23 24 #include <bpf/bpf_helpers.h> 25 #include <bpf/bpf_tracing.h> 26 #include <asm-generic/errno.h> 27 #include "user_exit_info.bpf.h" 28 #include "enum_defs.autogen.h" 29 30 #define PF_IDLE 0x00000002 /* I am an IDLE thread */ 31 #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ 32 #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ 33 #define PF_KCOMPACTD 0x00010000 /* I am kcompactd */ 34 #define PF_KSWAPD 0x00020000 /* I am kswapd */ 35 #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ 36 #define PF_EXITING 0x00000004 37 #define CLOCK_MONOTONIC 1 38 39 #ifndef NR_CPUS 40 #define NR_CPUS 1024 41 #endif 42 43 #ifndef NUMA_NO_NODE 44 #define NUMA_NO_NODE (-1) 45 #endif 46 47 extern int LINUX_KERNEL_VERSION __kconfig; 48 extern const char CONFIG_CC_VERSION_TEXT[64] __kconfig __weak; 49 extern const char CONFIG_LOCALVERSION[64] __kconfig __weak; 50 51 /* 52 * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can 53 * lead to really confusing misbehaviors. Let's trigger a build failure. 54 */ 55 static inline void ___vmlinux_h_sanity_check___(void) 56 { 57 _Static_assert(SCX_DSQ_FLAG_BUILTIN, 58 "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); 59 } 60 61 s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; 62 s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym; 63 s32 __scx_bpf_select_cpu_and(struct task_struct *p, const struct cpumask *cpus_allowed, 64 struct scx_bpf_select_cpu_and_args *args) __ksym __weak; 65 bool __scx_bpf_dsq_insert_vtime(struct task_struct *p, struct scx_bpf_dsq_insert_vtime_args *args) __ksym __weak; 66 u32 scx_bpf_dispatch_nr_slots(void) __ksym; 67 void scx_bpf_dispatch_cancel(void) __ksym; 68 void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; 69 s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; 70 void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; 71 struct task_struct *scx_bpf_dsq_peek(u64 dsq_id) __ksym __weak; 72 int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak; 73 struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak; 74 void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak; 75 void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak; 76 void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; 77 void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak; 78 u32 scx_bpf_cpuperf_cap(s32 cpu) __ksym __weak; 79 u32 scx_bpf_cpuperf_cur(s32 cpu) __ksym __weak; 80 void scx_bpf_cpuperf_set(s32 cpu, u32 perf) __ksym __weak; 81 u32 scx_bpf_nr_node_ids(void) __ksym __weak; 82 u32 scx_bpf_nr_cpu_ids(void) __ksym __weak; 83 int scx_bpf_cpu_node(s32 cpu) __ksym __weak; 84 const struct cpumask *scx_bpf_get_possible_cpumask(void) __ksym __weak; 85 const struct cpumask *scx_bpf_get_online_cpumask(void) __ksym __weak; 86 void scx_bpf_put_cpumask(const struct cpumask *cpumask) __ksym __weak; 87 const struct cpumask *scx_bpf_get_idle_cpumask_node(int node) __ksym __weak; 88 const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; 89 const struct cpumask *scx_bpf_get_idle_smtmask_node(int node) __ksym __weak; 90 const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; 91 void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; 92 bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; 93 s32 scx_bpf_pick_idle_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags) __ksym __weak; 94 s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; 95 s32 scx_bpf_pick_any_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags) __ksym __weak; 96 s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; 97 bool scx_bpf_task_running(const struct task_struct *p) __ksym; 98 s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; 99 struct rq *scx_bpf_locked_rq(void) __ksym; 100 struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak; 101 struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak; 102 u64 scx_bpf_now(void) __ksym __weak; 103 void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak; 104 s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; 105 s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; 106 void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out) __ksym __weak; 107 void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; 108 s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; 109 s32 scx_bpf_this_cid(void) __ksym __weak; 110 struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak; 111 u32 scx_bpf_nr_cids(void) __ksym __weak; 112 u32 scx_bpf_nr_online_cids(void) __ksym __weak; 113 u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak; 114 u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; 115 s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; 116 117 /* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */ 118 s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask, 119 struct scx_cmask *denied) __ksym __weak; 120 void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask) __ksym __weak; 121 s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out) __ksym __weak; 122 s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, 123 unsigned long long *data, u32 data__sz) __ksym __weak; 124 125 /* 126 * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from 127 * within bpf_for_each() loops. 128 */ 129 #define BPF_FOR_EACH_ITER (&___it) 130 131 #define scx_read_event(e, name) \ 132 (bpf_core_field_exists((e)->name) ? (e)->name : 0) 133 134 static inline __attribute__((format(printf, 1, 2))) 135 void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {} 136 137 #define SCX_STRINGIFY(x) #x 138 #define SCX_TOSTRING(x) SCX_STRINGIFY(x) 139 140 /* 141 * Helper macro for initializing the fmt and variadic argument inputs to both 142 * bstr exit kfuncs. Callers to this function should use ___fmt and ___param to 143 * refer to the initialized list of inputs to the bstr kfunc. 144 */ 145 #define scx_bpf_bstr_preamble(fmt, args...) \ 146 static char ___fmt[] = fmt; \ 147 /* \ 148 * Note that __param[] must have at least one \ 149 * element to keep the verifier happy. \ 150 */ \ 151 unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ 152 \ 153 _Pragma("GCC diagnostic push") \ 154 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ 155 ___bpf_fill(___param, args); \ 156 _Pragma("GCC diagnostic pop") 157 158 /* 159 * scx_bpf_exit() wraps the scx_bpf_exit_bstr() kfunc with variadic arguments 160 * instead of an array of u64. Using this macro will cause the scheduler to 161 * exit cleanly with the specified exit code being passed to user space. 162 */ 163 #define scx_bpf_exit(code, fmt, args...) \ 164 ({ \ 165 scx_bpf_bstr_preamble(fmt, args) \ 166 scx_bpf_exit_bstr(code, ___fmt, ___param, sizeof(___param)); \ 167 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 168 }) 169 170 /* 171 * scx_bpf_sub_kill() wraps the scx_bpf_sub_kill_bstr() kfunc with variadic 172 * arguments instead of an array of u64. It kills the direct child sub-scheduler 173 * @cgid, passing the formatted reason to its user space, and evaluates to the 174 * kfunc's return value. On a kernel without sub-scheduler support the kfunc is 175 * absent and it returns -EOPNOTSUPP. 176 */ 177 #define scx_bpf_sub_kill(cgid, fmt, args...) \ 178 ({ \ 179 scx_bpf_bstr_preamble(fmt, args) \ 180 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 181 bpf_ksym_exists(scx_bpf_sub_kill_bstr) ? \ 182 scx_bpf_sub_kill_bstr((cgid), ___fmt, ___param, \ 183 sizeof(___param)) : -EOPNOTSUPP; \ 184 }) 185 186 /* 187 * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments 188 * instead of an array of u64. Invoking this macro will cause the scheduler to 189 * exit in an erroneous state, with diagnostic information being passed to the 190 * user. It appends the file and line number to aid debugging. 191 */ 192 #define scx_bpf_error(fmt, args...) \ 193 ({ \ 194 scx_bpf_bstr_preamble( \ 195 __FILE__ ":" SCX_TOSTRING(__LINE__) ": " fmt, ##args) \ 196 scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ 197 ___scx_bpf_bstr_format_checker( \ 198 __FILE__ ":" SCX_TOSTRING(__LINE__) ": " fmt, ##args); \ 199 }) 200 201 /* 202 * scx_bpf_dump() wraps the scx_bpf_dump_bstr() kfunc with variadic arguments 203 * instead of an array of u64. To be used from ops.dump() and friends. 204 */ 205 #define scx_bpf_dump(fmt, args...) \ 206 ({ \ 207 scx_bpf_bstr_preamble(fmt, args) \ 208 scx_bpf_dump_bstr(___fmt, ___param, sizeof(___param)); \ 209 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 210 }) 211 212 /* 213 * scx_bpf_dump_header() is a wrapper around scx_bpf_dump that adds a header 214 * of system information for debugging. 215 */ 216 #define scx_bpf_dump_header() \ 217 ({ \ 218 scx_bpf_dump("kernel: %d.%d.%d %s\ncc: %s\n", \ 219 LINUX_KERNEL_VERSION >> 16, \ 220 LINUX_KERNEL_VERSION >> 8 & 0xFF, \ 221 LINUX_KERNEL_VERSION & 0xFF, \ 222 CONFIG_LOCALVERSION, \ 223 CONFIG_CC_VERSION_TEXT); \ 224 }) 225 226 #define BPF_STRUCT_OPS(name, args...) \ 227 SEC("struct_ops/"#name) \ 228 BPF_PROG(name, ##args) 229 230 #define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ 231 SEC("struct_ops.s/"#name) \ 232 BPF_PROG(name, ##args) 233 234 /** 235 * RESIZABLE_ARRAY - Generates annotations for an array that may be resized 236 * @elfsec: the data section of the BPF program in which to place the array 237 * @arr: the name of the array 238 * 239 * libbpf has an API for setting map value sizes. Since data sections (i.e. 240 * bss, data, rodata) themselves are maps, a data section can be resized. If 241 * a data section has an array as its last element, the BTF info for that 242 * array will be adjusted so that length of the array is extended to meet the 243 * new length of the data section. This macro annotates an array to have an 244 * element count of one with the assumption that this array can be resized 245 * within the userspace program. It also annotates the section specifier so 246 * this array exists in a custom sub data section which can be resized 247 * independently. 248 * 249 * See RESIZE_ARRAY() for the userspace convenience macro for resizing an 250 * array declared with RESIZABLE_ARRAY(). 251 */ 252 #define RESIZABLE_ARRAY(elfsec, arr) arr[1] SEC("."#elfsec"."#arr) 253 254 /** 255 * MEMBER_VPTR - Obtain the verified pointer to a struct or array member 256 * @base: struct or array to index 257 * @member: dereferenced member (e.g. .field, [idx0][idx1], .field[idx0] ...) 258 * 259 * The verifier often gets confused by the instruction sequence the compiler 260 * generates for indexing struct fields or arrays. This macro forces the 261 * compiler to generate a code sequence which first calculates the byte offset, 262 * checks it against the struct or array size and add that byte offset to 263 * generate the pointer to the member to help the verifier. 264 * 265 * Ideally, we want to abort if the calculated offset is out-of-bounds. However, 266 * BPF currently doesn't support abort, so evaluate to %NULL instead. The caller 267 * must check for %NULL and take appropriate action to appease the verifier. To 268 * avoid confusing the verifier, it's best to check for %NULL and dereference 269 * immediately. 270 * 271 * vptr = MEMBER_VPTR(my_array, [i][j]); 272 * if (!vptr) 273 * return error; 274 * *vptr = new_value; 275 * 276 * sizeof(@base) should encompass the memory area to be accessed and thus can't 277 * be a pointer to the area. Use `MEMBER_VPTR(*ptr, .member)` instead of 278 * `MEMBER_VPTR(ptr, ->member)`. 279 */ 280 #ifndef MEMBER_VPTR 281 #define MEMBER_VPTR(base, member) (typeof((base) member) *) \ 282 ({ \ 283 u64 __base = (u64)&(base); \ 284 u64 __addr = (u64)&((base) member) - __base; \ 285 _Static_assert(sizeof(base) >= sizeof((base) member), \ 286 "@base is smaller than @member, is @base a pointer?"); \ 287 asm volatile ( \ 288 "if %0 <= %[max] goto +2\n" \ 289 "%0 = 0\n" \ 290 "goto +1\n" \ 291 "%0 += %1\n" \ 292 : "+r"(__addr) \ 293 : "r"(__base), \ 294 [max]"i"(sizeof(base) - sizeof((base) member))); \ 295 __addr; \ 296 }) 297 #endif /* MEMBER_VPTR */ 298 299 /** 300 * ARRAY_ELEM_PTR - Obtain the verified pointer to an array element 301 * @arr: array to index into 302 * @i: array index 303 * @n: number of elements in array 304 * 305 * Similar to MEMBER_VPTR() but is intended for use with arrays where the 306 * element count needs to be explicit. 307 * It can be used in cases where a global array is defined with an initial 308 * size but is intended to be be resized before loading the BPF program. 309 * Without this version of the macro, MEMBER_VPTR() will use the compile time 310 * size of the array to compute the max, which will result in rejection by 311 * the verifier. 312 */ 313 #ifndef ARRAY_ELEM_PTR 314 #define ARRAY_ELEM_PTR(arr, i, n) (typeof(arr[i]) *) \ 315 ({ \ 316 u64 __base = (u64)arr; \ 317 u64 __addr = (u64)&(arr[i]) - __base; \ 318 asm volatile ( \ 319 "if %0 <= %[max] goto +2\n" \ 320 "%0 = 0\n" \ 321 "goto +1\n" \ 322 "%0 += %1\n" \ 323 : "+r"(__addr) \ 324 : "r"(__base), \ 325 [max]"r"(sizeof(arr[0]) * ((n) - 1))); \ 326 __addr; \ 327 }) 328 #endif /* ARRAY_ELEM_PTR */ 329 330 /** 331 * __sink - Hide @expr's value from the compiler and BPF verifier 332 * @expr: The expression whose value should be opacified 333 * 334 * No-op at runtime. The empty inline assembly with a read-write constraint 335 * ("+g") has two effects at compile/verify time: 336 * 337 * 1. Compiler: treats @expr as both read and written, preventing dead-code 338 * elimination and keeping @expr (and any side effects that produced it) 339 * alive. 340 * 341 * 2. BPF verifier: forgets the precise value/range of @expr ("makes it 342 * imprecise"). The verifier normally tracks exact ranges for every register 343 * and stack slot. While useful, precision means each distinct value creates a 344 * separate verifier state. Inside loops this leads to state explosion - each 345 * iteration carries different precise values so states never merge and the 346 * verifier explores every iteration individually. 347 * 348 * Example - preventing loop state explosion:: 349 * 350 * u32 nr_intersects = 0, nr_covered = 0; 351 * __sink(nr_intersects); 352 * __sink(nr_covered); 353 * bpf_for(i, 0, nr_nodes) { 354 * if (intersects(cpumask, node_mask[i])) 355 * nr_intersects++; 356 * if (covers(cpumask, node_mask[i])) 357 * nr_covered++; 358 * } 359 * 360 * Without __sink(), the verifier tracks every possible (nr_intersects, 361 * nr_covered) pair across iterations, causing "BPF program is too large". With 362 * __sink(), the values become unknown scalars so all iterations collapse into 363 * one reusable state. 364 * 365 * Example - keeping a reference alive:: 366 * 367 * struct task_struct *t = bpf_task_acquire(task); 368 * __sink(t); 369 * 370 * Follows the convention from BPF selftests (bpf_misc.h). 371 */ 372 #define __sink(expr) asm volatile ("" : "+g"(expr)) 373 374 /* 375 * BPF declarations and helpers 376 */ 377 378 /* list and rbtree */ 379 #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) 380 #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) 381 382 void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; 383 void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; 384 385 #define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) 386 #define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) 387 388 int bpf_list_push_front_impl(struct bpf_list_head *head, 389 struct bpf_list_node *node, 390 void *meta, __u64 off) __ksym; 391 #define bpf_list_push_front(head, node) bpf_list_push_front_impl(head, node, NULL, 0) 392 393 int bpf_list_push_back_impl(struct bpf_list_head *head, 394 struct bpf_list_node *node, 395 void *meta, __u64 off) __ksym; 396 #define bpf_list_push_back(head, node) bpf_list_push_back_impl(head, node, NULL, 0) 397 398 struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; 399 struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; 400 struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, 401 struct bpf_rb_node *node) __ksym; 402 int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, 403 bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b), 404 void *meta, __u64 off) __ksym; 405 #define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0) 406 407 struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; 408 409 void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym; 410 #define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL) 411 412 /* task */ 413 struct task_struct *bpf_task_from_pid(s32 pid) __ksym; 414 struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; 415 void bpf_task_release(struct task_struct *p) __ksym; 416 417 /* cgroup */ 418 struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; 419 struct cgroup *bpf_cgroup_acquire(struct cgroup *cgrp) __ksym; 420 void bpf_cgroup_release(struct cgroup *cgrp) __ksym; 421 struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; 422 423 /* css iteration */ 424 struct bpf_iter_css; 425 struct cgroup_subsys_state; 426 extern int bpf_iter_css_new(struct bpf_iter_css *it, 427 struct cgroup_subsys_state *start, 428 unsigned int flags) __weak __ksym; 429 extern struct cgroup_subsys_state * 430 bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym; 431 extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym; 432 433 /* cpumask */ 434 struct bpf_cpumask *bpf_cpumask_create(void) __ksym; 435 struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; 436 void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; 437 u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; 438 u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; 439 void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 440 void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 441 bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; 442 bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 443 bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 444 void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; 445 void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; 446 bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, 447 const struct cpumask *src2) __ksym; 448 void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, 449 const struct cpumask *src2) __ksym; 450 void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, 451 const struct cpumask *src2) __ksym; 452 bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; 453 bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; 454 bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; 455 bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; 456 bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; 457 void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; 458 u32 bpf_cpumask_any_distribute(const struct cpumask *cpumask) __ksym; 459 u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, 460 const struct cpumask *src2) __ksym; 461 u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym; 462 463 int bpf_iter_bits_new(struct bpf_iter_bits *it, const u64 *unsafe_ptr__ign, u32 nr_words) __ksym; 464 int *bpf_iter_bits_next(struct bpf_iter_bits *it) __ksym; 465 void bpf_iter_bits_destroy(struct bpf_iter_bits *it) __ksym; 466 467 #define def_iter_struct(name) \ 468 struct bpf_iter_##name { \ 469 struct bpf_iter_bits it; \ 470 const struct cpumask *bitmap; \ 471 }; 472 473 #define def_iter_new(name) \ 474 static inline int bpf_iter_##name##_new( \ 475 struct bpf_iter_##name *it, const u64 *unsafe_ptr__ign, u32 nr_words) \ 476 { \ 477 it->bitmap = scx_bpf_get_##name##_cpumask(); \ 478 return bpf_iter_bits_new(&it->it, (const u64 *)it->bitmap, \ 479 sizeof(struct cpumask) / 8); \ 480 } 481 482 #define def_iter_next(name) \ 483 static inline int *bpf_iter_##name##_next(struct bpf_iter_##name *it) { \ 484 return bpf_iter_bits_next(&it->it); \ 485 } 486 487 #define def_iter_destroy(name) \ 488 static inline void bpf_iter_##name##_destroy(struct bpf_iter_##name *it) { \ 489 scx_bpf_put_cpumask(it->bitmap); \ 490 bpf_iter_bits_destroy(&it->it); \ 491 } 492 #define def_for_each_cpu(cpu, name) for_each_##name##_cpu(cpu) 493 494 /// Provides iterator for possible and online cpus. 495 /// 496 /// # Example 497 /// 498 /// ``` 499 /// static inline void example_use() { 500 /// int *cpu; 501 /// 502 /// for_each_possible_cpu(cpu){ 503 /// bpf_printk("CPU %d is possible", *cpu); 504 /// } 505 /// 506 /// for_each_online_cpu(cpu){ 507 /// bpf_printk("CPU %d is online", *cpu); 508 /// } 509 /// } 510 /// ``` 511 def_iter_struct(possible); 512 def_iter_new(possible); 513 def_iter_next(possible); 514 def_iter_destroy(possible); 515 #define for_each_possible_cpu(cpu) bpf_for_each(possible, cpu, NULL, 0) 516 517 def_iter_struct(online); 518 def_iter_new(online); 519 def_iter_next(online); 520 def_iter_destroy(online); 521 #define for_each_online_cpu(cpu) bpf_for_each(online, cpu, NULL, 0) 522 523 /* 524 * Access a cpumask in read-only mode (typically to check bits). 525 */ 526 static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask) 527 { 528 return (const struct cpumask *)mask; 529 } 530 531 /* 532 * Return true if task @p cannot migrate to a different CPU, false 533 * otherwise. 534 */ 535 static inline bool is_migration_disabled(const struct task_struct *p) 536 { 537 /* 538 * Testing p->migration_disabled in a BPF code is tricky because the 539 * migration is _always_ disabled while running the BPF code. 540 * The prolog (__bpf_prog_enter) and epilog (__bpf_prog_exit) for BPF 541 * code execution disable and re-enable the migration of the current 542 * task, respectively. So, the _current_ task of the sched_ext ops is 543 * always migration-disabled. Moreover, p->migration_disabled could be 544 * two or greater when a sched_ext ops BPF code (e.g., ops.tick) is 545 * executed in the middle of the other BPF code execution. 546 * 547 * Therefore, we should decide that the _current_ task is 548 * migration-disabled only when its migration_disabled count is greater 549 * than one. In other words, when p->migration_disabled == 1, there is 550 * an ambiguity, so we should check if @p is the current task or not. 551 */ 552 if (bpf_core_field_exists(p->migration_disabled)) { 553 if (p->migration_disabled == 1) 554 return bpf_get_current_task_btf() != p; 555 else 556 return p->migration_disabled; 557 } 558 return false; 559 } 560 561 /* rcu */ 562 void bpf_rcu_read_lock(void) __ksym; 563 void bpf_rcu_read_unlock(void) __ksym; 564 565 /* resilient qspinlock */ 566 int bpf_res_spin_lock(struct bpf_res_spin_lock *lock) __ksym __weak; 567 void bpf_res_spin_unlock(struct bpf_res_spin_lock *lock) __ksym __weak; 568 569 /* 570 * Time helpers, most of which are from jiffies.h. 571 */ 572 573 /** 574 * time_delta - Calculate the delta between new and old time stamp 575 * @after: first comparable as u64 576 * @before: second comparable as u64 577 * 578 * Return: the time difference, which is >= 0 579 */ 580 static inline s64 time_delta(u64 after, u64 before) 581 { 582 return (s64)(after - before) > 0 ? (s64)(after - before) : 0; 583 } 584 585 /** 586 * time_after - returns true if the time a is after time b. 587 * @a: first comparable as u64 588 * @b: second comparable as u64 589 * 590 * Do this with "<0" and ">=0" to only test the sign of the result. A 591 * good compiler would generate better code (and a really good compiler 592 * wouldn't care). Gcc is currently neither. 593 * 594 * Return: %true is time a is after time b, otherwise %false. 595 */ 596 static inline bool time_after(u64 a, u64 b) 597 { 598 return (s64)(b - a) < 0; 599 } 600 601 /** 602 * time_before - returns true if the time a is before time b. 603 * @a: first comparable as u64 604 * @b: second comparable as u64 605 * 606 * Return: %true is time a is before time b, otherwise %false. 607 */ 608 static inline bool time_before(u64 a, u64 b) 609 { 610 return time_after(b, a); 611 } 612 613 /** 614 * time_after_eq - returns true if the time a is after or the same as time b. 615 * @a: first comparable as u64 616 * @b: second comparable as u64 617 * 618 * Return: %true is time a is after or the same as time b, otherwise %false. 619 */ 620 static inline bool time_after_eq(u64 a, u64 b) 621 { 622 return (s64)(a - b) >= 0; 623 } 624 625 /** 626 * time_before_eq - returns true if the time a is before or the same as time b. 627 * @a: first comparable as u64 628 * @b: second comparable as u64 629 * 630 * Return: %true is time a is before or the same as time b, otherwise %false. 631 */ 632 static inline bool time_before_eq(u64 a, u64 b) 633 { 634 return time_after_eq(b, a); 635 } 636 637 /** 638 * time_in_range - Calculate whether a is in the range of [b, c]. 639 * @a: time to test 640 * @b: beginning of the range 641 * @c: end of the range 642 * 643 * Return: %true is time a is in the range [b, c], otherwise %false. 644 */ 645 static inline bool time_in_range(u64 a, u64 b, u64 c) 646 { 647 return time_after_eq(a, b) && time_before_eq(a, c); 648 } 649 650 /** 651 * time_in_range_open - Calculate whether a is in the range of [b, c). 652 * @a: time to test 653 * @b: beginning of the range 654 * @c: end of the range 655 * 656 * Return: %true is time a is in the range [b, c), otherwise %false. 657 */ 658 static inline bool time_in_range_open(u64 a, u64 b, u64 c) 659 { 660 return time_after_eq(a, b) && time_before(a, c); 661 } 662 663 664 /* 665 * Other helpers 666 */ 667 668 /* useful compiler attributes */ 669 #ifndef likely 670 #define likely(x) __builtin_expect(!!(x), 1) 671 #endif 672 #ifndef unlikely 673 #define unlikely(x) __builtin_expect(!!(x), 0) 674 #endif 675 #ifndef __maybe_unused 676 #define __maybe_unused __attribute__((__unused__)) 677 #endif 678 679 /* 680 * READ/WRITE_ONCE() are from kernel (include/asm-generic/rwonce.h). They 681 * prevent compiler from caching, redoing or reordering reads or writes. 682 */ 683 typedef __u8 __attribute__((__may_alias__)) __u8_alias_t; 684 typedef __u16 __attribute__((__may_alias__)) __u16_alias_t; 685 typedef __u32 __attribute__((__may_alias__)) __u32_alias_t; 686 typedef __u64 __attribute__((__may_alias__)) __u64_alias_t; 687 688 static __always_inline void __read_once_size(const volatile void *p, void *res, int size) 689 { 690 switch (size) { 691 case 1: *(__u8_alias_t *) res = *(volatile __u8_alias_t *) p; break; 692 case 2: *(__u16_alias_t *) res = *(volatile __u16_alias_t *) p; break; 693 case 4: *(__u32_alias_t *) res = *(volatile __u32_alias_t *) p; break; 694 case 8: *(__u64_alias_t *) res = *(volatile __u64_alias_t *) p; break; 695 default: 696 barrier(); 697 __builtin_memcpy((void *)res, (const void *)p, size); 698 barrier(); 699 } 700 } 701 702 static __always_inline void __write_once_size(volatile void *p, void *res, int size) 703 { 704 switch (size) { 705 case 1: *(volatile __u8_alias_t *) p = *(__u8_alias_t *) res; break; 706 case 2: *(volatile __u16_alias_t *) p = *(__u16_alias_t *) res; break; 707 case 4: *(volatile __u32_alias_t *) p = *(__u32_alias_t *) res; break; 708 case 8: *(volatile __u64_alias_t *) p = *(__u64_alias_t *) res; break; 709 default: 710 barrier(); 711 __builtin_memcpy((void *)p, (const void *)res, size); 712 barrier(); 713 } 714 } 715 716 /* 717 * __unqual_typeof(x) - Declare an unqualified scalar type, leaving 718 * non-scalar types unchanged, 719 * 720 * Prefer C11 _Generic for better compile-times and simpler code. Note: 'char' 721 * is not type-compatible with 'signed char', and we define a separate case. 722 * 723 * This is copied verbatim from kernel's include/linux/compiler_types.h, but 724 * with default expression (for pointers) changed from (x) to (typeof(x)0). 725 * 726 * This is because LLVM has a bug where for lvalue (x), it does not get rid of 727 * an extra address_space qualifier, but does in case of rvalue (typeof(x)0). 728 * Hence, for pointers, we need to create an rvalue expression to get the 729 * desired type. See https://github.com/llvm/llvm-project/issues/53400. 730 */ 731 #define __scalar_type_to_expr_cases(type) \ 732 unsigned type : (unsigned type)0, signed type : (signed type)0 733 734 #define __unqual_typeof(x) \ 735 typeof(_Generic((x), \ 736 char: (char)0, \ 737 __scalar_type_to_expr_cases(char), \ 738 __scalar_type_to_expr_cases(short), \ 739 __scalar_type_to_expr_cases(int), \ 740 __scalar_type_to_expr_cases(long), \ 741 __scalar_type_to_expr_cases(long long), \ 742 default: (typeof(x))0)) 743 744 #define READ_ONCE(x) \ 745 ({ \ 746 union { __unqual_typeof(x) __val; char __c[1]; } __u = \ 747 { .__c = { 0 } }; \ 748 __read_once_size((__unqual_typeof(x) *)&(x), __u.__c, sizeof(x)); \ 749 __u.__val; \ 750 }) 751 752 #define WRITE_ONCE(x, val) \ 753 ({ \ 754 union { __unqual_typeof(x) __val; char __c[1]; } __u = \ 755 { .__val = (val) }; \ 756 __write_once_size((__unqual_typeof(x) *)&(x), __u.__c, sizeof(x)); \ 757 __u.__val; \ 758 }) 759 760 /* 761 * __calc_avg - Calculate exponential weighted moving average (EWMA) with 762 * @old and @new values. @decay represents how large the @old value remains. 763 * With a larger @decay value, the moving average changes slowly, exhibiting 764 * fewer fluctuations. 765 */ 766 #define __calc_avg(old, new, decay) ({ \ 767 typeof(decay) thr = 1 << (decay); \ 768 typeof(old) ret; \ 769 if (((old) < thr) || ((new) < thr)) { \ 770 if (((old) == 1) && ((new) == 0)) \ 771 ret = 0; \ 772 else \ 773 ret = ((old) - ((old) >> 1)) + ((new) >> 1); \ 774 } else { \ 775 ret = ((old) - ((old) >> (decay))) + ((new) >> (decay)); \ 776 } \ 777 ret; \ 778 }) 779 780 /* 781 * log2_u32 - Compute the base 2 logarithm of a 32-bit exponential value. 782 * @v: The value for which we're computing the base 2 logarithm. 783 */ 784 static inline u32 log2_u32(u32 v) 785 { 786 u32 r; 787 u32 shift; 788 789 r = (v > 0xFFFF) << 4; v >>= r; 790 shift = (v > 0xFF) << 3; v >>= shift; r |= shift; 791 shift = (v > 0xF) << 2; v >>= shift; r |= shift; 792 shift = (v > 0x3) << 1; v >>= shift; r |= shift; 793 r |= (v >> 1); 794 return r; 795 } 796 797 /* 798 * log2_u64 - Compute the base 2 logarithm of a 64-bit exponential value. 799 * @v: The value for which we're computing the base 2 logarithm. 800 */ 801 static inline u32 log2_u64(u64 v) 802 { 803 u32 hi = v >> 32; 804 if (hi) 805 return log2_u32(hi) + 32 + 1; 806 else 807 return log2_u32(v) + 1; 808 } 809 810 /* 811 * sqrt_u64 - Calculate the square root of value @x using Newton's method. 812 */ 813 static inline u64 __sqrt_u64(u64 x) 814 { 815 if (x == 0 || x == 1) 816 return x; 817 818 u64 r = ((1ULL << 32) > x) ? x : (1ULL << 32); 819 820 for (int i = 0; i < 8; ++i) { 821 u64 q = x / r; 822 if (r <= q) 823 break; 824 r = (r + q) >> 1; 825 } 826 return r; 827 } 828 829 /* 830 * ctzll -- Counts trailing zeros in an unsigned long long. If the input value 831 * is zero, the return value is undefined. 832 */ 833 static inline int ctzll(u64 v) 834 { 835 #if (!defined(__BPF__) && defined(__SCX_TARGET_ARCH_x86)) || \ 836 (defined(__BPF__) && defined(__clang_major__) && __clang_major__ >= 19) 837 /* 838 * Use the ctz builtin when: (1) building for native x86, or 839 * (2) building for BPF with clang >= 19 (BPF backend supports 840 * the intrinsic from clang 19 onward; earlier versions hit 841 * "unimplemented opcode" in the backend). 842 */ 843 return __builtin_ctzll(v); 844 #else 845 /* 846 * If neither the target architecture nor the toolchains support ctzll, 847 * use software-based emulation. Let's use the De Bruijn sequence-based 848 * approach to find LSB fastly. See the details of De Bruijn sequence: 849 * 850 * https://en.wikipedia.org/wiki/De_Bruijn_sequence 851 * https://www.chessprogramming.org/BitScan#De_Bruijn_Multiplication 852 */ 853 const int lookup_table[64] = { 854 0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4, 855 62, 55, 59, 36, 53, 51, 43, 22, 45, 39, 33, 30, 24, 18, 12, 5, 856 63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 44, 32, 23, 11, 857 46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, 858 }; 859 const u64 DEBRUIJN_CONSTANT = 0x03f79d71b4cb0a89ULL; 860 unsigned int index; 861 u64 lowest_bit; 862 const int *lt; 863 864 if (v == 0) 865 return -1; 866 867 /* 868 * Isolate the least significant bit (LSB). 869 * For example, if v = 0b...10100, then v & -v = 0b...00100 870 */ 871 lowest_bit = v & -v; 872 873 /* 874 * Each isolated bit produces a unique 6-bit value, guaranteed by the 875 * De Bruijn property. Calculate a unique index into the lookup table 876 * using the magic constant and a right shift. 877 * 878 * Multiplying by the 64-bit constant "spreads out" that 1-bit into a 879 * unique pattern in the top 6 bits. This uniqueness property is 880 * exactly what a De Bruijn sequence guarantees: Every possible 6-bit 881 * pattern (in top bits) occurs exactly once for each LSB position. So, 882 * the constant 0x03f79d71b4cb0a89ULL is carefully chosen to be a 883 * De Bruijn sequence, ensuring no collisions in the table index. 884 */ 885 index = (lowest_bit * DEBRUIJN_CONSTANT) >> 58; 886 887 /* 888 * Lookup in a precomputed table. No collision is guaranteed by the 889 * De Bruijn property. 890 */ 891 lt = MEMBER_VPTR(lookup_table, [index]); 892 return (lt)? *lt : -1; 893 #endif 894 } 895 896 /* 897 * Return a value proportionally scaled to the task's weight. 898 */ 899 static inline u64 scale_by_task_weight(const struct task_struct *p, u64 value) 900 { 901 return (value * p->scx.weight) / 100; 902 } 903 904 /* 905 * Return a value inversely proportional to the task's weight. 906 */ 907 static inline u64 scale_by_task_weight_inverse(const struct task_struct *p, u64 value) 908 { 909 return value * 100 / p->scx.weight; 910 } 911 912 913 /* 914 * Get a random u64 from the kernel's pseudo-random generator. 915 */ 916 static inline u64 get_prandom_u64() 917 { 918 return ((u64)bpf_get_prandom_u32() << 32) | bpf_get_prandom_u32(); 919 } 920 921 /* 922 * Define the shadow structure to avoid a compilation error when 923 * vmlinux.h does not enable necessary kernel configs. The ___local 924 * suffix is a CO-RE convention that tells the loader to match this 925 * against the base struct rq in the kernel. The attribute 926 * preserve_access_index tells the compiler to generate a CO-RE 927 * relocation for these fields. 928 */ 929 struct rq___local { 930 /* 931 * A monotonically increasing clock per CPU. It is rq->clock minus 932 * cumulative IRQ time and hypervisor steal time. Unlike rq->clock, 933 * it does not advance during IRQ processing or hypervisor preemption. 934 * It does advance during idle (the idle task counts as a running task 935 * for this purpose). 936 */ 937 u64 clock_task; 938 /* 939 * Invariant version of clock_task scaled by CPU capacity and 940 * frequency. For example, clock_pelt advances 2x slower on a CPU 941 * with half the capacity. 942 * 943 * At idle exit, rq->clock_pelt jumps forward to resync with 944 * clock_task. The kernel's rq_clock_pelt() corrects for this jump 945 * by subtracting lost_idle_time, yielding a clock that appears 946 * continuous across idle transitions. scx_clock_pelt() mirrors 947 * rq_clock_pelt() by performing the same subtraction. 948 */ 949 u64 clock_pelt; 950 /* 951 * Accumulates the magnitude of each clock_pelt jump at idle exit. 952 * Subtracting this from clock_pelt gives rq_clock_pelt(): a 953 * continuous, capacity-invariant clock suitable for both task 954 * execution time stamping and cross-idle measurements. 955 */ 956 unsigned long lost_idle_time; 957 /* 958 * Shadow of paravirt_steal_clock() (the hypervisor's cumulative 959 * stolen time counter). Stays frozen while the hypervisor preempts 960 * the vCPU; catches up the next time update_rq_clock_task() is 961 * called. The delta is the stolen time not yet subtracted from 962 * clock_task. 963 * 964 * Unlike irqtime->total (a plain kernel-side field), the live stolen 965 * time counter lives in hypervisor-specific shared memory and has no 966 * kernel-side equivalent readable from BPF in a hypervisor-agnostic 967 * way. This field is therefore the only portable BPF-accessible 968 * approximation of cumulative steal time. 969 * 970 * Available only when CONFIG_PARAVIRT_TIME_ACCOUNTING is on. 971 */ 972 u64 prev_steal_time_rq; 973 } __attribute__((preserve_access_index)); 974 975 extern struct rq runqueues __ksym; 976 977 /* 978 * Define the shadow structure to avoid a compilation error when 979 * vmlinux.h does not enable necessary kernel configs. 980 */ 981 struct irqtime___local { 982 /* 983 * Cumulative IRQ time counter for this CPU, in nanoseconds. Advances 984 * immediately at the exit of every hardirq and non-ksoftirqd softirq 985 * via irqtime_account_irq(). ksoftirqd time is counted as normal 986 * task time and is NOT included. NMI time is also NOT included. 987 * 988 * The companion field irqtime->sync (struct u64_stats_sync) protects 989 * against 64-bit tearing on 32-bit architectures. On 64-bit kernels, 990 * u64_stats_sync is an empty struct and all seqcount operations are 991 * no-ops, so a plain BPF_CORE_READ of this field is safe. 992 * 993 * Available only when CONFIG_IRQ_TIME_ACCOUNTING is on. 994 */ 995 u64 total; 996 } __attribute__((preserve_access_index)); 997 998 /* 999 * cpu_irqtime is a per-CPU variable defined only when 1000 * CONFIG_IRQ_TIME_ACCOUNTING is on. Declare it as __weak so the BPF 1001 * loader sets its address to 0 (rather than failing) when the symbol 1002 * is absent from the running kernel. 1003 */ 1004 extern struct irqtime___local cpu_irqtime __ksym __weak; 1005 1006 static inline struct rq___local *get_current_rq(u32 cpu) 1007 { 1008 /* 1009 * This is a workaround to get an rq pointer now that 1010 * scx_bpf_cpu_rq() has been removed. 1011 * 1012 * WARNING: The caller must hold the rq lock for @cpu. This is 1013 * guaranteed when called from scheduling callbacks (ops.running, 1014 * ops.stopping, ops.enqueue, ops.dequeue, ops.dispatch, etc.). 1015 * There is no runtime check available in BPF for kernel spinlock 1016 * state — correctness is enforced by calling context only. 1017 */ 1018 return (void *)bpf_per_cpu_ptr(&runqueues, cpu); 1019 } 1020 1021 static inline u64 scx_clock_task(u32 cpu) 1022 { 1023 struct rq___local *rq = get_current_rq(cpu); 1024 1025 /* Equivalent to the kernel's rq_clock_task(). */ 1026 return rq ? rq->clock_task : 0; 1027 } 1028 1029 static inline u64 scx_clock_pelt(u32 cpu) 1030 { 1031 struct rq___local *rq = get_current_rq(cpu); 1032 1033 /* 1034 * Equivalent to the kernel's rq_clock_pelt(): subtracts 1035 * lost_idle_time from clock_pelt to absorb the jump that occurs 1036 * when clock_pelt resyncs with clock_task at idle exit. The result 1037 * is a continuous, capacity-invariant clock safe for both task 1038 * execution time stamping and cross-idle measurements. 1039 */ 1040 return rq ? (rq->clock_pelt - rq->lost_idle_time) : 0; 1041 } 1042 1043 static inline u64 scx_clock_virt(u32 cpu) 1044 { 1045 struct rq___local *rq; 1046 1047 /* 1048 * Check field existence before calling get_current_rq() so we avoid 1049 * the per_cpu lookup entirely on kernels built without 1050 * CONFIG_PARAVIRT_TIME_ACCOUNTING. 1051 */ 1052 if (!bpf_core_field_exists(((struct rq___local *)0)->prev_steal_time_rq)) 1053 return 0; 1054 1055 /* Lagging shadow of the kernel's paravirt_steal_clock(). */ 1056 rq = get_current_rq(cpu); 1057 return rq ? BPF_CORE_READ(rq, prev_steal_time_rq) : 0; 1058 } 1059 1060 static inline u64 scx_clock_irq(u32 cpu) 1061 { 1062 struct irqtime___local *irqt; 1063 1064 /* 1065 * bpf_core_type_exists() resolves at load time: if struct irqtime is 1066 * absent from kernel BTF (CONFIG_IRQ_TIME_ACCOUNTING off), the loader 1067 * patches this into an unconditional return 0, making the 1068 * bpf_per_cpu_ptr() call below dead code that the verifier never sees. 1069 */ 1070 if (!bpf_core_type_exists(struct irqtime___local)) 1071 return 0; 1072 1073 /* Equivalent to the kernel's irq_time_read(). */ 1074 irqt = bpf_per_cpu_ptr(&cpu_irqtime, cpu); 1075 return irqt ? BPF_CORE_READ(irqt, total) : 0; 1076 } 1077 1078 /* Abbreviated forms of <linux/overflow.h>'s struct_size() family. */ 1079 #define flex_array_size(p, member, count) \ 1080 ((count) * sizeof(*(p)->member)) 1081 1082 #define struct_size(p, member, count) \ 1083 (offsetof(typeof(*(p)), member) + flex_array_size(p, member, count)) 1084 1085 #define struct_size_t(type, member, count) \ 1086 struct_size((type *)NULL, member, count) 1087 1088 #include "compat.bpf.h" 1089 #include "enums.bpf.h" 1090 #include "cid.bpf.h" 1091 1092 #endif /* __SCX_COMMON_BPF_H */ 1093