1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
4 * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
5 * Copyright (c) 2022 David Vernet <dvernet@meta.com>
6 */
7 #ifndef __SCX_COMMON_BPF_H
8 #define __SCX_COMMON_BPF_H
9
10 /*
11 * The generated kfunc prototypes in vmlinux.h are missing address space
12 * attributes which cause build failures. For now, suppress the generated
13 * prototypes. See https://github.com/sched-ext/scx/issues/1111.
14 */
15 #define BPF_NO_KFUNC_PROTOTYPES
16
17 #ifdef LSP
18 #define __bpf__
19 #include "../vmlinux.h"
20 #else
21 #include "vmlinux.h"
22 #endif
23
24 #include <bpf/bpf_helpers.h>
25 #include <bpf/bpf_tracing.h>
26 #include <asm-generic/errno.h>
27 #include "user_exit_info.bpf.h"
28 #include "enum_defs.autogen.h"
29 #include "bpf_arena_common.bpf.h"
30
31 #define PF_IDLE 0x00000002 /* I am an IDLE thread */
32 #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */
33 #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */
34 #define PF_KCOMPACTD 0x00010000 /* I am kcompactd */
35 #define PF_KSWAPD 0x00020000 /* I am kswapd */
36 #define PF_KTHREAD 0x00200000 /* I am a kernel thread */
37 #define PF_EXITING 0x00000004
38 #define CLOCK_MONOTONIC 1
39
40 #ifndef NR_CPUS
41 #define NR_CPUS 1024
42 #endif
43
44 #ifndef NUMA_NO_NODE
45 #define NUMA_NO_NODE (-1)
46 #endif
47
48 extern int LINUX_KERNEL_VERSION __kconfig;
49 extern const char CONFIG_CC_VERSION_TEXT[64] __kconfig __weak;
50 extern const char CONFIG_LOCALVERSION[64] __kconfig __weak;
51 extern bool CONFIG_PREEMPT_RCU __kconfig __weak;
52
53 /*
54 * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can
55 * lead to really confusing misbehaviors. Let's trigger a build failure.
56 */
___vmlinux_h_sanity_check___(void)57 static inline void ___vmlinux_h_sanity_check___(void)
58 {
59 _Static_assert(SCX_DSQ_FLAG_BUILTIN,
60 "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole");
61 }
62
63 s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym;
64 s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym;
65 s32 __scx_bpf_select_cpu_and(struct task_struct *p, const struct cpumask *cpus_allowed,
66 struct scx_bpf_select_cpu_and_args *args) __ksym __weak;
67 bool __scx_bpf_dsq_insert_vtime(struct task_struct *p, struct scx_bpf_dsq_insert_vtime_args *args) __ksym __weak;
68 u32 scx_bpf_dispatch_nr_slots(void) __ksym;
69 void scx_bpf_dispatch_cancel(void) __ksym;
70 void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym;
71 s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym;
72 void scx_bpf_destroy_dsq(u64 dsq_id) __ksym;
73 struct task_struct *scx_bpf_dsq_peek(u64 dsq_id) __ksym __weak;
74 int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak;
75 struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak;
76 void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak;
77 void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak;
78 void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym;
79 void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak;
80 u32 scx_bpf_cpuperf_cap(s32 cpu) __ksym __weak;
81 u32 scx_bpf_cpuperf_cur(s32 cpu) __ksym __weak;
82 void scx_bpf_cpuperf_set(s32 cpu, u32 perf) __ksym __weak;
83 u32 scx_bpf_nr_node_ids(void) __ksym __weak;
84 u32 scx_bpf_nr_cpu_ids(void) __ksym __weak;
85 int scx_bpf_cpu_node(s32 cpu) __ksym __weak;
86 const struct cpumask *scx_bpf_get_possible_cpumask(void) __ksym __weak;
87 const struct cpumask *scx_bpf_get_online_cpumask(void) __ksym __weak;
88 void scx_bpf_put_cpumask(const struct cpumask *cpumask) __ksym __weak;
89 const struct cpumask *scx_bpf_get_idle_cpumask_node(int node) __ksym __weak;
90 const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym;
91 const struct cpumask *scx_bpf_get_idle_smtmask_node(int node) __ksym __weak;
92 const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym;
93 void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym;
94 bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym;
95 s32 scx_bpf_pick_idle_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags) __ksym __weak;
96 s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym;
97 s32 scx_bpf_pick_any_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags) __ksym __weak;
98 s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym;
99 bool scx_bpf_task_running(const struct task_struct *p) __ksym;
100 s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym;
101 struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym __weak;
102 struct rq *scx_bpf_locked_rq(void) __ksym;
103 struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak;
104 struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak;
105 u64 scx_bpf_now(void) __ksym __weak;
106 void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak;
107 s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak;
108 s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak;
109 void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out) __ksym __weak;
110 void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak;
111 s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak;
112 s32 scx_bpf_this_cid(void) __ksym __weak;
113 struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak;
114 u32 scx_bpf_nr_cids(void) __ksym __weak;
115 u32 scx_bpf_nr_online_cids(void) __ksym __weak;
116 u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak;
117 u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak;
118 s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak;
119
120 /* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */
121 s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena, struct scx_cmask __arena *denied_out__arena__nullable) __ksym __weak;
122 void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena) __ksym __weak;
123 s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask __arena *out__arena) __ksym __weak;
124 s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak;
125
126 /*
127 * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from
128 * within bpf_for_each() loops.
129 */
130 #define BPF_FOR_EACH_ITER (&___it)
131
132 #define scx_read_event(e, name) \
133 (bpf_core_field_exists((e)->name) ? (e)->name : 0)
134
135 static inline __attribute__((format(printf, 1, 2)))
___scx_bpf_bstr_format_checker(const char * fmt,...)136 void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {}
137
138 #define SCX_STRINGIFY(x) #x
139 #define SCX_TOSTRING(x) SCX_STRINGIFY(x)
140
141 /*
142 * Helper macro for initializing the fmt and variadic argument inputs to both
143 * bstr exit kfuncs. Callers to this function should use ___fmt and ___param to
144 * refer to the initialized list of inputs to the bstr kfunc.
145 */
146 #define scx_bpf_bstr_preamble(fmt, args...) \
147 static char ___fmt[] = fmt; \
148 /* \
149 * Note that __param[] must have at least one \
150 * element to keep the verifier happy. \
151 */ \
152 unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \
153 \
154 _Pragma("GCC diagnostic push") \
155 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
156 ___bpf_fill(___param, args); \
157 _Pragma("GCC diagnostic pop")
158
159 /*
160 * scx_bpf_exit() wraps the scx_bpf_exit_bstr() kfunc with variadic arguments
161 * instead of an array of u64. Using this macro will cause the scheduler to
162 * exit cleanly with the specified exit code being passed to user space.
163 */
164 #define scx_bpf_exit(code, fmt, args...) \
165 ({ \
166 scx_bpf_bstr_preamble(fmt, args) \
167 scx_bpf_exit_bstr(code, ___fmt, ___param, sizeof(___param)); \
168 ___scx_bpf_bstr_format_checker(fmt, ##args); \
169 })
170
171 /*
172 * scx_bpf_sub_kill() wraps the scx_bpf_sub_kill_bstr() kfunc with variadic
173 * arguments instead of an array of u64. It kills the direct child sub-scheduler
174 * @cgid, passing the formatted reason to its user space, and evaluates to the
175 * kfunc's return value. On a kernel without sub-scheduler support the kfunc is
176 * absent and it returns -EOPNOTSUPP.
177 */
178 #define scx_bpf_sub_kill(cgid, fmt, args...) \
179 ({ \
180 scx_bpf_bstr_preamble(fmt, args) \
181 ___scx_bpf_bstr_format_checker(fmt, ##args); \
182 bpf_ksym_exists(scx_bpf_sub_kill_bstr) ? \
183 scx_bpf_sub_kill_bstr((cgid), ___fmt, ___param, \
184 sizeof(___param)) : -EOPNOTSUPP; \
185 })
186
187 /*
188 * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments
189 * instead of an array of u64. Invoking this macro will cause the scheduler to
190 * exit in an erroneous state, with diagnostic information being passed to the
191 * user. It appends the file and line number to aid debugging.
192 */
193 #define scx_bpf_error(fmt, args...) \
194 ({ \
195 scx_bpf_bstr_preamble( \
196 __FILE__ ":" SCX_TOSTRING(__LINE__) ": " fmt, ##args) \
197 scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \
198 ___scx_bpf_bstr_format_checker( \
199 __FILE__ ":" SCX_TOSTRING(__LINE__) ": " fmt, ##args); \
200 })
201
202 /*
203 * scx_bpf_dump() wraps the scx_bpf_dump_bstr() kfunc with variadic arguments
204 * instead of an array of u64. To be used from ops.dump() and friends.
205 */
206 #define scx_bpf_dump(fmt, args...) \
207 ({ \
208 scx_bpf_bstr_preamble(fmt, args) \
209 scx_bpf_dump_bstr(___fmt, ___param, sizeof(___param)); \
210 ___scx_bpf_bstr_format_checker(fmt, ##args); \
211 })
212
213 /*
214 * scx_bpf_dump_header() is a wrapper around scx_bpf_dump that adds a header
215 * of system information for debugging.
216 */
217 #define scx_bpf_dump_header() \
218 ({ \
219 scx_bpf_dump("kernel: %d.%d.%d %s\ncc: %s\n", \
220 LINUX_KERNEL_VERSION >> 16, \
221 LINUX_KERNEL_VERSION >> 8 & 0xFF, \
222 LINUX_KERNEL_VERSION & 0xFF, \
223 CONFIG_LOCALVERSION, \
224 CONFIG_CC_VERSION_TEXT); \
225 })
226
227 #define BPF_STRUCT_OPS(name, args...) \
228 SEC("struct_ops/"#name) \
229 BPF_PROG(name, ##args)
230
231 #define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \
232 SEC("struct_ops.s/"#name) \
233 BPF_PROG(name, ##args)
234
235 /**
236 * RESIZABLE_ARRAY - Generates annotations for an array that may be resized
237 * @elfsec: the data section of the BPF program in which to place the array
238 * @arr: the name of the array
239 *
240 * libbpf has an API for setting map value sizes. Since data sections (i.e.
241 * bss, data, rodata) themselves are maps, a data section can be resized. If
242 * a data section has an array as its last element, the BTF info for that
243 * array will be adjusted so that length of the array is extended to meet the
244 * new length of the data section. This macro annotates an array to have an
245 * element count of one with the assumption that this array can be resized
246 * within the userspace program. It also annotates the section specifier so
247 * this array exists in a custom sub data section which can be resized
248 * independently.
249 *
250 * See RESIZE_ARRAY() for the userspace convenience macro for resizing an
251 * array declared with RESIZABLE_ARRAY().
252 */
253 #define RESIZABLE_ARRAY(elfsec, arr) arr[1] SEC("."#elfsec"."#arr)
254
255 /**
256 * MEMBER_VPTR - Obtain the verified pointer to a struct or array member
257 * @base: struct or array to index
258 * @member: dereferenced member (e.g. .field, [idx0][idx1], .field[idx0] ...)
259 *
260 * The verifier often gets confused by the instruction sequence the compiler
261 * generates for indexing struct fields or arrays. This macro forces the
262 * compiler to generate a code sequence which first calculates the byte offset,
263 * checks it against the struct or array size and add that byte offset to
264 * generate the pointer to the member to help the verifier.
265 *
266 * Ideally, we want to abort if the calculated offset is out-of-bounds. However,
267 * BPF currently doesn't support abort, so evaluate to %NULL instead. The caller
268 * must check for %NULL and take appropriate action to appease the verifier. To
269 * avoid confusing the verifier, it's best to check for %NULL and dereference
270 * immediately.
271 *
272 * vptr = MEMBER_VPTR(my_array, [i][j]);
273 * if (!vptr)
274 * return error;
275 * *vptr = new_value;
276 *
277 * sizeof(@base) should encompass the memory area to be accessed and thus can't
278 * be a pointer to the area. Use `MEMBER_VPTR(*ptr, .member)` instead of
279 * `MEMBER_VPTR(ptr, ->member)`.
280 */
281 #ifndef MEMBER_VPTR
282 #define MEMBER_VPTR(base, member) (typeof((base) member) *) \
283 ({ \
284 u64 __base = (u64)&(base); \
285 u64 __addr = (u64)&((base) member) - __base; \
286 _Static_assert(sizeof(base) >= sizeof((base) member), \
287 "@base is smaller than @member, is @base a pointer?"); \
288 asm volatile ( \
289 "if %0 <= %[max] goto +2\n" \
290 "%0 = 0\n" \
291 "goto +1\n" \
292 "%0 += %1\n" \
293 : "+r"(__addr) \
294 : "r"(__base), \
295 [max]"i"(sizeof(base) - sizeof((base) member))); \
296 __addr; \
297 })
298 #endif /* MEMBER_VPTR */
299
300 /**
301 * ARRAY_ELEM_PTR - Obtain the verified pointer to an array element
302 * @arr: array to index into
303 * @i: array index
304 * @n: number of elements in array
305 *
306 * Similar to MEMBER_VPTR() but is intended for use with arrays where the
307 * element count needs to be explicit.
308 * It can be used in cases where a global array is defined with an initial
309 * size but is intended to be be resized before loading the BPF program.
310 * Without this version of the macro, MEMBER_VPTR() will use the compile time
311 * size of the array to compute the max, which will result in rejection by
312 * the verifier.
313 */
314 #ifndef ARRAY_ELEM_PTR
315 #define ARRAY_ELEM_PTR(arr, i, n) (typeof(arr[i]) *) \
316 ({ \
317 u64 __base = (u64)arr; \
318 u64 __addr = (u64)&(arr[i]) - __base; \
319 asm volatile ( \
320 "if %0 <= %[max] goto +2\n" \
321 "%0 = 0\n" \
322 "goto +1\n" \
323 "%0 += %1\n" \
324 : "+r"(__addr) \
325 : "r"(__base), \
326 [max]"r"(sizeof(arr[0]) * ((n) - 1))); \
327 __addr; \
328 })
329 #endif /* ARRAY_ELEM_PTR */
330
331 /**
332 * __sink - Hide @expr's value from the compiler and BPF verifier
333 * @expr: The expression whose value should be opacified
334 *
335 * No-op at runtime. The empty inline assembly with a read-write constraint
336 * ("+g") has two effects at compile/verify time:
337 *
338 * 1. Compiler: treats @expr as both read and written, preventing dead-code
339 * elimination and keeping @expr (and any side effects that produced it)
340 * alive.
341 *
342 * 2. BPF verifier: forgets the precise value/range of @expr ("makes it
343 * imprecise"). The verifier normally tracks exact ranges for every register
344 * and stack slot. While useful, precision means each distinct value creates a
345 * separate verifier state. Inside loops this leads to state explosion - each
346 * iteration carries different precise values so states never merge and the
347 * verifier explores every iteration individually.
348 *
349 * Example - preventing loop state explosion::
350 *
351 * u32 nr_intersects = 0, nr_covered = 0;
352 * __sink(nr_intersects);
353 * __sink(nr_covered);
354 * bpf_for(i, 0, nr_nodes) {
355 * if (intersects(cpumask, node_mask[i]))
356 * nr_intersects++;
357 * if (covers(cpumask, node_mask[i]))
358 * nr_covered++;
359 * }
360 *
361 * Without __sink(), the verifier tracks every possible (nr_intersects,
362 * nr_covered) pair across iterations, causing "BPF program is too large". With
363 * __sink(), the values become unknown scalars so all iterations collapse into
364 * one reusable state.
365 *
366 * Example - keeping a reference alive::
367 *
368 * struct task_struct *t = bpf_task_acquire(task);
369 * __sink(t);
370 *
371 * Follows the convention from BPF selftests (bpf_misc.h).
372 */
373 #define __sink(expr) asm volatile ("" : "+g"(expr))
374
375 /*
376 * BPF declarations and helpers
377 */
378
379 /* list and rbtree */
380 #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node)))
381 #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
382
383 void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym;
384 void bpf_obj_drop_impl(void *kptr, void *meta) __ksym;
385
386 #define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL))
387 #define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL)
388
389 int bpf_list_push_front_impl(struct bpf_list_head *head,
390 struct bpf_list_node *node,
391 void *meta, __u64 off) __ksym;
392 #define bpf_list_push_front(head, node) bpf_list_push_front_impl(head, node, NULL, 0)
393
394 int bpf_list_push_back_impl(struct bpf_list_head *head,
395 struct bpf_list_node *node,
396 void *meta, __u64 off) __ksym;
397 #define bpf_list_push_back(head, node) bpf_list_push_back_impl(head, node, NULL, 0)
398
399 struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym;
400 struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym;
401 struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root,
402 struct bpf_rb_node *node) __ksym;
403 int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
404 bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b),
405 void *meta, __u64 off) __ksym;
406 #define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0)
407
408 struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym;
409
410 void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym;
411 #define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL)
412
413 /* task */
414 struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
415 struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
416 void bpf_task_release(struct task_struct *p) __ksym;
417
418 /* cgroup */
419 struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym;
420 struct cgroup *bpf_cgroup_acquire(struct cgroup *cgrp) __ksym;
421 void bpf_cgroup_release(struct cgroup *cgrp) __ksym;
422 struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym;
423
424 /* css iteration */
425 struct bpf_iter_css;
426 struct cgroup_subsys_state;
427 extern int bpf_iter_css_new(struct bpf_iter_css *it,
428 struct cgroup_subsys_state *start,
429 unsigned int flags) __weak __ksym;
430 extern struct cgroup_subsys_state *
431 bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym;
432 extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym;
433
434 /* cpumask */
435 struct bpf_cpumask *bpf_cpumask_create(void) __ksym;
436 struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym;
437 void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym;
438 u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym;
439 u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym;
440 void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
441 void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
442 bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym;
443 bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
444 bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
445 void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym;
446 void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym;
447 bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1,
448 const struct cpumask *src2) __ksym;
449 void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1,
450 const struct cpumask *src2) __ksym;
451 void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1,
452 const struct cpumask *src2) __ksym;
453 bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym;
454 bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym;
455 bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym;
456 bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym;
457 bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym;
458 void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym;
459 u32 bpf_cpumask_any_distribute(const struct cpumask *cpumask) __ksym;
460 u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1,
461 const struct cpumask *src2) __ksym;
462 u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym;
463
464 int bpf_iter_bits_new(struct bpf_iter_bits *it, const u64 *unsafe_ptr__ign, u32 nr_words) __ksym;
465 int *bpf_iter_bits_next(struct bpf_iter_bits *it) __ksym;
466 void bpf_iter_bits_destroy(struct bpf_iter_bits *it) __ksym;
467
468 #define def_iter_struct(name) \
469 struct bpf_iter_##name { \
470 struct bpf_iter_bits it; \
471 const struct cpumask *bitmap; \
472 };
473
474 #define def_iter_new(name) \
475 static inline int bpf_iter_##name##_new( \
476 struct bpf_iter_##name *it, const u64 *unsafe_ptr__ign, u32 nr_words) \
477 { \
478 it->bitmap = scx_bpf_get_##name##_cpumask(); \
479 return bpf_iter_bits_new(&it->it, (const u64 *)it->bitmap, \
480 sizeof(struct cpumask) / 8); \
481 }
482
483 #define def_iter_next(name) \
484 static inline int *bpf_iter_##name##_next(struct bpf_iter_##name *it) { \
485 return bpf_iter_bits_next(&it->it); \
486 }
487
488 #define def_iter_destroy(name) \
489 static inline void bpf_iter_##name##_destroy(struct bpf_iter_##name *it) { \
490 scx_bpf_put_cpumask(it->bitmap); \
491 bpf_iter_bits_destroy(&it->it); \
492 }
493 #define def_for_each_cpu(cpu, name) for_each_##name##_cpu(cpu)
494
495 /// Provides iterator for possible and online cpus.
496 ///
497 /// # Example
498 ///
499 /// ```
500 /// static inline void example_use() {
501 /// int *cpu;
502 ///
503 /// for_each_possible_cpu(cpu){
504 /// bpf_printk("CPU %d is possible", *cpu);
505 /// }
506 ///
507 /// for_each_online_cpu(cpu){
508 /// bpf_printk("CPU %d is online", *cpu);
509 /// }
510 /// }
511 /// ```
512 def_iter_struct(possible);
513 def_iter_new(possible);
514 def_iter_next(possible);
515 def_iter_destroy(possible);
516 #define for_each_possible_cpu(cpu) bpf_for_each(possible, cpu, NULL, 0)
517
518 def_iter_struct(online);
519 def_iter_new(online);
520 def_iter_next(online);
521 def_iter_destroy(online);
522 #define for_each_online_cpu(cpu) bpf_for_each(online, cpu, NULL, 0)
523
524 /*
525 * Access a cpumask in read-only mode (typically to check bits).
526 */
cast_mask(struct bpf_cpumask * mask)527 static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask)
528 {
529 return (const struct cpumask *)mask;
530 }
531
532 /*
533 * True if the non-sleepable BPF trampoline prolog (__bpf_prog_enter) calls
534 * migrate_disable() for the current task. Recorded once by
535 * scx_lib_init_probe, an fentry program on bpf_scx_reg() that fires during
536 * the natural scheduler-attach call chain (auto-attached by scx_ops_attach!).
537 *
538 * Defaults to true (conservative). Over-reporting in is_migration_disabled()
539 * causes local-only dispatch, which is safe. Under-reporting can crash the
540 * scheduler, so we err high if the probe somehow fails to run.
541 */
542 bool __scx_prolog_disables_migration __weak = true;
543
544 /*
545 * scx_lib_init_probe - non-sleepable prolog probe.
546 *
547 * Attached to bpf_scx_reg(), the .reg callback in bpf_sched_ext_ops
548 * (kernel/sched/ext.c). The kernel's struct_ops machinery invokes
549 * bpf_scx_reg when userspace creates the scheduler link, before
550 * ops.init() fires. Its address is taken in the vtable, so the symbol
551 * is non-inlinable and has been stable since introduction.
552 *
553 * Entering via fentry runs us through __bpf_prog_enter -- the
554 * non-sleepable prolog that consumers of is_migration_disabled() live
555 * under.
556 *
557 * Loud warning: the prolog adds at most 1 to migration_disabled.
558 * Reading > 1 means something upstream in the
559 * bpf_struct_ops_link_create -> bpf_scx_reg path disabled migration
560 * before the prolog ran, invalidating the probe; audit and adjust.
561 */
562 SEC("fentry/bpf_scx_reg") __weak
scx_lib_init_probe(void * ctx)563 int scx_lib_init_probe(void *ctx)
564 {
565 if (bpf_core_field_exists(((struct task_struct *)0)->migration_disabled)) {
566 const struct task_struct *p = bpf_get_current_task_btf();
567 unsigned int md = p->migration_disabled;
568
569 if (md > 1)
570 bpf_printk("scx_lib_init_probe: unexpected migration_disabled=%u "
571 "upstream of BPF prolog; probe result unreliable",
572 md);
573
574 __scx_prolog_disables_migration = md > 0;
575 }
576 return 0;
577 }
578
579 /*
580 * Return true if task @p cannot migrate to a different CPU, false
581 * otherwise.
582 *
583 * IMPORTANT: designed for NON-SLEEPABLE BPF contexts only. Sleepable
584 * contexts (BPF_STRUCT_OPS_SLEEPABLE, SEC("syscall"),
585 * SEC("fentry.s/...")) enter via __bpf_prog_enter_sleepable() or
586 * __bpf_prog_enter_sleepable_recur(), both of which unconditionally
587 * call migrate_disable(); this helper can yield a false negative for
588 * p == current there, which can crash the scheduler.
589 */
is_migration_disabled(const struct task_struct * p)590 static inline bool is_migration_disabled(const struct task_struct *p)
591 {
592 /*
593 * Testing p->migration_disabled in BPF is tricky because the BPF prolog
594 * (__bpf_prog_enter) may call migrate_disable() for the current task,
595 * making migration_disabled == 1 even for tasks that are not truly
596 * migration-disabled.
597 *
598 * Since commit 8e4f0b1ebcf2 ("bpf: use rcu_read_lock_dont_migrate() for
599 * trampoline.c"), the BPF prolog calls migrate_disable() only when
600 * CONFIG_PREEMPT_RCU is enabled. Two fast paths cover the common cases:
601 *
602 * 1) CONFIG_PREEMPT_RCU: prolog always calls migrate_disable(), so
603 * migration_disabled == 1 for the current task is ambiguous.
604 * Disambiguate by checking p == current.
605 *
606 * 2) v6.18+ without CONFIG_PREEMPT_RCU: prolog never calls
607 * migrate_disable(), so migration_disabled == 1 is unambiguously
608 * a real migrate_disable() call.
609 *
610 * A slow path handles pre-v6.18 kernels without CONFIG_PREEMPT_RCU,
611 * where the prolog historically called migrate_disable() unconditionally
612 * but a cherry-picked downstream kernel may not. The runtime-probed flag
613 * __scx_prolog_disables_migration (set by scx_lib_init_probe) distinguishes
614 * the two cases without relying on the kernel version alone.
615 */
616 if (bpf_core_field_exists(p->migration_disabled)) {
617 if (p->migration_disabled == 1) {
618 /* Fast path: prolog always disables migration */
619 if (CONFIG_PREEMPT_RCU)
620 return bpf_get_current_task_btf() != p;
621 /* Fast path: prolog never disables migration */
622 if (LINUX_KERNEL_VERSION >= KERNEL_VERSION(6, 18, 0))
623 return true;
624 /* Slow path: pre-v6.18, !PREEMPT_RCU - use runtime flag */
625 return __scx_prolog_disables_migration ?
626 bpf_get_current_task_btf() != p : true;
627 }
628 return p->migration_disabled;
629 }
630 return false;
631 }
632
633 /* rcu */
634 void bpf_rcu_read_lock(void) __ksym;
635 void bpf_rcu_read_unlock(void) __ksym;
636
637 /* resilient qspinlock */
638 int bpf_res_spin_lock(struct bpf_res_spin_lock *lock) __ksym __weak;
639 void bpf_res_spin_unlock(struct bpf_res_spin_lock *lock) __ksym __weak;
640
641 /*
642 * Time helpers, most of which are from jiffies.h.
643 */
644
645 /**
646 * time_delta - Calculate the delta between new and old time stamp
647 * @after: first comparable as u64
648 * @before: second comparable as u64
649 *
650 * Return: the time difference, which is >= 0
651 */
time_delta(u64 after,u64 before)652 static inline s64 time_delta(u64 after, u64 before)
653 {
654 return (s64)(after - before) > 0 ? (s64)(after - before) : 0;
655 }
656
657 /**
658 * time_after - returns true if the time a is after time b.
659 * @a: first comparable as u64
660 * @b: second comparable as u64
661 *
662 * Do this with "<0" and ">=0" to only test the sign of the result. A
663 * good compiler would generate better code (and a really good compiler
664 * wouldn't care). Gcc is currently neither.
665 *
666 * Return: %true is time a is after time b, otherwise %false.
667 */
time_after(u64 a,u64 b)668 static inline bool time_after(u64 a, u64 b)
669 {
670 return (s64)(b - a) < 0;
671 }
672
673 /**
674 * time_before - returns true if the time a is before time b.
675 * @a: first comparable as u64
676 * @b: second comparable as u64
677 *
678 * Return: %true is time a is before time b, otherwise %false.
679 */
time_before(u64 a,u64 b)680 static inline bool time_before(u64 a, u64 b)
681 {
682 return time_after(b, a);
683 }
684
685 /**
686 * time_after_eq - returns true if the time a is after or the same as time b.
687 * @a: first comparable as u64
688 * @b: second comparable as u64
689 *
690 * Return: %true is time a is after or the same as time b, otherwise %false.
691 */
time_after_eq(u64 a,u64 b)692 static inline bool time_after_eq(u64 a, u64 b)
693 {
694 return (s64)(a - b) >= 0;
695 }
696
697 /**
698 * time_before_eq - returns true if the time a is before or the same as time b.
699 * @a: first comparable as u64
700 * @b: second comparable as u64
701 *
702 * Return: %true is time a is before or the same as time b, otherwise %false.
703 */
time_before_eq(u64 a,u64 b)704 static inline bool time_before_eq(u64 a, u64 b)
705 {
706 return time_after_eq(b, a);
707 }
708
709 /**
710 * time_in_range - Calculate whether a is in the range of [b, c].
711 * @a: time to test
712 * @b: beginning of the range
713 * @c: end of the range
714 *
715 * Return: %true is time a is in the range [b, c], otherwise %false.
716 */
time_in_range(u64 a,u64 b,u64 c)717 static inline bool time_in_range(u64 a, u64 b, u64 c)
718 {
719 return time_after_eq(a, b) && time_before_eq(a, c);
720 }
721
722 /**
723 * time_in_range_open - Calculate whether a is in the range of [b, c).
724 * @a: time to test
725 * @b: beginning of the range
726 * @c: end of the range
727 *
728 * Return: %true is time a is in the range [b, c), otherwise %false.
729 */
time_in_range_open(u64 a,u64 b,u64 c)730 static inline bool time_in_range_open(u64 a, u64 b, u64 c)
731 {
732 return time_after_eq(a, b) && time_before(a, c);
733 }
734
735
736 /*
737 * Other helpers
738 */
739
740 /* useful compiler attributes */
741 #ifndef likely
742 #define likely(x) __builtin_expect(!!(x), 1)
743 #endif
744 #ifndef unlikely
745 #define unlikely(x) __builtin_expect(!!(x), 0)
746 #endif
747 #ifndef __maybe_unused
748 #define __maybe_unused __attribute__((__unused__))
749 #endif
750
751 /*
752 * READ/WRITE_ONCE() are from kernel (include/asm-generic/rwonce.h). They
753 * prevent compiler from caching, redoing or reordering reads or writes.
754 */
755 typedef __u8 __attribute__((__may_alias__)) __u8_alias_t;
756 typedef __u16 __attribute__((__may_alias__)) __u16_alias_t;
757 typedef __u32 __attribute__((__may_alias__)) __u32_alias_t;
758 typedef __u64 __attribute__((__may_alias__)) __u64_alias_t;
759
__read_once_size(const volatile void * p,void * res,int size)760 static __always_inline void __read_once_size(const volatile void *p, void *res, int size)
761 {
762 switch (size) {
763 case 1: *(__u8_alias_t *) res = *(volatile __u8_alias_t *) p; break;
764 case 2: *(__u16_alias_t *) res = *(volatile __u16_alias_t *) p; break;
765 case 4: *(__u32_alias_t *) res = *(volatile __u32_alias_t *) p; break;
766 case 8: *(__u64_alias_t *) res = *(volatile __u64_alias_t *) p; break;
767 default:
768 barrier();
769 __builtin_memcpy((void *)res, (const void *)p, size);
770 barrier();
771 }
772 }
773
__write_once_size(volatile void * p,void * res,int size)774 static __always_inline void __write_once_size(volatile void *p, void *res, int size)
775 {
776 switch (size) {
777 case 1: *(volatile __u8_alias_t *) p = *(__u8_alias_t *) res; break;
778 case 2: *(volatile __u16_alias_t *) p = *(__u16_alias_t *) res; break;
779 case 4: *(volatile __u32_alias_t *) p = *(__u32_alias_t *) res; break;
780 case 8: *(volatile __u64_alias_t *) p = *(__u64_alias_t *) res; break;
781 default:
782 barrier();
783 __builtin_memcpy((void *)p, (const void *)res, size);
784 barrier();
785 }
786 }
787
788 /*
789 * __unqual_typeof(x) - Declare an unqualified scalar type, leaving
790 * non-scalar types unchanged,
791 *
792 * Prefer C11 _Generic for better compile-times and simpler code. Note: 'char'
793 * is not type-compatible with 'signed char', and we define a separate case.
794 *
795 * This is copied verbatim from kernel's include/linux/compiler_types.h, but
796 * with default expression (for pointers) changed from (x) to (typeof(x)0).
797 *
798 * This is because LLVM has a bug where for lvalue (x), it does not get rid of
799 * an extra address_space qualifier, but does in case of rvalue (typeof(x)0).
800 * Hence, for pointers, we need to create an rvalue expression to get the
801 * desired type. See https://github.com/llvm/llvm-project/issues/53400.
802 */
803 #define __scalar_type_to_expr_cases(type) \
804 unsigned type : (unsigned type)0, signed type : (signed type)0
805
806 #define __unqual_typeof(x) \
807 typeof(_Generic((x), \
808 char: (char)0, \
809 __scalar_type_to_expr_cases(char), \
810 __scalar_type_to_expr_cases(short), \
811 __scalar_type_to_expr_cases(int), \
812 __scalar_type_to_expr_cases(long), \
813 __scalar_type_to_expr_cases(long long), \
814 default: (typeof(x))0))
815
816 #define READ_ONCE(x) \
817 ({ \
818 union { __unqual_typeof(x) __val; char __c[1]; } __u = \
819 { .__c = { 0 } }; \
820 __read_once_size((__unqual_typeof(x) *)&(x), __u.__c, sizeof(x)); \
821 __u.__val; \
822 })
823
824 #define WRITE_ONCE(x, val) \
825 ({ \
826 union { __unqual_typeof(x) __val; char __c[1]; } __u = \
827 { .__val = (val) }; \
828 __write_once_size((__unqual_typeof(x) *)&(x), __u.__c, sizeof(x)); \
829 __u.__val; \
830 })
831
832 /*
833 * __calc_avg - Calculate exponential weighted moving average (EWMA) with
834 * @old and @new values. @decay represents how large the @old value remains.
835 * With a larger @decay value, the moving average changes slowly, exhibiting
836 * fewer fluctuations.
837 */
838 #define __calc_avg(old, new, decay) ({ \
839 typeof(decay) thr = 1 << (decay); \
840 typeof(old) ret; \
841 if (((old) < thr) || ((new) < thr)) { \
842 if (((old) == 1) && ((new) == 0)) \
843 ret = 0; \
844 else \
845 ret = ((old) - ((old) >> 1)) + ((new) >> 1); \
846 } else { \
847 ret = ((old) - ((old) >> (decay))) + ((new) >> (decay)); \
848 } \
849 ret; \
850 })
851
852 /*
853 * log2_u32 - Compute the base 2 logarithm of a 32-bit exponential value.
854 * @v: The value for which we're computing the base 2 logarithm.
855 */
log2_u32(u32 v)856 static inline u32 log2_u32(u32 v)
857 {
858 u32 r;
859 u32 shift;
860
861 r = (v > 0xFFFF) << 4; v >>= r;
862 shift = (v > 0xFF) << 3; v >>= shift; r |= shift;
863 shift = (v > 0xF) << 2; v >>= shift; r |= shift;
864 shift = (v > 0x3) << 1; v >>= shift; r |= shift;
865 r |= (v >> 1);
866 return r;
867 }
868
869 /*
870 * log2_u64 - Compute the base 2 logarithm of a 64-bit exponential value.
871 * @v: The value for which we're computing the base 2 logarithm.
872 */
log2_u64(u64 v)873 static inline u32 log2_u64(u64 v)
874 {
875 u32 hi = v >> 32;
876 if (hi)
877 return log2_u32(hi) + 32 + 1;
878 else
879 return log2_u32(v) + 1;
880 }
881
882 /*
883 * sqrt_u64 - Calculate the square root of value @x using Newton's method.
884 */
__sqrt_u64(u64 x)885 static inline u64 __sqrt_u64(u64 x)
886 {
887 if (x == 0 || x == 1)
888 return x;
889
890 u64 r = ((1ULL << 32) > x) ? x : (1ULL << 32);
891
892 for (int i = 0; i < 8; ++i) {
893 u64 q = x / r;
894 if (r <= q)
895 break;
896 r = (r + q) >> 1;
897 }
898 return r;
899 }
900
901 /*
902 * ctzll -- Counts trailing zeros in an unsigned long long. If the input value
903 * is zero, the return value is undefined.
904 */
ctzll(u64 v)905 static inline int ctzll(u64 v)
906 {
907 #if (!defined(__BPF__) && defined(__SCX_TARGET_ARCH_x86)) || \
908 (defined(__BPF__) && defined(__clang_major__) && __clang_major__ >= 19)
909 /*
910 * Use the ctz builtin when: (1) building for native x86, or
911 * (2) building for BPF with clang >= 19 (BPF backend supports
912 * the intrinsic from clang 19 onward; earlier versions hit
913 * "unimplemented opcode" in the backend).
914 */
915 return __builtin_ctzll(v);
916 #else
917 /*
918 * If neither the target architecture nor the toolchains support ctzll,
919 * use software-based emulation. Let's use the De Bruijn sequence-based
920 * approach to find LSB fastly. See the details of De Bruijn sequence:
921 *
922 * https://en.wikipedia.org/wiki/De_Bruijn_sequence
923 * https://www.chessprogramming.org/BitScan#De_Bruijn_Multiplication
924 */
925 const int lookup_table[64] = {
926 0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4,
927 62, 55, 59, 36, 53, 51, 43, 22, 45, 39, 33, 30, 24, 18, 12, 5,
928 63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 44, 32, 23, 11,
929 46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6,
930 };
931 const u64 DEBRUIJN_CONSTANT = 0x03f79d71b4cb0a89ULL;
932 unsigned int index;
933 u64 lowest_bit;
934 const int *lt;
935
936 if (v == 0)
937 return -1;
938
939 /*
940 * Isolate the least significant bit (LSB).
941 * For example, if v = 0b...10100, then v & -v = 0b...00100
942 */
943 lowest_bit = v & -v;
944
945 /*
946 * Each isolated bit produces a unique 6-bit value, guaranteed by the
947 * De Bruijn property. Calculate a unique index into the lookup table
948 * using the magic constant and a right shift.
949 *
950 * Multiplying by the 64-bit constant "spreads out" that 1-bit into a
951 * unique pattern in the top 6 bits. This uniqueness property is
952 * exactly what a De Bruijn sequence guarantees: Every possible 6-bit
953 * pattern (in top bits) occurs exactly once for each LSB position. So,
954 * the constant 0x03f79d71b4cb0a89ULL is carefully chosen to be a
955 * De Bruijn sequence, ensuring no collisions in the table index.
956 */
957 index = (lowest_bit * DEBRUIJN_CONSTANT) >> 58;
958
959 /*
960 * Lookup in a precomputed table. No collision is guaranteed by the
961 * De Bruijn property.
962 */
963 lt = MEMBER_VPTR(lookup_table, [index]);
964 return (lt)? *lt : -1;
965 #endif
966 }
967
968 /*
969 * Return a value proportionally scaled to the task's weight.
970 */
scale_by_task_weight(const struct task_struct * p,u64 value)971 static inline u64 scale_by_task_weight(const struct task_struct *p, u64 value)
972 {
973 return (value * p->scx.weight) / 100;
974 }
975
976 /*
977 * Return a value inversely proportional to the task's weight.
978 */
scale_by_task_weight_inverse(const struct task_struct * p,u64 value)979 static inline u64 scale_by_task_weight_inverse(const struct task_struct *p, u64 value)
980 {
981 return value * 100 / p->scx.weight;
982 }
983
984
985 /*
986 * Get a random u64 from the kernel's pseudo-random generator.
987 */
get_prandom_u64()988 static inline u64 get_prandom_u64()
989 {
990 return ((u64)bpf_get_prandom_u32() << 32) | bpf_get_prandom_u32();
991 }
992
993 /*
994 * Define the shadow structure to avoid a compilation error when
995 * vmlinux.h does not enable necessary kernel configs. The ___local
996 * suffix is a CO-RE convention that tells the loader to match this
997 * against the base struct rq in the kernel. The attribute
998 * preserve_access_index tells the compiler to generate a CO-RE
999 * relocation for these fields.
1000 */
1001 struct rq___local {
1002 /*
1003 * A monotonically increasing clock per CPU. It is rq->clock minus
1004 * cumulative IRQ time and hypervisor steal time. Unlike rq->clock,
1005 * it does not advance during IRQ processing or hypervisor preemption.
1006 * It does advance during idle (the idle task counts as a running task
1007 * for this purpose).
1008 */
1009 u64 clock_task;
1010 /*
1011 * Invariant version of clock_task scaled by CPU capacity and
1012 * frequency. For example, clock_pelt advances 2x slower on a CPU
1013 * with half the capacity.
1014 *
1015 * At idle exit, rq->clock_pelt jumps forward to resync with
1016 * clock_task. The kernel's rq_clock_pelt() corrects for this jump
1017 * by subtracting lost_idle_time, yielding a clock that appears
1018 * continuous across idle transitions. scx_clock_pelt() mirrors
1019 * rq_clock_pelt() by performing the same subtraction.
1020 */
1021 u64 clock_pelt;
1022 /*
1023 * Accumulates the magnitude of each clock_pelt jump at idle exit.
1024 * Subtracting this from clock_pelt gives rq_clock_pelt(): a
1025 * continuous, capacity-invariant clock suitable for both task
1026 * execution time stamping and cross-idle measurements.
1027 */
1028 unsigned long lost_idle_time;
1029 /*
1030 * Shadow of paravirt_steal_clock() (the hypervisor's cumulative
1031 * stolen time counter). Stays frozen while the hypervisor preempts
1032 * the vCPU; catches up the next time update_rq_clock_task() is
1033 * called. The delta is the stolen time not yet subtracted from
1034 * clock_task.
1035 *
1036 * Unlike irqtime->total (a plain kernel-side field), the live stolen
1037 * time counter lives in hypervisor-specific shared memory and has no
1038 * kernel-side equivalent readable from BPF in a hypervisor-agnostic
1039 * way. This field is therefore the only portable BPF-accessible
1040 * approximation of cumulative steal time.
1041 *
1042 * Available only when CONFIG_PARAVIRT_TIME_ACCOUNTING is on.
1043 */
1044 u64 prev_steal_time_rq;
1045 } __attribute__((preserve_access_index));
1046
1047 extern struct rq runqueues __ksym;
1048
1049 /*
1050 * Define the shadow structure to avoid a compilation error when
1051 * vmlinux.h does not enable necessary kernel configs.
1052 */
1053 struct irqtime___local {
1054 /*
1055 * Cumulative IRQ time counter for this CPU, in nanoseconds. Advances
1056 * immediately at the exit of every hardirq and non-ksoftirqd softirq
1057 * via irqtime_account_irq(). ksoftirqd time is counted as normal
1058 * task time and is NOT included. NMI time is also NOT included.
1059 *
1060 * The companion field irqtime->sync (struct u64_stats_sync) protects
1061 * against 64-bit tearing on 32-bit architectures. On 64-bit kernels,
1062 * u64_stats_sync is an empty struct and all seqcount operations are
1063 * no-ops, so a plain BPF_CORE_READ of this field is safe.
1064 *
1065 * Available only when CONFIG_IRQ_TIME_ACCOUNTING is on.
1066 */
1067 u64 total;
1068 } __attribute__((preserve_access_index));
1069
1070 /*
1071 * cpu_irqtime is a per-CPU variable defined only when
1072 * CONFIG_IRQ_TIME_ACCOUNTING is on. Declare it as __weak so the BPF
1073 * loader sets its address to 0 (rather than failing) when the symbol
1074 * is absent from the running kernel.
1075 */
1076 extern struct irqtime___local cpu_irqtime __ksym __weak;
1077
get_current_rq(u32 cpu)1078 static inline struct rq___local *get_current_rq(u32 cpu)
1079 {
1080 /*
1081 * This is a workaround to get an rq pointer now that
1082 * scx_bpf_cpu_rq() has been removed.
1083 *
1084 * WARNING: The caller must hold the rq lock for @cpu. This is
1085 * guaranteed when called from scheduling callbacks (ops.running,
1086 * ops.stopping, ops.enqueue, ops.dequeue, ops.dispatch, etc.).
1087 * There is no runtime check available in BPF for kernel spinlock
1088 * state — correctness is enforced by calling context only.
1089 */
1090 return (void *)bpf_per_cpu_ptr(&runqueues, cpu);
1091 }
1092
scx_clock_task(u32 cpu)1093 static inline u64 scx_clock_task(u32 cpu)
1094 {
1095 struct rq___local *rq = get_current_rq(cpu);
1096
1097 /*
1098 * Equivalent to the kernel's rq_clock_task(): wall-clock time minus
1099 * cumulative IRQ time (CONFIG_IRQ_TIME_ACCOUNTING) and hypervisor
1100 * steal time (CONFIG_PARAVIRT_TIME_ACCOUNTING). Without those configs,
1101 * it equals rq->clock.
1102 *
1103 * Conceptually this clock advances during idle (the idle task counts
1104 * as a running task), but rq->clock_task is only updated on scheduling
1105 * events. With NO_HZ_IDLE (the default), the periodic tick is stopped
1106 * on idle CPUs, so rq->clock_task is not refreshed while a CPU is
1107 * idle. Reading this clock for a remote idle CPU from a BPF timer
1108 * callback returns the value from when the CPU last went idle, making
1109 * the delta over an idle interval effectively zero.
1110 */
1111 return rq ? rq->clock_task : 0;
1112 }
1113
scx_clock_pelt(u32 cpu)1114 static inline u64 scx_clock_pelt(u32 cpu)
1115 {
1116 struct rq___local *rq = get_current_rq(cpu);
1117
1118 /*
1119 * Equivalent to the kernel's rq_clock_pelt(): subtracts
1120 * lost_idle_time from clock_pelt to absorb the jump that occurs
1121 * when clock_pelt resyncs with clock_task at idle exit. The intent
1122 * is a continuous, capacity- and frequency-invariant clock that is
1123 * frozen during idle, IRQ, and hypervisor steal.
1124 *
1125 * However, like scx_clock_task(), this clock has a stale-read issue
1126 * for remote idle CPUs with NO_HZ_IDLE (the default). clock_pelt
1127 * itself advances at wall-clock rate (hardware-clock based), but
1128 * lost_idle_time is only updated via update_rq_clock_pelt(), which
1129 * requires update_rq_clock() to be called. With NO_HZ_IDLE, the
1130 * periodic tick is stopped on idle CPUs, so lost_idle_time is not
1131 * refreshed during idle. Reading this clock for a remote idle CPU
1132 * from a BPF timer callback therefore returns a value that drifts
1133 * at wall-clock rate -- the same stale behaviour as scx_clock_task().
1134 *
1135 * Without NO_HZ_IDLE, periodic ticks keep lost_idle_time nearly in
1136 * sync (stale by at most one tick period, ~1 ms), so the result is
1137 * accurate.
1138 */
1139 return rq ? (rq->clock_pelt - rq->lost_idle_time) : 0;
1140 }
1141
scx_clock_virt(u32 cpu)1142 static inline u64 scx_clock_virt(u32 cpu)
1143 {
1144 struct rq___local *rq;
1145
1146 /*
1147 * Check field existence before calling get_current_rq() so we avoid
1148 * the per_cpu lookup entirely on kernels built without
1149 * CONFIG_PARAVIRT_TIME_ACCOUNTING.
1150 */
1151 if (!bpf_core_field_exists(((struct rq___local *)0)->prev_steal_time_rq))
1152 return 0;
1153
1154 /* Lagging shadow of the kernel's paravirt_steal_clock(). */
1155 rq = get_current_rq(cpu);
1156 return rq ? BPF_CORE_READ(rq, prev_steal_time_rq) : 0;
1157 }
1158
scx_clock_irq(u32 cpu)1159 static inline u64 scx_clock_irq(u32 cpu)
1160 {
1161 struct irqtime___local *irqt;
1162
1163 /*
1164 * bpf_core_type_exists() resolves at load time: if struct irqtime is
1165 * absent from kernel BTF (CONFIG_IRQ_TIME_ACCOUNTING off), the loader
1166 * patches this into an unconditional return 0, making the
1167 * bpf_per_cpu_ptr() call below dead code that the verifier never sees.
1168 */
1169 if (!bpf_core_type_exists(struct irqtime___local))
1170 return 0;
1171
1172 /* Equivalent to the kernel's irq_time_read(). */
1173 irqt = bpf_per_cpu_ptr(&cpu_irqtime, cpu);
1174 return irqt ? BPF_CORE_READ(irqt, total) : 0;
1175 }
1176
1177 /* Abbreviated forms of <linux/overflow.h>'s struct_size() family. */
1178 #define flex_array_size(p, member, count) \
1179 ((count) * sizeof(*(p)->member))
1180
1181 #define struct_size(p, member, count) \
1182 (offsetof(typeof(*(p)), member) + flex_array_size(p, member, count))
1183
1184 #define struct_size_t(type, member, count) \
1185 struct_size((type *)NULL, member, count)
1186
1187 #include "compat.bpf.h"
1188 #include "enums.bpf.h"
1189 #include "cid.bpf.h"
1190
1191 #endif /* __SCX_COMMON_BPF_H */
1192