1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. 4 * Copyright (c) 2022 Tejun Heo <tj@kernel.org> 5 * Copyright (c) 2022 David Vernet <dvernet@meta.com> 6 */ 7 #ifndef __SCX_COMMON_BPF_H 8 #define __SCX_COMMON_BPF_H 9 10 /* 11 * The generated kfunc prototypes in vmlinux.h are missing address space 12 * attributes which cause build failures. For now, suppress the generated 13 * prototypes. See https://github.com/sched-ext/scx/issues/1111. 14 */ 15 #define BPF_NO_KFUNC_PROTOTYPES 16 17 #ifdef LSP 18 #define __bpf__ 19 #include "../vmlinux.h" 20 #else 21 #include "vmlinux.h" 22 #endif 23 24 #include <bpf/bpf_helpers.h> 25 #include <bpf/bpf_tracing.h> 26 #include <asm-generic/errno.h> 27 #include "user_exit_info.h" 28 #include "enum_defs.autogen.h" 29 30 #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ 31 #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ 32 #define PF_EXITING 0x00000004 33 #define CLOCK_MONOTONIC 1 34 35 extern int LINUX_KERNEL_VERSION __kconfig; 36 extern const char CONFIG_CC_VERSION_TEXT[64] __kconfig __weak; 37 extern const char CONFIG_LOCALVERSION[64] __kconfig __weak; 38 39 /* 40 * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can 41 * lead to really confusing misbehaviors. Let's trigger a build failure. 42 */ 43 static inline void ___vmlinux_h_sanity_check___(void) 44 { 45 _Static_assert(SCX_DSQ_FLAG_BUILTIN, 46 "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); 47 } 48 49 s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; 50 s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym; 51 void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak; 52 void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak; 53 u32 scx_bpf_dispatch_nr_slots(void) __ksym; 54 void scx_bpf_dispatch_cancel(void) __ksym; 55 bool scx_bpf_dsq_move_to_local(u64 dsq_id) __ksym __weak; 56 void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak; 57 void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak; 58 bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 59 bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 60 u32 scx_bpf_reenqueue_local(void) __ksym; 61 void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; 62 s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; 63 void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; 64 int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak; 65 struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak; 66 void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak; 67 void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak; 68 void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; 69 void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak; 70 u32 scx_bpf_cpuperf_cap(s32 cpu) __ksym __weak; 71 u32 scx_bpf_cpuperf_cur(s32 cpu) __ksym __weak; 72 void scx_bpf_cpuperf_set(s32 cpu, u32 perf) __ksym __weak; 73 u32 scx_bpf_nr_node_ids(void) __ksym __weak; 74 u32 scx_bpf_nr_cpu_ids(void) __ksym __weak; 75 int scx_bpf_cpu_node(s32 cpu) __ksym __weak; 76 const struct cpumask *scx_bpf_get_possible_cpumask(void) __ksym __weak; 77 const struct cpumask *scx_bpf_get_online_cpumask(void) __ksym __weak; 78 void scx_bpf_put_cpumask(const struct cpumask *cpumask) __ksym __weak; 79 const struct cpumask *scx_bpf_get_idle_cpumask_node(int node) __ksym __weak; 80 const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; 81 const struct cpumask *scx_bpf_get_idle_smtmask_node(int node) __ksym __weak; 82 const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; 83 void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; 84 bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; 85 s32 scx_bpf_pick_idle_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags) __ksym __weak; 86 s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; 87 s32 scx_bpf_pick_any_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags) __ksym __weak; 88 s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; 89 bool scx_bpf_task_running(const struct task_struct *p) __ksym; 90 s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; 91 struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym; 92 struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym __weak; 93 u64 scx_bpf_now(void) __ksym __weak; 94 void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak; 95 96 /* 97 * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from 98 * within bpf_for_each() loops. 99 */ 100 #define BPF_FOR_EACH_ITER (&___it) 101 102 #define scx_read_event(e, name) \ 103 (bpf_core_field_exists((e)->name) ? (e)->name : 0) 104 105 static inline __attribute__((format(printf, 1, 2))) 106 void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {} 107 108 /* 109 * Helper macro for initializing the fmt and variadic argument inputs to both 110 * bstr exit kfuncs. Callers to this function should use ___fmt and ___param to 111 * refer to the initialized list of inputs to the bstr kfunc. 112 */ 113 #define scx_bpf_bstr_preamble(fmt, args...) \ 114 static char ___fmt[] = fmt; \ 115 /* \ 116 * Note that __param[] must have at least one \ 117 * element to keep the verifier happy. \ 118 */ \ 119 unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ 120 \ 121 _Pragma("GCC diagnostic push") \ 122 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ 123 ___bpf_fill(___param, args); \ 124 _Pragma("GCC diagnostic pop") 125 126 /* 127 * scx_bpf_exit() wraps the scx_bpf_exit_bstr() kfunc with variadic arguments 128 * instead of an array of u64. Using this macro will cause the scheduler to 129 * exit cleanly with the specified exit code being passed to user space. 130 */ 131 #define scx_bpf_exit(code, fmt, args...) \ 132 ({ \ 133 scx_bpf_bstr_preamble(fmt, args) \ 134 scx_bpf_exit_bstr(code, ___fmt, ___param, sizeof(___param)); \ 135 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 136 }) 137 138 /* 139 * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments 140 * instead of an array of u64. Invoking this macro will cause the scheduler to 141 * exit in an erroneous state, with diagnostic information being passed to the 142 * user. 143 */ 144 #define scx_bpf_error(fmt, args...) \ 145 ({ \ 146 scx_bpf_bstr_preamble(fmt, args) \ 147 scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ 148 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 149 }) 150 151 /* 152 * scx_bpf_dump() wraps the scx_bpf_dump_bstr() kfunc with variadic arguments 153 * instead of an array of u64. To be used from ops.dump() and friends. 154 */ 155 #define scx_bpf_dump(fmt, args...) \ 156 ({ \ 157 scx_bpf_bstr_preamble(fmt, args) \ 158 scx_bpf_dump_bstr(___fmt, ___param, sizeof(___param)); \ 159 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 160 }) 161 162 /* 163 * scx_bpf_dump_header() is a wrapper around scx_bpf_dump that adds a header 164 * of system information for debugging. 165 */ 166 #define scx_bpf_dump_header() \ 167 ({ \ 168 scx_bpf_dump("kernel: %d.%d.%d %s\ncc: %s\n", \ 169 LINUX_KERNEL_VERSION >> 16, \ 170 LINUX_KERNEL_VERSION >> 8 & 0xFF, \ 171 LINUX_KERNEL_VERSION & 0xFF, \ 172 CONFIG_LOCALVERSION, \ 173 CONFIG_CC_VERSION_TEXT); \ 174 }) 175 176 #define BPF_STRUCT_OPS(name, args...) \ 177 SEC("struct_ops/"#name) \ 178 BPF_PROG(name, ##args) 179 180 #define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ 181 SEC("struct_ops.s/"#name) \ 182 BPF_PROG(name, ##args) 183 184 /** 185 * RESIZABLE_ARRAY - Generates annotations for an array that may be resized 186 * @elfsec: the data section of the BPF program in which to place the array 187 * @arr: the name of the array 188 * 189 * libbpf has an API for setting map value sizes. Since data sections (i.e. 190 * bss, data, rodata) themselves are maps, a data section can be resized. If 191 * a data section has an array as its last element, the BTF info for that 192 * array will be adjusted so that length of the array is extended to meet the 193 * new length of the data section. This macro annotates an array to have an 194 * element count of one with the assumption that this array can be resized 195 * within the userspace program. It also annotates the section specifier so 196 * this array exists in a custom sub data section which can be resized 197 * independently. 198 * 199 * See RESIZE_ARRAY() for the userspace convenience macro for resizing an 200 * array declared with RESIZABLE_ARRAY(). 201 */ 202 #define RESIZABLE_ARRAY(elfsec, arr) arr[1] SEC("."#elfsec"."#arr) 203 204 /** 205 * MEMBER_VPTR - Obtain the verified pointer to a struct or array member 206 * @base: struct or array to index 207 * @member: dereferenced member (e.g. .field, [idx0][idx1], .field[idx0] ...) 208 * 209 * The verifier often gets confused by the instruction sequence the compiler 210 * generates for indexing struct fields or arrays. This macro forces the 211 * compiler to generate a code sequence which first calculates the byte offset, 212 * checks it against the struct or array size and add that byte offset to 213 * generate the pointer to the member to help the verifier. 214 * 215 * Ideally, we want to abort if the calculated offset is out-of-bounds. However, 216 * BPF currently doesn't support abort, so evaluate to %NULL instead. The caller 217 * must check for %NULL and take appropriate action to appease the verifier. To 218 * avoid confusing the verifier, it's best to check for %NULL and dereference 219 * immediately. 220 * 221 * vptr = MEMBER_VPTR(my_array, [i][j]); 222 * if (!vptr) 223 * return error; 224 * *vptr = new_value; 225 * 226 * sizeof(@base) should encompass the memory area to be accessed and thus can't 227 * be a pointer to the area. Use `MEMBER_VPTR(*ptr, .member)` instead of 228 * `MEMBER_VPTR(ptr, ->member)`. 229 */ 230 #define MEMBER_VPTR(base, member) (typeof((base) member) *) \ 231 ({ \ 232 u64 __base = (u64)&(base); \ 233 u64 __addr = (u64)&((base) member) - __base; \ 234 _Static_assert(sizeof(base) >= sizeof((base) member), \ 235 "@base is smaller than @member, is @base a pointer?"); \ 236 asm volatile ( \ 237 "if %0 <= %[max] goto +2\n" \ 238 "%0 = 0\n" \ 239 "goto +1\n" \ 240 "%0 += %1\n" \ 241 : "+r"(__addr) \ 242 : "r"(__base), \ 243 [max]"i"(sizeof(base) - sizeof((base) member))); \ 244 __addr; \ 245 }) 246 247 /** 248 * ARRAY_ELEM_PTR - Obtain the verified pointer to an array element 249 * @arr: array to index into 250 * @i: array index 251 * @n: number of elements in array 252 * 253 * Similar to MEMBER_VPTR() but is intended for use with arrays where the 254 * element count needs to be explicit. 255 * It can be used in cases where a global array is defined with an initial 256 * size but is intended to be be resized before loading the BPF program. 257 * Without this version of the macro, MEMBER_VPTR() will use the compile time 258 * size of the array to compute the max, which will result in rejection by 259 * the verifier. 260 */ 261 #define ARRAY_ELEM_PTR(arr, i, n) (typeof(arr[i]) *) \ 262 ({ \ 263 u64 __base = (u64)arr; \ 264 u64 __addr = (u64)&(arr[i]) - __base; \ 265 asm volatile ( \ 266 "if %0 <= %[max] goto +2\n" \ 267 "%0 = 0\n" \ 268 "goto +1\n" \ 269 "%0 += %1\n" \ 270 : "+r"(__addr) \ 271 : "r"(__base), \ 272 [max]"r"(sizeof(arr[0]) * ((n) - 1))); \ 273 __addr; \ 274 }) 275 276 277 /* 278 * BPF declarations and helpers 279 */ 280 281 /* list and rbtree */ 282 #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) 283 #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) 284 285 void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; 286 void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; 287 288 #define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) 289 #define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) 290 291 int bpf_list_push_front_impl(struct bpf_list_head *head, 292 struct bpf_list_node *node, 293 void *meta, __u64 off) __ksym; 294 #define bpf_list_push_front(head, node) bpf_list_push_front_impl(head, node, NULL, 0) 295 296 int bpf_list_push_back_impl(struct bpf_list_head *head, 297 struct bpf_list_node *node, 298 void *meta, __u64 off) __ksym; 299 #define bpf_list_push_back(head, node) bpf_list_push_back_impl(head, node, NULL, 0) 300 301 struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; 302 struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; 303 struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, 304 struct bpf_rb_node *node) __ksym; 305 int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, 306 bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b), 307 void *meta, __u64 off) __ksym; 308 #define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0) 309 310 struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; 311 312 void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym; 313 #define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL) 314 315 /* task */ 316 struct task_struct *bpf_task_from_pid(s32 pid) __ksym; 317 struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; 318 void bpf_task_release(struct task_struct *p) __ksym; 319 320 /* cgroup */ 321 struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; 322 void bpf_cgroup_release(struct cgroup *cgrp) __ksym; 323 struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; 324 325 /* css iteration */ 326 struct bpf_iter_css; 327 struct cgroup_subsys_state; 328 extern int bpf_iter_css_new(struct bpf_iter_css *it, 329 struct cgroup_subsys_state *start, 330 unsigned int flags) __weak __ksym; 331 extern struct cgroup_subsys_state * 332 bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym; 333 extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym; 334 335 /* cpumask */ 336 struct bpf_cpumask *bpf_cpumask_create(void) __ksym; 337 struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; 338 void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; 339 u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; 340 u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; 341 void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 342 void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 343 bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; 344 bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 345 bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 346 void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; 347 void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; 348 bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, 349 const struct cpumask *src2) __ksym; 350 void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, 351 const struct cpumask *src2) __ksym; 352 void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, 353 const struct cpumask *src2) __ksym; 354 bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; 355 bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; 356 bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; 357 bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; 358 bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; 359 void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; 360 u32 bpf_cpumask_any_distribute(const struct cpumask *cpumask) __ksym; 361 u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, 362 const struct cpumask *src2) __ksym; 363 u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym; 364 365 int bpf_iter_bits_new(struct bpf_iter_bits *it, const u64 *unsafe_ptr__ign, u32 nr_words) __ksym; 366 int *bpf_iter_bits_next(struct bpf_iter_bits *it) __ksym; 367 void bpf_iter_bits_destroy(struct bpf_iter_bits *it) __ksym; 368 369 #define def_iter_struct(name) \ 370 struct bpf_iter_##name { \ 371 struct bpf_iter_bits it; \ 372 const struct cpumask *bitmap; \ 373 }; 374 375 #define def_iter_new(name) \ 376 static inline int bpf_iter_##name##_new( \ 377 struct bpf_iter_##name *it, const u64 *unsafe_ptr__ign, u32 nr_words) \ 378 { \ 379 it->bitmap = scx_bpf_get_##name##_cpumask(); \ 380 return bpf_iter_bits_new(&it->it, (const u64 *)it->bitmap, \ 381 sizeof(struct cpumask) / 8); \ 382 } 383 384 #define def_iter_next(name) \ 385 static inline int *bpf_iter_##name##_next(struct bpf_iter_##name *it) { \ 386 return bpf_iter_bits_next(&it->it); \ 387 } 388 389 #define def_iter_destroy(name) \ 390 static inline void bpf_iter_##name##_destroy(struct bpf_iter_##name *it) { \ 391 scx_bpf_put_cpumask(it->bitmap); \ 392 bpf_iter_bits_destroy(&it->it); \ 393 } 394 #define def_for_each_cpu(cpu, name) for_each_##name##_cpu(cpu) 395 396 /// Provides iterator for possible and online cpus. 397 /// 398 /// # Example 399 /// 400 /// ``` 401 /// static inline void example_use() { 402 /// int *cpu; 403 /// 404 /// for_each_possible_cpu(cpu){ 405 /// bpf_printk("CPU %d is possible", *cpu); 406 /// } 407 /// 408 /// for_each_online_cpu(cpu){ 409 /// bpf_printk("CPU %d is online", *cpu); 410 /// } 411 /// } 412 /// ``` 413 def_iter_struct(possible); 414 def_iter_new(possible); 415 def_iter_next(possible); 416 def_iter_destroy(possible); 417 #define for_each_possible_cpu(cpu) bpf_for_each(possible, cpu, NULL, 0) 418 419 def_iter_struct(online); 420 def_iter_new(online); 421 def_iter_next(online); 422 def_iter_destroy(online); 423 #define for_each_online_cpu(cpu) bpf_for_each(online, cpu, NULL, 0) 424 425 /* 426 * Access a cpumask in read-only mode (typically to check bits). 427 */ 428 static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask) 429 { 430 return (const struct cpumask *)mask; 431 } 432 433 /* 434 * Return true if task @p cannot migrate to a different CPU, false 435 * otherwise. 436 */ 437 static inline bool is_migration_disabled(const struct task_struct *p) 438 { 439 if (bpf_core_field_exists(p->migration_disabled)) 440 return p->migration_disabled; 441 return false; 442 } 443 444 /* rcu */ 445 void bpf_rcu_read_lock(void) __ksym; 446 void bpf_rcu_read_unlock(void) __ksym; 447 448 /* 449 * Time helpers, most of which are from jiffies.h. 450 */ 451 452 /** 453 * time_delta - Calculate the delta between new and old time stamp 454 * @after: first comparable as u64 455 * @before: second comparable as u64 456 * 457 * Return: the time difference, which is >= 0 458 */ 459 static inline s64 time_delta(u64 after, u64 before) 460 { 461 return (s64)(after - before) > 0 ? (s64)(after - before) : 0; 462 } 463 464 /** 465 * time_after - returns true if the time a is after time b. 466 * @a: first comparable as u64 467 * @b: second comparable as u64 468 * 469 * Do this with "<0" and ">=0" to only test the sign of the result. A 470 * good compiler would generate better code (and a really good compiler 471 * wouldn't care). Gcc is currently neither. 472 * 473 * Return: %true is time a is after time b, otherwise %false. 474 */ 475 static inline bool time_after(u64 a, u64 b) 476 { 477 return (s64)(b - a) < 0; 478 } 479 480 /** 481 * time_before - returns true if the time a is before time b. 482 * @a: first comparable as u64 483 * @b: second comparable as u64 484 * 485 * Return: %true is time a is before time b, otherwise %false. 486 */ 487 static inline bool time_before(u64 a, u64 b) 488 { 489 return time_after(b, a); 490 } 491 492 /** 493 * time_after_eq - returns true if the time a is after or the same as time b. 494 * @a: first comparable as u64 495 * @b: second comparable as u64 496 * 497 * Return: %true is time a is after or the same as time b, otherwise %false. 498 */ 499 static inline bool time_after_eq(u64 a, u64 b) 500 { 501 return (s64)(a - b) >= 0; 502 } 503 504 /** 505 * time_before_eq - returns true if the time a is before or the same as time b. 506 * @a: first comparable as u64 507 * @b: second comparable as u64 508 * 509 * Return: %true is time a is before or the same as time b, otherwise %false. 510 */ 511 static inline bool time_before_eq(u64 a, u64 b) 512 { 513 return time_after_eq(b, a); 514 } 515 516 /** 517 * time_in_range - Calculate whether a is in the range of [b, c]. 518 * @a: time to test 519 * @b: beginning of the range 520 * @c: end of the range 521 * 522 * Return: %true is time a is in the range [b, c], otherwise %false. 523 */ 524 static inline bool time_in_range(u64 a, u64 b, u64 c) 525 { 526 return time_after_eq(a, b) && time_before_eq(a, c); 527 } 528 529 /** 530 * time_in_range_open - Calculate whether a is in the range of [b, c). 531 * @a: time to test 532 * @b: beginning of the range 533 * @c: end of the range 534 * 535 * Return: %true is time a is in the range [b, c), otherwise %false. 536 */ 537 static inline bool time_in_range_open(u64 a, u64 b, u64 c) 538 { 539 return time_after_eq(a, b) && time_before(a, c); 540 } 541 542 543 /* 544 * Other helpers 545 */ 546 547 /* useful compiler attributes */ 548 #define likely(x) __builtin_expect(!!(x), 1) 549 #define unlikely(x) __builtin_expect(!!(x), 0) 550 #define __maybe_unused __attribute__((__unused__)) 551 552 /* 553 * READ/WRITE_ONCE() are from kernel (include/asm-generic/rwonce.h). They 554 * prevent compiler from caching, redoing or reordering reads or writes. 555 */ 556 typedef __u8 __attribute__((__may_alias__)) __u8_alias_t; 557 typedef __u16 __attribute__((__may_alias__)) __u16_alias_t; 558 typedef __u32 __attribute__((__may_alias__)) __u32_alias_t; 559 typedef __u64 __attribute__((__may_alias__)) __u64_alias_t; 560 561 static __always_inline void __read_once_size(const volatile void *p, void *res, int size) 562 { 563 switch (size) { 564 case 1: *(__u8_alias_t *) res = *(volatile __u8_alias_t *) p; break; 565 case 2: *(__u16_alias_t *) res = *(volatile __u16_alias_t *) p; break; 566 case 4: *(__u32_alias_t *) res = *(volatile __u32_alias_t *) p; break; 567 case 8: *(__u64_alias_t *) res = *(volatile __u64_alias_t *) p; break; 568 default: 569 barrier(); 570 __builtin_memcpy((void *)res, (const void *)p, size); 571 barrier(); 572 } 573 } 574 575 static __always_inline void __write_once_size(volatile void *p, void *res, int size) 576 { 577 switch (size) { 578 case 1: *(volatile __u8_alias_t *) p = *(__u8_alias_t *) res; break; 579 case 2: *(volatile __u16_alias_t *) p = *(__u16_alias_t *) res; break; 580 case 4: *(volatile __u32_alias_t *) p = *(__u32_alias_t *) res; break; 581 case 8: *(volatile __u64_alias_t *) p = *(__u64_alias_t *) res; break; 582 default: 583 barrier(); 584 __builtin_memcpy((void *)p, (const void *)res, size); 585 barrier(); 586 } 587 } 588 589 /* 590 * __unqual_typeof(x) - Declare an unqualified scalar type, leaving 591 * non-scalar types unchanged, 592 * 593 * Prefer C11 _Generic for better compile-times and simpler code. Note: 'char' 594 * is not type-compatible with 'signed char', and we define a separate case. 595 * 596 * This is copied verbatim from kernel's include/linux/compiler_types.h, but 597 * with default expression (for pointers) changed from (x) to (typeof(x)0). 598 * 599 * This is because LLVM has a bug where for lvalue (x), it does not get rid of 600 * an extra address_space qualifier, but does in case of rvalue (typeof(x)0). 601 * Hence, for pointers, we need to create an rvalue expression to get the 602 * desired type. See https://github.com/llvm/llvm-project/issues/53400. 603 */ 604 #define __scalar_type_to_expr_cases(type) \ 605 unsigned type : (unsigned type)0, signed type : (signed type)0 606 607 #define __unqual_typeof(x) \ 608 typeof(_Generic((x), \ 609 char: (char)0, \ 610 __scalar_type_to_expr_cases(char), \ 611 __scalar_type_to_expr_cases(short), \ 612 __scalar_type_to_expr_cases(int), \ 613 __scalar_type_to_expr_cases(long), \ 614 __scalar_type_to_expr_cases(long long), \ 615 default: (typeof(x))0)) 616 617 #define READ_ONCE(x) \ 618 ({ \ 619 union { __unqual_typeof(x) __val; char __c[1]; } __u = \ 620 { .__c = { 0 } }; \ 621 __read_once_size((__unqual_typeof(x) *)&(x), __u.__c, sizeof(x)); \ 622 __u.__val; \ 623 }) 624 625 #define WRITE_ONCE(x, val) \ 626 ({ \ 627 union { __unqual_typeof(x) __val; char __c[1]; } __u = \ 628 { .__val = (val) }; \ 629 __write_once_size((__unqual_typeof(x) *)&(x), __u.__c, sizeof(x)); \ 630 __u.__val; \ 631 }) 632 633 /* 634 * log2_u32 - Compute the base 2 logarithm of a 32-bit exponential value. 635 * @v: The value for which we're computing the base 2 logarithm. 636 */ 637 static inline u32 log2_u32(u32 v) 638 { 639 u32 r; 640 u32 shift; 641 642 r = (v > 0xFFFF) << 4; v >>= r; 643 shift = (v > 0xFF) << 3; v >>= shift; r |= shift; 644 shift = (v > 0xF) << 2; v >>= shift; r |= shift; 645 shift = (v > 0x3) << 1; v >>= shift; r |= shift; 646 r |= (v >> 1); 647 return r; 648 } 649 650 /* 651 * log2_u64 - Compute the base 2 logarithm of a 64-bit exponential value. 652 * @v: The value for which we're computing the base 2 logarithm. 653 */ 654 static inline u32 log2_u64(u64 v) 655 { 656 u32 hi = v >> 32; 657 if (hi) 658 return log2_u32(hi) + 32 + 1; 659 else 660 return log2_u32(v) + 1; 661 } 662 663 /* 664 * Return a value proportionally scaled to the task's weight. 665 */ 666 static inline u64 scale_by_task_weight(const struct task_struct *p, u64 value) 667 { 668 return (value * p->scx.weight) / 100; 669 } 670 671 /* 672 * Return a value inversely proportional to the task's weight. 673 */ 674 static inline u64 scale_by_task_weight_inverse(const struct task_struct *p, u64 value) 675 { 676 return value * 100 / p->scx.weight; 677 } 678 679 680 #include "compat.bpf.h" 681 #include "enums.bpf.h" 682 683 #endif /* __SCX_COMMON_BPF_H */ 684