1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. 4 * Copyright (c) 2022 Tejun Heo <tj@kernel.org> 5 * Copyright (c) 2022 David Vernet <dvernet@meta.com> 6 */ 7 #ifndef __SCX_COMMON_BPF_H 8 #define __SCX_COMMON_BPF_H 9 10 #ifdef LSP 11 #define __bpf__ 12 #include "../vmlinux.h" 13 #else 14 #include "vmlinux.h" 15 #endif 16 17 #include <bpf/bpf_helpers.h> 18 #include <bpf/bpf_tracing.h> 19 #include <asm-generic/errno.h> 20 #include "user_exit_info.h" 21 22 #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ 23 #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ 24 #define PF_EXITING 0x00000004 25 #define CLOCK_MONOTONIC 1 26 27 extern int LINUX_KERNEL_VERSION __kconfig; 28 extern const char CONFIG_CC_VERSION_TEXT[64] __kconfig __weak; 29 extern const char CONFIG_LOCALVERSION[64] __kconfig __weak; 30 31 /* 32 * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can 33 * lead to really confusing misbehaviors. Let's trigger a build failure. 34 */ 35 static inline void ___vmlinux_h_sanity_check___(void) 36 { 37 _Static_assert(SCX_DSQ_FLAG_BUILTIN, 38 "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); 39 } 40 41 s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; 42 s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym; 43 void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak; 44 void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak; 45 u32 scx_bpf_dispatch_nr_slots(void) __ksym; 46 void scx_bpf_dispatch_cancel(void) __ksym; 47 bool scx_bpf_dsq_move_to_local(u64 dsq_id) __ksym __weak; 48 void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak; 49 void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak; 50 bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 51 bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 52 u32 scx_bpf_reenqueue_local(void) __ksym; 53 void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; 54 s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; 55 void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; 56 int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak; 57 struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak; 58 void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak; 59 void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak; 60 void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; 61 void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak; 62 u32 scx_bpf_cpuperf_cap(s32 cpu) __ksym __weak; 63 u32 scx_bpf_cpuperf_cur(s32 cpu) __ksym __weak; 64 void scx_bpf_cpuperf_set(s32 cpu, u32 perf) __ksym __weak; 65 u32 scx_bpf_nr_cpu_ids(void) __ksym __weak; 66 const struct cpumask *scx_bpf_get_possible_cpumask(void) __ksym __weak; 67 const struct cpumask *scx_bpf_get_online_cpumask(void) __ksym __weak; 68 void scx_bpf_put_cpumask(const struct cpumask *cpumask) __ksym __weak; 69 const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; 70 const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; 71 void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; 72 bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; 73 s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; 74 s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; 75 bool scx_bpf_task_running(const struct task_struct *p) __ksym; 76 s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; 77 struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym; 78 struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym __weak; 79 u64 scx_bpf_now(void) __ksym __weak; 80 81 /* 82 * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from 83 * within bpf_for_each() loops. 84 */ 85 #define BPF_FOR_EACH_ITER (&___it) 86 87 static inline __attribute__((format(printf, 1, 2))) 88 void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {} 89 90 /* 91 * Helper macro for initializing the fmt and variadic argument inputs to both 92 * bstr exit kfuncs. Callers to this function should use ___fmt and ___param to 93 * refer to the initialized list of inputs to the bstr kfunc. 94 */ 95 #define scx_bpf_bstr_preamble(fmt, args...) \ 96 static char ___fmt[] = fmt; \ 97 /* \ 98 * Note that __param[] must have at least one \ 99 * element to keep the verifier happy. \ 100 */ \ 101 unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ 102 \ 103 _Pragma("GCC diagnostic push") \ 104 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ 105 ___bpf_fill(___param, args); \ 106 _Pragma("GCC diagnostic pop") 107 108 /* 109 * scx_bpf_exit() wraps the scx_bpf_exit_bstr() kfunc with variadic arguments 110 * instead of an array of u64. Using this macro will cause the scheduler to 111 * exit cleanly with the specified exit code being passed to user space. 112 */ 113 #define scx_bpf_exit(code, fmt, args...) \ 114 ({ \ 115 scx_bpf_bstr_preamble(fmt, args) \ 116 scx_bpf_exit_bstr(code, ___fmt, ___param, sizeof(___param)); \ 117 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 118 }) 119 120 /* 121 * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments 122 * instead of an array of u64. Invoking this macro will cause the scheduler to 123 * exit in an erroneous state, with diagnostic information being passed to the 124 * user. 125 */ 126 #define scx_bpf_error(fmt, args...) \ 127 ({ \ 128 scx_bpf_bstr_preamble(fmt, args) \ 129 scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ 130 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 131 }) 132 133 /* 134 * scx_bpf_dump() wraps the scx_bpf_dump_bstr() kfunc with variadic arguments 135 * instead of an array of u64. To be used from ops.dump() and friends. 136 */ 137 #define scx_bpf_dump(fmt, args...) \ 138 ({ \ 139 scx_bpf_bstr_preamble(fmt, args) \ 140 scx_bpf_dump_bstr(___fmt, ___param, sizeof(___param)); \ 141 ___scx_bpf_bstr_format_checker(fmt, ##args); \ 142 }) 143 144 /* 145 * scx_bpf_dump_header() is a wrapper around scx_bpf_dump that adds a header 146 * of system information for debugging. 147 */ 148 #define scx_bpf_dump_header() \ 149 ({ \ 150 scx_bpf_dump("kernel: %d.%d.%d %s\ncc: %s\n", \ 151 LINUX_KERNEL_VERSION >> 16, \ 152 LINUX_KERNEL_VERSION >> 8 & 0xFF, \ 153 LINUX_KERNEL_VERSION & 0xFF, \ 154 CONFIG_LOCALVERSION, \ 155 CONFIG_CC_VERSION_TEXT); \ 156 }) 157 158 #define BPF_STRUCT_OPS(name, args...) \ 159 SEC("struct_ops/"#name) \ 160 BPF_PROG(name, ##args) 161 162 #define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ 163 SEC("struct_ops.s/"#name) \ 164 BPF_PROG(name, ##args) 165 166 /** 167 * RESIZABLE_ARRAY - Generates annotations for an array that may be resized 168 * @elfsec: the data section of the BPF program in which to place the array 169 * @arr: the name of the array 170 * 171 * libbpf has an API for setting map value sizes. Since data sections (i.e. 172 * bss, data, rodata) themselves are maps, a data section can be resized. If 173 * a data section has an array as its last element, the BTF info for that 174 * array will be adjusted so that length of the array is extended to meet the 175 * new length of the data section. This macro annotates an array to have an 176 * element count of one with the assumption that this array can be resized 177 * within the userspace program. It also annotates the section specifier so 178 * this array exists in a custom sub data section which can be resized 179 * independently. 180 * 181 * See RESIZE_ARRAY() for the userspace convenience macro for resizing an 182 * array declared with RESIZABLE_ARRAY(). 183 */ 184 #define RESIZABLE_ARRAY(elfsec, arr) arr[1] SEC("."#elfsec"."#arr) 185 186 /** 187 * MEMBER_VPTR - Obtain the verified pointer to a struct or array member 188 * @base: struct or array to index 189 * @member: dereferenced member (e.g. .field, [idx0][idx1], .field[idx0] ...) 190 * 191 * The verifier often gets confused by the instruction sequence the compiler 192 * generates for indexing struct fields or arrays. This macro forces the 193 * compiler to generate a code sequence which first calculates the byte offset, 194 * checks it against the struct or array size and add that byte offset to 195 * generate the pointer to the member to help the verifier. 196 * 197 * Ideally, we want to abort if the calculated offset is out-of-bounds. However, 198 * BPF currently doesn't support abort, so evaluate to %NULL instead. The caller 199 * must check for %NULL and take appropriate action to appease the verifier. To 200 * avoid confusing the verifier, it's best to check for %NULL and dereference 201 * immediately. 202 * 203 * vptr = MEMBER_VPTR(my_array, [i][j]); 204 * if (!vptr) 205 * return error; 206 * *vptr = new_value; 207 * 208 * sizeof(@base) should encompass the memory area to be accessed and thus can't 209 * be a pointer to the area. Use `MEMBER_VPTR(*ptr, .member)` instead of 210 * `MEMBER_VPTR(ptr, ->member)`. 211 */ 212 #define MEMBER_VPTR(base, member) (typeof((base) member) *) \ 213 ({ \ 214 u64 __base = (u64)&(base); \ 215 u64 __addr = (u64)&((base) member) - __base; \ 216 _Static_assert(sizeof(base) >= sizeof((base) member), \ 217 "@base is smaller than @member, is @base a pointer?"); \ 218 asm volatile ( \ 219 "if %0 <= %[max] goto +2\n" \ 220 "%0 = 0\n" \ 221 "goto +1\n" \ 222 "%0 += %1\n" \ 223 : "+r"(__addr) \ 224 : "r"(__base), \ 225 [max]"i"(sizeof(base) - sizeof((base) member))); \ 226 __addr; \ 227 }) 228 229 /** 230 * ARRAY_ELEM_PTR - Obtain the verified pointer to an array element 231 * @arr: array to index into 232 * @i: array index 233 * @n: number of elements in array 234 * 235 * Similar to MEMBER_VPTR() but is intended for use with arrays where the 236 * element count needs to be explicit. 237 * It can be used in cases where a global array is defined with an initial 238 * size but is intended to be be resized before loading the BPF program. 239 * Without this version of the macro, MEMBER_VPTR() will use the compile time 240 * size of the array to compute the max, which will result in rejection by 241 * the verifier. 242 */ 243 #define ARRAY_ELEM_PTR(arr, i, n) (typeof(arr[i]) *) \ 244 ({ \ 245 u64 __base = (u64)arr; \ 246 u64 __addr = (u64)&(arr[i]) - __base; \ 247 asm volatile ( \ 248 "if %0 <= %[max] goto +2\n" \ 249 "%0 = 0\n" \ 250 "goto +1\n" \ 251 "%0 += %1\n" \ 252 : "+r"(__addr) \ 253 : "r"(__base), \ 254 [max]"r"(sizeof(arr[0]) * ((n) - 1))); \ 255 __addr; \ 256 }) 257 258 259 /* 260 * BPF declarations and helpers 261 */ 262 263 /* list and rbtree */ 264 #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) 265 #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) 266 267 void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; 268 void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; 269 270 #define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) 271 #define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) 272 273 void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; 274 void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; 275 struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; 276 struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; 277 struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, 278 struct bpf_rb_node *node) __ksym; 279 int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, 280 bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b), 281 void *meta, __u64 off) __ksym; 282 #define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0) 283 284 struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; 285 286 void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym; 287 #define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL) 288 289 /* task */ 290 struct task_struct *bpf_task_from_pid(s32 pid) __ksym; 291 struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; 292 void bpf_task_release(struct task_struct *p) __ksym; 293 294 /* cgroup */ 295 struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; 296 void bpf_cgroup_release(struct cgroup *cgrp) __ksym; 297 struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; 298 299 /* css iteration */ 300 struct bpf_iter_css; 301 struct cgroup_subsys_state; 302 extern int bpf_iter_css_new(struct bpf_iter_css *it, 303 struct cgroup_subsys_state *start, 304 unsigned int flags) __weak __ksym; 305 extern struct cgroup_subsys_state * 306 bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym; 307 extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym; 308 309 /* cpumask */ 310 struct bpf_cpumask *bpf_cpumask_create(void) __ksym; 311 struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; 312 void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; 313 u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; 314 u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; 315 void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 316 void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 317 bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; 318 bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 319 bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; 320 void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; 321 void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; 322 bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, 323 const struct cpumask *src2) __ksym; 324 void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, 325 const struct cpumask *src2) __ksym; 326 void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, 327 const struct cpumask *src2) __ksym; 328 bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; 329 bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; 330 bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; 331 bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; 332 bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; 333 void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; 334 u32 bpf_cpumask_any_distribute(const struct cpumask *cpumask) __ksym; 335 u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, 336 const struct cpumask *src2) __ksym; 337 u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym; 338 339 int bpf_iter_bits_new(struct bpf_iter_bits *it, const u64 *unsafe_ptr__ign, u32 nr_words) __ksym; 340 int *bpf_iter_bits_next(struct bpf_iter_bits *it) __ksym; 341 void bpf_iter_bits_destroy(struct bpf_iter_bits *it) __ksym; 342 343 #define def_iter_struct(name) \ 344 struct bpf_iter_##name { \ 345 struct bpf_iter_bits it; \ 346 const struct cpumask *bitmap; \ 347 }; 348 349 #define def_iter_new(name) \ 350 static inline int bpf_iter_##name##_new( \ 351 struct bpf_iter_##name *it, const u64 *unsafe_ptr__ign, u32 nr_words) \ 352 { \ 353 it->bitmap = scx_bpf_get_##name##_cpumask(); \ 354 return bpf_iter_bits_new(&it->it, (const u64 *)it->bitmap, \ 355 sizeof(struct cpumask) / 8); \ 356 } 357 358 #define def_iter_next(name) \ 359 static inline int *bpf_iter_##name##_next(struct bpf_iter_##name *it) { \ 360 return bpf_iter_bits_next(&it->it); \ 361 } 362 363 #define def_iter_destroy(name) \ 364 static inline void bpf_iter_##name##_destroy(struct bpf_iter_##name *it) { \ 365 scx_bpf_put_cpumask(it->bitmap); \ 366 bpf_iter_bits_destroy(&it->it); \ 367 } 368 #define def_for_each_cpu(cpu, name) for_each_##name##_cpu(cpu) 369 370 /// Provides iterator for possible and online cpus. 371 /// 372 /// # Example 373 /// 374 /// ``` 375 /// static inline void example_use() { 376 /// int *cpu; 377 /// 378 /// for_each_possible_cpu(cpu){ 379 /// bpf_printk("CPU %d is possible", *cpu); 380 /// } 381 /// 382 /// for_each_online_cpu(cpu){ 383 /// bpf_printk("CPU %d is online", *cpu); 384 /// } 385 /// } 386 /// ``` 387 def_iter_struct(possible); 388 def_iter_new(possible); 389 def_iter_next(possible); 390 def_iter_destroy(possible); 391 #define for_each_possible_cpu(cpu) bpf_for_each(possible, cpu, NULL, 0) 392 393 def_iter_struct(online); 394 def_iter_new(online); 395 def_iter_next(online); 396 def_iter_destroy(online); 397 #define for_each_online_cpu(cpu) bpf_for_each(online, cpu, NULL, 0) 398 399 /* 400 * Access a cpumask in read-only mode (typically to check bits). 401 */ 402 static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask) 403 { 404 return (const struct cpumask *)mask; 405 } 406 407 /* rcu */ 408 void bpf_rcu_read_lock(void) __ksym; 409 void bpf_rcu_read_unlock(void) __ksym; 410 411 412 /* 413 * Other helpers 414 */ 415 416 /* useful compiler attributes */ 417 #define likely(x) __builtin_expect(!!(x), 1) 418 #define unlikely(x) __builtin_expect(!!(x), 0) 419 #define __maybe_unused __attribute__((__unused__)) 420 421 /* 422 * READ/WRITE_ONCE() are from kernel (include/asm-generic/rwonce.h). They 423 * prevent compiler from caching, redoing or reordering reads or writes. 424 */ 425 typedef __u8 __attribute__((__may_alias__)) __u8_alias_t; 426 typedef __u16 __attribute__((__may_alias__)) __u16_alias_t; 427 typedef __u32 __attribute__((__may_alias__)) __u32_alias_t; 428 typedef __u64 __attribute__((__may_alias__)) __u64_alias_t; 429 430 static __always_inline void __read_once_size(const volatile void *p, void *res, int size) 431 { 432 switch (size) { 433 case 1: *(__u8_alias_t *) res = *(volatile __u8_alias_t *) p; break; 434 case 2: *(__u16_alias_t *) res = *(volatile __u16_alias_t *) p; break; 435 case 4: *(__u32_alias_t *) res = *(volatile __u32_alias_t *) p; break; 436 case 8: *(__u64_alias_t *) res = *(volatile __u64_alias_t *) p; break; 437 default: 438 barrier(); 439 __builtin_memcpy((void *)res, (const void *)p, size); 440 barrier(); 441 } 442 } 443 444 static __always_inline void __write_once_size(volatile void *p, void *res, int size) 445 { 446 switch (size) { 447 case 1: *(volatile __u8_alias_t *) p = *(__u8_alias_t *) res; break; 448 case 2: *(volatile __u16_alias_t *) p = *(__u16_alias_t *) res; break; 449 case 4: *(volatile __u32_alias_t *) p = *(__u32_alias_t *) res; break; 450 case 8: *(volatile __u64_alias_t *) p = *(__u64_alias_t *) res; break; 451 default: 452 barrier(); 453 __builtin_memcpy((void *)p, (const void *)res, size); 454 barrier(); 455 } 456 } 457 458 #define READ_ONCE(x) \ 459 ({ \ 460 union { typeof(x) __val; char __c[1]; } __u = \ 461 { .__c = { 0 } }; \ 462 __read_once_size(&(x), __u.__c, sizeof(x)); \ 463 __u.__val; \ 464 }) 465 466 #define WRITE_ONCE(x, val) \ 467 ({ \ 468 union { typeof(x) __val; char __c[1]; } __u = \ 469 { .__val = (val) }; \ 470 __write_once_size(&(x), __u.__c, sizeof(x)); \ 471 __u.__val; \ 472 }) 473 474 /* 475 * log2_u32 - Compute the base 2 logarithm of a 32-bit exponential value. 476 * @v: The value for which we're computing the base 2 logarithm. 477 */ 478 static inline u32 log2_u32(u32 v) 479 { 480 u32 r; 481 u32 shift; 482 483 r = (v > 0xFFFF) << 4; v >>= r; 484 shift = (v > 0xFF) << 3; v >>= shift; r |= shift; 485 shift = (v > 0xF) << 2; v >>= shift; r |= shift; 486 shift = (v > 0x3) << 1; v >>= shift; r |= shift; 487 r |= (v >> 1); 488 return r; 489 } 490 491 /* 492 * log2_u64 - Compute the base 2 logarithm of a 64-bit exponential value. 493 * @v: The value for which we're computing the base 2 logarithm. 494 */ 495 static inline u32 log2_u64(u64 v) 496 { 497 u32 hi = v >> 32; 498 if (hi) 499 return log2_u32(hi) + 32 + 1; 500 else 501 return log2_u32(v) + 1; 502 } 503 504 #include "compat.bpf.h" 505 #include "enums.bpf.h" 506 507 #endif /* __SCX_COMMON_BPF_H */ 508