1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
4 * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
5 * Copyright (c) 2022 David Vernet <dvernet@meta.com>
6 */
7 #ifndef __SCX_COMMON_BPF_H
8 #define __SCX_COMMON_BPF_H
9
10 #ifdef LSP
11 #define __bpf__
12 #include "../vmlinux.h"
13 #else
14 #include "vmlinux.h"
15 #endif
16
17 #include <bpf/bpf_helpers.h>
18 #include <bpf/bpf_tracing.h>
19 #include <asm-generic/errno.h>
20 #include "user_exit_info.h"
21
22 #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */
23 #define PF_KTHREAD 0x00200000 /* I am a kernel thread */
24 #define PF_EXITING 0x00000004
25 #define CLOCK_MONOTONIC 1
26
27 extern int LINUX_KERNEL_VERSION __kconfig;
28 extern const char CONFIG_CC_VERSION_TEXT[64] __kconfig __weak;
29 extern const char CONFIG_LOCALVERSION[64] __kconfig __weak;
30
31 /*
32 * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can
33 * lead to really confusing misbehaviors. Let's trigger a build failure.
34 */
___vmlinux_h_sanity_check___(void)35 static inline void ___vmlinux_h_sanity_check___(void)
36 {
37 _Static_assert(SCX_DSQ_FLAG_BUILTIN,
38 "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole");
39 }
40
41 s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym;
42 s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym;
43 void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;
44 void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak;
45 u32 scx_bpf_dispatch_nr_slots(void) __ksym;
46 void scx_bpf_dispatch_cancel(void) __ksym;
47 bool scx_bpf_dsq_move_to_local(u64 dsq_id) __ksym __weak;
48 void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak;
49 void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak;
50 bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
51 bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
52 u32 scx_bpf_reenqueue_local(void) __ksym;
53 void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym;
54 s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym;
55 void scx_bpf_destroy_dsq(u64 dsq_id) __ksym;
56 int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak;
57 struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak;
58 void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak;
59 void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak;
60 void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym;
61 void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak;
62 u32 scx_bpf_cpuperf_cap(s32 cpu) __ksym __weak;
63 u32 scx_bpf_cpuperf_cur(s32 cpu) __ksym __weak;
64 void scx_bpf_cpuperf_set(s32 cpu, u32 perf) __ksym __weak;
65 u32 scx_bpf_nr_cpu_ids(void) __ksym __weak;
66 const struct cpumask *scx_bpf_get_possible_cpumask(void) __ksym __weak;
67 const struct cpumask *scx_bpf_get_online_cpumask(void) __ksym __weak;
68 void scx_bpf_put_cpumask(const struct cpumask *cpumask) __ksym __weak;
69 const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym;
70 const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym;
71 void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym;
72 bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym;
73 s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym;
74 s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym;
75 bool scx_bpf_task_running(const struct task_struct *p) __ksym;
76 s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym;
77 struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym;
78 struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym __weak;
79 u64 scx_bpf_now(void) __ksym __weak;
80
81 /*
82 * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from
83 * within bpf_for_each() loops.
84 */
85 #define BPF_FOR_EACH_ITER (&___it)
86
87 static inline __attribute__((format(printf, 1, 2)))
___scx_bpf_bstr_format_checker(const char * fmt,...)88 void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {}
89
90 /*
91 * Helper macro for initializing the fmt and variadic argument inputs to both
92 * bstr exit kfuncs. Callers to this function should use ___fmt and ___param to
93 * refer to the initialized list of inputs to the bstr kfunc.
94 */
95 #define scx_bpf_bstr_preamble(fmt, args...) \
96 static char ___fmt[] = fmt; \
97 /* \
98 * Note that __param[] must have at least one \
99 * element to keep the verifier happy. \
100 */ \
101 unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \
102 \
103 _Pragma("GCC diagnostic push") \
104 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
105 ___bpf_fill(___param, args); \
106 _Pragma("GCC diagnostic pop")
107
108 /*
109 * scx_bpf_exit() wraps the scx_bpf_exit_bstr() kfunc with variadic arguments
110 * instead of an array of u64. Using this macro will cause the scheduler to
111 * exit cleanly with the specified exit code being passed to user space.
112 */
113 #define scx_bpf_exit(code, fmt, args...) \
114 ({ \
115 scx_bpf_bstr_preamble(fmt, args) \
116 scx_bpf_exit_bstr(code, ___fmt, ___param, sizeof(___param)); \
117 ___scx_bpf_bstr_format_checker(fmt, ##args); \
118 })
119
120 /*
121 * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments
122 * instead of an array of u64. Invoking this macro will cause the scheduler to
123 * exit in an erroneous state, with diagnostic information being passed to the
124 * user.
125 */
126 #define scx_bpf_error(fmt, args...) \
127 ({ \
128 scx_bpf_bstr_preamble(fmt, args) \
129 scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \
130 ___scx_bpf_bstr_format_checker(fmt, ##args); \
131 })
132
133 /*
134 * scx_bpf_dump() wraps the scx_bpf_dump_bstr() kfunc with variadic arguments
135 * instead of an array of u64. To be used from ops.dump() and friends.
136 */
137 #define scx_bpf_dump(fmt, args...) \
138 ({ \
139 scx_bpf_bstr_preamble(fmt, args) \
140 scx_bpf_dump_bstr(___fmt, ___param, sizeof(___param)); \
141 ___scx_bpf_bstr_format_checker(fmt, ##args); \
142 })
143
144 /*
145 * scx_bpf_dump_header() is a wrapper around scx_bpf_dump that adds a header
146 * of system information for debugging.
147 */
148 #define scx_bpf_dump_header() \
149 ({ \
150 scx_bpf_dump("kernel: %d.%d.%d %s\ncc: %s\n", \
151 LINUX_KERNEL_VERSION >> 16, \
152 LINUX_KERNEL_VERSION >> 8 & 0xFF, \
153 LINUX_KERNEL_VERSION & 0xFF, \
154 CONFIG_LOCALVERSION, \
155 CONFIG_CC_VERSION_TEXT); \
156 })
157
158 #define BPF_STRUCT_OPS(name, args...) \
159 SEC("struct_ops/"#name) \
160 BPF_PROG(name, ##args)
161
162 #define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \
163 SEC("struct_ops.s/"#name) \
164 BPF_PROG(name, ##args)
165
166 /**
167 * RESIZABLE_ARRAY - Generates annotations for an array that may be resized
168 * @elfsec: the data section of the BPF program in which to place the array
169 * @arr: the name of the array
170 *
171 * libbpf has an API for setting map value sizes. Since data sections (i.e.
172 * bss, data, rodata) themselves are maps, a data section can be resized. If
173 * a data section has an array as its last element, the BTF info for that
174 * array will be adjusted so that length of the array is extended to meet the
175 * new length of the data section. This macro annotates an array to have an
176 * element count of one with the assumption that this array can be resized
177 * within the userspace program. It also annotates the section specifier so
178 * this array exists in a custom sub data section which can be resized
179 * independently.
180 *
181 * See RESIZE_ARRAY() for the userspace convenience macro for resizing an
182 * array declared with RESIZABLE_ARRAY().
183 */
184 #define RESIZABLE_ARRAY(elfsec, arr) arr[1] SEC("."#elfsec"."#arr)
185
186 /**
187 * MEMBER_VPTR - Obtain the verified pointer to a struct or array member
188 * @base: struct or array to index
189 * @member: dereferenced member (e.g. .field, [idx0][idx1], .field[idx0] ...)
190 *
191 * The verifier often gets confused by the instruction sequence the compiler
192 * generates for indexing struct fields or arrays. This macro forces the
193 * compiler to generate a code sequence which first calculates the byte offset,
194 * checks it against the struct or array size and add that byte offset to
195 * generate the pointer to the member to help the verifier.
196 *
197 * Ideally, we want to abort if the calculated offset is out-of-bounds. However,
198 * BPF currently doesn't support abort, so evaluate to %NULL instead. The caller
199 * must check for %NULL and take appropriate action to appease the verifier. To
200 * avoid confusing the verifier, it's best to check for %NULL and dereference
201 * immediately.
202 *
203 * vptr = MEMBER_VPTR(my_array, [i][j]);
204 * if (!vptr)
205 * return error;
206 * *vptr = new_value;
207 *
208 * sizeof(@base) should encompass the memory area to be accessed and thus can't
209 * be a pointer to the area. Use `MEMBER_VPTR(*ptr, .member)` instead of
210 * `MEMBER_VPTR(ptr, ->member)`.
211 */
212 #define MEMBER_VPTR(base, member) (typeof((base) member) *) \
213 ({ \
214 u64 __base = (u64)&(base); \
215 u64 __addr = (u64)&((base) member) - __base; \
216 _Static_assert(sizeof(base) >= sizeof((base) member), \
217 "@base is smaller than @member, is @base a pointer?"); \
218 asm volatile ( \
219 "if %0 <= %[max] goto +2\n" \
220 "%0 = 0\n" \
221 "goto +1\n" \
222 "%0 += %1\n" \
223 : "+r"(__addr) \
224 : "r"(__base), \
225 [max]"i"(sizeof(base) - sizeof((base) member))); \
226 __addr; \
227 })
228
229 /**
230 * ARRAY_ELEM_PTR - Obtain the verified pointer to an array element
231 * @arr: array to index into
232 * @i: array index
233 * @n: number of elements in array
234 *
235 * Similar to MEMBER_VPTR() but is intended for use with arrays where the
236 * element count needs to be explicit.
237 * It can be used in cases where a global array is defined with an initial
238 * size but is intended to be be resized before loading the BPF program.
239 * Without this version of the macro, MEMBER_VPTR() will use the compile time
240 * size of the array to compute the max, which will result in rejection by
241 * the verifier.
242 */
243 #define ARRAY_ELEM_PTR(arr, i, n) (typeof(arr[i]) *) \
244 ({ \
245 u64 __base = (u64)arr; \
246 u64 __addr = (u64)&(arr[i]) - __base; \
247 asm volatile ( \
248 "if %0 <= %[max] goto +2\n" \
249 "%0 = 0\n" \
250 "goto +1\n" \
251 "%0 += %1\n" \
252 : "+r"(__addr) \
253 : "r"(__base), \
254 [max]"r"(sizeof(arr[0]) * ((n) - 1))); \
255 __addr; \
256 })
257
258
259 /*
260 * BPF declarations and helpers
261 */
262
263 /* list and rbtree */
264 #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node)))
265 #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
266
267 void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym;
268 void bpf_obj_drop_impl(void *kptr, void *meta) __ksym;
269
270 #define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL))
271 #define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL)
272
273 int bpf_list_push_front_impl(struct bpf_list_head *head,
274 struct bpf_list_node *node,
275 void *meta, __u64 off) __ksym;
276 #define bpf_list_push_front(head, node) bpf_list_push_front_impl(head, node, NULL, 0)
277
278 int bpf_list_push_back_impl(struct bpf_list_head *head,
279 struct bpf_list_node *node,
280 void *meta, __u64 off) __ksym;
281 #define bpf_list_push_back(head, node) bpf_list_push_back_impl(head, node, NULL, 0)
282
283 struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym;
284 struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym;
285 struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root,
286 struct bpf_rb_node *node) __ksym;
287 int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
288 bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b),
289 void *meta, __u64 off) __ksym;
290 #define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0)
291
292 struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym;
293
294 void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym;
295 #define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL)
296
297 /* task */
298 struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
299 struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
300 void bpf_task_release(struct task_struct *p) __ksym;
301
302 /* cgroup */
303 struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym;
304 void bpf_cgroup_release(struct cgroup *cgrp) __ksym;
305 struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym;
306
307 /* css iteration */
308 struct bpf_iter_css;
309 struct cgroup_subsys_state;
310 extern int bpf_iter_css_new(struct bpf_iter_css *it,
311 struct cgroup_subsys_state *start,
312 unsigned int flags) __weak __ksym;
313 extern struct cgroup_subsys_state *
314 bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym;
315 extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym;
316
317 /* cpumask */
318 struct bpf_cpumask *bpf_cpumask_create(void) __ksym;
319 struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym;
320 void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym;
321 u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym;
322 u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym;
323 void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
324 void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
325 bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym;
326 bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
327 bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
328 void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym;
329 void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym;
330 bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1,
331 const struct cpumask *src2) __ksym;
332 void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1,
333 const struct cpumask *src2) __ksym;
334 void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1,
335 const struct cpumask *src2) __ksym;
336 bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym;
337 bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym;
338 bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym;
339 bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym;
340 bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym;
341 void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym;
342 u32 bpf_cpumask_any_distribute(const struct cpumask *cpumask) __ksym;
343 u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1,
344 const struct cpumask *src2) __ksym;
345 u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym;
346
347 int bpf_iter_bits_new(struct bpf_iter_bits *it, const u64 *unsafe_ptr__ign, u32 nr_words) __ksym;
348 int *bpf_iter_bits_next(struct bpf_iter_bits *it) __ksym;
349 void bpf_iter_bits_destroy(struct bpf_iter_bits *it) __ksym;
350
351 #define def_iter_struct(name) \
352 struct bpf_iter_##name { \
353 struct bpf_iter_bits it; \
354 const struct cpumask *bitmap; \
355 };
356
357 #define def_iter_new(name) \
358 static inline int bpf_iter_##name##_new( \
359 struct bpf_iter_##name *it, const u64 *unsafe_ptr__ign, u32 nr_words) \
360 { \
361 it->bitmap = scx_bpf_get_##name##_cpumask(); \
362 return bpf_iter_bits_new(&it->it, (const u64 *)it->bitmap, \
363 sizeof(struct cpumask) / 8); \
364 }
365
366 #define def_iter_next(name) \
367 static inline int *bpf_iter_##name##_next(struct bpf_iter_##name *it) { \
368 return bpf_iter_bits_next(&it->it); \
369 }
370
371 #define def_iter_destroy(name) \
372 static inline void bpf_iter_##name##_destroy(struct bpf_iter_##name *it) { \
373 scx_bpf_put_cpumask(it->bitmap); \
374 bpf_iter_bits_destroy(&it->it); \
375 }
376 #define def_for_each_cpu(cpu, name) for_each_##name##_cpu(cpu)
377
378 /// Provides iterator for possible and online cpus.
379 ///
380 /// # Example
381 ///
382 /// ```
383 /// static inline void example_use() {
384 /// int *cpu;
385 ///
386 /// for_each_possible_cpu(cpu){
387 /// bpf_printk("CPU %d is possible", *cpu);
388 /// }
389 ///
390 /// for_each_online_cpu(cpu){
391 /// bpf_printk("CPU %d is online", *cpu);
392 /// }
393 /// }
394 /// ```
395 def_iter_struct(possible);
396 def_iter_new(possible);
397 def_iter_next(possible);
398 def_iter_destroy(possible);
399 #define for_each_possible_cpu(cpu) bpf_for_each(possible, cpu, NULL, 0)
400
401 def_iter_struct(online);
402 def_iter_new(online);
403 def_iter_next(online);
404 def_iter_destroy(online);
405 #define for_each_online_cpu(cpu) bpf_for_each(online, cpu, NULL, 0)
406
407 /*
408 * Access a cpumask in read-only mode (typically to check bits).
409 */
cast_mask(struct bpf_cpumask * mask)410 static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask)
411 {
412 return (const struct cpumask *)mask;
413 }
414
415 /*
416 * Return true if task @p cannot migrate to a different CPU, false
417 * otherwise.
418 */
is_migration_disabled(const struct task_struct * p)419 static inline bool is_migration_disabled(const struct task_struct *p)
420 {
421 if (bpf_core_field_exists(p->migration_disabled))
422 return p->migration_disabled;
423 return false;
424 }
425
426 /* rcu */
427 void bpf_rcu_read_lock(void) __ksym;
428 void bpf_rcu_read_unlock(void) __ksym;
429
430 /*
431 * Time helpers, most of which are from jiffies.h.
432 */
433
434 /**
435 * time_delta - Calculate the delta between new and old time stamp
436 * @after: first comparable as u64
437 * @before: second comparable as u64
438 *
439 * Return: the time difference, which is >= 0
440 */
time_delta(u64 after,u64 before)441 static inline s64 time_delta(u64 after, u64 before)
442 {
443 return (s64)(after - before) > 0 ? (s64)(after - before) : 0;
444 }
445
446 /**
447 * time_after - returns true if the time a is after time b.
448 * @a: first comparable as u64
449 * @b: second comparable as u64
450 *
451 * Do this with "<0" and ">=0" to only test the sign of the result. A
452 * good compiler would generate better code (and a really good compiler
453 * wouldn't care). Gcc is currently neither.
454 *
455 * Return: %true is time a is after time b, otherwise %false.
456 */
time_after(u64 a,u64 b)457 static inline bool time_after(u64 a, u64 b)
458 {
459 return (s64)(b - a) < 0;
460 }
461
462 /**
463 * time_before - returns true if the time a is before time b.
464 * @a: first comparable as u64
465 * @b: second comparable as u64
466 *
467 * Return: %true is time a is before time b, otherwise %false.
468 */
time_before(u64 a,u64 b)469 static inline bool time_before(u64 a, u64 b)
470 {
471 return time_after(b, a);
472 }
473
474 /**
475 * time_after_eq - returns true if the time a is after or the same as time b.
476 * @a: first comparable as u64
477 * @b: second comparable as u64
478 *
479 * Return: %true is time a is after or the same as time b, otherwise %false.
480 */
time_after_eq(u64 a,u64 b)481 static inline bool time_after_eq(u64 a, u64 b)
482 {
483 return (s64)(a - b) >= 0;
484 }
485
486 /**
487 * time_before_eq - returns true if the time a is before or the same as time b.
488 * @a: first comparable as u64
489 * @b: second comparable as u64
490 *
491 * Return: %true is time a is before or the same as time b, otherwise %false.
492 */
time_before_eq(u64 a,u64 b)493 static inline bool time_before_eq(u64 a, u64 b)
494 {
495 return time_after_eq(b, a);
496 }
497
498 /**
499 * time_in_range - Calculate whether a is in the range of [b, c].
500 * @a: time to test
501 * @b: beginning of the range
502 * @c: end of the range
503 *
504 * Return: %true is time a is in the range [b, c], otherwise %false.
505 */
time_in_range(u64 a,u64 b,u64 c)506 static inline bool time_in_range(u64 a, u64 b, u64 c)
507 {
508 return time_after_eq(a, b) && time_before_eq(a, c);
509 }
510
511 /**
512 * time_in_range_open - Calculate whether a is in the range of [b, c).
513 * @a: time to test
514 * @b: beginning of the range
515 * @c: end of the range
516 *
517 * Return: %true is time a is in the range [b, c), otherwise %false.
518 */
time_in_range_open(u64 a,u64 b,u64 c)519 static inline bool time_in_range_open(u64 a, u64 b, u64 c)
520 {
521 return time_after_eq(a, b) && time_before(a, c);
522 }
523
524
525 /*
526 * Other helpers
527 */
528
529 /* useful compiler attributes */
530 #define likely(x) __builtin_expect(!!(x), 1)
531 #define unlikely(x) __builtin_expect(!!(x), 0)
532 #define __maybe_unused __attribute__((__unused__))
533
534 /*
535 * READ/WRITE_ONCE() are from kernel (include/asm-generic/rwonce.h). They
536 * prevent compiler from caching, redoing or reordering reads or writes.
537 */
538 typedef __u8 __attribute__((__may_alias__)) __u8_alias_t;
539 typedef __u16 __attribute__((__may_alias__)) __u16_alias_t;
540 typedef __u32 __attribute__((__may_alias__)) __u32_alias_t;
541 typedef __u64 __attribute__((__may_alias__)) __u64_alias_t;
542
__read_once_size(const volatile void * p,void * res,int size)543 static __always_inline void __read_once_size(const volatile void *p, void *res, int size)
544 {
545 switch (size) {
546 case 1: *(__u8_alias_t *) res = *(volatile __u8_alias_t *) p; break;
547 case 2: *(__u16_alias_t *) res = *(volatile __u16_alias_t *) p; break;
548 case 4: *(__u32_alias_t *) res = *(volatile __u32_alias_t *) p; break;
549 case 8: *(__u64_alias_t *) res = *(volatile __u64_alias_t *) p; break;
550 default:
551 barrier();
552 __builtin_memcpy((void *)res, (const void *)p, size);
553 barrier();
554 }
555 }
556
__write_once_size(volatile void * p,void * res,int size)557 static __always_inline void __write_once_size(volatile void *p, void *res, int size)
558 {
559 switch (size) {
560 case 1: *(volatile __u8_alias_t *) p = *(__u8_alias_t *) res; break;
561 case 2: *(volatile __u16_alias_t *) p = *(__u16_alias_t *) res; break;
562 case 4: *(volatile __u32_alias_t *) p = *(__u32_alias_t *) res; break;
563 case 8: *(volatile __u64_alias_t *) p = *(__u64_alias_t *) res; break;
564 default:
565 barrier();
566 __builtin_memcpy((void *)p, (const void *)res, size);
567 barrier();
568 }
569 }
570
571 #define READ_ONCE(x) \
572 ({ \
573 union { typeof(x) __val; char __c[1]; } __u = \
574 { .__c = { 0 } }; \
575 __read_once_size(&(x), __u.__c, sizeof(x)); \
576 __u.__val; \
577 })
578
579 #define WRITE_ONCE(x, val) \
580 ({ \
581 union { typeof(x) __val; char __c[1]; } __u = \
582 { .__val = (val) }; \
583 __write_once_size(&(x), __u.__c, sizeof(x)); \
584 __u.__val; \
585 })
586
587 /*
588 * log2_u32 - Compute the base 2 logarithm of a 32-bit exponential value.
589 * @v: The value for which we're computing the base 2 logarithm.
590 */
log2_u32(u32 v)591 static inline u32 log2_u32(u32 v)
592 {
593 u32 r;
594 u32 shift;
595
596 r = (v > 0xFFFF) << 4; v >>= r;
597 shift = (v > 0xFF) << 3; v >>= shift; r |= shift;
598 shift = (v > 0xF) << 2; v >>= shift; r |= shift;
599 shift = (v > 0x3) << 1; v >>= shift; r |= shift;
600 r |= (v >> 1);
601 return r;
602 }
603
604 /*
605 * log2_u64 - Compute the base 2 logarithm of a 64-bit exponential value.
606 * @v: The value for which we're computing the base 2 logarithm.
607 */
log2_u64(u64 v)608 static inline u32 log2_u64(u64 v)
609 {
610 u32 hi = v >> 32;
611 if (hi)
612 return log2_u32(hi) + 32 + 1;
613 else
614 return log2_u32(v) + 1;
615 }
616
617 #include "compat.bpf.h"
618 #include "enums.bpf.h"
619
620 #endif /* __SCX_COMMON_BPF_H */
621