xref: /linux/tools/sched_ext/include/scx/common.bpf.h (revision 5d6ba5ab8582aa35c1ee98e47af28e6f6772596c)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
4  * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
5  * Copyright (c) 2022 David Vernet <dvernet@meta.com>
6  */
7 #ifndef __SCX_COMMON_BPF_H
8 #define __SCX_COMMON_BPF_H
9 
10 /*
11  * The generated kfunc prototypes in vmlinux.h are missing address space
12  * attributes which cause build failures. For now, suppress the generated
13  * prototypes. See https://github.com/sched-ext/scx/issues/1111.
14  */
15 #define BPF_NO_KFUNC_PROTOTYPES
16 
17 #ifdef LSP
18 #define __bpf__
19 #include "../vmlinux.h"
20 #else
21 #include "vmlinux.h"
22 #endif
23 
24 #include <bpf/bpf_helpers.h>
25 #include <bpf/bpf_tracing.h>
26 #include <asm-generic/errno.h>
27 #include "user_exit_info.h"
28 #include "enum_defs.autogen.h"
29 
30 #define PF_WQ_WORKER			0x00000020	/* I'm a workqueue worker */
31 #define PF_KTHREAD			0x00200000	/* I am a kernel thread */
32 #define PF_EXITING			0x00000004
33 #define CLOCK_MONOTONIC			1
34 
35 extern int LINUX_KERNEL_VERSION __kconfig;
36 extern const char CONFIG_CC_VERSION_TEXT[64] __kconfig __weak;
37 extern const char CONFIG_LOCALVERSION[64] __kconfig __weak;
38 
39 /*
40  * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can
41  * lead to really confusing misbehaviors. Let's trigger a build failure.
42  */
___vmlinux_h_sanity_check___(void)43 static inline void ___vmlinux_h_sanity_check___(void)
44 {
45 	_Static_assert(SCX_DSQ_FLAG_BUILTIN,
46 		       "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole");
47 }
48 
49 s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym;
50 s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym;
51 void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;
52 void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak;
53 u32 scx_bpf_dispatch_nr_slots(void) __ksym;
54 void scx_bpf_dispatch_cancel(void) __ksym;
55 bool scx_bpf_dsq_move_to_local(u64 dsq_id) __ksym __weak;
56 void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak;
57 void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak;
58 bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
59 bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
60 u32 scx_bpf_reenqueue_local(void) __ksym;
61 void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym;
62 s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym;
63 void scx_bpf_destroy_dsq(u64 dsq_id) __ksym;
64 int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak;
65 struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak;
66 void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak;
67 void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak;
68 void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym;
69 void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak;
70 u32 scx_bpf_cpuperf_cap(s32 cpu) __ksym __weak;
71 u32 scx_bpf_cpuperf_cur(s32 cpu) __ksym __weak;
72 void scx_bpf_cpuperf_set(s32 cpu, u32 perf) __ksym __weak;
73 u32 scx_bpf_nr_node_ids(void) __ksym __weak;
74 u32 scx_bpf_nr_cpu_ids(void) __ksym __weak;
75 int scx_bpf_cpu_node(s32 cpu) __ksym __weak;
76 const struct cpumask *scx_bpf_get_possible_cpumask(void) __ksym __weak;
77 const struct cpumask *scx_bpf_get_online_cpumask(void) __ksym __weak;
78 void scx_bpf_put_cpumask(const struct cpumask *cpumask) __ksym __weak;
79 const struct cpumask *scx_bpf_get_idle_cpumask_node(int node) __ksym __weak;
80 const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym;
81 const struct cpumask *scx_bpf_get_idle_smtmask_node(int node) __ksym __weak;
82 const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym;
83 void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym;
84 bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym;
85 s32 scx_bpf_pick_idle_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags) __ksym __weak;
86 s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym;
87 s32 scx_bpf_pick_any_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags) __ksym __weak;
88 s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym;
89 bool scx_bpf_task_running(const struct task_struct *p) __ksym;
90 s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym;
91 struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym;
92 struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym __weak;
93 u64 scx_bpf_now(void) __ksym __weak;
94 void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak;
95 
96 /*
97  * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from
98  * within bpf_for_each() loops.
99  */
100 #define BPF_FOR_EACH_ITER	(&___it)
101 
102 #define scx_read_event(e, name)							\
103 	(bpf_core_field_exists((e)->name) ? (e)->name : 0)
104 
105 static inline __attribute__((format(printf, 1, 2)))
___scx_bpf_bstr_format_checker(const char * fmt,...)106 void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {}
107 
108 /*
109  * Helper macro for initializing the fmt and variadic argument inputs to both
110  * bstr exit kfuncs. Callers to this function should use ___fmt and ___param to
111  * refer to the initialized list of inputs to the bstr kfunc.
112  */
113 #define scx_bpf_bstr_preamble(fmt, args...)					\
114 	static char ___fmt[] = fmt;						\
115 	/*									\
116 	 * Note that __param[] must have at least one				\
117 	 * element to keep the verifier happy.					\
118 	 */									\
119 	unsigned long long ___param[___bpf_narg(args) ?: 1] = {};		\
120 										\
121 	_Pragma("GCC diagnostic push")						\
122 	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")			\
123 	___bpf_fill(___param, args);						\
124 	_Pragma("GCC diagnostic pop")
125 
126 /*
127  * scx_bpf_exit() wraps the scx_bpf_exit_bstr() kfunc with variadic arguments
128  * instead of an array of u64. Using this macro will cause the scheduler to
129  * exit cleanly with the specified exit code being passed to user space.
130  */
131 #define scx_bpf_exit(code, fmt, args...)					\
132 ({										\
133 	scx_bpf_bstr_preamble(fmt, args)					\
134 	scx_bpf_exit_bstr(code, ___fmt, ___param, sizeof(___param));		\
135 	___scx_bpf_bstr_format_checker(fmt, ##args);				\
136 })
137 
138 /*
139  * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments
140  * instead of an array of u64. Invoking this macro will cause the scheduler to
141  * exit in an erroneous state, with diagnostic information being passed to the
142  * user.
143  */
144 #define scx_bpf_error(fmt, args...)						\
145 ({										\
146 	scx_bpf_bstr_preamble(fmt, args)					\
147 	scx_bpf_error_bstr(___fmt, ___param, sizeof(___param));			\
148 	___scx_bpf_bstr_format_checker(fmt, ##args);				\
149 })
150 
151 /*
152  * scx_bpf_dump() wraps the scx_bpf_dump_bstr() kfunc with variadic arguments
153  * instead of an array of u64. To be used from ops.dump() and friends.
154  */
155 #define scx_bpf_dump(fmt, args...)						\
156 ({										\
157 	scx_bpf_bstr_preamble(fmt, args)					\
158 	scx_bpf_dump_bstr(___fmt, ___param, sizeof(___param));			\
159 	___scx_bpf_bstr_format_checker(fmt, ##args);				\
160 })
161 
162 /*
163  * scx_bpf_dump_header() is a wrapper around scx_bpf_dump that adds a header
164  * of system information for debugging.
165  */
166 #define scx_bpf_dump_header()							\
167 ({										\
168 	scx_bpf_dump("kernel: %d.%d.%d %s\ncc: %s\n",				\
169 		     LINUX_KERNEL_VERSION >> 16,				\
170 		     LINUX_KERNEL_VERSION >> 8 & 0xFF,				\
171 		     LINUX_KERNEL_VERSION & 0xFF,				\
172 		     CONFIG_LOCALVERSION,					\
173 		     CONFIG_CC_VERSION_TEXT);					\
174 })
175 
176 #define BPF_STRUCT_OPS(name, args...)						\
177 SEC("struct_ops/"#name)								\
178 BPF_PROG(name, ##args)
179 
180 #define BPF_STRUCT_OPS_SLEEPABLE(name, args...)					\
181 SEC("struct_ops.s/"#name)							\
182 BPF_PROG(name, ##args)
183 
184 /**
185  * RESIZABLE_ARRAY - Generates annotations for an array that may be resized
186  * @elfsec: the data section of the BPF program in which to place the array
187  * @arr: the name of the array
188  *
189  * libbpf has an API for setting map value sizes. Since data sections (i.e.
190  * bss, data, rodata) themselves are maps, a data section can be resized. If
191  * a data section has an array as its last element, the BTF info for that
192  * array will be adjusted so that length of the array is extended to meet the
193  * new length of the data section. This macro annotates an array to have an
194  * element count of one with the assumption that this array can be resized
195  * within the userspace program. It also annotates the section specifier so
196  * this array exists in a custom sub data section which can be resized
197  * independently.
198  *
199  * See RESIZE_ARRAY() for the userspace convenience macro for resizing an
200  * array declared with RESIZABLE_ARRAY().
201  */
202 #define RESIZABLE_ARRAY(elfsec, arr) arr[1] SEC("."#elfsec"."#arr)
203 
204 /**
205  * MEMBER_VPTR - Obtain the verified pointer to a struct or array member
206  * @base: struct or array to index
207  * @member: dereferenced member (e.g. .field, [idx0][idx1], .field[idx0] ...)
208  *
209  * The verifier often gets confused by the instruction sequence the compiler
210  * generates for indexing struct fields or arrays. This macro forces the
211  * compiler to generate a code sequence which first calculates the byte offset,
212  * checks it against the struct or array size and add that byte offset to
213  * generate the pointer to the member to help the verifier.
214  *
215  * Ideally, we want to abort if the calculated offset is out-of-bounds. However,
216  * BPF currently doesn't support abort, so evaluate to %NULL instead. The caller
217  * must check for %NULL and take appropriate action to appease the verifier. To
218  * avoid confusing the verifier, it's best to check for %NULL and dereference
219  * immediately.
220  *
221  *	vptr = MEMBER_VPTR(my_array, [i][j]);
222  *	if (!vptr)
223  *		return error;
224  *	*vptr = new_value;
225  *
226  * sizeof(@base) should encompass the memory area to be accessed and thus can't
227  * be a pointer to the area. Use `MEMBER_VPTR(*ptr, .member)` instead of
228  * `MEMBER_VPTR(ptr, ->member)`.
229  */
230 #define MEMBER_VPTR(base, member) (typeof((base) member) *)			\
231 ({										\
232 	u64 __base = (u64)&(base);						\
233 	u64 __addr = (u64)&((base) member) - __base;				\
234 	_Static_assert(sizeof(base) >= sizeof((base) member),			\
235 		       "@base is smaller than @member, is @base a pointer?");	\
236 	asm volatile (								\
237 		"if %0 <= %[max] goto +2\n"					\
238 		"%0 = 0\n"							\
239 		"goto +1\n"							\
240 		"%0 += %1\n"							\
241 		: "+r"(__addr)							\
242 		: "r"(__base),							\
243 		  [max]"i"(sizeof(base) - sizeof((base) member)));		\
244 	__addr;									\
245 })
246 
247 /**
248  * ARRAY_ELEM_PTR - Obtain the verified pointer to an array element
249  * @arr: array to index into
250  * @i: array index
251  * @n: number of elements in array
252  *
253  * Similar to MEMBER_VPTR() but is intended for use with arrays where the
254  * element count needs to be explicit.
255  * It can be used in cases where a global array is defined with an initial
256  * size but is intended to be be resized before loading the BPF program.
257  * Without this version of the macro, MEMBER_VPTR() will use the compile time
258  * size of the array to compute the max, which will result in rejection by
259  * the verifier.
260  */
261 #define ARRAY_ELEM_PTR(arr, i, n) (typeof(arr[i]) *)				\
262 ({										\
263 	u64 __base = (u64)arr;							\
264 	u64 __addr = (u64)&(arr[i]) - __base;					\
265 	asm volatile (								\
266 		"if %0 <= %[max] goto +2\n"					\
267 		"%0 = 0\n"							\
268 		"goto +1\n"							\
269 		"%0 += %1\n"							\
270 		: "+r"(__addr)							\
271 		: "r"(__base),							\
272 		  [max]"r"(sizeof(arr[0]) * ((n) - 1)));			\
273 	__addr;									\
274 })
275 
276 
277 /*
278  * BPF declarations and helpers
279  */
280 
281 /* list and rbtree */
282 #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node)))
283 #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
284 
285 void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym;
286 void bpf_obj_drop_impl(void *kptr, void *meta) __ksym;
287 
288 #define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL))
289 #define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL)
290 
291 int bpf_list_push_front_impl(struct bpf_list_head *head,
292 				    struct bpf_list_node *node,
293 				    void *meta, __u64 off) __ksym;
294 #define bpf_list_push_front(head, node) bpf_list_push_front_impl(head, node, NULL, 0)
295 
296 int bpf_list_push_back_impl(struct bpf_list_head *head,
297 				   struct bpf_list_node *node,
298 				   void *meta, __u64 off) __ksym;
299 #define bpf_list_push_back(head, node) bpf_list_push_back_impl(head, node, NULL, 0)
300 
301 struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym;
302 struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym;
303 struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root,
304 				      struct bpf_rb_node *node) __ksym;
305 int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
306 			bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b),
307 			void *meta, __u64 off) __ksym;
308 #define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0)
309 
310 struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym;
311 
312 void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym;
313 #define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL)
314 
315 /* task */
316 struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
317 struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
318 void bpf_task_release(struct task_struct *p) __ksym;
319 
320 /* cgroup */
321 struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym;
322 void bpf_cgroup_release(struct cgroup *cgrp) __ksym;
323 struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym;
324 
325 /* css iteration */
326 struct bpf_iter_css;
327 struct cgroup_subsys_state;
328 extern int bpf_iter_css_new(struct bpf_iter_css *it,
329 			    struct cgroup_subsys_state *start,
330 			    unsigned int flags) __weak __ksym;
331 extern struct cgroup_subsys_state *
332 bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym;
333 extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym;
334 
335 /* cpumask */
336 struct bpf_cpumask *bpf_cpumask_create(void) __ksym;
337 struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym;
338 void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym;
339 u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym;
340 u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym;
341 void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
342 void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
343 bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym;
344 bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
345 bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
346 void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym;
347 void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym;
348 bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1,
349 		     const struct cpumask *src2) __ksym;
350 void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1,
351 		    const struct cpumask *src2) __ksym;
352 void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1,
353 		     const struct cpumask *src2) __ksym;
354 bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym;
355 bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym;
356 bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym;
357 bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym;
358 bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym;
359 void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym;
360 u32 bpf_cpumask_any_distribute(const struct cpumask *cpumask) __ksym;
361 u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1,
362 				   const struct cpumask *src2) __ksym;
363 u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym;
364 
365 int bpf_iter_bits_new(struct bpf_iter_bits *it, const u64 *unsafe_ptr__ign, u32 nr_words) __ksym;
366 int *bpf_iter_bits_next(struct bpf_iter_bits *it) __ksym;
367 void bpf_iter_bits_destroy(struct bpf_iter_bits *it) __ksym;
368 
369 #define def_iter_struct(name)							\
370 struct bpf_iter_##name {							\
371     struct bpf_iter_bits it;							\
372     const struct cpumask *bitmap;						\
373 };
374 
375 #define def_iter_new(name)							\
376 static inline int bpf_iter_##name##_new(					\
377 	struct bpf_iter_##name *it, const u64 *unsafe_ptr__ign, u32 nr_words)	\
378 {										\
379 	it->bitmap = scx_bpf_get_##name##_cpumask();				\
380 	return bpf_iter_bits_new(&it->it, (const u64 *)it->bitmap,		\
381 				 sizeof(struct cpumask) / 8);			\
382 }
383 
384 #define def_iter_next(name)							\
385 static inline int *bpf_iter_##name##_next(struct bpf_iter_##name *it) {		\
386 	return bpf_iter_bits_next(&it->it);					\
387 }
388 
389 #define def_iter_destroy(name)							\
390 static inline void bpf_iter_##name##_destroy(struct bpf_iter_##name *it) {	\
391 	scx_bpf_put_cpumask(it->bitmap);					\
392 	bpf_iter_bits_destroy(&it->it);						\
393 }
394 #define def_for_each_cpu(cpu, name) for_each_##name##_cpu(cpu)
395 
396 /// Provides iterator for possible and online cpus.
397 ///
398 /// # Example
399 ///
400 /// ```
401 /// static inline void example_use() {
402 ///     int *cpu;
403 ///
404 ///     for_each_possible_cpu(cpu){
405 ///         bpf_printk("CPU %d is possible", *cpu);
406 ///     }
407 ///
408 ///     for_each_online_cpu(cpu){
409 ///         bpf_printk("CPU %d is online", *cpu);
410 ///     }
411 /// }
412 /// ```
413 def_iter_struct(possible);
414 def_iter_new(possible);
415 def_iter_next(possible);
416 def_iter_destroy(possible);
417 #define for_each_possible_cpu(cpu) bpf_for_each(possible, cpu, NULL, 0)
418 
419 def_iter_struct(online);
420 def_iter_new(online);
421 def_iter_next(online);
422 def_iter_destroy(online);
423 #define for_each_online_cpu(cpu) bpf_for_each(online, cpu, NULL, 0)
424 
425 /*
426  * Access a cpumask in read-only mode (typically to check bits).
427  */
cast_mask(struct bpf_cpumask * mask)428 static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask)
429 {
430 	return (const struct cpumask *)mask;
431 }
432 
433 /*
434  * Return true if task @p cannot migrate to a different CPU, false
435  * otherwise.
436  */
is_migration_disabled(const struct task_struct * p)437 static inline bool is_migration_disabled(const struct task_struct *p)
438 {
439 	if (bpf_core_field_exists(p->migration_disabled))
440 		return p->migration_disabled;
441 	return false;
442 }
443 
444 /* rcu */
445 void bpf_rcu_read_lock(void) __ksym;
446 void bpf_rcu_read_unlock(void) __ksym;
447 
448 /*
449  * Time helpers, most of which are from jiffies.h.
450  */
451 
452 /**
453  * time_delta - Calculate the delta between new and old time stamp
454  * @after: first comparable as u64
455  * @before: second comparable as u64
456  *
457  * Return: the time difference, which is >= 0
458  */
time_delta(u64 after,u64 before)459 static inline s64 time_delta(u64 after, u64 before)
460 {
461 	return (s64)(after - before) > 0 ? (s64)(after - before) : 0;
462 }
463 
464 /**
465  * time_after - returns true if the time a is after time b.
466  * @a: first comparable as u64
467  * @b: second comparable as u64
468  *
469  * Do this with "<0" and ">=0" to only test the sign of the result. A
470  * good compiler would generate better code (and a really good compiler
471  * wouldn't care). Gcc is currently neither.
472  *
473  * Return: %true is time a is after time b, otherwise %false.
474  */
time_after(u64 a,u64 b)475 static inline bool time_after(u64 a, u64 b)
476 {
477 	 return (s64)(b - a) < 0;
478 }
479 
480 /**
481  * time_before - returns true if the time a is before time b.
482  * @a: first comparable as u64
483  * @b: second comparable as u64
484  *
485  * Return: %true is time a is before time b, otherwise %false.
486  */
time_before(u64 a,u64 b)487 static inline bool time_before(u64 a, u64 b)
488 {
489 	return time_after(b, a);
490 }
491 
492 /**
493  * time_after_eq - returns true if the time a is after or the same as time b.
494  * @a: first comparable as u64
495  * @b: second comparable as u64
496  *
497  * Return: %true is time a is after or the same as time b, otherwise %false.
498  */
time_after_eq(u64 a,u64 b)499 static inline bool time_after_eq(u64 a, u64 b)
500 {
501 	 return (s64)(a - b) >= 0;
502 }
503 
504 /**
505  * time_before_eq - returns true if the time a is before or the same as time b.
506  * @a: first comparable as u64
507  * @b: second comparable as u64
508  *
509  * Return: %true is time a is before or the same as time b, otherwise %false.
510  */
time_before_eq(u64 a,u64 b)511 static inline bool time_before_eq(u64 a, u64 b)
512 {
513 	return time_after_eq(b, a);
514 }
515 
516 /**
517  * time_in_range - Calculate whether a is in the range of [b, c].
518  * @a: time to test
519  * @b: beginning of the range
520  * @c: end of the range
521  *
522  * Return: %true is time a is in the range [b, c], otherwise %false.
523  */
time_in_range(u64 a,u64 b,u64 c)524 static inline bool time_in_range(u64 a, u64 b, u64 c)
525 {
526 	return time_after_eq(a, b) && time_before_eq(a, c);
527 }
528 
529 /**
530  * time_in_range_open - Calculate whether a is in the range of [b, c).
531  * @a: time to test
532  * @b: beginning of the range
533  * @c: end of the range
534  *
535  * Return: %true is time a is in the range [b, c), otherwise %false.
536  */
time_in_range_open(u64 a,u64 b,u64 c)537 static inline bool time_in_range_open(u64 a, u64 b, u64 c)
538 {
539 	return time_after_eq(a, b) && time_before(a, c);
540 }
541 
542 
543 /*
544  * Other helpers
545  */
546 
547 /* useful compiler attributes */
548 #define likely(x) __builtin_expect(!!(x), 1)
549 #define unlikely(x) __builtin_expect(!!(x), 0)
550 #define __maybe_unused __attribute__((__unused__))
551 
552 /*
553  * READ/WRITE_ONCE() are from kernel (include/asm-generic/rwonce.h). They
554  * prevent compiler from caching, redoing or reordering reads or writes.
555  */
556 typedef __u8  __attribute__((__may_alias__))  __u8_alias_t;
557 typedef __u16 __attribute__((__may_alias__)) __u16_alias_t;
558 typedef __u32 __attribute__((__may_alias__)) __u32_alias_t;
559 typedef __u64 __attribute__((__may_alias__)) __u64_alias_t;
560 
__read_once_size(const volatile void * p,void * res,int size)561 static __always_inline void __read_once_size(const volatile void *p, void *res, int size)
562 {
563 	switch (size) {
564 	case 1: *(__u8_alias_t  *) res = *(volatile __u8_alias_t  *) p; break;
565 	case 2: *(__u16_alias_t *) res = *(volatile __u16_alias_t *) p; break;
566 	case 4: *(__u32_alias_t *) res = *(volatile __u32_alias_t *) p; break;
567 	case 8: *(__u64_alias_t *) res = *(volatile __u64_alias_t *) p; break;
568 	default:
569 		barrier();
570 		__builtin_memcpy((void *)res, (const void *)p, size);
571 		barrier();
572 	}
573 }
574 
__write_once_size(volatile void * p,void * res,int size)575 static __always_inline void __write_once_size(volatile void *p, void *res, int size)
576 {
577 	switch (size) {
578 	case 1: *(volatile  __u8_alias_t *) p = *(__u8_alias_t  *) res; break;
579 	case 2: *(volatile __u16_alias_t *) p = *(__u16_alias_t *) res; break;
580 	case 4: *(volatile __u32_alias_t *) p = *(__u32_alias_t *) res; break;
581 	case 8: *(volatile __u64_alias_t *) p = *(__u64_alias_t *) res; break;
582 	default:
583 		barrier();
584 		__builtin_memcpy((void *)p, (const void *)res, size);
585 		barrier();
586 	}
587 }
588 
589 #define READ_ONCE(x)					\
590 ({							\
591 	union { typeof(x) __val; char __c[1]; } __u =	\
592 		{ .__c = { 0 } };			\
593 	__read_once_size(&(x), __u.__c, sizeof(x));	\
594 	__u.__val;					\
595 })
596 
597 #define WRITE_ONCE(x, val)				\
598 ({							\
599 	union { typeof(x) __val; char __c[1]; } __u =	\
600 		{ .__val = (val) }; 			\
601 	__write_once_size(&(x), __u.__c, sizeof(x));	\
602 	__u.__val;					\
603 })
604 
605 #define READ_ONCE_ARENA(type, x)				\
606 ({								\
607 	union { type __val; char __c[1]; } __u =		\
608 		{ .__c = { 0 } };				\
609 	__read_once_size((void *)&(x), __u.__c, sizeof(x));	\
610 	__u.__val;						\
611 })
612 
613 #define WRITE_ONCE_ARENA(type, x, val)				\
614 ({								\
615 	union { type __val; char __c[1]; } __u =		\
616 		{ .__val = (val) }; 				\
617 	__write_once_size((void *)&(x), __u.__c, sizeof(x));	\
618 	__u.__val;						\
619 })
620 
621 /*
622  * log2_u32 - Compute the base 2 logarithm of a 32-bit exponential value.
623  * @v: The value for which we're computing the base 2 logarithm.
624  */
log2_u32(u32 v)625 static inline u32 log2_u32(u32 v)
626 {
627         u32 r;
628         u32 shift;
629 
630         r = (v > 0xFFFF) << 4; v >>= r;
631         shift = (v > 0xFF) << 3; v >>= shift; r |= shift;
632         shift = (v > 0xF) << 2; v >>= shift; r |= shift;
633         shift = (v > 0x3) << 1; v >>= shift; r |= shift;
634         r |= (v >> 1);
635         return r;
636 }
637 
638 /*
639  * log2_u64 - Compute the base 2 logarithm of a 64-bit exponential value.
640  * @v: The value for which we're computing the base 2 logarithm.
641  */
log2_u64(u64 v)642 static inline u32 log2_u64(u64 v)
643 {
644         u32 hi = v >> 32;
645         if (hi)
646                 return log2_u32(hi) + 32 + 1;
647         else
648                 return log2_u32(v) + 1;
649 }
650 
651 #include "compat.bpf.h"
652 #include "enums.bpf.h"
653 
654 #endif	/* __SCX_COMMON_BPF_H */
655