1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ 3 #ifndef BPF_ATOMIC_H 4 #define BPF_ATOMIC_H 5 6 #include <vmlinux.h> 7 #include <bpf/bpf_helpers.h> 8 #include <bpf_may_goto.h> 9 10 extern bool CONFIG_X86_64 __kconfig __weak; 11 12 /* 13 * __unqual_typeof(x) - Declare an unqualified scalar type, leaving 14 * non-scalar types unchanged, 15 * 16 * Prefer C11 _Generic for better compile-times and simpler code. Note: 'char' 17 * is not type-compatible with 'signed char', and we define a separate case. 18 * 19 * This is copied verbatim from kernel's include/linux/compiler_types.h, but 20 * with default expression (for pointers) changed from (x) to (typeof(x)0). 21 * 22 * This is because LLVM has a bug where for lvalue (x), it does not get rid of 23 * an extra address_space qualifier, but does in case of rvalue (typeof(x)0). 24 * Hence, for pointers, we need to create an rvalue expression to get the 25 * desired type. See https://github.com/llvm/llvm-project/issues/53400. 26 */ 27 #define __scalar_type_to_expr_cases(type) \ 28 unsigned type : (unsigned type)0, signed type : (signed type)0 29 30 #define __unqual_typeof(x) \ 31 typeof(_Generic((x), \ 32 char: (char)0, \ 33 __scalar_type_to_expr_cases(char), \ 34 __scalar_type_to_expr_cases(short), \ 35 __scalar_type_to_expr_cases(int), \ 36 __scalar_type_to_expr_cases(long), \ 37 __scalar_type_to_expr_cases(long long), \ 38 default: (typeof(x))0)) 39 40 /* No-op for BPF */ 41 #define cpu_relax() ({}) 42 43 #define READ_ONCE(x) (*(volatile typeof(x) *)&(x)) 44 45 #ifndef WRITE_ONCE 46 #define WRITE_ONCE(x, val) ((*(volatile typeof(x) *)&(x)) = (val)) 47 #endif 48 49 #define cmpxchg(p, old, new) __sync_val_compare_and_swap((p), old, new) 50 51 #define try_cmpxchg(p, pold, new) \ 52 ({ \ 53 __unqual_typeof(*(pold)) __o = *(pold); \ 54 __unqual_typeof(*(p)) __r = cmpxchg(p, __o, new); \ 55 if (__r != __o) \ 56 *(pold) = __r; \ 57 __r == __o; \ 58 }) 59 60 #define try_cmpxchg_relaxed(p, pold, new) try_cmpxchg(p, pold, new) 61 62 #define try_cmpxchg_acquire(p, pold, new) try_cmpxchg(p, pold, new) 63 64 #define smp_mb() \ 65 ({ \ 66 volatile unsigned long __val; \ 67 __sync_fetch_and_add(&__val, 0); \ 68 }) 69 70 #define smp_rmb() \ 71 ({ \ 72 if (!CONFIG_X86_64) \ 73 smp_mb(); \ 74 else \ 75 barrier(); \ 76 }) 77 78 #define smp_wmb() \ 79 ({ \ 80 if (!CONFIG_X86_64) \ 81 smp_mb(); \ 82 else \ 83 barrier(); \ 84 }) 85 86 /* Control dependency provides LOAD->STORE, provide LOAD->LOAD */ 87 #define smp_acquire__after_ctrl_dep() ({ smp_rmb(); }) 88 89 #define smp_load_acquire(p) \ 90 ({ \ 91 __unqual_typeof(*(p)) __v = READ_ONCE(*(p)); \ 92 if (!CONFIG_X86_64) \ 93 smp_mb(); \ 94 barrier(); \ 95 __v; \ 96 }) 97 98 #define smp_store_release(p, val) \ 99 ({ \ 100 if (!CONFIG_X86_64) \ 101 smp_mb(); \ 102 barrier(); \ 103 WRITE_ONCE(*(p), val); \ 104 }) 105 106 #define smp_cond_load_relaxed_label(p, cond_expr, label) \ 107 ({ \ 108 typeof(p) __ptr = (p); \ 109 __unqual_typeof(*(p)) VAL; \ 110 for (;;) { \ 111 VAL = (__unqual_typeof(*(p)))READ_ONCE(*__ptr); \ 112 if (cond_expr) \ 113 break; \ 114 cond_break_label(label); \ 115 cpu_relax(); \ 116 } \ 117 (typeof(*(p)))VAL; \ 118 }) 119 120 #define smp_cond_load_acquire_label(p, cond_expr, label) \ 121 ({ \ 122 __unqual_typeof(*p) __val = \ 123 smp_cond_load_relaxed_label(p, cond_expr, label); \ 124 smp_acquire__after_ctrl_dep(); \ 125 (typeof(*(p)))__val; \ 126 }) 127 128 #define atomic_read(p) READ_ONCE((p)->counter) 129 130 #define atomic_cond_read_relaxed_label(p, cond_expr, label) \ 131 smp_cond_load_relaxed_label(&(p)->counter, cond_expr, label) 132 133 #define atomic_cond_read_acquire_label(p, cond_expr, label) \ 134 smp_cond_load_acquire_label(&(p)->counter, cond_expr, label) 135 136 #define atomic_try_cmpxchg_relaxed(p, pold, new) \ 137 try_cmpxchg_relaxed(&(p)->counter, pold, new) 138 139 #define atomic_try_cmpxchg_acquire(p, pold, new) \ 140 try_cmpxchg_acquire(&(p)->counter, pold, new) 141 142 #endif /* BPF_ATOMIC_H */ 143