1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ 3 #ifndef BPF_ATOMIC_H 4 #define BPF_ATOMIC_H 5 6 #include <vmlinux.h> 7 #include <bpf/bpf_helpers.h> 8 #include <bpf_may_goto.h> 9 10 extern bool CONFIG_X86_64 __kconfig __weak; 11 12 /* 13 * __unqual_typeof(x) - Declare an unqualified scalar type, leaving 14 * non-scalar types unchanged, 15 * 16 * Prefer C11 _Generic for better compile-times and simpler code. Note: 'char' 17 * is not type-compatible with 'signed char', and we define a separate case. 18 * 19 * This is copied verbatim from kernel's include/linux/compiler_types.h, but 20 * with default expression (for pointers) changed from (x) to (typeof(x)0). 21 * 22 * This is because LLVM has a bug where for lvalue (x), it does not get rid of 23 * an extra address_space qualifier, but does in case of rvalue (typeof(x)0). 24 * Hence, for pointers, we need to create an rvalue expression to get the 25 * desired type. See https://github.com/llvm/llvm-project/issues/53400. 26 */ 27 #define __scalar_type_to_expr_cases(type) \ 28 unsigned type : (unsigned type)0, signed type : (signed type)0 29 30 #define __unqual_typeof(x) \ 31 typeof(_Generic((x), \ 32 char: (char)0, \ 33 __scalar_type_to_expr_cases(char), \ 34 __scalar_type_to_expr_cases(short), \ 35 __scalar_type_to_expr_cases(int), \ 36 __scalar_type_to_expr_cases(long), \ 37 __scalar_type_to_expr_cases(long long), \ 38 default: (typeof(x))0)) 39 40 /* No-op for BPF */ 41 #define cpu_relax() ({}) 42 43 #define READ_ONCE(x) (*(volatile typeof(x) *)&(x)) 44 45 #ifndef WRITE_ONCE 46 #define WRITE_ONCE(x, val) ((*(volatile typeof(x) *)&(x)) = (val)) 47 #endif 48 49 #define cmpxchg(p, old, new) __sync_val_compare_and_swap((p), old, new) 50 51 #define try_cmpxchg(p, pold, new) \ 52 ({ \ 53 __unqual_typeof(*(pold)) __o = *(pold); \ 54 __unqual_typeof(*(p)) __r = cmpxchg(p, __o, new); \ 55 if (__r != __o) \ 56 *(pold) = __r; \ 57 __r == __o; \ 58 }) 59 60 #define try_cmpxchg_relaxed(p, pold, new) try_cmpxchg(p, pold, new) 61 62 #define try_cmpxchg_acquire(p, pold, new) try_cmpxchg(p, pold, new) 63 64 #define smp_mb() \ 65 ({ \ 66 volatile unsigned long __val; \ 67 __sync_fetch_and_add(&__val, 0); \ 68 }) 69 70 #define smp_rmb() \ 71 ({ \ 72 if (!CONFIG_X86_64) \ 73 smp_mb(); \ 74 else \ 75 barrier(); \ 76 }) 77 78 #define smp_wmb() \ 79 ({ \ 80 if (!CONFIG_X86_64) \ 81 smp_mb(); \ 82 else \ 83 barrier(); \ 84 }) 85 86 /* Control dependency provides LOAD->STORE, provide LOAD->LOAD */ 87 #define smp_acquire__after_ctrl_dep() ({ smp_rmb(); }) 88 89 #if defined(__BPF_FEATURE_LOAD_ACQ_STORE_REL) 90 /* 91 * Clang advertises this feature when it can lower acquire/release atomic 92 * builtins to BPF_LOAD_ACQ/BPF_STORE_REL. Older compilers keep using the 93 * barrier-based fallback below. The generated instructions require kernel 94 * verifier/JIT support added in Linux 6.15; compile for an older BPF CPU to 95 * keep using the fallback when targeting older kernels. 96 */ 97 #define smp_load_acquire(p) \ 98 ({ \ 99 __unqual_typeof(*(p)) ___p1 = __atomic_load_n((p), __ATOMIC_ACQUIRE); \ 100 (typeof(*(p)))___p1; \ 101 }) 102 103 #define smp_store_release(p, val) \ 104 ({ \ 105 __atomic_store_n((p), (val), __ATOMIC_RELEASE); \ 106 }) 107 #else 108 #define smp_load_acquire(p) \ 109 ({ \ 110 __unqual_typeof(*(p)) __v = READ_ONCE(*(p)); \ 111 if (!CONFIG_X86_64) \ 112 smp_mb(); \ 113 barrier(); \ 114 __v; \ 115 }) 116 117 #define smp_store_release(p, val) \ 118 ({ \ 119 if (!CONFIG_X86_64) \ 120 smp_mb(); \ 121 barrier(); \ 122 WRITE_ONCE(*(p), val); \ 123 }) 124 #endif 125 126 #define smp_cond_load_relaxed_label(p, cond_expr, label) \ 127 ({ \ 128 typeof(p) __ptr = (p); \ 129 __unqual_typeof(*(p)) VAL; \ 130 for (;;) { \ 131 VAL = (__unqual_typeof(*(p)))READ_ONCE(*__ptr); \ 132 if (cond_expr) \ 133 break; \ 134 cond_break_label(label); \ 135 cpu_relax(); \ 136 } \ 137 (typeof(*(p)))VAL; \ 138 }) 139 140 #define smp_cond_load_acquire_label(p, cond_expr, label) \ 141 ({ \ 142 __unqual_typeof(*p) __val = \ 143 smp_cond_load_relaxed_label(p, cond_expr, label); \ 144 smp_acquire__after_ctrl_dep(); \ 145 (typeof(*(p)))__val; \ 146 }) 147 148 #define atomic_read(p) READ_ONCE((p)->counter) 149 150 #define atomic_cond_read_relaxed_label(p, cond_expr, label) \ 151 smp_cond_load_relaxed_label(&(p)->counter, cond_expr, label) 152 153 #define atomic_cond_read_acquire_label(p, cond_expr, label) \ 154 smp_cond_load_acquire_label(&(p)->counter, cond_expr, label) 155 156 #define atomic_try_cmpxchg_relaxed(p, pold, new) \ 157 try_cmpxchg_relaxed(&(p)->counter, pold, new) 158 159 #define atomic_try_cmpxchg_acquire(p, pold, new) \ 160 try_cmpxchg_acquire(&(p)->counter, pold, new) 161 162 #endif /* BPF_ATOMIC_H */ 163