1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2025 Google LLC. */ 3 4 #include <linux/bpf.h> 5 #include <bpf/bpf_helpers.h> 6 #include <bpf/bpf_core_read.h> 7 #include "../../../include/linux/filter.h" 8 #include "bpf_misc.h" 9 10 #ifdef CAN_USE_LOAD_ACQ_STORE_REL 11 12 SEC("socket") 13 __description("load-acquire, 8-bit") 14 __success __success_unpriv __retval(0) 15 __naked void load_acquire_8(void) 16 { 17 asm volatile ( 18 "r0 = 0;" 19 "w1 = 0xfe;" 20 "*(u8 *)(r10 - 1) = w1;" 21 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1)); 22 "if r2 == r1 goto 1f;" 23 "r0 = 1;" 24 "1:" 25 "exit;" 26 : 27 : __imm_insn(load_acquire_insn, 28 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -1)) 29 : __clobber_all); 30 } 31 32 SEC("socket") 33 __description("load-acquire, 16-bit") 34 __success __success_unpriv __retval(0) 35 __naked void load_acquire_16(void) 36 { 37 asm volatile ( 38 "r0 = 0;" 39 "w1 = 0xfedc;" 40 "*(u16 *)(r10 - 2) = w1;" 41 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2)); 42 "if r2 == r1 goto 1f;" 43 "r0 = 1;" 44 "1:" 45 "exit;" 46 : 47 : __imm_insn(load_acquire_insn, 48 BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -2)) 49 : __clobber_all); 50 } 51 52 SEC("socket") 53 __description("load-acquire, 32-bit") 54 __success __success_unpriv __retval(0) 55 __naked void load_acquire_32(void) 56 { 57 asm volatile ( 58 "r0 = 0;" 59 "w1 = 0xfedcba09;" 60 "*(u32 *)(r10 - 4) = w1;" 61 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4)); 62 "if r2 == r1 goto 1f;" 63 "r0 = 1;" 64 "1:" 65 "exit;" 66 : 67 : __imm_insn(load_acquire_insn, 68 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -4)) 69 : __clobber_all); 70 } 71 72 SEC("socket") 73 __description("load-acquire, 64-bit") 74 __success __success_unpriv __retval(0) 75 __naked void load_acquire_64(void) 76 { 77 asm volatile ( 78 "r0 = 0;" 79 "r1 = 0xfedcba0987654321 ll;" 80 "*(u64 *)(r10 - 8) = r1;" 81 ".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8)); 82 "if r2 == r1 goto 1f;" 83 "r0 = 1;" 84 "1:" 85 "exit;" 86 : 87 : __imm_insn(load_acquire_insn, 88 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8)) 89 : __clobber_all); 90 } 91 92 SEC("socket") 93 __description("load-acquire with uninitialized src_reg") 94 __failure __failure_unpriv __msg("R2 !read_ok") 95 __naked void load_acquire_with_uninitialized_src_reg(void) 96 { 97 asm volatile ( 98 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r2 + 0)); 99 "exit;" 100 : 101 : __imm_insn(load_acquire_insn, 102 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0)) 103 : __clobber_all); 104 } 105 106 SEC("socket") 107 __description("load-acquire with non-pointer src_reg") 108 __failure __failure_unpriv __msg("R1 invalid mem access 'scalar'") 109 __naked void load_acquire_with_non_pointer_src_reg(void) 110 { 111 asm volatile ( 112 "r1 = 0;" 113 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0)); 114 "exit;" 115 : 116 : __imm_insn(load_acquire_insn, 117 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0)) 118 : __clobber_all); 119 } 120 121 SEC("socket") 122 __description("misaligned load-acquire") 123 __failure __failure_unpriv __msg("misaligned stack access off") 124 __flag(BPF_F_ANY_ALIGNMENT) 125 __naked void load_acquire_misaligned(void) 126 { 127 asm volatile ( 128 "r1 = 0;" 129 "*(u64 *)(r10 - 8) = r1;" 130 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 5)); 131 "exit;" 132 : 133 : __imm_insn(load_acquire_insn, 134 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -5)) 135 : __clobber_all); 136 } 137 138 SEC("socket") 139 __description("load-acquire from ctx pointer") 140 __failure __failure_unpriv __msg("BPF_ATOMIC loads from R1 ctx is not allowed") 141 __naked void load_acquire_from_ctx_pointer(void) 142 { 143 asm volatile ( 144 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r1 + 0)); 145 "exit;" 146 : 147 : __imm_insn(load_acquire_insn, 148 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0)) 149 : __clobber_all); 150 } 151 152 SEC("socket") 153 __description("load-acquire from ctx pointer, same dst and src register") 154 __failure __failure_unpriv __msg("BPF_ATOMIC loads from R6 ctx is not allowed") 155 __naked void load_acquire_ctx_same_dst_src(void) 156 { 157 asm volatile ( 158 "r6 = r1;" 159 ".8byte %[load_acquire_insn];" // w6 = load_acquire((u32 *)(r6 + 0)); 160 "r0 = 0;" 161 "exit;" 162 : 163 : __imm_insn(load_acquire_insn, 164 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_6, BPF_REG_6, 0)) 165 : __clobber_all); 166 } 167 168 SEC("xdp") 169 __description("load-acquire from pkt pointer") 170 __failure __msg("BPF_ATOMIC loads from R2 pkt is not allowed") 171 __naked void load_acquire_from_pkt_pointer(void) 172 { 173 asm volatile ( 174 "r2 = *(u32 *)(r1 + %[xdp_md_data]);" 175 "r3 = *(u32 *)(r1 + %[xdp_md_data_end]);" 176 "r1 = r2;" 177 "r1 += 8;" 178 "if r1 >= r3 goto l0_%=;" 179 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0)); 180 "l0_%=: r0 = 0;" 181 "exit;" 182 : 183 : __imm_const(xdp_md_data, offsetof(struct xdp_md, data)), 184 __imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)), 185 __imm_insn(load_acquire_insn, 186 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0)) 187 : __clobber_all); 188 } 189 190 SEC("flow_dissector") 191 __description("load-acquire from flow_keys pointer") 192 __failure __msg("BPF_ATOMIC loads from R2 flow_keys is not allowed") 193 __naked void load_acquire_from_flow_keys_pointer(void) 194 { 195 asm volatile ( 196 "r2 = *(u64 *)(r1 + %[__sk_buff_flow_keys]);" 197 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0)); 198 "exit;" 199 : 200 : __imm_const(__sk_buff_flow_keys, 201 offsetof(struct __sk_buff, flow_keys)), 202 __imm_insn(load_acquire_insn, 203 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0)) 204 : __clobber_all); 205 } 206 207 SEC("sk_reuseport") 208 __description("load-acquire from sock pointer") 209 __failure __msg("BPF_ATOMIC loads from R2 sock is not allowed") 210 __naked void load_acquire_from_sock_pointer(void) 211 { 212 asm volatile ( 213 "r2 = *(u64 *)(r1 + %[sk_reuseport_md_sk]);" 214 // w0 = load_acquire((u8 *)(r2 + offsetof(struct bpf_sock, family))); 215 ".8byte %[load_acquire_insn];" 216 "exit;" 217 : 218 : __imm_const(sk_reuseport_md_sk, offsetof(struct sk_reuseport_md, sk)), 219 __imm_insn(load_acquire_insn, 220 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 221 offsetof(struct bpf_sock, family))) 222 : __clobber_all); 223 } 224 225 SEC("socket") 226 __description("load-acquire from rdonly_untrusted_mem pointer") 227 __failure __msg("BPF_ATOMIC loads from R{{[0-9]+}} rdonly_untrusted_mem is not allowed") 228 int load_acquire_from_rdonly_untrusted_mem(void *ctx) 229 { 230 __u64 val = 0; 231 void *p; 232 233 /* 234 * bpf_rdonly_cast(x, 0) yields PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED. 235 * A regular BPF_LDX from it is rewritten to BPF_PROBE_MEM, but a 236 * load-acquire is not, so it must be rejected, otherwise the JIT emits 237 * a plain load with no exception table entry and a fault would crash 238 * the kernel. 239 */ 240 p = bpf_rdonly_cast(&val, 0); 241 asm volatile ( 242 "r1 = %[p];" 243 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0)); 244 : 245 : [p] "r" (p), 246 __imm_insn(load_acquire_insn, 247 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0)) 248 : "r0", "r1"); 249 return 0; 250 } 251 252 SEC("socket") 253 __description("load-acquire with invalid register R15") 254 __failure __failure_unpriv __msg("R15 is invalid") 255 __naked void load_acquire_with_invalid_reg(void) 256 { 257 asm volatile ( 258 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r15 + 0)); 259 "exit;" 260 : 261 : __imm_insn(load_acquire_insn, 262 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, 15 /* invalid reg */, 0)) 263 : __clobber_all); 264 } 265 266 #else /* CAN_USE_LOAD_ACQ_STORE_REL */ 267 268 SEC("socket") 269 __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test") 270 __success 271 int dummy_test(void) 272 { 273 return 0; 274 } 275 276 #endif /* CAN_USE_LOAD_ACQ_STORE_REL */ 277 278 char _license[] SEC("license") = "GPL"; 279