1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2025 Google LLC. */ 3 4 #include <linux/bpf.h> 5 #include <bpf/bpf_helpers.h> 6 #include "../../../include/linux/filter.h" 7 #include "bpf_misc.h" 8 9 #ifdef CAN_USE_LOAD_ACQ_STORE_REL 10 11 SEC("socket") 12 __description("load-acquire, 8-bit") 13 __success __success_unpriv __retval(0) 14 __naked void load_acquire_8(void) 15 { 16 asm volatile ( 17 "r0 = 0;" 18 "w1 = 0xfe;" 19 "*(u8 *)(r10 - 1) = w1;" 20 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1)); 21 "if r2 == r1 goto 1f;" 22 "r0 = 1;" 23 "1:" 24 "exit;" 25 : 26 : __imm_insn(load_acquire_insn, 27 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -1)) 28 : __clobber_all); 29 } 30 31 SEC("socket") 32 __description("load-acquire, 16-bit") 33 __success __success_unpriv __retval(0) 34 __naked void load_acquire_16(void) 35 { 36 asm volatile ( 37 "r0 = 0;" 38 "w1 = 0xfedc;" 39 "*(u16 *)(r10 - 2) = w1;" 40 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2)); 41 "if r2 == r1 goto 1f;" 42 "r0 = 1;" 43 "1:" 44 "exit;" 45 : 46 : __imm_insn(load_acquire_insn, 47 BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -2)) 48 : __clobber_all); 49 } 50 51 SEC("socket") 52 __description("load-acquire, 32-bit") 53 __success __success_unpriv __retval(0) 54 __naked void load_acquire_32(void) 55 { 56 asm volatile ( 57 "r0 = 0;" 58 "w1 = 0xfedcba09;" 59 "*(u32 *)(r10 - 4) = w1;" 60 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4)); 61 "if r2 == r1 goto 1f;" 62 "r0 = 1;" 63 "1:" 64 "exit;" 65 : 66 : __imm_insn(load_acquire_insn, 67 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -4)) 68 : __clobber_all); 69 } 70 71 SEC("socket") 72 __description("load-acquire, 64-bit") 73 __success __success_unpriv __retval(0) 74 __naked void load_acquire_64(void) 75 { 76 asm volatile ( 77 "r0 = 0;" 78 "r1 = 0xfedcba0987654321 ll;" 79 "*(u64 *)(r10 - 8) = r1;" 80 ".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8)); 81 "if r2 == r1 goto 1f;" 82 "r0 = 1;" 83 "1:" 84 "exit;" 85 : 86 : __imm_insn(load_acquire_insn, 87 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8)) 88 : __clobber_all); 89 } 90 91 SEC("socket") 92 __description("load-acquire with uninitialized src_reg") 93 __failure __failure_unpriv __msg("R2 !read_ok") 94 __naked void load_acquire_with_uninitialized_src_reg(void) 95 { 96 asm volatile ( 97 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r2 + 0)); 98 "exit;" 99 : 100 : __imm_insn(load_acquire_insn, 101 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0)) 102 : __clobber_all); 103 } 104 105 SEC("socket") 106 __description("load-acquire with non-pointer src_reg") 107 __failure __failure_unpriv __msg("R1 invalid mem access 'scalar'") 108 __naked void load_acquire_with_non_pointer_src_reg(void) 109 { 110 asm volatile ( 111 "r1 = 0;" 112 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0)); 113 "exit;" 114 : 115 : __imm_insn(load_acquire_insn, 116 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0)) 117 : __clobber_all); 118 } 119 120 SEC("socket") 121 __description("misaligned load-acquire") 122 __failure __failure_unpriv __msg("misaligned stack access off") 123 __flag(BPF_F_ANY_ALIGNMENT) 124 __naked void load_acquire_misaligned(void) 125 { 126 asm volatile ( 127 "r1 = 0;" 128 "*(u64 *)(r10 - 8) = r1;" 129 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 5)); 130 "exit;" 131 : 132 : __imm_insn(load_acquire_insn, 133 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -5)) 134 : __clobber_all); 135 } 136 137 SEC("socket") 138 __description("load-acquire from ctx pointer") 139 __failure __failure_unpriv __msg("BPF_ATOMIC loads from R1 ctx is not allowed") 140 __naked void load_acquire_from_ctx_pointer(void) 141 { 142 asm volatile ( 143 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r1 + 0)); 144 "exit;" 145 : 146 : __imm_insn(load_acquire_insn, 147 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0)) 148 : __clobber_all); 149 } 150 151 SEC("xdp") 152 __description("load-acquire from pkt pointer") 153 __failure __msg("BPF_ATOMIC loads from R2 pkt is not allowed") 154 __naked void load_acquire_from_pkt_pointer(void) 155 { 156 asm volatile ( 157 "r2 = *(u32 *)(r1 + %[xdp_md_data]);" 158 "r3 = *(u32 *)(r1 + %[xdp_md_data_end]);" 159 "r1 = r2;" 160 "r1 += 8;" 161 "if r1 >= r3 goto l0_%=;" 162 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0)); 163 "l0_%=: r0 = 0;" 164 "exit;" 165 : 166 : __imm_const(xdp_md_data, offsetof(struct xdp_md, data)), 167 __imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)), 168 __imm_insn(load_acquire_insn, 169 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0)) 170 : __clobber_all); 171 } 172 173 SEC("flow_dissector") 174 __description("load-acquire from flow_keys pointer") 175 __failure __msg("BPF_ATOMIC loads from R2 flow_keys is not allowed") 176 __naked void load_acquire_from_flow_keys_pointer(void) 177 { 178 asm volatile ( 179 "r2 = *(u64 *)(r1 + %[__sk_buff_flow_keys]);" 180 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0)); 181 "exit;" 182 : 183 : __imm_const(__sk_buff_flow_keys, 184 offsetof(struct __sk_buff, flow_keys)), 185 __imm_insn(load_acquire_insn, 186 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0)) 187 : __clobber_all); 188 } 189 190 SEC("sk_reuseport") 191 __description("load-acquire from sock pointer") 192 __failure __msg("BPF_ATOMIC loads from R2 sock is not allowed") 193 __naked void load_acquire_from_sock_pointer(void) 194 { 195 asm volatile ( 196 "r2 = *(u64 *)(r1 + %[sk_reuseport_md_sk]);" 197 // w0 = load_acquire((u8 *)(r2 + offsetof(struct bpf_sock, family))); 198 ".8byte %[load_acquire_insn];" 199 "exit;" 200 : 201 : __imm_const(sk_reuseport_md_sk, offsetof(struct sk_reuseport_md, sk)), 202 __imm_insn(load_acquire_insn, 203 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 204 offsetof(struct bpf_sock, family))) 205 : __clobber_all); 206 } 207 208 SEC("socket") 209 __description("load-acquire with invalid register R15") 210 __failure __failure_unpriv __msg("R15 is invalid") 211 __naked void load_acquire_with_invalid_reg(void) 212 { 213 asm volatile ( 214 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r15 + 0)); 215 "exit;" 216 : 217 : __imm_insn(load_acquire_insn, 218 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, 15 /* invalid reg */, 0)) 219 : __clobber_all); 220 } 221 222 #else /* CAN_USE_LOAD_ACQ_STORE_REL */ 223 224 SEC("socket") 225 __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test") 226 __success 227 int dummy_test(void) 228 { 229 return 0; 230 } 231 232 #endif /* CAN_USE_LOAD_ACQ_STORE_REL */ 233 234 char _license[] SEC("license") = "GPL"; 235