1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2025 Google LLC. */ 3 4 #include <linux/bpf.h> 5 #include <bpf/bpf_helpers.h> 6 #include "../../../include/linux/filter.h" 7 #include "bpf_misc.h" 8 9 #ifdef CAN_USE_LOAD_ACQ_STORE_REL 10 11 SEC("socket") 12 __description("load-acquire, 8-bit") 13 __success __success_unpriv __retval(0x12) 14 __naked void load_acquire_8(void) 15 { 16 asm volatile ( 17 "w1 = 0x12;" 18 "*(u8 *)(r10 - 1) = w1;" 19 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r10 - 1)); 20 "exit;" 21 : 22 : __imm_insn(load_acquire_insn, 23 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -1)) 24 : __clobber_all); 25 } 26 27 SEC("socket") 28 __description("load-acquire, 16-bit") 29 __success __success_unpriv __retval(0x1234) 30 __naked void load_acquire_16(void) 31 { 32 asm volatile ( 33 "w1 = 0x1234;" 34 "*(u16 *)(r10 - 2) = w1;" 35 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u16 *)(r10 - 2)); 36 "exit;" 37 : 38 : __imm_insn(load_acquire_insn, 39 BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -2)) 40 : __clobber_all); 41 } 42 43 SEC("socket") 44 __description("load-acquire, 32-bit") 45 __success __success_unpriv __retval(0x12345678) 46 __naked void load_acquire_32(void) 47 { 48 asm volatile ( 49 "w1 = 0x12345678;" 50 "*(u32 *)(r10 - 4) = w1;" 51 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 4)); 52 "exit;" 53 : 54 : __imm_insn(load_acquire_insn, 55 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -4)) 56 : __clobber_all); 57 } 58 59 SEC("socket") 60 __description("load-acquire, 64-bit") 61 __success __success_unpriv __retval(0x1234567890abcdef) 62 __naked void load_acquire_64(void) 63 { 64 asm volatile ( 65 "r1 = 0x1234567890abcdef ll;" 66 "*(u64 *)(r10 - 8) = r1;" 67 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r10 - 8)); 68 "exit;" 69 : 70 : __imm_insn(load_acquire_insn, 71 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8)) 72 : __clobber_all); 73 } 74 75 SEC("socket") 76 __description("load-acquire with uninitialized src_reg") 77 __failure __failure_unpriv __msg("R2 !read_ok") 78 __naked void load_acquire_with_uninitialized_src_reg(void) 79 { 80 asm volatile ( 81 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r2 + 0)); 82 "exit;" 83 : 84 : __imm_insn(load_acquire_insn, 85 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0)) 86 : __clobber_all); 87 } 88 89 SEC("socket") 90 __description("load-acquire with non-pointer src_reg") 91 __failure __failure_unpriv __msg("R1 invalid mem access 'scalar'") 92 __naked void load_acquire_with_non_pointer_src_reg(void) 93 { 94 asm volatile ( 95 "r1 = 0;" 96 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0)); 97 "exit;" 98 : 99 : __imm_insn(load_acquire_insn, 100 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0)) 101 : __clobber_all); 102 } 103 104 SEC("socket") 105 __description("misaligned load-acquire") 106 __failure __failure_unpriv __msg("misaligned stack access off") 107 __flag(BPF_F_ANY_ALIGNMENT) 108 __naked void load_acquire_misaligned(void) 109 { 110 asm volatile ( 111 "r1 = 0;" 112 "*(u64 *)(r10 - 8) = r1;" 113 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 5)); 114 "exit;" 115 : 116 : __imm_insn(load_acquire_insn, 117 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -5)) 118 : __clobber_all); 119 } 120 121 SEC("socket") 122 __description("load-acquire from ctx pointer") 123 __failure __failure_unpriv __msg("BPF_ATOMIC loads from R1 ctx is not allowed") 124 __naked void load_acquire_from_ctx_pointer(void) 125 { 126 asm volatile ( 127 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r1 + 0)); 128 "exit;" 129 : 130 : __imm_insn(load_acquire_insn, 131 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0)) 132 : __clobber_all); 133 } 134 135 SEC("xdp") 136 __description("load-acquire from pkt pointer") 137 __failure __msg("BPF_ATOMIC loads from R2 pkt is not allowed") 138 __naked void load_acquire_from_pkt_pointer(void) 139 { 140 asm volatile ( 141 "r2 = *(u32 *)(r1 + %[xdp_md_data]);" 142 "r3 = *(u32 *)(r1 + %[xdp_md_data_end]);" 143 "r1 = r2;" 144 "r1 += 8;" 145 "if r1 >= r3 goto l0_%=;" 146 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0)); 147 "l0_%=: r0 = 0;" 148 "exit;" 149 : 150 : __imm_const(xdp_md_data, offsetof(struct xdp_md, data)), 151 __imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)), 152 __imm_insn(load_acquire_insn, 153 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0)) 154 : __clobber_all); 155 } 156 157 SEC("flow_dissector") 158 __description("load-acquire from flow_keys pointer") 159 __failure __msg("BPF_ATOMIC loads from R2 flow_keys is not allowed") 160 __naked void load_acquire_from_flow_keys_pointer(void) 161 { 162 asm volatile ( 163 "r2 = *(u64 *)(r1 + %[__sk_buff_flow_keys]);" 164 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0)); 165 "exit;" 166 : 167 : __imm_const(__sk_buff_flow_keys, 168 offsetof(struct __sk_buff, flow_keys)), 169 __imm_insn(load_acquire_insn, 170 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0)) 171 : __clobber_all); 172 } 173 174 SEC("sk_reuseport") 175 __description("load-acquire from sock pointer") 176 __failure __msg("BPF_ATOMIC loads from R2 sock is not allowed") 177 __naked void load_acquire_from_sock_pointer(void) 178 { 179 asm volatile ( 180 "r2 = *(u64 *)(r1 + %[sk_reuseport_md_sk]);" 181 // w0 = load_acquire((u8 *)(r2 + offsetof(struct bpf_sock, family))); 182 ".8byte %[load_acquire_insn];" 183 "exit;" 184 : 185 : __imm_const(sk_reuseport_md_sk, offsetof(struct sk_reuseport_md, sk)), 186 __imm_insn(load_acquire_insn, 187 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 188 offsetof(struct bpf_sock, family))) 189 : __clobber_all); 190 } 191 192 SEC("socket") 193 __description("load-acquire with invalid register R15") 194 __failure __failure_unpriv __msg("R15 is invalid") 195 __naked void load_acquire_with_invalid_reg(void) 196 { 197 asm volatile ( 198 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r15 + 0)); 199 "exit;" 200 : 201 : __imm_insn(load_acquire_insn, 202 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, 15 /* invalid reg */, 0)) 203 : __clobber_all); 204 } 205 206 #else /* CAN_USE_LOAD_ACQ_STORE_REL */ 207 208 SEC("socket") 209 __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test") 210 __success 211 int dummy_test(void) 212 { 213 return 0; 214 } 215 216 #endif /* CAN_USE_LOAD_ACQ_STORE_REL */ 217 218 char _license[] SEC("license") = "GPL"; 219