1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2025 Google LLC. */
3
4 #include <linux/bpf.h>
5 #include <bpf/bpf_helpers.h>
6 #include <bpf/bpf_core_read.h>
7 #include "../../../include/linux/filter.h"
8 #include "bpf_misc.h"
9
10 #ifdef CAN_USE_LOAD_ACQ_STORE_REL
11
12 SEC("socket")
13 __description("load-acquire, 8-bit")
14 __success __success_unpriv __retval(0)
load_acquire_8(void)15 __naked void load_acquire_8(void)
16 {
17 asm volatile (
18 "r0 = 0;"
19 "w1 = 0xfe;"
20 "*(u8 *)(r10 - 1) = w1;"
21 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1));
22 "if r2 == r1 goto 1f;"
23 "r0 = 1;"
24 "1:"
25 "exit;"
26 :
27 : __imm_insn(load_acquire_insn,
28 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -1))
29 : __clobber_all);
30 }
31
32 SEC("socket")
33 __description("load-acquire, 16-bit")
34 __success __success_unpriv __retval(0)
load_acquire_16(void)35 __naked void load_acquire_16(void)
36 {
37 asm volatile (
38 "r0 = 0;"
39 "w1 = 0xfedc;"
40 "*(u16 *)(r10 - 2) = w1;"
41 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2));
42 "if r2 == r1 goto 1f;"
43 "r0 = 1;"
44 "1:"
45 "exit;"
46 :
47 : __imm_insn(load_acquire_insn,
48 BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -2))
49 : __clobber_all);
50 }
51
52 SEC("socket")
53 __description("load-acquire, 32-bit")
54 __success __success_unpriv __retval(0)
load_acquire_32(void)55 __naked void load_acquire_32(void)
56 {
57 asm volatile (
58 "r0 = 0;"
59 "w1 = 0xfedcba09;"
60 "*(u32 *)(r10 - 4) = w1;"
61 ".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4));
62 "if r2 == r1 goto 1f;"
63 "r0 = 1;"
64 "1:"
65 "exit;"
66 :
67 : __imm_insn(load_acquire_insn,
68 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -4))
69 : __clobber_all);
70 }
71
72 SEC("socket")
73 __description("load-acquire, 64-bit")
74 __success __success_unpriv __retval(0)
load_acquire_64(void)75 __naked void load_acquire_64(void)
76 {
77 asm volatile (
78 "r0 = 0;"
79 "r1 = 0xfedcba0987654321 ll;"
80 "*(u64 *)(r10 - 8) = r1;"
81 ".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8));
82 "if r2 == r1 goto 1f;"
83 "r0 = 1;"
84 "1:"
85 "exit;"
86 :
87 : __imm_insn(load_acquire_insn,
88 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8))
89 : __clobber_all);
90 }
91
92 SEC("socket")
93 __description("load-acquire with uninitialized src_reg")
94 __failure __failure_unpriv __msg("R2 !read_ok")
load_acquire_with_uninitialized_src_reg(void)95 __naked void load_acquire_with_uninitialized_src_reg(void)
96 {
97 asm volatile (
98 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r2 + 0));
99 "exit;"
100 :
101 : __imm_insn(load_acquire_insn,
102 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
103 : __clobber_all);
104 }
105
106 SEC("socket")
107 __description("load-acquire with non-pointer src_reg")
108 __failure __failure_unpriv __msg("R1 invalid mem access 'scalar'")
load_acquire_with_non_pointer_src_reg(void)109 __naked void load_acquire_with_non_pointer_src_reg(void)
110 {
111 asm volatile (
112 "r1 = 0;"
113 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0));
114 "exit;"
115 :
116 : __imm_insn(load_acquire_insn,
117 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0))
118 : __clobber_all);
119 }
120
121 SEC("socket")
122 __description("misaligned load-acquire")
123 __failure __failure_unpriv __msg("misaligned stack access off")
__flag(BPF_F_ANY_ALIGNMENT)124 __flag(BPF_F_ANY_ALIGNMENT)
125 __naked void load_acquire_misaligned(void)
126 {
127 asm volatile (
128 "r1 = 0;"
129 "*(u64 *)(r10 - 8) = r1;"
130 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 5));
131 "exit;"
132 :
133 : __imm_insn(load_acquire_insn,
134 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -5))
135 : __clobber_all);
136 }
137
138 SEC("socket")
139 __description("load-acquire from ctx pointer")
140 __failure __failure_unpriv __msg("BPF_ATOMIC loads from R1 ctx is not allowed")
load_acquire_from_ctx_pointer(void)141 __naked void load_acquire_from_ctx_pointer(void)
142 {
143 asm volatile (
144 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r1 + 0));
145 "exit;"
146 :
147 : __imm_insn(load_acquire_insn,
148 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0))
149 : __clobber_all);
150 }
151
152 SEC("socket")
153 __description("load-acquire from ctx pointer, same dst and src register")
154 __failure __failure_unpriv __msg("BPF_ATOMIC loads from R6 ctx is not allowed")
load_acquire_ctx_same_dst_src(void)155 __naked void load_acquire_ctx_same_dst_src(void)
156 {
157 asm volatile (
158 "r6 = r1;"
159 ".8byte %[load_acquire_insn];" // w6 = load_acquire((u32 *)(r6 + 0));
160 "r0 = 0;"
161 "exit;"
162 :
163 : __imm_insn(load_acquire_insn,
164 BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_6, BPF_REG_6, 0))
165 : __clobber_all);
166 }
167
168 SEC("xdp")
169 __description("load-acquire from pkt pointer")
170 __failure __msg("BPF_ATOMIC loads from R2 pkt is not allowed")
load_acquire_from_pkt_pointer(void)171 __naked void load_acquire_from_pkt_pointer(void)
172 {
173 asm volatile (
174 "r2 = *(u32 *)(r1 + %[xdp_md_data]);"
175 "r3 = *(u32 *)(r1 + %[xdp_md_data_end]);"
176 "r1 = r2;"
177 "r1 += 8;"
178 "if r1 >= r3 goto l0_%=;"
179 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0));
180 "l0_%=: r0 = 0;"
181 "exit;"
182 :
183 : __imm_const(xdp_md_data, offsetof(struct xdp_md, data)),
184 __imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)),
185 __imm_insn(load_acquire_insn,
186 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
187 : __clobber_all);
188 }
189
190 SEC("flow_dissector")
191 __description("load-acquire from flow_keys pointer")
192 __failure __msg("BPF_ATOMIC loads from R2 flow_keys is not allowed")
load_acquire_from_flow_keys_pointer(void)193 __naked void load_acquire_from_flow_keys_pointer(void)
194 {
195 asm volatile (
196 "r2 = *(u64 *)(r1 + %[__sk_buff_flow_keys]);"
197 ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0));
198 "exit;"
199 :
200 : __imm_const(__sk_buff_flow_keys,
201 offsetof(struct __sk_buff, flow_keys)),
202 __imm_insn(load_acquire_insn,
203 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
204 : __clobber_all);
205 }
206
207 SEC("sk_reuseport")
208 __description("load-acquire from sock pointer")
209 __failure __msg("BPF_ATOMIC loads from R2 sock is not allowed")
load_acquire_from_sock_pointer(void)210 __naked void load_acquire_from_sock_pointer(void)
211 {
212 asm volatile (
213 "r2 = *(u64 *)(r1 + %[sk_reuseport_md_sk]);"
214 // w0 = load_acquire((u8 *)(r2 + offsetof(struct bpf_sock, family)));
215 ".8byte %[load_acquire_insn];"
216 "exit;"
217 :
218 : __imm_const(sk_reuseport_md_sk, offsetof(struct sk_reuseport_md, sk)),
219 __imm_insn(load_acquire_insn,
220 BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2,
221 offsetof(struct bpf_sock, family)))
222 : __clobber_all);
223 }
224
225 SEC("socket")
226 __description("load-acquire from rdonly_untrusted_mem pointer")
227 __failure __msg("BPF_ATOMIC loads from R{{[0-9]+}} rdonly_untrusted_mem is not allowed")
load_acquire_from_rdonly_untrusted_mem(void * ctx)228 int load_acquire_from_rdonly_untrusted_mem(void *ctx)
229 {
230 __u64 val = 0;
231 void *p;
232
233 /*
234 * bpf_rdonly_cast(x, 0) yields PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED.
235 * A regular BPF_LDX from it is rewritten to BPF_PROBE_MEM, but a
236 * load-acquire is not, so it must be rejected, otherwise the JIT emits
237 * a plain load with no exception table entry and a fault would crash
238 * the kernel.
239 */
240 p = bpf_rdonly_cast(&val, 0);
241 asm volatile (
242 "r1 = %[p];"
243 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0));
244 :
245 : [p] "r" (p),
246 __imm_insn(load_acquire_insn,
247 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0))
248 : "r0", "r1");
249 return 0;
250 }
251
252 SEC("socket")
253 __description("load-acquire with invalid register R15")
254 __failure __failure_unpriv __msg("R15 is invalid")
load_acquire_with_invalid_reg(void)255 __naked void load_acquire_with_invalid_reg(void)
256 {
257 asm volatile (
258 ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r15 + 0));
259 "exit;"
260 :
261 : __imm_insn(load_acquire_insn,
262 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, 15 /* invalid reg */, 0))
263 : __clobber_all);
264 }
265
266 #else /* CAN_USE_LOAD_ACQ_STORE_REL */
267
268 SEC("socket")
269 __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test")
270 __success
dummy_test(void)271 int dummy_test(void)
272 {
273 return 0;
274 }
275
276 #endif /* CAN_USE_LOAD_ACQ_STORE_REL */
277
278 char _license[] SEC("license") = "GPL";
279