1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3 #include <linux/bpf.h>
4 #include <bpf/bpf_helpers.h>
5 #include <bpf/bpf_tracing.h>
6 #include <stdbool.h>
7 #include <stdatomic.h>
8 #include <bpf_arena_common.h>
9 #include "../../../include/linux/filter.h"
10 #include "bpf_misc.h"
11
12 struct {
13 __uint(type, BPF_MAP_TYPE_ARENA);
14 __uint(map_flags, BPF_F_MMAPABLE);
15 __uint(max_entries, 10); /* number of pages */
16 #ifdef __TARGET_ARCH_arm64
17 __ulong(map_extra, 0x1ull << 32); /* start of mmap() region */
18 #else
19 __ulong(map_extra, 0x1ull << 44); /* start of mmap() region */
20 #endif
21 } arena SEC(".maps");
22
23 #if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
24 bool skip_all_tests __attribute((__section__(".data"))) = false;
25 #else
26 bool skip_all_tests = true;
27 #endif
28
29 #if defined(ENABLE_ATOMICS_TESTS) && \
30 defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
31 (defined(__TARGET_ARCH_arm64) || \
32 defined(__TARGET_ARCH_x86) || \
33 (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
34 defined(__TARGET_ARCH_s390))
35 bool skip_lacq_srel_tests __attribute((__section__(".data"))) = false;
36 #else
37 bool skip_lacq_srel_tests = true;
38 #endif
39
40 __u32 pid = 0;
41
42 __u64 __arena_global add64_value = 1;
43 __u64 __arena_global add64_result = 0;
44 __u32 __arena_global add32_value = 1;
45 __u32 __arena_global add32_result = 0;
46 __u64 __arena_global add_stack_value_copy = 0;
47 __u64 __arena_global add_stack_result = 0;
48 __u64 __arena_global add_noreturn_value = 1;
49
50 SEC("raw_tp/sys_enter")
add(const void * ctx)51 int add(const void *ctx)
52 {
53 if (pid != (bpf_get_current_pid_tgid() >> 32))
54 return 0;
55 #ifdef ENABLE_ATOMICS_TESTS
56 __u64 add_stack_value = 1;
57
58 add64_result = __sync_fetch_and_add(&add64_value, 2);
59 add32_result = __sync_fetch_and_add(&add32_value, 2);
60 add_stack_result = __sync_fetch_and_add(&add_stack_value, 2);
61 add_stack_value_copy = add_stack_value;
62 __sync_fetch_and_add(&add_noreturn_value, 2);
63 #endif
64
65 return 0;
66 }
67
68 __s64 __arena_global sub64_value = 1;
69 __s64 __arena_global sub64_result = 0;
70 __s32 __arena_global sub32_value = 1;
71 __s32 __arena_global sub32_result = 0;
72 __s64 __arena_global sub_stack_value_copy = 0;
73 __s64 __arena_global sub_stack_result = 0;
74 __s64 __arena_global sub_noreturn_value = 1;
75
76 SEC("raw_tp/sys_enter")
sub(const void * ctx)77 int sub(const void *ctx)
78 {
79 if (pid != (bpf_get_current_pid_tgid() >> 32))
80 return 0;
81 #ifdef ENABLE_ATOMICS_TESTS
82 __u64 sub_stack_value = 1;
83
84 sub64_result = __sync_fetch_and_sub(&sub64_value, 2);
85 sub32_result = __sync_fetch_and_sub(&sub32_value, 2);
86 sub_stack_result = __sync_fetch_and_sub(&sub_stack_value, 2);
87 sub_stack_value_copy = sub_stack_value;
88 __sync_fetch_and_sub(&sub_noreturn_value, 2);
89 #endif
90
91 return 0;
92 }
93
94 #ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
95 _Atomic __u64 __arena_global and64_value = (0x110ull << 32);
96 _Atomic __u32 __arena_global and32_value = 0x110;
97 #else
98 __u64 __arena_global and64_value = (0x110ull << 32);
99 __u32 __arena_global and32_value = 0x110;
100 #endif
101
102 SEC("raw_tp/sys_enter")
and(const void * ctx)103 int and(const void *ctx)
104 {
105 if (pid != (bpf_get_current_pid_tgid() >> 32))
106 return 0;
107 #ifdef ENABLE_ATOMICS_TESTS
108 #ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
109 __c11_atomic_fetch_and(&and64_value, 0x011ull << 32, memory_order_relaxed);
110 __c11_atomic_fetch_and(&and32_value, 0x011, memory_order_relaxed);
111 #else
112 __sync_fetch_and_and(&and64_value, 0x011ull << 32);
113 __sync_fetch_and_and(&and32_value, 0x011);
114 #endif
115 #endif
116
117 return 0;
118 }
119
120 #ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
121 _Atomic __u32 __arena_global or32_value = 0x110;
122 _Atomic __u64 __arena_global or64_value = (0x110ull << 32);
123 #else
124 __u32 __arena_global or32_value = 0x110;
125 __u64 __arena_global or64_value = (0x110ull << 32);
126 #endif
127
128 SEC("raw_tp/sys_enter")
or(const void * ctx)129 int or(const void *ctx)
130 {
131 if (pid != (bpf_get_current_pid_tgid() >> 32))
132 return 0;
133 #ifdef ENABLE_ATOMICS_TESTS
134 #ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
135 __c11_atomic_fetch_or(&or64_value, 0x011ull << 32, memory_order_relaxed);
136 __c11_atomic_fetch_or(&or32_value, 0x011, memory_order_relaxed);
137 #else
138 __sync_fetch_and_or(&or64_value, 0x011ull << 32);
139 __sync_fetch_and_or(&or32_value, 0x011);
140 #endif
141 #endif
142
143 return 0;
144 }
145
146 #ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
147 _Atomic __u64 __arena_global xor64_value = (0x110ull << 32);
148 _Atomic __u32 __arena_global xor32_value = 0x110;
149 #else
150 __u64 __arena_global xor64_value = (0x110ull << 32);
151 __u32 __arena_global xor32_value = 0x110;
152 #endif
153
154 SEC("raw_tp/sys_enter")
xor(const void * ctx)155 int xor(const void *ctx)
156 {
157 if (pid != (bpf_get_current_pid_tgid() >> 32))
158 return 0;
159 #ifdef ENABLE_ATOMICS_TESTS
160 #ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
161 __c11_atomic_fetch_xor(&xor64_value, 0x011ull << 32, memory_order_relaxed);
162 __c11_atomic_fetch_xor(&xor32_value, 0x011, memory_order_relaxed);
163 #else
164 __sync_fetch_and_xor(&xor64_value, 0x011ull << 32);
165 __sync_fetch_and_xor(&xor32_value, 0x011);
166 #endif
167 #endif
168
169 return 0;
170 }
171
172 __u32 __arena_global cmpxchg32_value = 1;
173 __u32 __arena_global cmpxchg32_result_fail = 0;
174 __u32 __arena_global cmpxchg32_result_succeed = 0;
175 __u64 __arena_global cmpxchg64_value = 1;
176 __u64 __arena_global cmpxchg64_result_fail = 0;
177 __u64 __arena_global cmpxchg64_result_succeed = 0;
178
179 SEC("raw_tp/sys_enter")
cmpxchg(const void * ctx)180 int cmpxchg(const void *ctx)
181 {
182 if (pid != (bpf_get_current_pid_tgid() >> 32))
183 return 0;
184 #ifdef ENABLE_ATOMICS_TESTS
185 cmpxchg64_result_fail = __sync_val_compare_and_swap(&cmpxchg64_value, 0, 3);
186 cmpxchg64_result_succeed = __sync_val_compare_and_swap(&cmpxchg64_value, 1, 2);
187
188 cmpxchg32_result_fail = __sync_val_compare_and_swap(&cmpxchg32_value, 0, 3);
189 cmpxchg32_result_succeed = __sync_val_compare_and_swap(&cmpxchg32_value, 1, 2);
190 #endif
191
192 return 0;
193 }
194
195 __u64 __arena_global xchg64_value = 1;
196 __u64 __arena_global xchg64_result = 0;
197 __u32 __arena_global xchg32_value = 1;
198 __u32 __arena_global xchg32_result = 0;
199
200 SEC("raw_tp/sys_enter")
xchg(const void * ctx)201 int xchg(const void *ctx)
202 {
203 if (pid != (bpf_get_current_pid_tgid() >> 32))
204 return 0;
205 #ifdef ENABLE_ATOMICS_TESTS
206 __u64 val64 = 2;
207 __u32 val32 = 2;
208
209 xchg64_result = __sync_lock_test_and_set(&xchg64_value, val64);
210 xchg32_result = __sync_lock_test_and_set(&xchg32_value, val32);
211 #endif
212
213 return 0;
214 }
215
216 __u64 __arena_global uaf_sink;
217 volatile __u64 __arena_global uaf_recovery_fails;
218
219 SEC("syscall")
uaf(const void * ctx)220 int uaf(const void *ctx)
221 {
222 if (pid != (bpf_get_current_pid_tgid() >> 32))
223 return 0;
224 #if defined(ENABLE_ATOMICS_TESTS) && !defined(__TARGET_ARCH_arm64) && \
225 !defined(__TARGET_ARCH_x86)
226 __u32 __arena *page32;
227 __u64 __arena *page64;
228 void __arena *page;
229
230 page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
231 bpf_arena_free_pages(&arena, page, 1);
232 uaf_recovery_fails = 24;
233
234 page32 = (__u32 __arena *)page;
235 uaf_sink += __sync_fetch_and_add(page32, 1);
236 uaf_recovery_fails -= 1;
237 __sync_add_and_fetch(page32, 1);
238 uaf_recovery_fails -= 1;
239 uaf_sink += __sync_fetch_and_sub(page32, 1);
240 uaf_recovery_fails -= 1;
241 __sync_sub_and_fetch(page32, 1);
242 uaf_recovery_fails -= 1;
243 uaf_sink += __sync_fetch_and_and(page32, 1);
244 uaf_recovery_fails -= 1;
245 __sync_and_and_fetch(page32, 1);
246 uaf_recovery_fails -= 1;
247 uaf_sink += __sync_fetch_and_or(page32, 1);
248 uaf_recovery_fails -= 1;
249 __sync_or_and_fetch(page32, 1);
250 uaf_recovery_fails -= 1;
251 uaf_sink += __sync_fetch_and_xor(page32, 1);
252 uaf_recovery_fails -= 1;
253 __sync_xor_and_fetch(page32, 1);
254 uaf_recovery_fails -= 1;
255 uaf_sink += __sync_val_compare_and_swap(page32, 0, 1);
256 uaf_recovery_fails -= 1;
257 uaf_sink += __sync_lock_test_and_set(page32, 1);
258 uaf_recovery_fails -= 1;
259
260 page64 = (__u64 __arena *)page;
261 uaf_sink += __sync_fetch_and_add(page64, 1);
262 uaf_recovery_fails -= 1;
263 __sync_add_and_fetch(page64, 1);
264 uaf_recovery_fails -= 1;
265 uaf_sink += __sync_fetch_and_sub(page64, 1);
266 uaf_recovery_fails -= 1;
267 __sync_sub_and_fetch(page64, 1);
268 uaf_recovery_fails -= 1;
269 uaf_sink += __sync_fetch_and_and(page64, 1);
270 uaf_recovery_fails -= 1;
271 __sync_and_and_fetch(page64, 1);
272 uaf_recovery_fails -= 1;
273 uaf_sink += __sync_fetch_and_or(page64, 1);
274 uaf_recovery_fails -= 1;
275 __sync_or_and_fetch(page64, 1);
276 uaf_recovery_fails -= 1;
277 uaf_sink += __sync_fetch_and_xor(page64, 1);
278 uaf_recovery_fails -= 1;
279 __sync_xor_and_fetch(page64, 1);
280 uaf_recovery_fails -= 1;
281 uaf_sink += __sync_val_compare_and_swap(page64, 0, 1);
282 uaf_recovery_fails -= 1;
283 uaf_sink += __sync_lock_test_and_set(page64, 1);
284 uaf_recovery_fails -= 1;
285 #endif
286
287 return 0;
288 }
289
290 #if __clang_major__ >= 18
291 __u8 __arena_global load_acquire8_value = 0x12;
292 __u16 __arena_global load_acquire16_value = 0x1234;
293 __u32 __arena_global load_acquire32_value = 0x12345678;
294 __u64 __arena_global load_acquire64_value = 0x1234567890abcdef;
295
296 __u8 __arena_global load_acquire8_result = 0;
297 __u16 __arena_global load_acquire16_result = 0;
298 __u32 __arena_global load_acquire32_result = 0;
299 __u64 __arena_global load_acquire64_result = 0;
300 #else
301 /* clang-17 crashes if the .addr_space.1 ELF section has holes. Work around
302 * this issue by defining the below variables as 64-bit.
303 */
304 __u64 __arena_global load_acquire8_value;
305 __u64 __arena_global load_acquire16_value;
306 __u64 __arena_global load_acquire32_value;
307 __u64 __arena_global load_acquire64_value;
308
309 __u64 __arena_global load_acquire8_result;
310 __u64 __arena_global load_acquire16_result;
311 __u64 __arena_global load_acquire32_result;
312 __u64 __arena_global load_acquire64_result;
313 #endif
314
315 SEC("raw_tp/sys_enter")
load_acquire(const void * ctx)316 int load_acquire(const void *ctx)
317 {
318 #if defined(ENABLE_ATOMICS_TESTS) && \
319 defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
320 (defined(__TARGET_ARCH_arm64) || \
321 defined(__TARGET_ARCH_x86) || \
322 (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
323 defined(__TARGET_ARCH_s390))
324
325 #define LOAD_ACQUIRE_ARENA(SIZEOP, SIZE, SRC, DST) \
326 { asm volatile ( \
327 "r1 = %[" #SRC "] ll;" \
328 "r1 = addr_space_cast(r1, 0x0, 0x1);" \
329 ".8byte %[load_acquire_insn];" \
330 "r3 = %[" #DST "] ll;" \
331 "r3 = addr_space_cast(r3, 0x0, 0x1);" \
332 "*(" #SIZE " *)(r3 + 0) = r2;" \
333 : \
334 : __imm_addr(SRC), \
335 __imm_insn(load_acquire_insn, \
336 BPF_ATOMIC_OP(BPF_##SIZEOP, BPF_LOAD_ACQ, \
337 BPF_REG_2, BPF_REG_1, 0)), \
338 __imm_addr(DST) \
339 : __clobber_all); } \
340
341 LOAD_ACQUIRE_ARENA(B, u8, load_acquire8_value, load_acquire8_result)
342 LOAD_ACQUIRE_ARENA(H, u16, load_acquire16_value,
343 load_acquire16_result)
344 LOAD_ACQUIRE_ARENA(W, u32, load_acquire32_value,
345 load_acquire32_result)
346 LOAD_ACQUIRE_ARENA(DW, u64, load_acquire64_value,
347 load_acquire64_result)
348 #undef LOAD_ACQUIRE_ARENA
349
350 #endif
351 return 0;
352 }
353
354 #if __clang_major__ >= 18
355 __u8 __arena_global store_release8_result = 0;
356 __u16 __arena_global store_release16_result = 0;
357 __u32 __arena_global store_release32_result = 0;
358 __u64 __arena_global store_release64_result = 0;
359 #else
360 /* clang-17 crashes if the .addr_space.1 ELF section has holes. Work around
361 * this issue by defining the below variables as 64-bit.
362 */
363 __u64 __arena_global store_release8_result;
364 __u64 __arena_global store_release16_result;
365 __u64 __arena_global store_release32_result;
366 __u64 __arena_global store_release64_result;
367 #endif
368
369 SEC("raw_tp/sys_enter")
store_release(const void * ctx)370 int store_release(const void *ctx)
371 {
372 #if defined(ENABLE_ATOMICS_TESTS) && \
373 defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
374 (defined(__TARGET_ARCH_arm64) || \
375 defined(__TARGET_ARCH_x86) || \
376 (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
377 defined(__TARGET_ARCH_s390))
378
379 #define STORE_RELEASE_ARENA(SIZEOP, DST, VAL) \
380 { asm volatile ( \
381 "r1 = " VAL ";" \
382 "r2 = %[" #DST "] ll;" \
383 "r2 = addr_space_cast(r2, 0x0, 0x1);" \
384 ".8byte %[store_release_insn];" \
385 : \
386 : __imm_addr(DST), \
387 __imm_insn(store_release_insn, \
388 BPF_ATOMIC_OP(BPF_##SIZEOP, BPF_STORE_REL, \
389 BPF_REG_2, BPF_REG_1, 0)) \
390 : __clobber_all); } \
391
392 STORE_RELEASE_ARENA(B, store_release8_result, "0x12")
393 STORE_RELEASE_ARENA(H, store_release16_result, "0x1234")
394 STORE_RELEASE_ARENA(W, store_release32_result, "0x12345678")
395 STORE_RELEASE_ARENA(DW, store_release64_result,
396 "0x1234567890abcdef ll")
397 #undef STORE_RELEASE_ARENA
398
399 #endif
400 return 0;
401 }
402
403 char _license[] SEC("license") = "GPL";
404