1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2023 SUSE LLC */
3 #include <linux/bpf.h>
4 #include <bpf/bpf_helpers.h>
5 #include <stdbool.h>
6 #include "../../../include/linux/filter.h"
7 #include "bpf_misc.h"
8 #include "bpf_kfuncs.h"
9
10 struct {
11 __uint(type, BPF_MAP_TYPE_ARRAY);
12 __uint(max_entries, 1);
13 __type(key, __u32);
14 __type(value, __u64);
15 } precision_map SEC(".maps");
16
17 SEC("?raw_tp")
18 __success __log_level(2)
19 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r1 = r10")
20 __msg("mark_precise: frame0: regs=r2 stack= before 2: (55) if r2 != 0xfffffff8 goto pc+2")
21 __msg("mark_precise: frame0: regs=r2 stack= before 1: (87) r2 = -r2")
22 __msg("mark_precise: frame0: regs=r2 stack= before 0: (b7) r2 = 8")
bpf_neg(void)23 __naked int bpf_neg(void)
24 {
25 asm volatile (
26 "r2 = 8;"
27 "r2 = -r2;"
28 "if r2 != -8 goto 1f;"
29 "r1 = r10;"
30 "r1 += r2;"
31 "1:"
32 "r0 = 0;"
33 "exit;"
34 ::: __clobber_all);
35 }
36
37 SEC("?raw_tp")
38 __success __log_level(2)
39 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r1 = r10")
40 __msg("mark_precise: frame0: regs=r2 stack= before 2: (55) if r2 != 0x0 goto pc+2")
41 __msg("mark_precise: frame0: regs=r2 stack= before 1: (d4) r2 = le16 r2")
42 __msg("mark_precise: frame0: regs=r2 stack= before 0: (b7) r2 = 0")
bpf_end_to_le(void)43 __naked int bpf_end_to_le(void)
44 {
45 asm volatile (
46 "r2 = 0;"
47 "r2 = le16 r2;"
48 "if r2 != 0 goto 1f;"
49 "r1 = r10;"
50 "r1 += r2;"
51 "1:"
52 "r0 = 0;"
53 "exit;"
54 ::: __clobber_all);
55 }
56
57
58 SEC("?raw_tp")
59 __success __log_level(2)
60 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r1 = r10")
61 __msg("mark_precise: frame0: regs=r2 stack= before 2: (55) if r2 != 0x0 goto pc+2")
62 __msg("mark_precise: frame0: regs=r2 stack= before 1: (dc) r2 = be16 r2")
63 __msg("mark_precise: frame0: regs=r2 stack= before 0: (b7) r2 = 0")
bpf_end_to_be(void)64 __naked int bpf_end_to_be(void)
65 {
66 asm volatile (
67 "r2 = 0;"
68 "r2 = be16 r2;"
69 "if r2 != 0 goto 1f;"
70 "r1 = r10;"
71 "r1 += r2;"
72 "1:"
73 "r0 = 0;"
74 "exit;"
75 ::: __clobber_all);
76 }
77
78 #if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
79 (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
80 defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \
81 defined(__TARGET_ARCH_loongarch)) && __clang_major__ >= 18
82
83 SEC("?raw_tp")
84 __success __log_level(2)
85 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r1 = r10")
86 __msg("mark_precise: frame0: regs=r2 stack= before 2: (55) if r2 != 0x0 goto pc+2")
87 __msg("mark_precise: frame0: regs=r2 stack= before 1: (d7) r2 = bswap16 r2")
88 __msg("mark_precise: frame0: regs=r2 stack= before 0: (b7) r2 = 0")
bpf_end_bswap(void)89 __naked int bpf_end_bswap(void)
90 {
91 asm volatile (
92 "r2 = 0;"
93 "r2 = bswap16 r2;"
94 "if r2 != 0 goto 1f;"
95 "r1 = r10;"
96 "r1 += r2;"
97 "1:"
98 "r0 = 0;"
99 "exit;"
100 ::: __clobber_all);
101 }
102
103 #ifdef CAN_USE_LOAD_ACQ_STORE_REL
104
105 SEC("?raw_tp")
106 __success __log_level(2)
107 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r3 = r10")
108 __msg("mark_precise: frame0: regs=r2 stack= before 2: (db) r2 = load_acquire((u64 *)(r10 -8))")
109 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
110 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
bpf_load_acquire(void)111 __naked int bpf_load_acquire(void)
112 {
113 asm volatile (
114 "r1 = 8;"
115 "*(u64 *)(r10 - 8) = r1;"
116 ".8byte %[load_acquire_insn];" /* r2 = load_acquire((u64 *)(r10 - 8)); */
117 "r3 = r10;"
118 "r3 += r2;" /* mark_precise */
119 "r0 = 0;"
120 "exit;"
121 :
122 : __imm_insn(load_acquire_insn,
123 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8))
124 : __clobber_all);
125 }
126
127 SEC("?raw_tp")
128 __success __log_level(2)
129 __msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r2 = r10")
130 __msg("mark_precise: frame0: regs=r1 stack= before 2: (79) r1 = *(u64 *)(r10 -8)")
131 __msg("mark_precise: frame0: regs= stack=-8 before 1: (db) store_release((u64 *)(r10 -8), r1)")
132 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
bpf_store_release(void)133 __naked int bpf_store_release(void)
134 {
135 asm volatile (
136 "r1 = 8;"
137 ".8byte %[store_release_insn];" /* store_release((u64 *)(r10 - 8), r1); */
138 "r1 = *(u64 *)(r10 - 8);"
139 "r2 = r10;"
140 "r2 += r1;" /* mark_precise */
141 "r0 = 0;"
142 "exit;"
143 :
144 : __imm_insn(store_release_insn,
145 BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8))
146 : __clobber_all);
147 }
148
149 #endif /* CAN_USE_LOAD_ACQ_STORE_REL */
150 #endif /* v4 instruction */
151
152 SEC("?raw_tp")
153 __success __log_level(2)
154 /*
155 * Without the bug fix there will be no history between "last_idx 3 first_idx 3"
156 * and "parent state regs=" lines. "R0=6" parts are here to help anchor
157 * expected log messages to the one specific mark_chain_precision operation.
158 *
159 * This is quite fragile: if verifier checkpointing heuristic changes, this
160 * might need adjusting.
161 */
162 __msg("2: (07) r0 += 1 ; R0=6")
163 __msg("3: (35) if r0 >= 0xa goto pc+1")
164 __msg("mark_precise: frame0: last_idx 3 first_idx 3 subseq_idx -1")
165 __msg("mark_precise: frame0: regs=r0 stack= before 2: (07) r0 += 1")
166 __msg("mark_precise: frame0: regs=r0 stack= before 1: (07) r0 += 1")
167 __msg("mark_precise: frame0: regs=r0 stack= before 4: (05) goto pc-4")
168 __msg("mark_precise: frame0: regs=r0 stack= before 3: (35) if r0 >= 0xa goto pc+1")
169 __msg("mark_precise: frame0: parent state regs= stack=: R0=P4")
170 __msg("3: R0=6")
state_loop_first_last_equal(void)171 __naked int state_loop_first_last_equal(void)
172 {
173 asm volatile (
174 "r0 = 0;"
175 "l0_%=:"
176 "r0 += 1;"
177 "r0 += 1;"
178 /* every few iterations we'll have a checkpoint here with
179 * first_idx == last_idx, potentially confusing precision
180 * backtracking logic
181 */
182 "if r0 >= 10 goto l1_%=;" /* checkpoint + mark_precise */
183 "goto l0_%=;"
184 "l1_%=:"
185 "exit;"
186 ::: __clobber_common
187 );
188 }
189
__bpf_cond_op_r10(void)190 __used __naked static void __bpf_cond_op_r10(void)
191 {
192 asm volatile (
193 "r2 = 2314885393468386424 ll;"
194 "goto +0;"
195 "if r2 <= r10 goto +3;"
196 "if r1 >= -1835016 goto +0;"
197 "if r2 <= 8 goto +0;"
198 "if r3 <= 0 goto +0;"
199 "exit;"
200 ::: __clobber_all);
201 }
202
203 SEC("?raw_tp")
204 __success __log_level(2)
205 __msg("8: (bd) if r2 <= r10 goto pc+3")
206 __msg("9: (35) if r1 >= 0xffe3fff8 goto pc+0")
207 __msg("10: (b5) if r2 <= 0x8 goto pc+0")
208 __msg("mark_precise: frame1: last_idx 10 first_idx 0 subseq_idx -1")
209 __msg("mark_precise: frame1: regs=r2 stack= before 9: (35) if r1 >= 0xffe3fff8 goto pc+0")
210 __msg("mark_precise: frame1: regs=r2 stack= before 8: (bd) if r2 <= r10 goto pc+3")
211 __msg("mark_precise: frame1: regs=r2 stack= before 7: (05) goto pc+0")
bpf_cond_op_r10(void)212 __naked void bpf_cond_op_r10(void)
213 {
214 asm volatile (
215 "r3 = 0 ll;"
216 "call __bpf_cond_op_r10;"
217 "r0 = 0;"
218 "exit;"
219 ::: __clobber_all);
220 }
221
222 SEC("?raw_tp")
223 __success __log_level(2)
224 __msg("3: (bf) r3 = r10")
225 __msg("4: (bd) if r3 <= r2 goto pc+1")
226 __msg("5: (b5) if r2 <= 0x8 goto pc+2")
227 __msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1")
228 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bd) if r3 <= r2 goto pc+1")
229 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r3 = r10")
bpf_cond_op_not_r10(void)230 __naked void bpf_cond_op_not_r10(void)
231 {
232 asm volatile (
233 "r0 = 0;"
234 "r2 = 2314885393468386424 ll;"
235 "r3 = r10;"
236 "if r3 <= r2 goto +1;"
237 "if r2 <= 8 goto +2;"
238 "r0 = 2 ll;"
239 "exit;"
240 ::: __clobber_all);
241 }
242
243 SEC("lsm.s/socket_connect")
244 __success __log_level(2)
245 __msg("0: (b7) r0 = 1 ; R0=1")
246 __msg("1: (84) w0 = -w0 ; R0=0xffffffff")
247 __msg("mark_precise: frame0: last_idx 2 first_idx 0 subseq_idx -1")
248 __msg("mark_precise: frame0: regs=r0 stack= before 1: (84) w0 = -w0")
249 __msg("mark_precise: frame0: regs=r0 stack= before 0: (b7) r0 = 1")
bpf_neg_2(void)250 __naked int bpf_neg_2(void)
251 {
252 /*
253 * lsm.s/socket_connect requires a return value within [-4095, 0].
254 * Returning -1 is allowed
255 */
256 asm volatile (
257 "r0 = 1;"
258 "w0 = -w0;"
259 "exit;"
260 ::: __clobber_all);
261 }
262
263 SEC("lsm.s/socket_connect")
264 __failure __msg("At program exit the register R0 has")
bpf_neg_3(void)265 __naked int bpf_neg_3(void)
266 {
267 /*
268 * lsm.s/socket_connect requires a return value within [-4095, 0].
269 * Returning -10000 is not allowed.
270 */
271 asm volatile (
272 "r0 = 10000;"
273 "w0 = -w0;"
274 "exit;"
275 ::: __clobber_all);
276 }
277
278 SEC("lsm.s/socket_connect")
279 __success __log_level(2)
280 __msg("0: (b7) r0 = 1 ; R0=1")
281 __msg("1: (87) r0 = -r0 ; R0=-1")
282 __msg("mark_precise: frame0: last_idx 2 first_idx 0 subseq_idx -1")
283 __msg("mark_precise: frame0: regs=r0 stack= before 1: (87) r0 = -r0")
284 __msg("mark_precise: frame0: regs=r0 stack= before 0: (b7) r0 = 1")
bpf_neg_4(void)285 __naked int bpf_neg_4(void)
286 {
287 /*
288 * lsm.s/socket_connect requires a return value within [-4095, 0].
289 * Returning -1 is allowed
290 */
291 asm volatile (
292 "r0 = 1;"
293 "r0 = -r0;"
294 "exit;"
295 ::: __clobber_all);
296 }
297
298 SEC("lsm.s/socket_connect")
299 __failure __msg("At program exit the register R0 has")
bpf_neg_5(void)300 __naked int bpf_neg_5(void)
301 {
302 /*
303 * lsm.s/socket_connect requires a return value within [-4095, 0].
304 * Returning -10000 is not allowed.
305 */
306 asm volatile (
307 "r0 = 10000;"
308 "r0 = -r0;"
309 "exit;"
310 ::: __clobber_all);
311 }
312
313 SEC("?raw_tp")
314 __success __log_level(2)
315 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10")
316 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_add((u64 *)(r10 -8), r2)")
317 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
318 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
319 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
bpf_atomic_fetch_add_precision(void)320 __naked int bpf_atomic_fetch_add_precision(void)
321 {
322 asm volatile (
323 "r1 = 8;"
324 "*(u64 *)(r10 - 8) = r1;"
325 "r2 = 0;"
326 ".8byte %[fetch_add_insn];" /* r2 = atomic_fetch_add(*(u64 *)(r10 - 8), r2) */
327 "r3 = r10;"
328 "r3 += r2;" /* mark_precise */
329 "r0 = 0;"
330 "exit;"
331 :
332 : __imm_insn(fetch_add_insn,
333 BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8))
334 : __clobber_all);
335 }
336
337 SEC("?raw_tp")
338 __success __log_level(2)
339 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10")
340 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_xchg((u64 *)(r10 -8), r2)")
341 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
342 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
343 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
bpf_atomic_xchg_precision(void)344 __naked int bpf_atomic_xchg_precision(void)
345 {
346 asm volatile (
347 "r1 = 8;"
348 "*(u64 *)(r10 - 8) = r1;"
349 "r2 = 0;"
350 ".8byte %[xchg_insn];" /* r2 = atomic_xchg(*(u64 *)(r10 - 8), r2) */
351 "r3 = r10;"
352 "r3 += r2;" /* mark_precise */
353 "r0 = 0;"
354 "exit;"
355 :
356 : __imm_insn(xchg_insn,
357 BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_2, -8))
358 : __clobber_all);
359 }
360
361 SEC("?raw_tp")
362 __success __log_level(2)
363 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10")
364 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_or((u64 *)(r10 -8), r2)")
365 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
366 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
367 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
bpf_atomic_fetch_or_precision(void)368 __naked int bpf_atomic_fetch_or_precision(void)
369 {
370 asm volatile (
371 "r1 = 8;"
372 "*(u64 *)(r10 - 8) = r1;"
373 "r2 = 0;"
374 ".8byte %[fetch_or_insn];" /* r2 = atomic_fetch_or(*(u64 *)(r10 - 8), r2) */
375 "r3 = r10;"
376 "r3 += r2;" /* mark_precise */
377 "r0 = 0;"
378 "exit;"
379 :
380 : __imm_insn(fetch_or_insn,
381 BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8))
382 : __clobber_all);
383 }
384
385 SEC("?raw_tp")
386 __success __log_level(2)
387 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10")
388 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_and((u64 *)(r10 -8), r2)")
389 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
390 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
391 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
bpf_atomic_fetch_and_precision(void)392 __naked int bpf_atomic_fetch_and_precision(void)
393 {
394 asm volatile (
395 "r1 = 8;"
396 "*(u64 *)(r10 - 8) = r1;"
397 "r2 = 0;"
398 ".8byte %[fetch_and_insn];" /* r2 = atomic_fetch_and(*(u64 *)(r10 - 8), r2) */
399 "r3 = r10;"
400 "r3 += r2;" /* mark_precise */
401 "r0 = 0;"
402 "exit;"
403 :
404 : __imm_insn(fetch_and_insn,
405 BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8))
406 : __clobber_all);
407 }
408
409 SEC("?raw_tp")
410 __success __log_level(2)
411 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10")
412 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_xor((u64 *)(r10 -8), r2)")
413 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
414 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
415 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
bpf_atomic_fetch_xor_precision(void)416 __naked int bpf_atomic_fetch_xor_precision(void)
417 {
418 asm volatile (
419 "r1 = 8;"
420 "*(u64 *)(r10 - 8) = r1;"
421 "r2 = 0;"
422 ".8byte %[fetch_xor_insn];" /* r2 = atomic_fetch_xor(*(u64 *)(r10 - 8), r2) */
423 "r3 = r10;"
424 "r3 += r2;" /* mark_precise */
425 "r0 = 0;"
426 "exit;"
427 :
428 : __imm_insn(fetch_xor_insn,
429 BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8))
430 : __clobber_all);
431 }
432
433 SEC("?raw_tp")
434 __success __log_level(2)
435 __msg("mark_precise: frame0: regs=r0 stack= before 5: (bf) r3 = r10")
436 __msg("mark_precise: frame0: regs=r0 stack= before 4: (db) r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2)")
437 __msg("mark_precise: frame0: regs= stack=-8 before 3: (b7) r2 = 0")
438 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r0 = 0")
439 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
440 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
bpf_atomic_cmpxchg_precision(void)441 __naked int bpf_atomic_cmpxchg_precision(void)
442 {
443 asm volatile (
444 "r1 = 8;"
445 "*(u64 *)(r10 - 8) = r1;"
446 "r0 = 0;"
447 "r2 = 0;"
448 ".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u64 *)(r10 - 8), r0, r2) */
449 "r3 = r10;"
450 "r3 += r0;" /* mark_precise */
451 "r0 = 0;"
452 "exit;"
453 :
454 : __imm_insn(cmpxchg_insn,
455 BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, -8))
456 : __clobber_all);
457 }
458
459 /* Regression test for dual precision: Both the fetched value (r2) and
460 * a reread of the same stack slot (r3) are tracked for precision. After
461 * the atomic operation, the stack slot is STACK_MISC. Thus, the ldx at
462 * insn 4 does NOT set INSN_F_STACK_ACCESS. Precision for the stack slot
463 * propagates solely through the atomic fetch's load side (insn 3).
464 */
465 SEC("?raw_tp")
466 __success __log_level(2)
467 __msg("mark_precise: frame0: regs=r2,r3 stack= before 4: (79) r3 = *(u64 *)(r10 -8)")
468 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_add((u64 *)(r10 -8), r2)")
469 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
470 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
471 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
bpf_atomic_fetch_add_dual_precision(void)472 __naked int bpf_atomic_fetch_add_dual_precision(void)
473 {
474 asm volatile (
475 "r1 = 8;"
476 "*(u64 *)(r10 - 8) = r1;"
477 "r2 = 0;"
478 ".8byte %[fetch_add_insn];" /* r2 = atomic_fetch_add(*(u64 *)(r10 - 8), r2) */
479 "r3 = *(u64 *)(r10 - 8);"
480 "r4 = r2;"
481 "r4 += r3;"
482 "r4 &= 7;"
483 "r5 = r10;"
484 "r5 += r4;" /* mark_precise */
485 "r0 = 0;"
486 "exit;"
487 :
488 : __imm_insn(fetch_add_insn,
489 BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8))
490 : __clobber_all);
491 }
492
493 SEC("?raw_tp")
494 __success __log_level(2)
495 __msg("mark_precise: frame0: regs=r0,r3 stack= before 5: (79) r3 = *(u64 *)(r10 -8)")
496 __msg("mark_precise: frame0: regs=r0 stack= before 4: (db) r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2)")
497 __msg("mark_precise: frame0: regs= stack=-8 before 3: (b7) r2 = 0")
498 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r0 = 8")
499 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
500 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
bpf_atomic_cmpxchg_dual_precision(void)501 __naked int bpf_atomic_cmpxchg_dual_precision(void)
502 {
503 asm volatile (
504 "r1 = 8;"
505 "*(u64 *)(r10 - 8) = r1;"
506 "r0 = 8;"
507 "r2 = 0;"
508 ".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u64 *)(r10 - 8), r0, r2) */
509 "r3 = *(u64 *)(r10 - 8);"
510 "r4 = r0;"
511 "r4 += r3;"
512 "r4 &= 7;"
513 "r5 = r10;"
514 "r5 += r4;" /* mark_precise */
515 "r0 = 0;"
516 "exit;"
517 :
518 : __imm_insn(cmpxchg_insn,
519 BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, -8))
520 : __clobber_all);
521 }
522
523 SEC("?raw_tp")
524 __success __log_level(2)
525 __msg("mark_precise: frame0: regs=r1 stack= before 10: (57) r1 &= 7")
526 __msg("mark_precise: frame0: regs=r1 stack= before 9: (db) r1 = atomic64_fetch_add((u64 *)(r0 +0), r1)")
527 __not_msg("falling back to forcing all scalars precise")
bpf_atomic_fetch_add_map_precision(void)528 __naked int bpf_atomic_fetch_add_map_precision(void)
529 {
530 asm volatile (
531 "r1 = 0;"
532 "*(u64 *)(r10 - 8) = r1;"
533 "r2 = r10;"
534 "r2 += -8;"
535 "r1 = %[precision_map] ll;"
536 "call %[bpf_map_lookup_elem];"
537 "if r0 == 0 goto 1f;"
538 "r1 = 0;"
539 ".8byte %[fetch_add_insn];" /* r1 = atomic_fetch_add(*(u64 *)(r0 + 0), r1) */
540 "r1 &= 7;"
541 "r2 = r10;"
542 "r2 += r1;" /* mark_precise */
543 "1: r0 = 0;"
544 "exit;"
545 :
546 : __imm_addr(precision_map),
547 __imm(bpf_map_lookup_elem),
548 __imm_insn(fetch_add_insn,
549 BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0))
550 : __clobber_all);
551 }
552
553 SEC("?raw_tp")
554 __success __log_level(2)
555 __msg("mark_precise: frame0: regs=r0 stack= before 12: (57) r0 &= 7")
556 __msg("mark_precise: frame0: regs=r0 stack= before 11: (db) r0 = atomic64_cmpxchg((u64 *)(r6 +0), r0, r1)")
557 __not_msg("falling back to forcing all scalars precise")
bpf_atomic_cmpxchg_map_precision(void)558 __naked int bpf_atomic_cmpxchg_map_precision(void)
559 {
560 asm volatile (
561 "r1 = 0;"
562 "*(u64 *)(r10 - 8) = r1;"
563 "r2 = r10;"
564 "r2 += -8;"
565 "r1 = %[precision_map] ll;"
566 "call %[bpf_map_lookup_elem];"
567 "if r0 == 0 goto 1f;"
568 "r6 = r0;"
569 "r0 = 0;"
570 "r1 = 0;"
571 ".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u64 *)(r6 + 0), r0, r1) */
572 "r0 &= 7;"
573 "r2 = r10;"
574 "r2 += r0;" /* mark_precise */
575 "1: r0 = 0;"
576 "exit;"
577 :
578 : __imm_addr(precision_map),
579 __imm(bpf_map_lookup_elem),
580 __imm_insn(cmpxchg_insn,
581 BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 0))
582 : __clobber_all);
583 }
584
585 SEC("?raw_tp")
586 __success __log_level(2)
587 __msg("mark_precise: frame0: regs=r1 stack= before 10: (57) r1 &= 7")
588 __msg("mark_precise: frame0: regs=r1 stack= before 9: (c3) r1 = atomic_fetch_add((u32 *)(r0 +0), r1)")
589 __not_msg("falling back to forcing all scalars precise")
bpf_atomic_fetch_add_32bit_precision(void)590 __naked int bpf_atomic_fetch_add_32bit_precision(void)
591 {
592 asm volatile (
593 "r1 = 0;"
594 "*(u64 *)(r10 - 8) = r1;"
595 "r2 = r10;"
596 "r2 += -8;"
597 "r1 = %[precision_map] ll;"
598 "call %[bpf_map_lookup_elem];"
599 "if r0 == 0 goto 1f;"
600 "r1 = 0;"
601 ".8byte %[fetch_add_insn];" /* r1 = atomic_fetch_add(*(u32 *)(r0 + 0), r1) */
602 "r1 &= 7;"
603 "r2 = r10;"
604 "r2 += r1;" /* mark_precise */
605 "1: r0 = 0;"
606 "exit;"
607 :
608 : __imm_addr(precision_map),
609 __imm(bpf_map_lookup_elem),
610 __imm_insn(fetch_add_insn,
611 BPF_ATOMIC_OP(BPF_W, BPF_ADD | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0))
612 : __clobber_all);
613 }
614
615 SEC("?raw_tp")
616 __success __log_level(2)
617 __msg("mark_precise: frame0: regs=r0 stack= before 12: (57) r0 &= 7")
618 __msg("mark_precise: frame0: regs=r0 stack= before 11: (c3) r0 = atomic_cmpxchg((u32 *)(r6 +0), r0, r1)")
619 __not_msg("falling back to forcing all scalars precise")
bpf_atomic_cmpxchg_32bit_precision(void)620 __naked int bpf_atomic_cmpxchg_32bit_precision(void)
621 {
622 asm volatile (
623 "r1 = 0;"
624 "*(u64 *)(r10 - 8) = r1;"
625 "r2 = r10;"
626 "r2 += -8;"
627 "r1 = %[precision_map] ll;"
628 "call %[bpf_map_lookup_elem];"
629 "if r0 == 0 goto 1f;"
630 "r6 = r0;"
631 "r0 = 0;"
632 "r1 = 0;"
633 ".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u32 *)(r6 + 0), r0, r1) */
634 "r0 &= 7;"
635 "r2 = r10;"
636 "r2 += r0;" /* mark_precise */
637 "1: r0 = 0;"
638 "exit;"
639 :
640 : __imm_addr(precision_map),
641 __imm(bpf_map_lookup_elem),
642 __imm_insn(cmpxchg_insn,
643 BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 0))
644 : __clobber_all);
645 }
646
647 /*
648 * Verification takes two paths: with r1 being scalar zero on path (1)
649 * and with r1 being some other scalar on path (2).
650 * Check that the verifier does not use checkpoints created
651 * on path (1) to prune path (2).
652 */
653 SEC("?tc")
__flag(BPF_F_TEST_STATE_FREQ)654 __flag(BPF_F_TEST_STATE_FREQ)
655 __failure __msg("R1 type=scalar expected=fp")
656 __naked int null_mem_arg_zero_size(void)
657 {
658 asm volatile (
659 "call %[bpf_get_prandom_u32];"
660 "r1 = 42;"
661 "if r0 > 42 goto 1f;"
662 "r1 = 0;"
663 "1:"
664 "r2 = 0;"
665 "r3 = 0;"
666 "r4 = 0;"
667 "r5 = 0;"
668 /*
669 * ARG_PTR_TO_MEM | PTR_MAYBE_NULL parameter can be NULL,
670 * but can't be some other scalar value.
671 */
672 "call %[bpf_csum_diff];"
673 "r0 = 0;"
674 "exit;"
675 :
676 : __imm(bpf_get_prandom_u32),
677 __imm(bpf_csum_diff)
678 : __clobber_all);
679 }
680
subprog_mem_arg(int * p)681 __weak int subprog_mem_arg(int *p)
682 {
683 if (p)
684 return *p;
685 return 0;
686 }
687
688 /*
689 * Verification takes two paths: with r1 being scalar zero on path (1)
690 * and with r1 being some other scalar on path (2).
691 * Check that the verifier does not use checkpoints created
692 * on path (1) to prune path (2).
693 */
694 SEC("?raw_tp")
__flag(BPF_F_TEST_STATE_FREQ)695 __flag(BPF_F_TEST_STATE_FREQ)
696 __failure __msg("R1 type=scalar expected=fp")
697 __naked int null_mem_arg_global_subprog(void)
698 {
699 asm volatile (
700 "call %[bpf_get_prandom_u32];"
701 "r1 = 42;"
702 "if r0 > 42 goto 1f;"
703 "r1 = 0;"
704 "1:"
705 "call subprog_mem_arg;"
706 "r0 = 0;"
707 "exit;"
708 :
709 : __imm(bpf_get_prandom_u32)
710 : __clobber_all);
711 }
712
713 /* Same as above, check that path with r3 == 0 does not prune the path with r3 != 0 */
714 SEC("?tc")
__flag(BPF_F_TEST_STATE_FREQ)715 __flag(BPF_F_TEST_STATE_FREQ)
716 __failure __msg("R3 type=scalar expected=fp")
717 int null_kfunc_arg_dynptr_slice(struct __sk_buff *skb)
718 {
719 struct bpf_dynptr ptr;
720
721 bpf_dynptr_from_skb(skb, 0, &ptr);
722 asm volatile (
723 "call %[bpf_get_prandom_u32];"
724 "r3 = 42;"
725 "if r0 > 42 goto 1f;"
726 "r3 = 0;"
727 "1:"
728 "r1 = %[ptr];"
729 "r2 = 0;"
730 "r4 = 8;"
731 "call %[bpf_dynptr_slice];"
732 :
733 : __imm_ptr(ptr),
734 __imm(bpf_get_prandom_u32),
735 __imm(bpf_dynptr_slice)
736 : __clobber_common);
737 return 0;
738 }
739
__kfunc_btf_root(void)740 void __kfunc_btf_root(void)
741 {
742 bpf_dynptr_slice(0, 0, 0, 0);
743 }
744
745 char _license[] SEC("license") = "GPL";
746