1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2023 SUSE LLC */ 3 #include <linux/bpf.h> 4 #include <bpf/bpf_helpers.h> 5 #include <stdbool.h> 6 #include "../../../include/linux/filter.h" 7 #include "bpf_misc.h" 8 #include "bpf_kfuncs.h" 9 10 struct { 11 __uint(type, BPF_MAP_TYPE_ARRAY); 12 __uint(max_entries, 1); 13 __type(key, __u32); 14 __type(value, __u64); 15 } precision_map SEC(".maps"); 16 17 SEC("?raw_tp") 18 __success __log_level(2) 19 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r1 = r10") 20 __msg("mark_precise: frame0: regs=r2 stack= before 2: (55) if r2 != 0xfffffff8 goto pc+2") 21 __msg("mark_precise: frame0: regs=r2 stack= before 1: (87) r2 = -r2") 22 __msg("mark_precise: frame0: regs=r2 stack= before 0: (b7) r2 = 8") 23 __naked int bpf_neg(void) 24 { 25 asm volatile ( 26 "r2 = 8;" 27 "r2 = -r2;" 28 "if r2 != -8 goto 1f;" 29 "r1 = r10;" 30 "r1 += r2;" 31 "1:" 32 "r0 = 0;" 33 "exit;" 34 ::: __clobber_all); 35 } 36 37 SEC("?raw_tp") 38 __success __log_level(2) 39 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r1 = r10") 40 __msg("mark_precise: frame0: regs=r2 stack= before 2: (55) if r2 != 0x0 goto pc+2") 41 __msg("mark_precise: frame0: regs=r2 stack= before 1: (d4) r2 = le16 r2") 42 __msg("mark_precise: frame0: regs=r2 stack= before 0: (b7) r2 = 0") 43 __naked int bpf_end_to_le(void) 44 { 45 asm volatile ( 46 "r2 = 0;" 47 "r2 = le16 r2;" 48 "if r2 != 0 goto 1f;" 49 "r1 = r10;" 50 "r1 += r2;" 51 "1:" 52 "r0 = 0;" 53 "exit;" 54 ::: __clobber_all); 55 } 56 57 58 SEC("?raw_tp") 59 __success __log_level(2) 60 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r1 = r10") 61 __msg("mark_precise: frame0: regs=r2 stack= before 2: (55) if r2 != 0x0 goto pc+2") 62 __msg("mark_precise: frame0: regs=r2 stack= before 1: (dc) r2 = be16 r2") 63 __msg("mark_precise: frame0: regs=r2 stack= before 0: (b7) r2 = 0") 64 __naked int bpf_end_to_be(void) 65 { 66 asm volatile ( 67 "r2 = 0;" 68 "r2 = be16 r2;" 69 "if r2 != 0 goto 1f;" 70 "r1 = r10;" 71 "r1 += r2;" 72 "1:" 73 "r0 = 0;" 74 "exit;" 75 ::: __clobber_all); 76 } 77 78 #if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ 79 (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ 80 defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ 81 defined(__TARGET_ARCH_loongarch)) && __clang_major__ >= 18 82 83 SEC("?raw_tp") 84 __success __log_level(2) 85 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r1 = r10") 86 __msg("mark_precise: frame0: regs=r2 stack= before 2: (55) if r2 != 0x0 goto pc+2") 87 __msg("mark_precise: frame0: regs=r2 stack= before 1: (d7) r2 = bswap16 r2") 88 __msg("mark_precise: frame0: regs=r2 stack= before 0: (b7) r2 = 0") 89 __naked int bpf_end_bswap(void) 90 { 91 asm volatile ( 92 "r2 = 0;" 93 "r2 = bswap16 r2;" 94 "if r2 != 0 goto 1f;" 95 "r1 = r10;" 96 "r1 += r2;" 97 "1:" 98 "r0 = 0;" 99 "exit;" 100 ::: __clobber_all); 101 } 102 103 #ifdef CAN_USE_LOAD_ACQ_STORE_REL 104 105 SEC("?raw_tp") 106 __success __log_level(2) 107 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r3 = r10") 108 __msg("mark_precise: frame0: regs=r2 stack= before 2: (db) r2 = load_acquire((u64 *)(r10 -8))") 109 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1") 110 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8") 111 __naked int bpf_load_acquire(void) 112 { 113 asm volatile ( 114 "r1 = 8;" 115 "*(u64 *)(r10 - 8) = r1;" 116 ".8byte %[load_acquire_insn];" /* r2 = load_acquire((u64 *)(r10 - 8)); */ 117 "r3 = r10;" 118 "r3 += r2;" /* mark_precise */ 119 "r0 = 0;" 120 "exit;" 121 : 122 : __imm_insn(load_acquire_insn, 123 BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8)) 124 : __clobber_all); 125 } 126 127 SEC("?raw_tp") 128 __success __log_level(2) 129 __msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r2 = r10") 130 __msg("mark_precise: frame0: regs=r1 stack= before 2: (79) r1 = *(u64 *)(r10 -8)") 131 __msg("mark_precise: frame0: regs= stack=-8 before 1: (db) store_release((u64 *)(r10 -8), r1)") 132 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8") 133 __naked int bpf_store_release(void) 134 { 135 asm volatile ( 136 "r1 = 8;" 137 ".8byte %[store_release_insn];" /* store_release((u64 *)(r10 - 8), r1); */ 138 "r1 = *(u64 *)(r10 - 8);" 139 "r2 = r10;" 140 "r2 += r1;" /* mark_precise */ 141 "r0 = 0;" 142 "exit;" 143 : 144 : __imm_insn(store_release_insn, 145 BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)) 146 : __clobber_all); 147 } 148 149 #endif /* CAN_USE_LOAD_ACQ_STORE_REL */ 150 #endif /* v4 instruction */ 151 152 SEC("?raw_tp") 153 __success __log_level(2) 154 /* 155 * Without the bug fix there will be no history between "last_idx 3 first_idx 3" 156 * and "parent state regs=" lines. "R0=6" parts are here to help anchor 157 * expected log messages to the one specific mark_chain_precision operation. 158 * 159 * This is quite fragile: if verifier checkpointing heuristic changes, this 160 * might need adjusting. 161 */ 162 __msg("2: (07) r0 += 1 ; R0=6") 163 __msg("3: (35) if r0 >= 0xa goto pc+1") 164 __msg("mark_precise: frame0: last_idx 3 first_idx 3 subseq_idx -1") 165 __msg("mark_precise: frame0: regs=r0 stack= before 2: (07) r0 += 1") 166 __msg("mark_precise: frame0: regs=r0 stack= before 1: (07) r0 += 1") 167 __msg("mark_precise: frame0: regs=r0 stack= before 4: (05) goto pc-4") 168 __msg("mark_precise: frame0: regs=r0 stack= before 3: (35) if r0 >= 0xa goto pc+1") 169 __msg("mark_precise: frame0: parent state regs= stack=: R0=P4") 170 __msg("3: R0=6") 171 __naked int state_loop_first_last_equal(void) 172 { 173 asm volatile ( 174 "r0 = 0;" 175 "l0_%=:" 176 "r0 += 1;" 177 "r0 += 1;" 178 /* every few iterations we'll have a checkpoint here with 179 * first_idx == last_idx, potentially confusing precision 180 * backtracking logic 181 */ 182 "if r0 >= 10 goto l1_%=;" /* checkpoint + mark_precise */ 183 "goto l0_%=;" 184 "l1_%=:" 185 "exit;" 186 ::: __clobber_common 187 ); 188 } 189 190 __used __naked static void __bpf_cond_op_r10(void) 191 { 192 asm volatile ( 193 "r2 = 2314885393468386424 ll;" 194 "goto +0;" 195 "if r2 <= r10 goto +3;" 196 "if r1 >= -1835016 goto +0;" 197 "if r2 <= 8 goto +0;" 198 "if r3 <= 0 goto +0;" 199 "exit;" 200 ::: __clobber_all); 201 } 202 203 SEC("?raw_tp") 204 __success __log_level(2) 205 __msg("8: (bd) if r2 <= r10 goto pc+3") 206 __msg("9: (35) if r1 >= 0xffe3fff8 goto pc+0") 207 __msg("10: (b5) if r2 <= 0x8 goto pc+0") 208 __msg("mark_precise: frame1: last_idx 10 first_idx 0 subseq_idx -1") 209 __msg("mark_precise: frame1: regs=r2 stack= before 9: (35) if r1 >= 0xffe3fff8 goto pc+0") 210 __msg("mark_precise: frame1: regs=r2 stack= before 8: (bd) if r2 <= r10 goto pc+3") 211 __msg("mark_precise: frame1: regs=r2 stack= before 7: (05) goto pc+0") 212 __naked void bpf_cond_op_r10(void) 213 { 214 asm volatile ( 215 "r3 = 0 ll;" 216 "call __bpf_cond_op_r10;" 217 "r0 = 0;" 218 "exit;" 219 ::: __clobber_all); 220 } 221 222 SEC("?raw_tp") 223 __success __log_level(2) 224 __msg("3: (bf) r3 = r10") 225 __msg("4: (bd) if r3 <= r2 goto pc+1") 226 __msg("5: (b5) if r2 <= 0x8 goto pc+2") 227 __msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") 228 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bd) if r3 <= r2 goto pc+1") 229 __msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r3 = r10") 230 __naked void bpf_cond_op_not_r10(void) 231 { 232 asm volatile ( 233 "r0 = 0;" 234 "r2 = 2314885393468386424 ll;" 235 "r3 = r10;" 236 "if r3 <= r2 goto +1;" 237 "if r2 <= 8 goto +2;" 238 "r0 = 2 ll;" 239 "exit;" 240 ::: __clobber_all); 241 } 242 243 SEC("lsm.s/socket_connect") 244 __success __log_level(2) 245 __msg("0: (b7) r0 = 1 ; R0=1") 246 __msg("1: (84) w0 = -w0 ; R0=0xffffffff") 247 __msg("mark_precise: frame0: last_idx 2 first_idx 0 subseq_idx -1") 248 __msg("mark_precise: frame0: regs=r0 stack= before 1: (84) w0 = -w0") 249 __msg("mark_precise: frame0: regs=r0 stack= before 0: (b7) r0 = 1") 250 __naked int bpf_neg_2(void) 251 { 252 /* 253 * lsm.s/socket_connect requires a return value within [-4095, 0]. 254 * Returning -1 is allowed 255 */ 256 asm volatile ( 257 "r0 = 1;" 258 "w0 = -w0;" 259 "exit;" 260 ::: __clobber_all); 261 } 262 263 SEC("lsm.s/socket_connect") 264 __failure __msg("At program exit the register R0 has") 265 __naked int bpf_neg_3(void) 266 { 267 /* 268 * lsm.s/socket_connect requires a return value within [-4095, 0]. 269 * Returning -10000 is not allowed. 270 */ 271 asm volatile ( 272 "r0 = 10000;" 273 "w0 = -w0;" 274 "exit;" 275 ::: __clobber_all); 276 } 277 278 SEC("lsm.s/socket_connect") 279 __success __log_level(2) 280 __msg("0: (b7) r0 = 1 ; R0=1") 281 __msg("1: (87) r0 = -r0 ; R0=-1") 282 __msg("mark_precise: frame0: last_idx 2 first_idx 0 subseq_idx -1") 283 __msg("mark_precise: frame0: regs=r0 stack= before 1: (87) r0 = -r0") 284 __msg("mark_precise: frame0: regs=r0 stack= before 0: (b7) r0 = 1") 285 __naked int bpf_neg_4(void) 286 { 287 /* 288 * lsm.s/socket_connect requires a return value within [-4095, 0]. 289 * Returning -1 is allowed 290 */ 291 asm volatile ( 292 "r0 = 1;" 293 "r0 = -r0;" 294 "exit;" 295 ::: __clobber_all); 296 } 297 298 SEC("lsm.s/socket_connect") 299 __failure __msg("At program exit the register R0 has") 300 __naked int bpf_neg_5(void) 301 { 302 /* 303 * lsm.s/socket_connect requires a return value within [-4095, 0]. 304 * Returning -10000 is not allowed. 305 */ 306 asm volatile ( 307 "r0 = 10000;" 308 "r0 = -r0;" 309 "exit;" 310 ::: __clobber_all); 311 } 312 313 SEC("?raw_tp") 314 __success __log_level(2) 315 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10") 316 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_add((u64 *)(r10 -8), r2)") 317 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0") 318 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1") 319 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8") 320 __naked int bpf_atomic_fetch_add_precision(void) 321 { 322 asm volatile ( 323 "r1 = 8;" 324 "*(u64 *)(r10 - 8) = r1;" 325 "r2 = 0;" 326 ".8byte %[fetch_add_insn];" /* r2 = atomic_fetch_add(*(u64 *)(r10 - 8), r2) */ 327 "r3 = r10;" 328 "r3 += r2;" /* mark_precise */ 329 "r0 = 0;" 330 "exit;" 331 : 332 : __imm_insn(fetch_add_insn, 333 BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8)) 334 : __clobber_all); 335 } 336 337 SEC("?raw_tp") 338 __success __log_level(2) 339 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10") 340 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_xchg((u64 *)(r10 -8), r2)") 341 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0") 342 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1") 343 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8") 344 __naked int bpf_atomic_xchg_precision(void) 345 { 346 asm volatile ( 347 "r1 = 8;" 348 "*(u64 *)(r10 - 8) = r1;" 349 "r2 = 0;" 350 ".8byte %[xchg_insn];" /* r2 = atomic_xchg(*(u64 *)(r10 - 8), r2) */ 351 "r3 = r10;" 352 "r3 += r2;" /* mark_precise */ 353 "r0 = 0;" 354 "exit;" 355 : 356 : __imm_insn(xchg_insn, 357 BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_2, -8)) 358 : __clobber_all); 359 } 360 361 SEC("?raw_tp") 362 __success __log_level(2) 363 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10") 364 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_or((u64 *)(r10 -8), r2)") 365 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0") 366 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1") 367 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8") 368 __naked int bpf_atomic_fetch_or_precision(void) 369 { 370 asm volatile ( 371 "r1 = 8;" 372 "*(u64 *)(r10 - 8) = r1;" 373 "r2 = 0;" 374 ".8byte %[fetch_or_insn];" /* r2 = atomic_fetch_or(*(u64 *)(r10 - 8), r2) */ 375 "r3 = r10;" 376 "r3 += r2;" /* mark_precise */ 377 "r0 = 0;" 378 "exit;" 379 : 380 : __imm_insn(fetch_or_insn, 381 BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8)) 382 : __clobber_all); 383 } 384 385 SEC("?raw_tp") 386 __success __log_level(2) 387 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10") 388 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_and((u64 *)(r10 -8), r2)") 389 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0") 390 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1") 391 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8") 392 __naked int bpf_atomic_fetch_and_precision(void) 393 { 394 asm volatile ( 395 "r1 = 8;" 396 "*(u64 *)(r10 - 8) = r1;" 397 "r2 = 0;" 398 ".8byte %[fetch_and_insn];" /* r2 = atomic_fetch_and(*(u64 *)(r10 - 8), r2) */ 399 "r3 = r10;" 400 "r3 += r2;" /* mark_precise */ 401 "r0 = 0;" 402 "exit;" 403 : 404 : __imm_insn(fetch_and_insn, 405 BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8)) 406 : __clobber_all); 407 } 408 409 SEC("?raw_tp") 410 __success __log_level(2) 411 __msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10") 412 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_xor((u64 *)(r10 -8), r2)") 413 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0") 414 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1") 415 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8") 416 __naked int bpf_atomic_fetch_xor_precision(void) 417 { 418 asm volatile ( 419 "r1 = 8;" 420 "*(u64 *)(r10 - 8) = r1;" 421 "r2 = 0;" 422 ".8byte %[fetch_xor_insn];" /* r2 = atomic_fetch_xor(*(u64 *)(r10 - 8), r2) */ 423 "r3 = r10;" 424 "r3 += r2;" /* mark_precise */ 425 "r0 = 0;" 426 "exit;" 427 : 428 : __imm_insn(fetch_xor_insn, 429 BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8)) 430 : __clobber_all); 431 } 432 433 SEC("?raw_tp") 434 __success __log_level(2) 435 __msg("mark_precise: frame0: regs=r0 stack= before 5: (bf) r3 = r10") 436 __msg("mark_precise: frame0: regs=r0 stack= before 4: (db) r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2)") 437 __msg("mark_precise: frame0: regs= stack=-8 before 3: (b7) r2 = 0") 438 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r0 = 0") 439 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1") 440 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8") 441 __naked int bpf_atomic_cmpxchg_precision(void) 442 { 443 asm volatile ( 444 "r1 = 8;" 445 "*(u64 *)(r10 - 8) = r1;" 446 "r0 = 0;" 447 "r2 = 0;" 448 ".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u64 *)(r10 - 8), r0, r2) */ 449 "r3 = r10;" 450 "r3 += r0;" /* mark_precise */ 451 "r0 = 0;" 452 "exit;" 453 : 454 : __imm_insn(cmpxchg_insn, 455 BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, -8)) 456 : __clobber_all); 457 } 458 459 /* Regression test for dual precision: Both the fetched value (r2) and 460 * a reread of the same stack slot (r3) are tracked for precision. After 461 * the atomic operation, the stack slot is STACK_MISC. Thus, the ldx at 462 * insn 4 does NOT set INSN_F_STACK_ACCESS. Precision for the stack slot 463 * propagates solely through the atomic fetch's load side (insn 3). 464 */ 465 SEC("?raw_tp") 466 __success __log_level(2) 467 __msg("mark_precise: frame0: regs=r2,r3 stack= before 4: (79) r3 = *(u64 *)(r10 -8)") 468 __msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_add((u64 *)(r10 -8), r2)") 469 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0") 470 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1") 471 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8") 472 __naked int bpf_atomic_fetch_add_dual_precision(void) 473 { 474 asm volatile ( 475 "r1 = 8;" 476 "*(u64 *)(r10 - 8) = r1;" 477 "r2 = 0;" 478 ".8byte %[fetch_add_insn];" /* r2 = atomic_fetch_add(*(u64 *)(r10 - 8), r2) */ 479 "r3 = *(u64 *)(r10 - 8);" 480 "r4 = r2;" 481 "r4 += r3;" 482 "r4 &= 7;" 483 "r5 = r10;" 484 "r5 += r4;" /* mark_precise */ 485 "r0 = 0;" 486 "exit;" 487 : 488 : __imm_insn(fetch_add_insn, 489 BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8)) 490 : __clobber_all); 491 } 492 493 SEC("?raw_tp") 494 __success __log_level(2) 495 __msg("mark_precise: frame0: regs=r0,r3 stack= before 5: (79) r3 = *(u64 *)(r10 -8)") 496 __msg("mark_precise: frame0: regs=r0 stack= before 4: (db) r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2)") 497 __msg("mark_precise: frame0: regs= stack=-8 before 3: (b7) r2 = 0") 498 __msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r0 = 8") 499 __msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1") 500 __msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8") 501 __naked int bpf_atomic_cmpxchg_dual_precision(void) 502 { 503 asm volatile ( 504 "r1 = 8;" 505 "*(u64 *)(r10 - 8) = r1;" 506 "r0 = 8;" 507 "r2 = 0;" 508 ".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u64 *)(r10 - 8), r0, r2) */ 509 "r3 = *(u64 *)(r10 - 8);" 510 "r4 = r0;" 511 "r4 += r3;" 512 "r4 &= 7;" 513 "r5 = r10;" 514 "r5 += r4;" /* mark_precise */ 515 "r0 = 0;" 516 "exit;" 517 : 518 : __imm_insn(cmpxchg_insn, 519 BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, -8)) 520 : __clobber_all); 521 } 522 523 SEC("?raw_tp") 524 __success __log_level(2) 525 __msg("mark_precise: frame0: regs=r1 stack= before 10: (57) r1 &= 7") 526 __msg("mark_precise: frame0: regs=r1 stack= before 9: (db) r1 = atomic64_fetch_add((u64 *)(r0 +0), r1)") 527 __not_msg("falling back to forcing all scalars precise") 528 __naked int bpf_atomic_fetch_add_map_precision(void) 529 { 530 asm volatile ( 531 "r1 = 0;" 532 "*(u64 *)(r10 - 8) = r1;" 533 "r2 = r10;" 534 "r2 += -8;" 535 "r1 = %[precision_map] ll;" 536 "call %[bpf_map_lookup_elem];" 537 "if r0 == 0 goto 1f;" 538 "r1 = 0;" 539 ".8byte %[fetch_add_insn];" /* r1 = atomic_fetch_add(*(u64 *)(r0 + 0), r1) */ 540 "r1 &= 7;" 541 "r2 = r10;" 542 "r2 += r1;" /* mark_precise */ 543 "1: r0 = 0;" 544 "exit;" 545 : 546 : __imm_addr(precision_map), 547 __imm(bpf_map_lookup_elem), 548 __imm_insn(fetch_add_insn, 549 BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0)) 550 : __clobber_all); 551 } 552 553 SEC("?raw_tp") 554 __success __log_level(2) 555 __msg("mark_precise: frame0: regs=r0 stack= before 12: (57) r0 &= 7") 556 __msg("mark_precise: frame0: regs=r0 stack= before 11: (db) r0 = atomic64_cmpxchg((u64 *)(r6 +0), r0, r1)") 557 __not_msg("falling back to forcing all scalars precise") 558 __naked int bpf_atomic_cmpxchg_map_precision(void) 559 { 560 asm volatile ( 561 "r1 = 0;" 562 "*(u64 *)(r10 - 8) = r1;" 563 "r2 = r10;" 564 "r2 += -8;" 565 "r1 = %[precision_map] ll;" 566 "call %[bpf_map_lookup_elem];" 567 "if r0 == 0 goto 1f;" 568 "r6 = r0;" 569 "r0 = 0;" 570 "r1 = 0;" 571 ".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u64 *)(r6 + 0), r0, r1) */ 572 "r0 &= 7;" 573 "r2 = r10;" 574 "r2 += r0;" /* mark_precise */ 575 "1: r0 = 0;" 576 "exit;" 577 : 578 : __imm_addr(precision_map), 579 __imm(bpf_map_lookup_elem), 580 __imm_insn(cmpxchg_insn, 581 BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 0)) 582 : __clobber_all); 583 } 584 585 SEC("?raw_tp") 586 __success __log_level(2) 587 __msg("mark_precise: frame0: regs=r1 stack= before 10: (57) r1 &= 7") 588 __msg("mark_precise: frame0: regs=r1 stack= before 9: (c3) r1 = atomic_fetch_add((u32 *)(r0 +0), r1)") 589 __not_msg("falling back to forcing all scalars precise") 590 __naked int bpf_atomic_fetch_add_32bit_precision(void) 591 { 592 asm volatile ( 593 "r1 = 0;" 594 "*(u64 *)(r10 - 8) = r1;" 595 "r2 = r10;" 596 "r2 += -8;" 597 "r1 = %[precision_map] ll;" 598 "call %[bpf_map_lookup_elem];" 599 "if r0 == 0 goto 1f;" 600 "r1 = 0;" 601 ".8byte %[fetch_add_insn];" /* r1 = atomic_fetch_add(*(u32 *)(r0 + 0), r1) */ 602 "r1 &= 7;" 603 "r2 = r10;" 604 "r2 += r1;" /* mark_precise */ 605 "1: r0 = 0;" 606 "exit;" 607 : 608 : __imm_addr(precision_map), 609 __imm(bpf_map_lookup_elem), 610 __imm_insn(fetch_add_insn, 611 BPF_ATOMIC_OP(BPF_W, BPF_ADD | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0)) 612 : __clobber_all); 613 } 614 615 SEC("?raw_tp") 616 __success __log_level(2) 617 __msg("mark_precise: frame0: regs=r0 stack= before 12: (57) r0 &= 7") 618 __msg("mark_precise: frame0: regs=r0 stack= before 11: (c3) r0 = atomic_cmpxchg((u32 *)(r6 +0), r0, r1)") 619 __not_msg("falling back to forcing all scalars precise") 620 __naked int bpf_atomic_cmpxchg_32bit_precision(void) 621 { 622 asm volatile ( 623 "r1 = 0;" 624 "*(u64 *)(r10 - 8) = r1;" 625 "r2 = r10;" 626 "r2 += -8;" 627 "r1 = %[precision_map] ll;" 628 "call %[bpf_map_lookup_elem];" 629 "if r0 == 0 goto 1f;" 630 "r6 = r0;" 631 "r0 = 0;" 632 "r1 = 0;" 633 ".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u32 *)(r6 + 0), r0, r1) */ 634 "r0 &= 7;" 635 "r2 = r10;" 636 "r2 += r0;" /* mark_precise */ 637 "1: r0 = 0;" 638 "exit;" 639 : 640 : __imm_addr(precision_map), 641 __imm(bpf_map_lookup_elem), 642 __imm_insn(cmpxchg_insn, 643 BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 0)) 644 : __clobber_all); 645 } 646 647 /* 648 * Verification takes two paths: with r1 being scalar zero on path (1) 649 * and with r1 being some other scalar on path (2). 650 * Check that the verifier does not use checkpoints created 651 * on path (1) to prune path (2). 652 */ 653 SEC("?tc") 654 __flag(BPF_F_TEST_STATE_FREQ) 655 __failure __msg("R1 type=scalar expected=fp") 656 __naked int null_mem_arg_zero_size(void) 657 { 658 asm volatile ( 659 "call %[bpf_get_prandom_u32];" 660 "r1 = 42;" 661 "if r0 > 42 goto 1f;" 662 "r1 = 0;" 663 "1:" 664 "r2 = 0;" 665 "r3 = 0;" 666 "r4 = 0;" 667 "r5 = 0;" 668 /* 669 * ARG_PTR_TO_MEM | PTR_MAYBE_NULL parameter can be NULL, 670 * but can't be some other scalar value. 671 */ 672 "call %[bpf_csum_diff];" 673 "r0 = 0;" 674 "exit;" 675 : 676 : __imm(bpf_get_prandom_u32), 677 __imm(bpf_csum_diff) 678 : __clobber_all); 679 } 680 681 __weak int subprog_mem_arg(int *p) 682 { 683 if (p) 684 return *p; 685 return 0; 686 } 687 688 /* 689 * Verification takes two paths: with r1 being scalar zero on path (1) 690 * and with r1 being some other scalar on path (2). 691 * Check that the verifier does not use checkpoints created 692 * on path (1) to prune path (2). 693 */ 694 SEC("?raw_tp") 695 __flag(BPF_F_TEST_STATE_FREQ) 696 __failure __msg("R1 type=scalar expected=fp") 697 __naked int null_mem_arg_global_subprog(void) 698 { 699 asm volatile ( 700 "call %[bpf_get_prandom_u32];" 701 "r1 = 42;" 702 "if r0 > 42 goto 1f;" 703 "r1 = 0;" 704 "1:" 705 "call subprog_mem_arg;" 706 "r0 = 0;" 707 "exit;" 708 : 709 : __imm(bpf_get_prandom_u32) 710 : __clobber_all); 711 } 712 713 /* Same as above, check that path with r3 == 0 does not prune the path with r3 != 0 */ 714 SEC("?tc") 715 __flag(BPF_F_TEST_STATE_FREQ) 716 __failure __msg("R3 type=scalar expected=fp") 717 int null_kfunc_arg_dynptr_slice(struct __sk_buff *skb) 718 { 719 struct bpf_dynptr ptr; 720 721 bpf_dynptr_from_skb(skb, 0, &ptr); 722 asm volatile ( 723 "call %[bpf_get_prandom_u32];" 724 "r3 = 42;" 725 "if r0 > 42 goto 1f;" 726 "r3 = 0;" 727 "1:" 728 "r1 = %[ptr];" 729 "r2 = 0;" 730 "r4 = 8;" 731 "call %[bpf_dynptr_slice];" 732 : 733 : __imm_ptr(ptr), 734 __imm(bpf_get_prandom_u32), 735 __imm(bpf_dynptr_slice) 736 : __clobber_common); 737 return 0; 738 } 739 740 void __kfunc_btf_root(void) 741 { 742 bpf_dynptr_slice(0, 0, 0, 0); 743 } 744 745 char _license[] SEC("license") = "GPL"; 746