1 // SPDX-License-Identifier: GPL-2.0 2 /* Converted from tools/testing/selftests/bpf/verifier/spill_fill.c */ 3 4 #include <linux/bpf.h> 5 #include <bpf/bpf_helpers.h> 6 #include "bpf_misc.h" 7 #include <../../../tools/include/linux/filter.h> 8 9 struct { 10 __uint(type, BPF_MAP_TYPE_RINGBUF); 11 __uint(max_entries, 4096); 12 } map_ringbuf SEC(".maps"); 13 14 SEC("socket") 15 __description("check valid spill/fill") 16 __success __failure_unpriv __msg_unpriv("R0 leaks addr") 17 __retval(POINTER_VALUE) 18 __naked void check_valid_spill_fill(void) 19 { 20 asm volatile (" \ 21 /* spill R1(ctx) into stack */ \ 22 *(u64*)(r10 - 8) = r1; \ 23 /* fill it back into R2 */ \ 24 r2 = *(u64*)(r10 - 8); \ 25 /* should be able to access R0 = *(R2 + 8) */ \ 26 /* BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 8), */\ 27 r0 = r2; \ 28 exit; \ 29 " ::: __clobber_all); 30 } 31 32 SEC("socket") 33 __description("check valid spill/fill, skb mark") 34 __success __success_unpriv __retval(0) 35 __naked void valid_spill_fill_skb_mark(void) 36 { 37 asm volatile (" \ 38 r6 = r1; \ 39 *(u64*)(r10 - 8) = r6; \ 40 r0 = *(u64*)(r10 - 8); \ 41 r0 = *(u32*)(r0 + %[__sk_buff_mark]); \ 42 exit; \ 43 " : 44 : __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) 45 : __clobber_all); 46 } 47 48 SEC("socket") 49 __description("check valid spill/fill, ptr to mem") 50 __success __success_unpriv __retval(0) 51 __naked void spill_fill_ptr_to_mem(void) 52 { 53 asm volatile (" \ 54 /* reserve 8 byte ringbuf memory */ \ 55 r1 = 0; \ 56 *(u64*)(r10 - 8) = r1; \ 57 r1 = %[map_ringbuf] ll; \ 58 r2 = 8; \ 59 r3 = 0; \ 60 call %[bpf_ringbuf_reserve]; \ 61 /* store a pointer to the reserved memory in R6 */\ 62 r6 = r0; \ 63 /* check whether the reservation was successful */\ 64 if r0 == 0 goto l0_%=; \ 65 /* spill R6(mem) into the stack */ \ 66 *(u64*)(r10 - 8) = r6; \ 67 /* fill it back in R7 */ \ 68 r7 = *(u64*)(r10 - 8); \ 69 /* should be able to access *(R7) = 0 */ \ 70 r1 = 0; \ 71 *(u64*)(r7 + 0) = r1; \ 72 /* submit the reserved ringbuf memory */ \ 73 r1 = r7; \ 74 r2 = 0; \ 75 call %[bpf_ringbuf_submit]; \ 76 l0_%=: r0 = 0; \ 77 exit; \ 78 " : 79 : __imm(bpf_ringbuf_reserve), 80 __imm(bpf_ringbuf_submit), 81 __imm_addr(map_ringbuf) 82 : __clobber_all); 83 } 84 85 SEC("socket") 86 __description("check with invalid reg offset 0") 87 __failure __msg("R0 pointer arithmetic on ringbuf_mem_or_null prohibited") 88 __failure_unpriv 89 __naked void with_invalid_reg_offset_0(void) 90 { 91 asm volatile (" \ 92 /* reserve 8 byte ringbuf memory */ \ 93 r1 = 0; \ 94 *(u64*)(r10 - 8) = r1; \ 95 r1 = %[map_ringbuf] ll; \ 96 r2 = 8; \ 97 r3 = 0; \ 98 call %[bpf_ringbuf_reserve]; \ 99 /* store a pointer to the reserved memory in R6 */\ 100 r6 = r0; \ 101 /* add invalid offset to memory or NULL */ \ 102 r0 += 1; \ 103 /* check whether the reservation was successful */\ 104 if r0 == 0 goto l0_%=; \ 105 /* should not be able to access *(R7) = 0 */ \ 106 r1 = 0; \ 107 *(u32*)(r6 + 0) = r1; \ 108 /* submit the reserved ringbuf memory */ \ 109 r1 = r6; \ 110 r2 = 0; \ 111 call %[bpf_ringbuf_submit]; \ 112 l0_%=: r0 = 0; \ 113 exit; \ 114 " : 115 : __imm(bpf_ringbuf_reserve), 116 __imm(bpf_ringbuf_submit), 117 __imm_addr(map_ringbuf) 118 : __clobber_all); 119 } 120 121 SEC("socket") 122 __description("check corrupted spill/fill") 123 __failure __msg("R0 invalid mem access 'scalar'") 124 __msg_unpriv("attempt to corrupt spilled") 125 __flag(BPF_F_ANY_ALIGNMENT) 126 __naked void check_corrupted_spill_fill(void) 127 { 128 asm volatile (" \ 129 /* spill R1(ctx) into stack */ \ 130 *(u64*)(r10 - 8) = r1; \ 131 /* mess up with R1 pointer on stack */ \ 132 r0 = 0x23; \ 133 *(u8*)(r10 - 7) = r0; \ 134 /* fill back into R0 is fine for priv. \ 135 * R0 now becomes SCALAR_VALUE. \ 136 */ \ 137 r0 = *(u64*)(r10 - 8); \ 138 /* Load from R0 should fail. */ \ 139 r0 = *(u64*)(r0 + 8); \ 140 exit; \ 141 " ::: __clobber_all); 142 } 143 144 SEC("socket") 145 __description("check corrupted spill/fill, LSB") 146 __success __failure_unpriv __msg_unpriv("attempt to corrupt spilled") 147 __retval(POINTER_VALUE) 148 __naked void check_corrupted_spill_fill_lsb(void) 149 { 150 asm volatile (" \ 151 *(u64*)(r10 - 8) = r1; \ 152 r0 = 0xcafe; \ 153 *(u16*)(r10 - 8) = r0; \ 154 r0 = *(u64*)(r10 - 8); \ 155 exit; \ 156 " ::: __clobber_all); 157 } 158 159 SEC("socket") 160 __description("check corrupted spill/fill, MSB") 161 __success __failure_unpriv __msg_unpriv("attempt to corrupt spilled") 162 __retval(POINTER_VALUE) 163 __naked void check_corrupted_spill_fill_msb(void) 164 { 165 asm volatile (" \ 166 *(u64*)(r10 - 8) = r1; \ 167 r0 = 0x12345678; \ 168 *(u32*)(r10 - 4) = r0; \ 169 r0 = *(u64*)(r10 - 8); \ 170 exit; \ 171 " ::: __clobber_all); 172 } 173 174 SEC("tc") 175 __description("Spill and refill a u32 const scalar. Offset to skb->data") 176 __success __retval(0) 177 __naked void scalar_offset_to_skb_data_1(void) 178 { 179 asm volatile (" \ 180 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 181 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 182 w4 = 20; \ 183 *(u32*)(r10 - 8) = r4; \ 184 r4 = *(u32*)(r10 - 8); \ 185 r0 = r2; \ 186 /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=20 */ \ 187 r0 += r4; \ 188 /* if (r0 > r3) R0=pkt,off=20 R2=pkt R3=pkt_end R4=20 */\ 189 if r0 > r3 goto l0_%=; \ 190 /* r0 = *(u32 *)r2 R0=pkt,off=20,r=20 R2=pkt,r=20 R3=pkt_end R4=20 */\ 191 r0 = *(u32*)(r2 + 0); \ 192 l0_%=: r0 = 0; \ 193 exit; \ 194 " : 195 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 196 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) 197 : __clobber_all); 198 } 199 200 SEC("socket") 201 __description("Spill a u32 const, refill from another half of the uninit u32 from the stack") 202 /* in privileged mode reads from uninitialized stack locations are permitted */ 203 __success __failure_unpriv 204 __msg_unpriv("invalid read from stack off -4+0 size 4") 205 __retval(0) 206 __naked void uninit_u32_from_the_stack(void) 207 { 208 asm volatile (" \ 209 w4 = 20; \ 210 *(u32*)(r10 - 8) = r4; \ 211 /* r4 = *(u32 *)(r10 -4) fp-8=????rrrr*/ \ 212 r4 = *(u32*)(r10 - 4); \ 213 r0 = 0; \ 214 exit; \ 215 " ::: __clobber_all); 216 } 217 218 SEC("tc") 219 __description("Spill a u32 const scalar. Refill as u16. Offset to skb->data") 220 __failure __msg("invalid access to packet") 221 __naked void u16_offset_to_skb_data(void) 222 { 223 asm volatile (" \ 224 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 225 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 226 w4 = 20; \ 227 *(u32*)(r10 - 8) = r4; \ 228 r4 = *(u16*)(r10 - 8); \ 229 r0 = r2; \ 230 /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=umax=65535 */\ 231 r0 += r4; \ 232 /* if (r0 > r3) R0=pkt,umax=65535 R2=pkt R3=pkt_end R4=umax=65535 */\ 233 if r0 > r3 goto l0_%=; \ 234 /* r0 = *(u32 *)r2 R0=pkt,umax=65535 R2=pkt R3=pkt_end R4=20 */\ 235 r0 = *(u32*)(r2 + 0); \ 236 l0_%=: r0 = 0; \ 237 exit; \ 238 " : 239 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 240 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) 241 : __clobber_all); 242 } 243 244 SEC("tc") 245 __description("Spill u32 const scalars. Refill as u64. Offset to skb->data") 246 __failure __msg("invalid access to packet") 247 __naked void u64_offset_to_skb_data(void) 248 { 249 asm volatile (" \ 250 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 251 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 252 w6 = 0; \ 253 w7 = 20; \ 254 *(u32*)(r10 - 4) = r6; \ 255 *(u32*)(r10 - 8) = r7; \ 256 r4 = *(u16*)(r10 - 8); \ 257 r0 = r2; \ 258 /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=umax=65535 */\ 259 r0 += r4; \ 260 /* if (r0 > r3) R0=pkt,umax=65535 R2=pkt R3=pkt_end R4=umax=65535 */\ 261 if r0 > r3 goto l0_%=; \ 262 /* r0 = *(u32 *)r2 R0=pkt,umax=65535 R2=pkt R3=pkt_end R4=20 */\ 263 r0 = *(u32*)(r2 + 0); \ 264 l0_%=: r0 = 0; \ 265 exit; \ 266 " : 267 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 268 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) 269 : __clobber_all); 270 } 271 272 SEC("tc") 273 __description("Spill a u32 const scalar. Refill as u16 from fp-6. Offset to skb->data") 274 __failure __msg("invalid access to packet") 275 __naked void _6_offset_to_skb_data(void) 276 { 277 asm volatile (" \ 278 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 279 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 280 w4 = 20; \ 281 *(u32*)(r10 - 8) = r4; \ 282 r4 = *(u16*)(r10 - 6); \ 283 r0 = r2; \ 284 /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=umax=65535 */\ 285 r0 += r4; \ 286 /* if (r0 > r3) R0=pkt,umax=65535 R2=pkt R3=pkt_end R4=umax=65535 */\ 287 if r0 > r3 goto l0_%=; \ 288 /* r0 = *(u32 *)r2 R0=pkt,umax=65535 R2=pkt R3=pkt_end R4=20 */\ 289 r0 = *(u32*)(r2 + 0); \ 290 l0_%=: r0 = 0; \ 291 exit; \ 292 " : 293 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 294 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) 295 : __clobber_all); 296 } 297 298 SEC("tc") 299 __description("Spill and refill a u32 const scalar at non 8byte aligned stack addr. Offset to skb->data") 300 __failure __msg("invalid access to packet") 301 __naked void addr_offset_to_skb_data(void) 302 { 303 asm volatile (" \ 304 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 305 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 306 w4 = 20; \ 307 *(u32*)(r10 - 8) = r4; \ 308 *(u32*)(r10 - 4) = r4; \ 309 r4 = *(u32*)(r10 - 4); \ 310 r0 = r2; \ 311 /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=umax=U32_MAX */\ 312 r0 += r4; \ 313 /* if (r0 > r3) R0=pkt,umax=U32_MAX R2=pkt R3=pkt_end R4= */\ 314 if r0 > r3 goto l0_%=; \ 315 /* r0 = *(u32 *)r2 R0=pkt,umax=U32_MAX R2=pkt R3=pkt_end R4= */\ 316 r0 = *(u32*)(r2 + 0); \ 317 l0_%=: r0 = 0; \ 318 exit; \ 319 " : 320 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 321 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) 322 : __clobber_all); 323 } 324 325 SEC("tc") 326 __description("Spill and refill a umax=40 bounded scalar. Offset to skb->data") 327 __success __retval(0) 328 __naked void scalar_offset_to_skb_data_2(void) 329 { 330 asm volatile (" \ 331 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 332 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 333 r4 = *(u64*)(r1 + %[__sk_buff_tstamp]); \ 334 if r4 <= 40 goto l0_%=; \ 335 r0 = 0; \ 336 exit; \ 337 l0_%=: /* *(u32 *)(r10 -8) = r4 R4=umax=40 */ \ 338 *(u32*)(r10 - 8) = r4; \ 339 /* r4 = (*u32 *)(r10 - 8) */ \ 340 r4 = *(u32*)(r10 - 8); \ 341 /* r2 += r4 R2=pkt R4=umax=40 */ \ 342 r2 += r4; \ 343 /* r0 = r2 R2=pkt,umax=40 R4=umax=40 */ \ 344 r0 = r2; \ 345 /* r2 += 20 R0=pkt,umax=40 R2=pkt,umax=40 */ \ 346 r2 += 20; \ 347 /* if (r2 > r3) R0=pkt,umax=40 R2=pkt,off=20,umax=40 */\ 348 if r2 > r3 goto l1_%=; \ 349 /* r0 = *(u32 *)r0 R0=pkt,r=20,umax=40 R2=pkt,off=20,r=20,umax=40 */\ 350 r0 = *(u32*)(r0 + 0); \ 351 l1_%=: r0 = 0; \ 352 exit; \ 353 " : 354 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 355 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), 356 __imm_const(__sk_buff_tstamp, offsetof(struct __sk_buff, tstamp)) 357 : __clobber_all); 358 } 359 360 SEC("tc") 361 __description("Spill a u32 scalar at fp-4 and then at fp-8") 362 __success __retval(0) 363 __naked void and_then_at_fp_8(void) 364 { 365 asm volatile (" \ 366 w4 = 4321; \ 367 *(u32*)(r10 - 4) = r4; \ 368 *(u32*)(r10 - 8) = r4; \ 369 r4 = *(u64*)(r10 - 8); \ 370 r0 = 0; \ 371 exit; \ 372 " ::: __clobber_all); 373 } 374 375 SEC("xdp") 376 __description("32-bit spill of 64-bit reg should clear ID") 377 __failure __msg("math between ctx pointer and 4294967295 is not allowed") 378 __naked void spill_32bit_of_64bit_fail(void) 379 { 380 asm volatile (" \ 381 r6 = r1; \ 382 /* Roll one bit to force the verifier to track both branches. */\ 383 call %[bpf_get_prandom_u32]; \ 384 r0 &= 0x8; \ 385 /* Put a large number into r1. */ \ 386 r1 = 0xffffffff; \ 387 r1 <<= 32; \ 388 r1 += r0; \ 389 /* Assign an ID to r1. */ \ 390 r2 = r1; \ 391 /* 32-bit spill r1 to stack - should clear the ID! */\ 392 *(u32*)(r10 - 8) = r1; \ 393 /* 32-bit fill r2 from stack. */ \ 394 r2 = *(u32*)(r10 - 8); \ 395 /* Compare r2 with another register to trigger find_equal_scalars.\ 396 * Having one random bit is important here, otherwise the verifier cuts\ 397 * the corners. If the ID was mistakenly preserved on spill, this would\ 398 * cause the verifier to think that r1 is also equal to zero in one of\ 399 * the branches, and equal to eight on the other branch.\ 400 */ \ 401 r3 = 0; \ 402 if r2 != r3 goto l0_%=; \ 403 l0_%=: r1 >>= 32; \ 404 /* At this point, if the verifier thinks that r1 is 0, an out-of-bounds\ 405 * read will happen, because it actually contains 0xffffffff.\ 406 */ \ 407 r6 += r1; \ 408 r0 = *(u32*)(r6 + 0); \ 409 exit; \ 410 " : 411 : __imm(bpf_get_prandom_u32) 412 : __clobber_all); 413 } 414 415 SEC("xdp") 416 __description("16-bit spill of 32-bit reg should clear ID") 417 __failure __msg("dereference of modified ctx ptr R6 off=65535 disallowed") 418 __naked void spill_16bit_of_32bit_fail(void) 419 { 420 asm volatile (" \ 421 r6 = r1; \ 422 /* Roll one bit to force the verifier to track both branches. */\ 423 call %[bpf_get_prandom_u32]; \ 424 r0 &= 0x8; \ 425 /* Put a large number into r1. */ \ 426 w1 = 0xffff0000; \ 427 r1 += r0; \ 428 /* Assign an ID to r1. */ \ 429 r2 = r1; \ 430 /* 16-bit spill r1 to stack - should clear the ID! */\ 431 *(u16*)(r10 - 8) = r1; \ 432 /* 16-bit fill r2 from stack. */ \ 433 r2 = *(u16*)(r10 - 8); \ 434 /* Compare r2 with another register to trigger find_equal_scalars.\ 435 * Having one random bit is important here, otherwise the verifier cuts\ 436 * the corners. If the ID was mistakenly preserved on spill, this would\ 437 * cause the verifier to think that r1 is also equal to zero in one of\ 438 * the branches, and equal to eight on the other branch.\ 439 */ \ 440 r3 = 0; \ 441 if r2 != r3 goto l0_%=; \ 442 l0_%=: r1 >>= 16; \ 443 /* At this point, if the verifier thinks that r1 is 0, an out-of-bounds\ 444 * read will happen, because it actually contains 0xffff.\ 445 */ \ 446 r6 += r1; \ 447 r0 = *(u32*)(r6 + 0); \ 448 exit; \ 449 " : 450 : __imm(bpf_get_prandom_u32) 451 : __clobber_all); 452 } 453 454 SEC("raw_tp") 455 __log_level(2) 456 __success 457 __msg("fp-8=0m??mmmm") 458 __msg("fp-16=00mm??mm") 459 __msg("fp-24=00mm???m") 460 __naked void spill_subregs_preserve_stack_zero(void) 461 { 462 asm volatile ( 463 "call %[bpf_get_prandom_u32];" 464 465 /* 32-bit subreg spill with ZERO, MISC, and INVALID */ 466 ".8byte %[fp1_u8_st_zero];" /* ZERO, LLVM-18+: *(u8 *)(r10 -1) = 0; */ 467 "*(u8 *)(r10 -2) = r0;" /* MISC */ 468 /* fp-3 and fp-4 stay INVALID */ 469 "*(u32 *)(r10 -8) = r0;" 470 471 /* 16-bit subreg spill with ZERO, MISC, and INVALID */ 472 ".8byte %[fp10_u16_st_zero];" /* ZERO, LLVM-18+: *(u16 *)(r10 -10) = 0; */ 473 "*(u16 *)(r10 -12) = r0;" /* MISC */ 474 /* fp-13 and fp-14 stay INVALID */ 475 "*(u16 *)(r10 -16) = r0;" 476 477 /* 8-bit subreg spill with ZERO, MISC, and INVALID */ 478 ".8byte %[fp18_u16_st_zero];" /* ZERO, LLVM-18+: *(u16 *)(r18 -10) = 0; */ 479 "*(u16 *)(r10 -20) = r0;" /* MISC */ 480 /* fp-21, fp-22, and fp-23 stay INVALID */ 481 "*(u8 *)(r10 -24) = r0;" 482 483 "r0 = 0;" 484 "exit;" 485 : 486 : __imm(bpf_get_prandom_u32), 487 __imm_insn(fp1_u8_st_zero, BPF_ST_MEM(BPF_B, BPF_REG_FP, -1, 0)), 488 __imm_insn(fp10_u16_st_zero, BPF_ST_MEM(BPF_H, BPF_REG_FP, -10, 0)), 489 __imm_insn(fp18_u16_st_zero, BPF_ST_MEM(BPF_H, BPF_REG_FP, -18, 0)) 490 : __clobber_all); 491 } 492 493 char single_byte_buf[1] SEC(".data.single_byte_buf"); 494 495 SEC("raw_tp") 496 __log_level(2) 497 __success 498 /* make sure fp-8 is all STACK_ZERO */ 499 __msg("2: (7a) *(u64 *)(r10 -8) = 0 ; R10=fp0 fp-8_w=00000000") 500 /* but fp-16 is spilled IMPRECISE zero const reg */ 501 __msg("4: (7b) *(u64 *)(r10 -16) = r0 ; R0_w=0 R10=fp0 fp-16_w=0") 502 /* validate that assigning R2 from STACK_ZERO doesn't mark register 503 * precise immediately; if necessary, it will be marked precise later 504 */ 505 __msg("6: (71) r2 = *(u8 *)(r10 -1) ; R2_w=0 R10=fp0 fp-8_w=00000000") 506 /* similarly, when R2 is assigned from spilled register, it is initially 507 * imprecise, but will be marked precise later once it is used in precise context 508 */ 509 __msg("10: (71) r2 = *(u8 *)(r10 -9) ; R2_w=0 R10=fp0 fp-16_w=0") 510 __msg("11: (0f) r1 += r2") 511 __msg("mark_precise: frame0: last_idx 11 first_idx 0 subseq_idx -1") 512 __msg("mark_precise: frame0: regs=r2 stack= before 10: (71) r2 = *(u8 *)(r10 -9)") 513 __msg("mark_precise: frame0: regs= stack=-16 before 9: (bf) r1 = r6") 514 __msg("mark_precise: frame0: regs= stack=-16 before 8: (73) *(u8 *)(r1 +0) = r2") 515 __msg("mark_precise: frame0: regs= stack=-16 before 7: (0f) r1 += r2") 516 __msg("mark_precise: frame0: regs= stack=-16 before 6: (71) r2 = *(u8 *)(r10 -1)") 517 __msg("mark_precise: frame0: regs= stack=-16 before 5: (bf) r1 = r6") 518 __msg("mark_precise: frame0: regs= stack=-16 before 4: (7b) *(u64 *)(r10 -16) = r0") 519 __msg("mark_precise: frame0: regs=r0 stack= before 3: (b7) r0 = 0") 520 __naked void partial_stack_load_preserves_zeros(void) 521 { 522 asm volatile ( 523 /* fp-8 is all STACK_ZERO */ 524 ".8byte %[fp8_st_zero];" /* LLVM-18+: *(u64 *)(r10 -8) = 0; */ 525 526 /* fp-16 is const zero register */ 527 "r0 = 0;" 528 "*(u64 *)(r10 -16) = r0;" 529 530 /* load single U8 from non-aligned STACK_ZERO slot */ 531 "r1 = %[single_byte_buf];" 532 "r2 = *(u8 *)(r10 -1);" 533 "r1 += r2;" 534 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 535 536 /* load single U8 from non-aligned ZERO REG slot */ 537 "r1 = %[single_byte_buf];" 538 "r2 = *(u8 *)(r10 -9);" 539 "r1 += r2;" 540 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 541 542 /* load single U16 from non-aligned STACK_ZERO slot */ 543 "r1 = %[single_byte_buf];" 544 "r2 = *(u16 *)(r10 -2);" 545 "r1 += r2;" 546 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 547 548 /* load single U16 from non-aligned ZERO REG slot */ 549 "r1 = %[single_byte_buf];" 550 "r2 = *(u16 *)(r10 -10);" 551 "r1 += r2;" 552 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 553 554 /* load single U32 from non-aligned STACK_ZERO slot */ 555 "r1 = %[single_byte_buf];" 556 "r2 = *(u32 *)(r10 -4);" 557 "r1 += r2;" 558 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 559 560 /* load single U32 from non-aligned ZERO REG slot */ 561 "r1 = %[single_byte_buf];" 562 "r2 = *(u32 *)(r10 -12);" 563 "r1 += r2;" 564 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 565 566 /* for completeness, load U64 from STACK_ZERO slot */ 567 "r1 = %[single_byte_buf];" 568 "r2 = *(u64 *)(r10 -8);" 569 "r1 += r2;" 570 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 571 572 /* for completeness, load U64 from ZERO REG slot */ 573 "r1 = %[single_byte_buf];" 574 "r2 = *(u64 *)(r10 -16);" 575 "r1 += r2;" 576 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 577 578 "r0 = 0;" 579 "exit;" 580 : 581 : __imm_ptr(single_byte_buf), 582 __imm_insn(fp8_st_zero, BPF_ST_MEM(BPF_DW, BPF_REG_FP, -8, 0)) 583 : __clobber_common); 584 } 585 586 char two_byte_buf[2] SEC(".data.two_byte_buf"); 587 588 SEC("raw_tp") 589 __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) 590 __success 591 /* make sure fp-8 is IMPRECISE fake register spill */ 592 __msg("3: (7a) *(u64 *)(r10 -8) = 1 ; R10=fp0 fp-8_w=1") 593 /* and fp-16 is spilled IMPRECISE const reg */ 594 __msg("5: (7b) *(u64 *)(r10 -16) = r0 ; R0_w=1 R10=fp0 fp-16_w=1") 595 /* validate load from fp-8, which was initialized using BPF_ST_MEM */ 596 __msg("8: (79) r2 = *(u64 *)(r10 -8) ; R2_w=1 R10=fp0 fp-8=1") 597 __msg("9: (0f) r1 += r2") 598 __msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx -1") 599 __msg("mark_precise: frame0: regs=r2 stack= before 8: (79) r2 = *(u64 *)(r10 -8)") 600 __msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r1 = r6") 601 /* note, fp-8 is precise, fp-16 is not yet precise, we'll get there */ 602 __msg("mark_precise: frame0: parent state regs= stack=-8: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_rw=P1 fp-16_w=1") 603 __msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") 604 __msg("mark_precise: frame0: regs= stack=-8 before 6: (05) goto pc+0") 605 __msg("mark_precise: frame0: regs= stack=-8 before 5: (7b) *(u64 *)(r10 -16) = r0") 606 __msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r0 = 1") 607 __msg("mark_precise: frame0: regs= stack=-8 before 3: (7a) *(u64 *)(r10 -8) = 1") 608 __msg("10: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") 609 /* validate load from fp-16, which was initialized using BPF_STX_MEM */ 610 __msg("12: (79) r2 = *(u64 *)(r10 -16) ; R2_w=1 R10=fp0 fp-16=1") 611 __msg("13: (0f) r1 += r2") 612 __msg("mark_precise: frame0: last_idx 13 first_idx 7 subseq_idx -1") 613 __msg("mark_precise: frame0: regs=r2 stack= before 12: (79) r2 = *(u64 *)(r10 -16)") 614 __msg("mark_precise: frame0: regs= stack=-16 before 11: (bf) r1 = r6") 615 __msg("mark_precise: frame0: regs= stack=-16 before 10: (73) *(u8 *)(r1 +0) = r2") 616 __msg("mark_precise: frame0: regs= stack=-16 before 9: (0f) r1 += r2") 617 __msg("mark_precise: frame0: regs= stack=-16 before 8: (79) r2 = *(u64 *)(r10 -8)") 618 __msg("mark_precise: frame0: regs= stack=-16 before 7: (bf) r1 = r6") 619 /* now both fp-8 and fp-16 are precise, very good */ 620 __msg("mark_precise: frame0: parent state regs= stack=-16: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_rw=P1 fp-16_rw=P1") 621 __msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") 622 __msg("mark_precise: frame0: regs= stack=-16 before 6: (05) goto pc+0") 623 __msg("mark_precise: frame0: regs= stack=-16 before 5: (7b) *(u64 *)(r10 -16) = r0") 624 __msg("mark_precise: frame0: regs=r0 stack= before 4: (b7) r0 = 1") 625 __msg("14: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") 626 __naked void stack_load_preserves_const_precision(void) 627 { 628 asm volatile ( 629 /* establish checkpoint with state that has no stack slots; 630 * if we bubble up to this state without finding desired stack 631 * slot, then it's a bug and should be caught 632 */ 633 "goto +0;" 634 635 /* fp-8 is const 1 *fake* register */ 636 ".8byte %[fp8_st_one];" /* LLVM-18+: *(u64 *)(r10 -8) = 1; */ 637 638 /* fp-16 is const 1 register */ 639 "r0 = 1;" 640 "*(u64 *)(r10 -16) = r0;" 641 642 /* force checkpoint to check precision marks preserved in parent states */ 643 "goto +0;" 644 645 /* load single U64 from aligned FAKE_REG=1 slot */ 646 "r1 = %[two_byte_buf];" 647 "r2 = *(u64 *)(r10 -8);" 648 "r1 += r2;" 649 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 650 651 /* load single U64 from aligned REG=1 slot */ 652 "r1 = %[two_byte_buf];" 653 "r2 = *(u64 *)(r10 -16);" 654 "r1 += r2;" 655 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 656 657 "r0 = 0;" 658 "exit;" 659 : 660 : __imm_ptr(two_byte_buf), 661 __imm_insn(fp8_st_one, BPF_ST_MEM(BPF_DW, BPF_REG_FP, -8, 1)) 662 : __clobber_common); 663 } 664 665 SEC("raw_tp") 666 __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) 667 __success 668 /* make sure fp-8 is 32-bit FAKE subregister spill */ 669 __msg("3: (62) *(u32 *)(r10 -8) = 1 ; R10=fp0 fp-8=????1") 670 /* but fp-16 is spilled IMPRECISE zero const reg */ 671 __msg("5: (63) *(u32 *)(r10 -16) = r0 ; R0_w=1 R10=fp0 fp-16=????1") 672 /* validate load from fp-8, which was initialized using BPF_ST_MEM */ 673 __msg("8: (61) r2 = *(u32 *)(r10 -8) ; R2_w=1 R10=fp0 fp-8=????1") 674 __msg("9: (0f) r1 += r2") 675 __msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx -1") 676 __msg("mark_precise: frame0: regs=r2 stack= before 8: (61) r2 = *(u32 *)(r10 -8)") 677 __msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r1 = r6") 678 __msg("mark_precise: frame0: parent state regs= stack=-8: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_r=????P1 fp-16=????1") 679 __msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") 680 __msg("mark_precise: frame0: regs= stack=-8 before 6: (05) goto pc+0") 681 __msg("mark_precise: frame0: regs= stack=-8 before 5: (63) *(u32 *)(r10 -16) = r0") 682 __msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r0 = 1") 683 __msg("mark_precise: frame0: regs= stack=-8 before 3: (62) *(u32 *)(r10 -8) = 1") 684 __msg("10: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") 685 /* validate load from fp-16, which was initialized using BPF_STX_MEM */ 686 __msg("12: (61) r2 = *(u32 *)(r10 -16) ; R2_w=1 R10=fp0 fp-16=????1") 687 __msg("13: (0f) r1 += r2") 688 __msg("mark_precise: frame0: last_idx 13 first_idx 7 subseq_idx -1") 689 __msg("mark_precise: frame0: regs=r2 stack= before 12: (61) r2 = *(u32 *)(r10 -16)") 690 __msg("mark_precise: frame0: regs= stack=-16 before 11: (bf) r1 = r6") 691 __msg("mark_precise: frame0: regs= stack=-16 before 10: (73) *(u8 *)(r1 +0) = r2") 692 __msg("mark_precise: frame0: regs= stack=-16 before 9: (0f) r1 += r2") 693 __msg("mark_precise: frame0: regs= stack=-16 before 8: (61) r2 = *(u32 *)(r10 -8)") 694 __msg("mark_precise: frame0: regs= stack=-16 before 7: (bf) r1 = r6") 695 __msg("mark_precise: frame0: parent state regs= stack=-16: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_r=????P1 fp-16_r=????P1") 696 __msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") 697 __msg("mark_precise: frame0: regs= stack=-16 before 6: (05) goto pc+0") 698 __msg("mark_precise: frame0: regs= stack=-16 before 5: (63) *(u32 *)(r10 -16) = r0") 699 __msg("mark_precise: frame0: regs=r0 stack= before 4: (b7) r0 = 1") 700 __msg("14: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") 701 __naked void stack_load_preserves_const_precision_subreg(void) 702 { 703 asm volatile ( 704 /* establish checkpoint with state that has no stack slots; 705 * if we bubble up to this state without finding desired stack 706 * slot, then it's a bug and should be caught 707 */ 708 "goto +0;" 709 710 /* fp-8 is const 1 *fake* SUB-register */ 711 ".8byte %[fp8_st_one];" /* LLVM-18+: *(u32 *)(r10 -8) = 1; */ 712 713 /* fp-16 is const 1 SUB-register */ 714 "r0 = 1;" 715 "*(u32 *)(r10 -16) = r0;" 716 717 /* force checkpoint to check precision marks preserved in parent states */ 718 "goto +0;" 719 720 /* load single U32 from aligned FAKE_REG=1 slot */ 721 "r1 = %[two_byte_buf];" 722 "r2 = *(u32 *)(r10 -8);" 723 "r1 += r2;" 724 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 725 726 /* load single U32 from aligned REG=1 slot */ 727 "r1 = %[two_byte_buf];" 728 "r2 = *(u32 *)(r10 -16);" 729 "r1 += r2;" 730 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 731 732 "r0 = 0;" 733 "exit;" 734 : 735 : __imm_ptr(two_byte_buf), 736 __imm_insn(fp8_st_one, BPF_ST_MEM(BPF_W, BPF_REG_FP, -8, 1)) /* 32-bit spill */ 737 : __clobber_common); 738 } 739 740 char _license[] SEC("license") = "GPL"; 741