1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ 3 4 #include <errno.h> 5 #include <string.h> 6 #include <linux/bpf.h> 7 #include <bpf/bpf_helpers.h> 8 #include "bpf_misc.h" 9 #include <../../../tools/include/linux/filter.h> 10 11 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) 12 13 int vals[] SEC(".data.vals") = {1, 2, 3, 4}; 14 15 __naked __noinline __used 16 static unsigned long identity_subprog() 17 { 18 /* the simplest *static* 64-bit identity function */ 19 asm volatile ( 20 "r0 = r1;" 21 "exit;" 22 ); 23 } 24 25 __noinline __used 26 unsigned long global_identity_subprog(__u64 x) 27 { 28 /* the simplest *global* 64-bit identity function */ 29 return x; 30 } 31 32 __naked __noinline __used 33 static unsigned long callback_subprog() 34 { 35 /* the simplest callback function */ 36 asm volatile ( 37 "r0 = 0;" 38 "exit;" 39 ); 40 } 41 42 SEC("?raw_tp") 43 __success __log_level(2) 44 __msg("7: (0f) r1 += r0") 45 __msg("mark_precise: frame0: regs=r0 stack= before 6: (bf) r1 = r7") 46 __msg("mark_precise: frame0: regs=r0 stack= before 5: (27) r0 *= 4") 47 __msg("mark_precise: frame0: regs=r0 stack= before 11: (95) exit") 48 __msg("mark_precise: frame1: regs=r0 stack= before 10: (bf) r0 = r1") 49 __msg("mark_precise: frame1: regs=r1 stack= before 4: (85) call pc+5") 50 __msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r1 = r6") 51 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 52 __naked int subprog_result_precise(void) 53 { 54 asm volatile ( 55 "r6 = 3;" 56 /* pass r6 through r1 into subprog to get it back as r0; 57 * this whole chain will have to be marked as precise later 58 */ 59 "r1 = r6;" 60 "call identity_subprog;" 61 /* now use subprog's returned value (which is a 62 * r6 -> r1 -> r0 chain), as index into vals array, forcing 63 * all of that to be known precisely 64 */ 65 "r0 *= 4;" 66 "r1 = %[vals];" 67 /* here r0->r1->r6 chain is forced to be precise and has to be 68 * propagated back to the beginning, including through the 69 * subprog call 70 */ 71 "r1 += r0;" 72 "r0 = *(u32 *)(r1 + 0);" 73 "exit;" 74 : 75 : __imm_ptr(vals) 76 : __clobber_common, "r6" 77 ); 78 } 79 80 __naked __noinline __used 81 static unsigned long fp_leaking_subprog() 82 { 83 asm volatile ( 84 ".8byte %[r0_eq_r10_cast_s8];" 85 "exit;" 86 :: __imm_insn(r0_eq_r10_cast_s8, BPF_MOVSX64_REG(BPF_REG_0, BPF_REG_10, 8)) 87 ); 88 } 89 90 __naked __noinline __used 91 static unsigned long sneaky_fp_leaking_subprog() 92 { 93 asm volatile ( 94 "r1 = r10;" 95 ".8byte %[r0_eq_r1_cast_s8];" 96 "exit;" 97 :: __imm_insn(r0_eq_r1_cast_s8, BPF_MOVSX64_REG(BPF_REG_0, BPF_REG_1, 8)) 98 ); 99 } 100 101 SEC("?raw_tp") 102 __success __log_level(2) 103 __msg("6: (0f) r1 += r0") 104 __msg("mark_precise: frame0: last_idx 6 first_idx 0 subseq_idx -1") 105 __msg("mark_precise: frame0: regs=r0 stack= before 5: (bf) r1 = r6") 106 __msg("mark_precise: frame0: regs=r0 stack= before 4: (27) r0 *= 4") 107 __msg("mark_precise: frame0: regs=r0 stack= before 3: (57) r0 &= 3") 108 __msg("mark_precise: frame0: regs=r0 stack= before 10: (95) exit") 109 __msg("mark_precise: frame1: regs=r0 stack= before 9: (bf) r0 = (s8)r10") 110 __msg("7: R0_w=scalar") 111 __naked int fp_precise_subprog_result(void) 112 { 113 asm volatile ( 114 "call fp_leaking_subprog;" 115 /* use subprog's returned value (which is derived from r10=fp 116 * register), as index into vals array, forcing all of that to 117 * be known precisely 118 */ 119 "r0 &= 3;" 120 "r0 *= 4;" 121 "r1 = %[vals];" 122 /* force precision marking */ 123 "r1 += r0;" 124 "r0 = *(u32 *)(r1 + 0);" 125 "exit;" 126 : 127 : __imm_ptr(vals) 128 : __clobber_common 129 ); 130 } 131 132 SEC("?raw_tp") 133 __success __log_level(2) 134 __msg("6: (0f) r1 += r0") 135 __msg("mark_precise: frame0: last_idx 6 first_idx 0 subseq_idx -1") 136 __msg("mark_precise: frame0: regs=r0 stack= before 5: (bf) r1 = r6") 137 __msg("mark_precise: frame0: regs=r0 stack= before 4: (27) r0 *= 4") 138 __msg("mark_precise: frame0: regs=r0 stack= before 3: (57) r0 &= 3") 139 __msg("mark_precise: frame0: regs=r0 stack= before 11: (95) exit") 140 __msg("mark_precise: frame1: regs=r0 stack= before 10: (bf) r0 = (s8)r1") 141 /* here r1 is marked precise, even though it's fp register, but that's fine 142 * because by the time we get out of subprogram it has to be derived from r10 143 * anyways, at which point we'll break precision chain 144 */ 145 __msg("mark_precise: frame1: regs=r1 stack= before 9: (bf) r1 = r10") 146 __msg("7: R0_w=scalar") 147 __naked int sneaky_fp_precise_subprog_result(void) 148 { 149 asm volatile ( 150 "call sneaky_fp_leaking_subprog;" 151 /* use subprog's returned value (which is derived from r10=fp 152 * register), as index into vals array, forcing all of that to 153 * be known precisely 154 */ 155 "r0 &= 3;" 156 "r0 *= 4;" 157 "r1 = %[vals];" 158 /* force precision marking */ 159 "r1 += r0;" 160 "r0 = *(u32 *)(r1 + 0);" 161 "exit;" 162 : 163 : __imm_ptr(vals) 164 : __clobber_common 165 ); 166 } 167 168 SEC("?raw_tp") 169 __success __log_level(2) 170 __msg("9: (0f) r1 += r0") 171 __msg("mark_precise: frame0: last_idx 9 first_idx 0") 172 __msg("mark_precise: frame0: regs=r0 stack= before 8: (bf) r1 = r7") 173 __msg("mark_precise: frame0: regs=r0 stack= before 7: (27) r0 *= 4") 174 __msg("mark_precise: frame0: regs=r0 stack= before 5: (a5) if r0 < 0x4 goto pc+1") 175 __msg("mark_precise: frame0: regs=r0 stack= before 4: (85) call pc+7") 176 __naked int global_subprog_result_precise(void) 177 { 178 asm volatile ( 179 "r6 = 3;" 180 /* pass r6 through r1 into subprog to get it back as r0; 181 * given global_identity_subprog is global, precision won't 182 * propagate all the way back to r6 183 */ 184 "r1 = r6;" 185 "call global_identity_subprog;" 186 /* now use subprog's returned value (which is unknown now, so 187 * we need to clamp it), as index into vals array, forcing r0 188 * to be marked precise (with no effect on r6, though) 189 */ 190 "if r0 < %[vals_arr_sz] goto 1f;" 191 "r0 = %[vals_arr_sz] - 1;" 192 "1:" 193 "r0 *= 4;" 194 "r1 = %[vals];" 195 /* here r0 is forced to be precise and has to be 196 * propagated back to the global subprog call, but it 197 * shouldn't go all the way to mark r6 as precise 198 */ 199 "r1 += r0;" 200 "r0 = *(u32 *)(r1 + 0);" 201 "exit;" 202 : 203 : __imm_ptr(vals), 204 __imm_const(vals_arr_sz, ARRAY_SIZE(vals)) 205 : __clobber_common, "r6" 206 ); 207 } 208 209 __naked __noinline __used 210 static unsigned long loop_callback_bad() 211 { 212 /* bpf_loop() callback that can return values outside of [0, 1] range */ 213 asm volatile ( 214 "call %[bpf_get_prandom_u32];" 215 "if r0 s> 1000 goto 1f;" 216 "r0 = 0;" 217 "1:" 218 "goto +0;" /* checkpoint */ 219 /* bpf_loop() expects [0, 1] values, so branch above skipping 220 * r0 = 0; should lead to a failure, but if exit instruction 221 * doesn't enforce r0's precision, this callback will be 222 * successfully verified 223 */ 224 "exit;" 225 : 226 : __imm(bpf_get_prandom_u32) 227 : __clobber_common 228 ); 229 } 230 231 SEC("?raw_tp") 232 __failure __log_level(2) 233 __flag(BPF_F_TEST_STATE_FREQ) 234 /* check that fallthrough code path marks r0 as precise */ 235 __msg("mark_precise: frame1: regs=r0 stack= before 11: (b7) r0 = 0") 236 /* check that we have branch code path doing its own validation */ 237 __msg("from 10 to 12: frame1: R0=scalar(smin=umin=1001") 238 /* check that branch code path marks r0 as precise, before failing */ 239 __msg("mark_precise: frame1: regs=r0 stack= before 9: (85) call bpf_get_prandom_u32#7") 240 __msg("At callback return the register R0 has smin=1001 should have been in [0, 1]") 241 __naked int callback_precise_return_fail(void) 242 { 243 asm volatile ( 244 "r1 = 1;" /* nr_loops */ 245 "r2 = %[loop_callback_bad];" /* callback_fn */ 246 "r3 = 0;" /* callback_ctx */ 247 "r4 = 0;" /* flags */ 248 "call %[bpf_loop];" 249 250 "r0 = 0;" 251 "exit;" 252 : 253 : __imm_ptr(loop_callback_bad), 254 __imm(bpf_loop) 255 : __clobber_common 256 ); 257 } 258 259 SEC("?raw_tp") 260 __success __log_level(2) 261 /* First simulated path does not include callback body, 262 * r1 and r4 are always precise for bpf_loop() calls. 263 */ 264 __msg("9: (85) call bpf_loop#181") 265 __msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") 266 __msg("mark_precise: frame0: parent state regs=r4 stack=:") 267 __msg("mark_precise: frame0: last_idx 8 first_idx 0 subseq_idx 9") 268 __msg("mark_precise: frame0: regs=r4 stack= before 8: (b7) r4 = 0") 269 __msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") 270 __msg("mark_precise: frame0: parent state regs=r1 stack=:") 271 __msg("mark_precise: frame0: last_idx 8 first_idx 0 subseq_idx 9") 272 __msg("mark_precise: frame0: regs=r1 stack= before 8: (b7) r4 = 0") 273 __msg("mark_precise: frame0: regs=r1 stack= before 7: (b7) r3 = 0") 274 __msg("mark_precise: frame0: regs=r1 stack= before 6: (bf) r2 = r8") 275 __msg("mark_precise: frame0: regs=r1 stack= before 5: (bf) r1 = r6") 276 __msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3") 277 /* r6 precision propagation */ 278 __msg("14: (0f) r1 += r6") 279 __msg("mark_precise: frame0: last_idx 14 first_idx 9") 280 __msg("mark_precise: frame0: regs=r6 stack= before 13: (bf) r1 = r7") 281 __msg("mark_precise: frame0: regs=r6 stack= before 12: (27) r6 *= 4") 282 __msg("mark_precise: frame0: regs=r6 stack= before 11: (25) if r6 > 0x3 goto pc+4") 283 __msg("mark_precise: frame0: regs=r6 stack= before 10: (bf) r6 = r0") 284 __msg("mark_precise: frame0: regs=r0 stack= before 9: (85) call bpf_loop") 285 /* State entering callback body popped from states stack */ 286 __msg("from 9 to 17: frame1:") 287 __msg("17: frame1: R1=scalar() R2=0 R10=fp0 cb") 288 __msg("17: (b7) r0 = 0") 289 __msg("18: (95) exit") 290 __msg("returning from callee:") 291 __msg("to caller at 9:") 292 __msg("frame 0: propagating r1,r4") 293 __msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") 294 __msg("mark_precise: frame0: regs=r1,r4 stack= before 18: (95) exit") 295 __msg("from 18 to 9: safe") 296 __naked int callback_result_precise(void) 297 { 298 asm volatile ( 299 "r6 = 3;" 300 301 /* call subprog and use result; r0 shouldn't propagate back to 302 * callback_subprog 303 */ 304 "r1 = r6;" /* nr_loops */ 305 "r2 = %[callback_subprog];" /* callback_fn */ 306 "r3 = 0;" /* callback_ctx */ 307 "r4 = 0;" /* flags */ 308 "call %[bpf_loop];" 309 310 "r6 = r0;" 311 "if r6 > 3 goto 1f;" 312 "r6 *= 4;" 313 "r1 = %[vals];" 314 /* here r6 is forced to be precise and has to be propagated 315 * back to the bpf_loop() call, but not beyond 316 */ 317 "r1 += r6;" 318 "r0 = *(u32 *)(r1 + 0);" 319 "1:" 320 "exit;" 321 : 322 : __imm_ptr(vals), 323 __imm_ptr(callback_subprog), 324 __imm(bpf_loop) 325 : __clobber_common, "r6" 326 ); 327 } 328 329 SEC("?raw_tp") 330 __success __log_level(2) 331 __msg("7: (0f) r1 += r6") 332 __msg("mark_precise: frame0: last_idx 7 first_idx 0") 333 __msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r1 = r7") 334 __msg("mark_precise: frame0: regs=r6 stack= before 5: (27) r6 *= 4") 335 __msg("mark_precise: frame0: regs=r6 stack= before 11: (95) exit") 336 __msg("mark_precise: frame1: regs= stack= before 10: (bf) r0 = r1") 337 __msg("mark_precise: frame1: regs= stack= before 4: (85) call pc+5") 338 __msg("mark_precise: frame0: regs=r6 stack= before 3: (b7) r1 = 0") 339 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 340 __naked int parent_callee_saved_reg_precise(void) 341 { 342 asm volatile ( 343 "r6 = 3;" 344 345 /* call subprog and ignore result; we need this call only to 346 * complicate jump history 347 */ 348 "r1 = 0;" 349 "call identity_subprog;" 350 351 "r6 *= 4;" 352 "r1 = %[vals];" 353 /* here r6 is forced to be precise and has to be propagated 354 * back to the beginning, handling (and ignoring) subprog call 355 */ 356 "r1 += r6;" 357 "r0 = *(u32 *)(r1 + 0);" 358 "exit;" 359 : 360 : __imm_ptr(vals) 361 : __clobber_common, "r6" 362 ); 363 } 364 365 SEC("?raw_tp") 366 __success __log_level(2) 367 __msg("7: (0f) r1 += r6") 368 __msg("mark_precise: frame0: last_idx 7 first_idx 0") 369 __msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r1 = r7") 370 __msg("mark_precise: frame0: regs=r6 stack= before 5: (27) r6 *= 4") 371 __msg("mark_precise: frame0: regs=r6 stack= before 4: (85) call pc+5") 372 __msg("mark_precise: frame0: regs=r6 stack= before 3: (b7) r1 = 0") 373 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 374 __naked int parent_callee_saved_reg_precise_global(void) 375 { 376 asm volatile ( 377 "r6 = 3;" 378 379 /* call subprog and ignore result; we need this call only to 380 * complicate jump history 381 */ 382 "r1 = 0;" 383 "call global_identity_subprog;" 384 385 "r6 *= 4;" 386 "r1 = %[vals];" 387 /* here r6 is forced to be precise and has to be propagated 388 * back to the beginning, handling (and ignoring) subprog call 389 */ 390 "r1 += r6;" 391 "r0 = *(u32 *)(r1 + 0);" 392 "exit;" 393 : 394 : __imm_ptr(vals) 395 : __clobber_common, "r6" 396 ); 397 } 398 399 SEC("?raw_tp") 400 __success __log_level(2) 401 /* First simulated path does not include callback body */ 402 __msg("12: (0f) r1 += r6") 403 __msg("mark_precise: frame0: last_idx 12 first_idx 9") 404 __msg("mark_precise: frame0: regs=r6 stack= before 11: (bf) r1 = r7") 405 __msg("mark_precise: frame0: regs=r6 stack= before 10: (27) r6 *= 4") 406 __msg("mark_precise: frame0: regs=r6 stack= before 9: (85) call bpf_loop") 407 __msg("mark_precise: frame0: parent state regs=r6 stack=:") 408 __msg("mark_precise: frame0: last_idx 8 first_idx 0 subseq_idx 9") 409 __msg("mark_precise: frame0: regs=r6 stack= before 8: (b7) r4 = 0") 410 __msg("mark_precise: frame0: regs=r6 stack= before 7: (b7) r3 = 0") 411 __msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r2 = r8") 412 __msg("mark_precise: frame0: regs=r6 stack= before 5: (b7) r1 = 1") 413 __msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3") 414 /* State entering callback body popped from states stack */ 415 __msg("from 9 to 15: frame1:") 416 __msg("15: frame1: R1=scalar() R2=0 R10=fp0 cb") 417 __msg("15: (b7) r0 = 0") 418 __msg("16: (95) exit") 419 __msg("returning from callee:") 420 __msg("to caller at 9:") 421 /* r1, r4 are always precise for bpf_loop(), 422 * r6 was marked before backtracking to callback body. 423 */ 424 __msg("frame 0: propagating r1,r4,r6") 425 __msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") 426 __msg("mark_precise: frame0: regs=r1,r4,r6 stack= before 16: (95) exit") 427 __msg("mark_precise: frame1: regs= stack= before 15: (b7) r0 = 0") 428 __msg("mark_precise: frame1: regs= stack= before 9: (85) call bpf_loop") 429 __msg("mark_precise: frame0: parent state regs= stack=:") 430 __msg("from 16 to 9: safe") 431 __naked int parent_callee_saved_reg_precise_with_callback(void) 432 { 433 asm volatile ( 434 "r6 = 3;" 435 436 /* call subprog and ignore result; we need this call only to 437 * complicate jump history 438 */ 439 "r1 = 1;" /* nr_loops */ 440 "r2 = %[callback_subprog];" /* callback_fn */ 441 "r3 = 0;" /* callback_ctx */ 442 "r4 = 0;" /* flags */ 443 "call %[bpf_loop];" 444 445 "r6 *= 4;" 446 "r1 = %[vals];" 447 /* here r6 is forced to be precise and has to be propagated 448 * back to the beginning, handling (and ignoring) callback call 449 */ 450 "r1 += r6;" 451 "r0 = *(u32 *)(r1 + 0);" 452 "exit;" 453 : 454 : __imm_ptr(vals), 455 __imm_ptr(callback_subprog), 456 __imm(bpf_loop) 457 : __clobber_common, "r6" 458 ); 459 } 460 461 SEC("?raw_tp") 462 __success __log_level(2) 463 __msg("9: (0f) r1 += r6") 464 __msg("mark_precise: frame0: last_idx 9 first_idx 6") 465 __msg("mark_precise: frame0: regs=r6 stack= before 8: (bf) r1 = r7") 466 __msg("mark_precise: frame0: regs=r6 stack= before 7: (27) r6 *= 4") 467 __msg("mark_precise: frame0: regs=r6 stack= before 6: (79) r6 = *(u64 *)(r10 -8)") 468 __msg("mark_precise: frame0: parent state regs= stack=-8:") 469 __msg("mark_precise: frame0: last_idx 13 first_idx 0") 470 __msg("mark_precise: frame0: regs= stack=-8 before 13: (95) exit") 471 __msg("mark_precise: frame1: regs= stack= before 12: (bf) r0 = r1") 472 __msg("mark_precise: frame1: regs= stack= before 5: (85) call pc+6") 473 __msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r1 = 0") 474 __msg("mark_precise: frame0: regs= stack=-8 before 3: (7b) *(u64 *)(r10 -8) = r6") 475 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 476 __naked int parent_stack_slot_precise(void) 477 { 478 asm volatile ( 479 /* spill reg */ 480 "r6 = 3;" 481 "*(u64 *)(r10 - 8) = r6;" 482 483 /* call subprog and ignore result; we need this call only to 484 * complicate jump history 485 */ 486 "r1 = 0;" 487 "call identity_subprog;" 488 489 /* restore reg from stack; in this case we'll be carrying 490 * stack mask when going back into subprog through jump 491 * history 492 */ 493 "r6 = *(u64 *)(r10 - 8);" 494 495 "r6 *= 4;" 496 "r1 = %[vals];" 497 /* here r6 is forced to be precise and has to be propagated 498 * back to the beginning, handling (and ignoring) subprog call 499 */ 500 "r1 += r6;" 501 "r0 = *(u32 *)(r1 + 0);" 502 "exit;" 503 : 504 : __imm_ptr(vals) 505 : __clobber_common, "r6" 506 ); 507 } 508 509 SEC("?raw_tp") 510 __success __log_level(2) 511 __msg("9: (0f) r1 += r6") 512 __msg("mark_precise: frame0: last_idx 9 first_idx 0") 513 __msg("mark_precise: frame0: regs=r6 stack= before 8: (bf) r1 = r7") 514 __msg("mark_precise: frame0: regs=r6 stack= before 7: (27) r6 *= 4") 515 __msg("mark_precise: frame0: regs=r6 stack= before 6: (79) r6 = *(u64 *)(r10 -8)") 516 __msg("mark_precise: frame0: regs= stack=-8 before 5: (85) call pc+6") 517 __msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r1 = 0") 518 __msg("mark_precise: frame0: regs= stack=-8 before 3: (7b) *(u64 *)(r10 -8) = r6") 519 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 520 __naked int parent_stack_slot_precise_global(void) 521 { 522 asm volatile ( 523 /* spill reg */ 524 "r6 = 3;" 525 "*(u64 *)(r10 - 8) = r6;" 526 527 /* call subprog and ignore result; we need this call only to 528 * complicate jump history 529 */ 530 "r1 = 0;" 531 "call global_identity_subprog;" 532 533 /* restore reg from stack; in this case we'll be carrying 534 * stack mask when going back into subprog through jump 535 * history 536 */ 537 "r6 = *(u64 *)(r10 - 8);" 538 539 "r6 *= 4;" 540 "r1 = %[vals];" 541 /* here r6 is forced to be precise and has to be propagated 542 * back to the beginning, handling (and ignoring) subprog call 543 */ 544 "r1 += r6;" 545 "r0 = *(u32 *)(r1 + 0);" 546 "exit;" 547 : 548 : __imm_ptr(vals) 549 : __clobber_common, "r6" 550 ); 551 } 552 553 SEC("?raw_tp") 554 __success __log_level(2) 555 /* First simulated path does not include callback body */ 556 __msg("14: (0f) r1 += r6") 557 __msg("mark_precise: frame0: last_idx 14 first_idx 10") 558 __msg("mark_precise: frame0: regs=r6 stack= before 13: (bf) r1 = r7") 559 __msg("mark_precise: frame0: regs=r6 stack= before 12: (27) r6 *= 4") 560 __msg("mark_precise: frame0: regs=r6 stack= before 11: (79) r6 = *(u64 *)(r10 -8)") 561 __msg("mark_precise: frame0: regs= stack=-8 before 10: (85) call bpf_loop") 562 __msg("mark_precise: frame0: parent state regs= stack=-8:") 563 __msg("mark_precise: frame0: last_idx 9 first_idx 0 subseq_idx 10") 564 __msg("mark_precise: frame0: regs= stack=-8 before 9: (b7) r4 = 0") 565 __msg("mark_precise: frame0: regs= stack=-8 before 8: (b7) r3 = 0") 566 __msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r2 = r8") 567 __msg("mark_precise: frame0: regs= stack=-8 before 6: (bf) r1 = r6") 568 __msg("mark_precise: frame0: regs= stack=-8 before 5: (7b) *(u64 *)(r10 -8) = r6") 569 __msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3") 570 /* State entering callback body popped from states stack */ 571 __msg("from 10 to 17: frame1:") 572 __msg("17: frame1: R1=scalar() R2=0 R10=fp0 cb") 573 __msg("17: (b7) r0 = 0") 574 __msg("18: (95) exit") 575 __msg("returning from callee:") 576 __msg("to caller at 10:") 577 /* r1, r4 are always precise for bpf_loop(), 578 * fp-8 was marked before backtracking to callback body. 579 */ 580 __msg("frame 0: propagating r1,r4,fp-8") 581 __msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx -1") 582 __msg("mark_precise: frame0: regs=r1,r4 stack=-8 before 18: (95) exit") 583 __msg("mark_precise: frame1: regs= stack= before 17: (b7) r0 = 0") 584 __msg("mark_precise: frame1: regs= stack= before 10: (85) call bpf_loop#181") 585 __msg("mark_precise: frame0: parent state regs= stack=:") 586 __msg("from 18 to 10: safe") 587 __naked int parent_stack_slot_precise_with_callback(void) 588 { 589 asm volatile ( 590 /* spill reg */ 591 "r6 = 3;" 592 "*(u64 *)(r10 - 8) = r6;" 593 594 /* ensure we have callback frame in jump history */ 595 "r1 = r6;" /* nr_loops */ 596 "r2 = %[callback_subprog];" /* callback_fn */ 597 "r3 = 0;" /* callback_ctx */ 598 "r4 = 0;" /* flags */ 599 "call %[bpf_loop];" 600 601 /* restore reg from stack; in this case we'll be carrying 602 * stack mask when going back into subprog through jump 603 * history 604 */ 605 "r6 = *(u64 *)(r10 - 8);" 606 607 "r6 *= 4;" 608 "r1 = %[vals];" 609 /* here r6 is forced to be precise and has to be propagated 610 * back to the beginning, handling (and ignoring) subprog call 611 */ 612 "r1 += r6;" 613 "r0 = *(u32 *)(r1 + 0);" 614 "exit;" 615 : 616 : __imm_ptr(vals), 617 __imm_ptr(callback_subprog), 618 __imm(bpf_loop) 619 : __clobber_common, "r6" 620 ); 621 } 622 623 __noinline __used 624 static __u64 subprog_with_precise_arg(__u64 x) 625 { 626 return vals[x]; /* x is forced to be precise */ 627 } 628 629 SEC("?raw_tp") 630 __success __log_level(2) 631 __msg("8: (0f) r2 += r1") 632 __msg("mark_precise: frame1: last_idx 8 first_idx 0") 633 __msg("mark_precise: frame1: regs=r1 stack= before 6: (18) r2 = ") 634 __msg("mark_precise: frame1: regs=r1 stack= before 5: (67) r1 <<= 2") 635 __msg("mark_precise: frame1: regs=r1 stack= before 2: (85) call pc+2") 636 __msg("mark_precise: frame0: regs=r1 stack= before 1: (bf) r1 = r6") 637 __msg("mark_precise: frame0: regs=r6 stack= before 0: (b7) r6 = 3") 638 __naked int subprog_arg_precise(void) 639 { 640 asm volatile ( 641 "r6 = 3;" 642 "r1 = r6;" 643 /* subprog_with_precise_arg expects its argument to be 644 * precise, so r1->r6 will be marked precise from inside the 645 * subprog 646 */ 647 "call subprog_with_precise_arg;" 648 "r0 += r6;" 649 "exit;" 650 : 651 : 652 : __clobber_common, "r6" 653 ); 654 } 655 656 /* r1 is pointer to stack slot; 657 * r2 is a register to spill into that slot 658 * subprog also spills r2 into its own stack slot 659 */ 660 __naked __noinline __used 661 static __u64 subprog_spill_reg_precise(void) 662 { 663 asm volatile ( 664 /* spill to parent stack */ 665 "*(u64 *)(r1 + 0) = r2;" 666 /* spill to subprog stack (we use -16 offset to avoid 667 * accidental confusion with parent's -8 stack slot in 668 * verifier log output) 669 */ 670 "*(u64 *)(r10 - 16) = r2;" 671 /* use both spills as return result to propagete precision everywhere */ 672 "r0 = *(u64 *)(r10 - 16);" 673 "r2 = *(u64 *)(r1 + 0);" 674 "r0 += r2;" 675 "exit;" 676 ); 677 } 678 679 SEC("?raw_tp") 680 __success __log_level(2) 681 __msg("10: (0f) r1 += r7") 682 __msg("mark_precise: frame0: last_idx 10 first_idx 7 subseq_idx -1") 683 __msg("mark_precise: frame0: regs=r7 stack= before 9: (bf) r1 = r8") 684 __msg("mark_precise: frame0: regs=r7 stack= before 8: (27) r7 *= 4") 685 __msg("mark_precise: frame0: regs=r7 stack= before 7: (79) r7 = *(u64 *)(r10 -8)") 686 __msg("mark_precise: frame0: parent state regs= stack=-8: R0_w=2 R6_w=1 R8_rw=map_value(map=.data.vals,ks=4,vs=16) R10=fp0 fp-8_rw=P1") 687 __msg("mark_precise: frame0: last_idx 18 first_idx 0 subseq_idx 7") 688 __msg("mark_precise: frame0: regs= stack=-8 before 18: (95) exit") 689 __msg("mark_precise: frame1: regs= stack= before 17: (0f) r0 += r2") 690 __msg("mark_precise: frame1: regs= stack= before 16: (79) r2 = *(u64 *)(r1 +0)") 691 __msg("mark_precise: frame1: regs= stack= before 15: (79) r0 = *(u64 *)(r10 -16)") 692 __msg("mark_precise: frame1: regs= stack= before 14: (7b) *(u64 *)(r10 -16) = r2") 693 __msg("mark_precise: frame1: regs= stack= before 13: (7b) *(u64 *)(r1 +0) = r2") 694 __msg("mark_precise: frame1: regs=r2 stack= before 6: (85) call pc+6") 695 __msg("mark_precise: frame0: regs=r2 stack= before 5: (bf) r2 = r6") 696 __msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") 697 __msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") 698 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 1") 699 __naked int subprog_spill_into_parent_stack_slot_precise(void) 700 { 701 asm volatile ( 702 "r6 = 1;" 703 704 /* pass pointer to stack slot and r6 to subprog; 705 * r6 will be marked precise and spilled into fp-8 slot, which 706 * also should be marked precise 707 */ 708 "r1 = r10;" 709 "r1 += -8;" 710 "r2 = r6;" 711 "call subprog_spill_reg_precise;" 712 713 /* restore reg from stack; in this case we'll be carrying 714 * stack mask when going back into subprog through jump 715 * history 716 */ 717 "r7 = *(u64 *)(r10 - 8);" 718 719 "r7 *= 4;" 720 "r1 = %[vals];" 721 /* here r7 is forced to be precise and has to be propagated 722 * back to the beginning, handling subprog call and logic 723 */ 724 "r1 += r7;" 725 "r0 = *(u32 *)(r1 + 0);" 726 "exit;" 727 : 728 : __imm_ptr(vals) 729 : __clobber_common, "r6", "r7" 730 ); 731 } 732 733 SEC("?raw_tp") 734 __success __log_level(2) 735 __msg("17: (0f) r1 += r0") 736 __msg("mark_precise: frame0: last_idx 17 first_idx 0 subseq_idx -1") 737 __msg("mark_precise: frame0: regs=r0 stack= before 16: (bf) r1 = r7") 738 __msg("mark_precise: frame0: regs=r0 stack= before 15: (27) r0 *= 4") 739 __msg("mark_precise: frame0: regs=r0 stack= before 14: (79) r0 = *(u64 *)(r10 -16)") 740 __msg("mark_precise: frame0: regs= stack=-16 before 13: (7b) *(u64 *)(r7 -8) = r0") 741 __msg("mark_precise: frame0: regs=r0 stack= before 12: (79) r0 = *(u64 *)(r8 +16)") 742 __msg("mark_precise: frame0: regs= stack=-16 before 11: (7b) *(u64 *)(r8 +16) = r0") 743 __msg("mark_precise: frame0: regs=r0 stack= before 10: (79) r0 = *(u64 *)(r7 -8)") 744 __msg("mark_precise: frame0: regs= stack=-16 before 9: (7b) *(u64 *)(r10 -16) = r0") 745 __msg("mark_precise: frame0: regs=r0 stack= before 8: (07) r8 += -32") 746 __msg("mark_precise: frame0: regs=r0 stack= before 7: (bf) r8 = r10") 747 __msg("mark_precise: frame0: regs=r0 stack= before 6: (07) r7 += -8") 748 __msg("mark_precise: frame0: regs=r0 stack= before 5: (bf) r7 = r10") 749 __msg("mark_precise: frame0: regs=r0 stack= before 21: (95) exit") 750 __msg("mark_precise: frame1: regs=r0 stack= before 20: (bf) r0 = r1") 751 __msg("mark_precise: frame1: regs=r1 stack= before 4: (85) call pc+15") 752 __msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r1 = r6") 753 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 1") 754 __naked int stack_slot_aliases_precision(void) 755 { 756 asm volatile ( 757 "r6 = 1;" 758 /* pass r6 through r1 into subprog to get it back as r0; 759 * this whole chain will have to be marked as precise later 760 */ 761 "r1 = r6;" 762 "call identity_subprog;" 763 /* let's setup two registers that are aliased to r10 */ 764 "r7 = r10;" 765 "r7 += -8;" /* r7 = r10 - 8 */ 766 "r8 = r10;" 767 "r8 += -32;" /* r8 = r10 - 32 */ 768 /* now spill subprog's return value (a r6 -> r1 -> r0 chain) 769 * a few times through different stack pointer regs, making 770 * sure to use r10, r7, and r8 both in LDX and STX insns, and 771 * *importantly* also using a combination of const var_off and 772 * insn->off to validate that we record final stack slot 773 * correctly, instead of relying on just insn->off derivation, 774 * which is only valid for r10-based stack offset 775 */ 776 "*(u64 *)(r10 - 16) = r0;" 777 "r0 = *(u64 *)(r7 - 8);" /* r7 - 8 == r10 - 16 */ 778 "*(u64 *)(r8 + 16) = r0;" /* r8 + 16 = r10 - 16 */ 779 "r0 = *(u64 *)(r8 + 16);" 780 "*(u64 *)(r7 - 8) = r0;" 781 "r0 = *(u64 *)(r10 - 16);" 782 /* get ready to use r0 as an index into array to force precision */ 783 "r0 *= 4;" 784 "r1 = %[vals];" 785 /* here r0->r1->r6 chain is forced to be precise and has to be 786 * propagated back to the beginning, including through the 787 * subprog call and all the stack spills and loads 788 */ 789 "r1 += r0;" 790 "r0 = *(u32 *)(r1 + 0);" 791 "exit;" 792 : 793 : __imm_ptr(vals) 794 : __clobber_common, "r6" 795 ); 796 } 797 798 char _license[] SEC("license") = "GPL"; 799