1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ 3 4 #include <errno.h> 5 #include <string.h> 6 #include <linux/bpf.h> 7 #include <bpf/bpf_helpers.h> 8 #include "bpf_misc.h" 9 10 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) 11 12 int vals[] SEC(".data.vals") = {1, 2, 3, 4}; 13 14 __naked __noinline __used 15 static unsigned long identity_subprog() 16 { 17 /* the simplest *static* 64-bit identity function */ 18 asm volatile ( 19 "r0 = r1;" 20 "exit;" 21 ); 22 } 23 24 __noinline __used 25 unsigned long global_identity_subprog(__u64 x) 26 { 27 /* the simplest *global* 64-bit identity function */ 28 return x; 29 } 30 31 __naked __noinline __used 32 static unsigned long callback_subprog() 33 { 34 /* the simplest callback function */ 35 asm volatile ( 36 "r0 = 0;" 37 "exit;" 38 ); 39 } 40 41 SEC("?raw_tp") 42 __success __log_level(2) 43 __msg("7: (0f) r1 += r0") 44 __msg("mark_precise: frame0: regs=r0 stack= before 6: (bf) r1 = r7") 45 __msg("mark_precise: frame0: regs=r0 stack= before 5: (27) r0 *= 4") 46 __msg("mark_precise: frame0: regs=r0 stack= before 11: (95) exit") 47 __msg("mark_precise: frame1: regs=r0 stack= before 10: (bf) r0 = r1") 48 __msg("mark_precise: frame1: regs=r1 stack= before 4: (85) call pc+5") 49 __msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r1 = r6") 50 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 51 __naked int subprog_result_precise(void) 52 { 53 asm volatile ( 54 "r6 = 3;" 55 /* pass r6 through r1 into subprog to get it back as r0; 56 * this whole chain will have to be marked as precise later 57 */ 58 "r1 = r6;" 59 "call identity_subprog;" 60 /* now use subprog's returned value (which is a 61 * r6 -> r1 -> r0 chain), as index into vals array, forcing 62 * all of that to be known precisely 63 */ 64 "r0 *= 4;" 65 "r1 = %[vals];" 66 /* here r0->r1->r6 chain is forced to be precise and has to be 67 * propagated back to the beginning, including through the 68 * subprog call 69 */ 70 "r1 += r0;" 71 "r0 = *(u32 *)(r1 + 0);" 72 "exit;" 73 : 74 : __imm_ptr(vals) 75 : __clobber_common, "r6" 76 ); 77 } 78 79 SEC("?raw_tp") 80 __success __log_level(2) 81 __msg("9: (0f) r1 += r0") 82 __msg("mark_precise: frame0: last_idx 9 first_idx 0") 83 __msg("mark_precise: frame0: regs=r0 stack= before 8: (bf) r1 = r7") 84 __msg("mark_precise: frame0: regs=r0 stack= before 7: (27) r0 *= 4") 85 __msg("mark_precise: frame0: regs=r0 stack= before 5: (a5) if r0 < 0x4 goto pc+1") 86 __msg("mark_precise: frame0: regs=r0 stack= before 4: (85) call pc+7") 87 __naked int global_subprog_result_precise(void) 88 { 89 asm volatile ( 90 "r6 = 3;" 91 /* pass r6 through r1 into subprog to get it back as r0; 92 * given global_identity_subprog is global, precision won't 93 * propagate all the way back to r6 94 */ 95 "r1 = r6;" 96 "call global_identity_subprog;" 97 /* now use subprog's returned value (which is unknown now, so 98 * we need to clamp it), as index into vals array, forcing r0 99 * to be marked precise (with no effect on r6, though) 100 */ 101 "if r0 < %[vals_arr_sz] goto 1f;" 102 "r0 = %[vals_arr_sz] - 1;" 103 "1:" 104 "r0 *= 4;" 105 "r1 = %[vals];" 106 /* here r0 is forced to be precise and has to be 107 * propagated back to the global subprog call, but it 108 * shouldn't go all the way to mark r6 as precise 109 */ 110 "r1 += r0;" 111 "r0 = *(u32 *)(r1 + 0);" 112 "exit;" 113 : 114 : __imm_ptr(vals), 115 __imm_const(vals_arr_sz, ARRAY_SIZE(vals)) 116 : __clobber_common, "r6" 117 ); 118 } 119 120 SEC("?raw_tp") 121 __success __log_level(2) 122 __msg("14: (0f) r1 += r6") 123 __msg("mark_precise: frame0: last_idx 14 first_idx 10") 124 __msg("mark_precise: frame0: regs=r6 stack= before 13: (bf) r1 = r7") 125 __msg("mark_precise: frame0: regs=r6 stack= before 12: (27) r6 *= 4") 126 __msg("mark_precise: frame0: regs=r6 stack= before 11: (25) if r6 > 0x3 goto pc+4") 127 __msg("mark_precise: frame0: regs=r6 stack= before 10: (bf) r6 = r0") 128 __msg("mark_precise: frame0: parent state regs=r0 stack=:") 129 __msg("mark_precise: frame0: last_idx 18 first_idx 0") 130 __msg("mark_precise: frame0: regs=r0 stack= before 18: (95) exit") 131 __naked int callback_result_precise(void) 132 { 133 asm volatile ( 134 "r6 = 3;" 135 136 /* call subprog and use result; r0 shouldn't propagate back to 137 * callback_subprog 138 */ 139 "r1 = r6;" /* nr_loops */ 140 "r2 = %[callback_subprog];" /* callback_fn */ 141 "r3 = 0;" /* callback_ctx */ 142 "r4 = 0;" /* flags */ 143 "call %[bpf_loop];" 144 145 "r6 = r0;" 146 "if r6 > 3 goto 1f;" 147 "r6 *= 4;" 148 "r1 = %[vals];" 149 /* here r6 is forced to be precise and has to be propagated 150 * back to the bpf_loop() call, but not beyond 151 */ 152 "r1 += r6;" 153 "r0 = *(u32 *)(r1 + 0);" 154 "1:" 155 "exit;" 156 : 157 : __imm_ptr(vals), 158 __imm_ptr(callback_subprog), 159 __imm(bpf_loop) 160 : __clobber_common, "r6" 161 ); 162 } 163 164 SEC("?raw_tp") 165 __success __log_level(2) 166 __msg("7: (0f) r1 += r6") 167 __msg("mark_precise: frame0: last_idx 7 first_idx 0") 168 __msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r1 = r7") 169 __msg("mark_precise: frame0: regs=r6 stack= before 5: (27) r6 *= 4") 170 __msg("mark_precise: frame0: regs=r6 stack= before 11: (95) exit") 171 __msg("mark_precise: frame1: regs= stack= before 10: (bf) r0 = r1") 172 __msg("mark_precise: frame1: regs= stack= before 4: (85) call pc+5") 173 __msg("mark_precise: frame0: regs=r6 stack= before 3: (b7) r1 = 0") 174 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 175 __naked int parent_callee_saved_reg_precise(void) 176 { 177 asm volatile ( 178 "r6 = 3;" 179 180 /* call subprog and ignore result; we need this call only to 181 * complicate jump history 182 */ 183 "r1 = 0;" 184 "call identity_subprog;" 185 186 "r6 *= 4;" 187 "r1 = %[vals];" 188 /* here r6 is forced to be precise and has to be propagated 189 * back to the beginning, handling (and ignoring) subprog call 190 */ 191 "r1 += r6;" 192 "r0 = *(u32 *)(r1 + 0);" 193 "exit;" 194 : 195 : __imm_ptr(vals) 196 : __clobber_common, "r6" 197 ); 198 } 199 200 SEC("?raw_tp") 201 __success __log_level(2) 202 __msg("7: (0f) r1 += r6") 203 __msg("mark_precise: frame0: last_idx 7 first_idx 0") 204 __msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r1 = r7") 205 __msg("mark_precise: frame0: regs=r6 stack= before 5: (27) r6 *= 4") 206 __msg("mark_precise: frame0: regs=r6 stack= before 4: (85) call pc+5") 207 __msg("mark_precise: frame0: regs=r6 stack= before 3: (b7) r1 = 0") 208 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 209 __naked int parent_callee_saved_reg_precise_global(void) 210 { 211 asm volatile ( 212 "r6 = 3;" 213 214 /* call subprog and ignore result; we need this call only to 215 * complicate jump history 216 */ 217 "r1 = 0;" 218 "call global_identity_subprog;" 219 220 "r6 *= 4;" 221 "r1 = %[vals];" 222 /* here r6 is forced to be precise and has to be propagated 223 * back to the beginning, handling (and ignoring) subprog call 224 */ 225 "r1 += r6;" 226 "r0 = *(u32 *)(r1 + 0);" 227 "exit;" 228 : 229 : __imm_ptr(vals) 230 : __clobber_common, "r6" 231 ); 232 } 233 234 SEC("?raw_tp") 235 __success __log_level(2) 236 __msg("12: (0f) r1 += r6") 237 __msg("mark_precise: frame0: last_idx 12 first_idx 10") 238 __msg("mark_precise: frame0: regs=r6 stack= before 11: (bf) r1 = r7") 239 __msg("mark_precise: frame0: regs=r6 stack= before 10: (27) r6 *= 4") 240 __msg("mark_precise: frame0: parent state regs=r6 stack=:") 241 __msg("mark_precise: frame0: last_idx 16 first_idx 0") 242 __msg("mark_precise: frame0: regs=r6 stack= before 16: (95) exit") 243 __msg("mark_precise: frame1: regs= stack= before 15: (b7) r0 = 0") 244 __msg("mark_precise: frame1: regs= stack= before 9: (85) call bpf_loop#181") 245 __msg("mark_precise: frame0: regs=r6 stack= before 8: (b7) r4 = 0") 246 __msg("mark_precise: frame0: regs=r6 stack= before 7: (b7) r3 = 0") 247 __msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r2 = r8") 248 __msg("mark_precise: frame0: regs=r6 stack= before 5: (b7) r1 = 1") 249 __msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3") 250 __naked int parent_callee_saved_reg_precise_with_callback(void) 251 { 252 asm volatile ( 253 "r6 = 3;" 254 255 /* call subprog and ignore result; we need this call only to 256 * complicate jump history 257 */ 258 "r1 = 1;" /* nr_loops */ 259 "r2 = %[callback_subprog];" /* callback_fn */ 260 "r3 = 0;" /* callback_ctx */ 261 "r4 = 0;" /* flags */ 262 "call %[bpf_loop];" 263 264 "r6 *= 4;" 265 "r1 = %[vals];" 266 /* here r6 is forced to be precise and has to be propagated 267 * back to the beginning, handling (and ignoring) callback call 268 */ 269 "r1 += r6;" 270 "r0 = *(u32 *)(r1 + 0);" 271 "exit;" 272 : 273 : __imm_ptr(vals), 274 __imm_ptr(callback_subprog), 275 __imm(bpf_loop) 276 : __clobber_common, "r6" 277 ); 278 } 279 280 SEC("?raw_tp") 281 __success __log_level(2) 282 __msg("9: (0f) r1 += r6") 283 __msg("mark_precise: frame0: last_idx 9 first_idx 6") 284 __msg("mark_precise: frame0: regs=r6 stack= before 8: (bf) r1 = r7") 285 __msg("mark_precise: frame0: regs=r6 stack= before 7: (27) r6 *= 4") 286 __msg("mark_precise: frame0: regs=r6 stack= before 6: (79) r6 = *(u64 *)(r10 -8)") 287 __msg("mark_precise: frame0: parent state regs= stack=-8:") 288 __msg("mark_precise: frame0: last_idx 13 first_idx 0") 289 __msg("mark_precise: frame0: regs= stack=-8 before 13: (95) exit") 290 __msg("mark_precise: frame1: regs= stack= before 12: (bf) r0 = r1") 291 __msg("mark_precise: frame1: regs= stack= before 5: (85) call pc+6") 292 __msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r1 = 0") 293 __msg("mark_precise: frame0: regs= stack=-8 before 3: (7b) *(u64 *)(r10 -8) = r6") 294 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 295 __naked int parent_stack_slot_precise(void) 296 { 297 asm volatile ( 298 /* spill reg */ 299 "r6 = 3;" 300 "*(u64 *)(r10 - 8) = r6;" 301 302 /* call subprog and ignore result; we need this call only to 303 * complicate jump history 304 */ 305 "r1 = 0;" 306 "call identity_subprog;" 307 308 /* restore reg from stack; in this case we'll be carrying 309 * stack mask when going back into subprog through jump 310 * history 311 */ 312 "r6 = *(u64 *)(r10 - 8);" 313 314 "r6 *= 4;" 315 "r1 = %[vals];" 316 /* here r6 is forced to be precise and has to be propagated 317 * back to the beginning, handling (and ignoring) subprog call 318 */ 319 "r1 += r6;" 320 "r0 = *(u32 *)(r1 + 0);" 321 "exit;" 322 : 323 : __imm_ptr(vals) 324 : __clobber_common, "r6" 325 ); 326 } 327 328 SEC("?raw_tp") 329 __success __log_level(2) 330 __msg("9: (0f) r1 += r6") 331 __msg("mark_precise: frame0: last_idx 9 first_idx 6") 332 __msg("mark_precise: frame0: regs=r6 stack= before 8: (bf) r1 = r7") 333 __msg("mark_precise: frame0: regs=r6 stack= before 7: (27) r6 *= 4") 334 __msg("mark_precise: frame0: regs=r6 stack= before 6: (79) r6 = *(u64 *)(r10 -8)") 335 __msg("mark_precise: frame0: parent state regs= stack=-8:") 336 __msg("mark_precise: frame0: last_idx 5 first_idx 0") 337 __msg("mark_precise: frame0: regs= stack=-8 before 5: (85) call pc+6") 338 __msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r1 = 0") 339 __msg("mark_precise: frame0: regs= stack=-8 before 3: (7b) *(u64 *)(r10 -8) = r6") 340 __msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 341 __naked int parent_stack_slot_precise_global(void) 342 { 343 asm volatile ( 344 /* spill reg */ 345 "r6 = 3;" 346 "*(u64 *)(r10 - 8) = r6;" 347 348 /* call subprog and ignore result; we need this call only to 349 * complicate jump history 350 */ 351 "r1 = 0;" 352 "call global_identity_subprog;" 353 354 /* restore reg from stack; in this case we'll be carrying 355 * stack mask when going back into subprog through jump 356 * history 357 */ 358 "r6 = *(u64 *)(r10 - 8);" 359 360 "r6 *= 4;" 361 "r1 = %[vals];" 362 /* here r6 is forced to be precise and has to be propagated 363 * back to the beginning, handling (and ignoring) subprog call 364 */ 365 "r1 += r6;" 366 "r0 = *(u32 *)(r1 + 0);" 367 "exit;" 368 : 369 : __imm_ptr(vals) 370 : __clobber_common, "r6" 371 ); 372 } 373 374 SEC("?raw_tp") 375 __success __log_level(2) 376 __msg("14: (0f) r1 += r6") 377 __msg("mark_precise: frame0: last_idx 14 first_idx 11") 378 __msg("mark_precise: frame0: regs=r6 stack= before 13: (bf) r1 = r7") 379 __msg("mark_precise: frame0: regs=r6 stack= before 12: (27) r6 *= 4") 380 __msg("mark_precise: frame0: regs=r6 stack= before 11: (79) r6 = *(u64 *)(r10 -8)") 381 __msg("mark_precise: frame0: parent state regs= stack=-8:") 382 __msg("mark_precise: frame0: last_idx 18 first_idx 0") 383 __msg("mark_precise: frame0: regs= stack=-8 before 18: (95) exit") 384 __msg("mark_precise: frame1: regs= stack= before 17: (b7) r0 = 0") 385 __msg("mark_precise: frame1: regs= stack= before 10: (85) call bpf_loop#181") 386 __msg("mark_precise: frame0: regs= stack=-8 before 9: (b7) r4 = 0") 387 __msg("mark_precise: frame0: regs= stack=-8 before 8: (b7) r3 = 0") 388 __msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r2 = r8") 389 __msg("mark_precise: frame0: regs= stack=-8 before 6: (bf) r1 = r6") 390 __msg("mark_precise: frame0: regs= stack=-8 before 5: (7b) *(u64 *)(r10 -8) = r6") 391 __msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3") 392 __naked int parent_stack_slot_precise_with_callback(void) 393 { 394 asm volatile ( 395 /* spill reg */ 396 "r6 = 3;" 397 "*(u64 *)(r10 - 8) = r6;" 398 399 /* ensure we have callback frame in jump history */ 400 "r1 = r6;" /* nr_loops */ 401 "r2 = %[callback_subprog];" /* callback_fn */ 402 "r3 = 0;" /* callback_ctx */ 403 "r4 = 0;" /* flags */ 404 "call %[bpf_loop];" 405 406 /* restore reg from stack; in this case we'll be carrying 407 * stack mask when going back into subprog through jump 408 * history 409 */ 410 "r6 = *(u64 *)(r10 - 8);" 411 412 "r6 *= 4;" 413 "r1 = %[vals];" 414 /* here r6 is forced to be precise and has to be propagated 415 * back to the beginning, handling (and ignoring) subprog call 416 */ 417 "r1 += r6;" 418 "r0 = *(u32 *)(r1 + 0);" 419 "exit;" 420 : 421 : __imm_ptr(vals), 422 __imm_ptr(callback_subprog), 423 __imm(bpf_loop) 424 : __clobber_common, "r6" 425 ); 426 } 427 428 __noinline __used 429 static __u64 subprog_with_precise_arg(__u64 x) 430 { 431 return vals[x]; /* x is forced to be precise */ 432 } 433 434 SEC("?raw_tp") 435 __success __log_level(2) 436 __msg("8: (0f) r2 += r1") 437 __msg("mark_precise: frame1: last_idx 8 first_idx 0") 438 __msg("mark_precise: frame1: regs=r1 stack= before 6: (18) r2 = ") 439 __msg("mark_precise: frame1: regs=r1 stack= before 5: (67) r1 <<= 2") 440 __msg("mark_precise: frame1: regs=r1 stack= before 2: (85) call pc+2") 441 __msg("mark_precise: frame0: regs=r1 stack= before 1: (bf) r1 = r6") 442 __msg("mark_precise: frame0: regs=r6 stack= before 0: (b7) r6 = 3") 443 __naked int subprog_arg_precise(void) 444 { 445 asm volatile ( 446 "r6 = 3;" 447 "r1 = r6;" 448 /* subprog_with_precise_arg expects its argument to be 449 * precise, so r1->r6 will be marked precise from inside the 450 * subprog 451 */ 452 "call subprog_with_precise_arg;" 453 "r0 += r6;" 454 "exit;" 455 : 456 : 457 : __clobber_common, "r6" 458 ); 459 } 460 461 /* r1 is pointer to stack slot; 462 * r2 is a register to spill into that slot 463 * subprog also spills r2 into its own stack slot 464 */ 465 __naked __noinline __used 466 static __u64 subprog_spill_reg_precise(void) 467 { 468 asm volatile ( 469 /* spill to parent stack */ 470 "*(u64 *)(r1 + 0) = r2;" 471 /* spill to subprog stack (we use -16 offset to avoid 472 * accidental confusion with parent's -8 stack slot in 473 * verifier log output) 474 */ 475 "*(u64 *)(r10 - 16) = r2;" 476 /* use both spills as return result to propagete precision everywhere */ 477 "r0 = *(u64 *)(r10 - 16);" 478 "r2 = *(u64 *)(r1 + 0);" 479 "r0 += r2;" 480 "exit;" 481 ); 482 } 483 484 SEC("?raw_tp") 485 __success __log_level(2) 486 /* precision backtracking can't currently handle stack access not through r10, 487 * so we won't be able to mark stack slot fp-8 as precise, and so will 488 * fallback to forcing all as precise 489 */ 490 __msg("mark_precise: frame0: falling back to forcing all scalars precise") 491 __naked int subprog_spill_into_parent_stack_slot_precise(void) 492 { 493 asm volatile ( 494 "r6 = 1;" 495 496 /* pass pointer to stack slot and r6 to subprog; 497 * r6 will be marked precise and spilled into fp-8 slot, which 498 * also should be marked precise 499 */ 500 "r1 = r10;" 501 "r1 += -8;" 502 "r2 = r6;" 503 "call subprog_spill_reg_precise;" 504 505 /* restore reg from stack; in this case we'll be carrying 506 * stack mask when going back into subprog through jump 507 * history 508 */ 509 "r7 = *(u64 *)(r10 - 8);" 510 511 "r7 *= 4;" 512 "r1 = %[vals];" 513 /* here r7 is forced to be precise and has to be propagated 514 * back to the beginning, handling subprog call and logic 515 */ 516 "r1 += r7;" 517 "r0 = *(u32 *)(r1 + 0);" 518 "exit;" 519 : 520 : __imm_ptr(vals) 521 : __clobber_common, "r6", "r7" 522 ); 523 } 524 525 __naked __noinline __used 526 static __u64 subprog_with_checkpoint(void) 527 { 528 asm volatile ( 529 "r0 = 0;" 530 /* guaranteed checkpoint if BPF_F_TEST_STATE_FREQ is used */ 531 "goto +0;" 532 "exit;" 533 ); 534 } 535 536 char _license[] SEC("license") = "GPL"; 537