1 // SPDX-License-Identifier: GPL-2.0-only 2 // Copyright (c) 2026 Meta Platforms, Inc. and affiliates. 3 4 #include <linux/bpf.h> 5 #include <linux/bpf_verifier.h> 6 #include <linux/btf.h> 7 #include <linux/ctype.h> 8 #include <linux/kernel.h> 9 #include <linux/list.h> 10 #include <linux/seq_buf.h> 11 #include <linux/slab.h> 12 #include <linux/stdarg.h> 13 #include <linux/string.h> 14 15 #include "disasm.h" 16 #include "diagnostics.h" 17 18 #define BPF_DIAG_TEXT_WIDTH 100 19 #define BPF_DIAG_CONTEXT 2 20 #define BPF_DIAG_CONTEXT_CNT (1 + BPF_DIAG_CONTEXT * 2) 21 #define BPF_DIAG_SOURCE_LANE_WIDTH 88 22 #define BPF_DIAG_TAB_WIDTH 8 23 #define BPF_DIAG_FMT_CHUNK_SIZE (PAGE_SIZE - sizeof(struct diag_fmt_chunk)) 24 #define BPF_DIAG_FMT_BUF_SIZE 256 25 #define BPF_DIAG_EVENT_LOG_MAX_SIZE (64U << 20) 26 #define DISASM_LINE_LEN 160 27 28 enum bpf_diag_mod_target_kind { 29 BPF_DIAG_MOD_TARGET_NONE, 30 BPF_DIAG_MOD_TARGET_REG, 31 BPF_DIAG_MOD_TARGET_STACK_ARG, 32 BPF_DIAG_MOD_TARGET_STACK_SLOT, 33 BPF_DIAG_MOD_TARGET_STACK_RANGE, 34 }; 35 36 struct bpf_diag_mod_target { 37 u32 frame_id; 38 union { 39 struct { 40 s16 min_off; 41 s16 max_off; 42 } range; 43 u16 spi; 44 u8 regno; 45 u8 stack_arg; 46 }; 47 u8 frameno; 48 u8 kind; 49 }; 50 51 static struct bpf_diag_mod_target diag_reg_target(u32 frame_id, u8 frameno, u8 regno) 52 { 53 return (struct bpf_diag_mod_target){ 54 .frame_id = frame_id, 55 .frameno = frameno, 56 .kind = BPF_DIAG_MOD_TARGET_REG, 57 .regno = regno, 58 }; 59 } 60 61 static struct bpf_diag_mod_target diag_stack_arg_target(u32 frame_id, u8 frameno, u8 slot) 62 { 63 return (struct bpf_diag_mod_target){ 64 .frame_id = frame_id, 65 .frameno = frameno, 66 .kind = BPF_DIAG_MOD_TARGET_STACK_ARG, 67 .stack_arg = slot, 68 }; 69 } 70 71 static struct bpf_diag_mod_target diag_stack_slot_target(u32 frame_id, u8 frameno, u16 spi) 72 { 73 return (struct bpf_diag_mod_target){ 74 .frame_id = frame_id, 75 .frameno = frameno, 76 .kind = BPF_DIAG_MOD_TARGET_STACK_SLOT, 77 .spi = spi, 78 }; 79 } 80 81 static struct bpf_diag_mod_target diag_stack_range_target(u32 frame_id, u8 frameno, 82 s16 min_off, s16 max_off) 83 { 84 return (struct bpf_diag_mod_target){ 85 .frame_id = frame_id, 86 .frameno = frameno, 87 .kind = BPF_DIAG_MOD_TARGET_STACK_RANGE, 88 .range.min_off = min_off, 89 .range.max_off = max_off, 90 }; 91 } 92 93 struct bpf_diag_reg_snapshot { 94 u32 type; 95 u32 btf_id; 96 const struct bpf_map *map_ptr; 97 const struct btf *btf; 98 struct tnum var_off; 99 struct cnum64 r64; 100 }; 101 102 enum bpf_diag_history_kind { 103 BPF_DIAG_HISTORY_BRANCH, 104 BPF_DIAG_HISTORY_MOD, 105 BPF_DIAG_HISTORY_REF_ACQUIRE, 106 BPF_DIAG_HISTORY_REF_RELEASE, 107 BPF_DIAG_HISTORY_CONTEXT, 108 }; 109 110 struct bpf_diag_history_event { 111 u32 insn_idx : 24; 112 u32 kind : 8; 113 u8 in_lineage : 1; 114 union { 115 struct { 116 bool cond_true; 117 } branch; 118 struct { 119 struct bpf_diag_mod_target target; 120 struct bpf_diag_mod_target origin; 121 struct bpf_diag_reg_snapshot old, new; 122 u8 reason; 123 bool origin_valid; 124 } mod; 125 struct { 126 u32 ref_id; 127 } ref; 128 struct { 129 u32 depth; 130 u8 kind; 131 bool enter; 132 } ctx; 133 }; 134 }; 135 136 struct disasm_line { 137 char text[DISASM_LINE_LEN]; 138 int idx; 139 bool valid; 140 }; 141 142 struct disasm_ctx { 143 struct bpf_verifier_env *env; 144 struct seq_buf seq; 145 }; 146 147 struct diag_fmt_chunk { 148 struct list_head node; 149 struct seq_buf seq; 150 char data[]; 151 }; 152 153 struct diag_fmt_mark { 154 struct diag_fmt_chunk *chunk; 155 size_t len; 156 }; 157 158 struct bpf_diag_log { 159 struct bpf_diag_history_event *events; 160 /* Sequence number of the oldest retained event on the active path. */ 161 u64 first_seq; 162 u32 cnt; 163 u32 cap; 164 u32 head; 165 bool growth_failed; 166 }; 167 168 struct bpf_diag_scratch { 169 struct bpf_linfo_source source_lines[BPF_DIAG_CONTEXT_CNT]; 170 struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT]; 171 }; 172 173 struct bpf_diag_mod_scope { 174 struct bpf_reg_state target_reg_snapshot; 175 struct bpf_diag_mod_target target; 176 struct bpf_diag_mod_target origin; 177 enum bpf_diag_mod_reason reason; 178 u32 insn_idx; 179 bool active; 180 bool origin_valid; 181 }; 182 183 struct bpf_diag { 184 struct bpf_diag_log log; 185 struct bpf_diag_scratch scratch; 186 struct list_head fmt_chunks; 187 struct bpf_diag_mod_scope mod; 188 u32 frame_id_gen; 189 }; 190 191 bool bpf_diag_enabled(const struct bpf_verifier_env *env) 192 { 193 return env->log.level & BPF_LOG_LEVEL; 194 } 195 196 static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); 197 198 int bpf_diag_init(struct bpf_verifier_env *env) 199 { 200 if (!bpf_diag_enabled(env)) 201 return 0; 202 203 env->diag = kzalloc_obj(struct bpf_diag, GFP_KERNEL_ACCOUNT); 204 if (!env->diag) 205 return -ENOMEM; 206 207 INIT_LIST_HEAD(&env->diag->fmt_chunks); 208 return 0; 209 } 210 211 void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state) 212 { 213 if (env->diag) 214 state->diag_frame_id = ++env->diag->frame_id_gen; 215 } 216 217 static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size) 218 { 219 struct bpf_diag *diag = env->diag; 220 struct diag_fmt_chunk *chunk; 221 size_t capacity, available; 222 char *buf; 223 224 if (!diag || !size || size > INT_MAX) 225 return NULL; 226 227 if (!list_empty(&diag->fmt_chunks)) { 228 chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); 229 available = seq_buf_get_buf(&chunk->seq, &buf); 230 if (available >= size) 231 goto commit; 232 } 233 234 capacity = max_t(size_t, BPF_DIAG_FMT_CHUNK_SIZE, size); 235 chunk = kmalloc(struct_size(chunk, data, capacity), GFP_KERNEL_ACCOUNT); 236 if (!chunk) 237 return NULL; 238 239 seq_buf_init(&chunk->seq, chunk->data, capacity); 240 list_add_tail(&chunk->node, &diag->fmt_chunks); 241 available = seq_buf_get_buf(&chunk->seq, &buf); 242 if (WARN_ON_ONCE(available < size)) 243 return NULL; 244 245 commit: 246 seq_buf_commit(&chunk->seq, size); 247 return buf; 248 } 249 250 char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size) 251 { 252 char *buf; 253 254 buf = diag_fmt_alloc(env, size); 255 if (buf) 256 buf[0] = '\0'; 257 return buf; 258 } 259 260 const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) 261 { 262 va_list copy; 263 char *buf; 264 int len; 265 266 va_copy(copy, args); 267 len = vsnprintf(NULL, 0, fmt, copy); 268 va_end(copy); 269 if (len < 0 || len == INT_MAX) 270 return ""; 271 272 buf = diag_fmt_alloc(env, len + 1); 273 if (buf) 274 vsnprintf(buf, len + 1, fmt, args); 275 return buf ?: ""; 276 } 277 278 const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) 279 { 280 const char *buf; 281 va_list args; 282 283 va_start(args, fmt); 284 buf = bpf_diag_vfmt(env, fmt, args); 285 va_end(args); 286 return buf; 287 } 288 289 static struct diag_fmt_mark diag_fmt_save(struct bpf_verifier_env *env) 290 { 291 struct bpf_diag *diag = env->diag; 292 struct diag_fmt_mark mark = {}; 293 294 if (!diag || list_empty(&diag->fmt_chunks)) 295 return mark; 296 297 mark.chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); 298 mark.len = mark.chunk->seq.len; 299 return mark; 300 } 301 302 static void diag_fmt_restore(struct bpf_verifier_env *env, struct diag_fmt_mark mark) 303 { 304 struct bpf_diag *diag = env->diag; 305 struct diag_fmt_chunk *chunk; 306 307 if (!diag) 308 return; 309 310 while (!list_empty(&diag->fmt_chunks)) { 311 chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); 312 if (chunk == mark.chunk) 313 break; 314 list_del(&chunk->node); 315 kfree(chunk); 316 } 317 318 if (mark.chunk) { 319 mark.chunk->seq.len = mark.len; 320 seq_buf_str(&mark.chunk->seq); 321 } 322 } 323 324 void bpf_diag_free(struct bpf_verifier_env *env) 325 { 326 struct bpf_diag *diag = env->diag; 327 328 if (!diag) 329 return; 330 331 diag_fmt_restore(env, (struct diag_fmt_mark){}); 332 kvfree(diag->log.events); 333 kfree(diag); 334 env->diag = NULL; 335 } 336 337 static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) 338 { 339 va_list args; 340 341 if (!bpf_diag_enabled(env)) 342 return; 343 344 va_start(args, fmt); 345 bpf_verifier_vlog(&env->log, fmt, args); 346 va_end(args); 347 } 348 349 static u64 log_end(const struct bpf_diag_log *log) 350 { 351 return log->first_seq + log->cnt; 352 } 353 354 static u32 log_pos(const struct bpf_diag_log *log, u32 idx) 355 { 356 u32 pos = log->head + idx; 357 358 return pos < log->cap ? pos : pos - log->cap; 359 } 360 361 u64 bpf_diag_event_log_save(struct bpf_verifier_env *env) 362 { 363 struct bpf_diag *diag = env->diag; 364 365 return diag ? log_end(&diag->log) : 0; 366 } 367 368 void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos) 369 { 370 struct bpf_diag *diag = env->diag; 371 struct bpf_diag_log *log; 372 u64 end_seq; 373 374 if (!diag) 375 return; 376 377 log = &diag->log; 378 end_seq = log_end(log); 379 if (WARN_ON_ONCE(log_pos > end_seq)) 380 log_pos = end_seq; 381 382 /* 383 * A deep abandoned path may have rotated away the shared prefix. In 384 * that case, restart with an empty retained suffix and remember that 385 * every event before the restored mark is unavailable. 386 */ 387 if (log_pos <= log->first_seq) { 388 log->first_seq = log_pos; 389 log->head = 0; 390 log->cnt = 0; 391 return; 392 } 393 394 log->cnt = log_pos - log->first_seq; 395 } 396 397 u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state) 398 { 399 u32 depth = 0; 400 int i; 401 402 for (i = 0; i < state->acquired_refs; i++) { 403 if (state->refs[i].type == REF_TYPE_IRQ) 404 depth++; 405 } 406 407 return depth; 408 } 409 410 static void diag_append_history(struct bpf_verifier_env *env, 411 const struct bpf_diag_history_event *event) 412 { 413 struct bpf_diag_history_event *events; 414 struct bpf_diag *diag = env->diag; 415 struct bpf_diag_log *log; 416 u32 cap, max_events; 417 418 if (!diag) 419 return; 420 log = &diag->log; 421 422 if (log->cnt < log->cap) { 423 log->events[log_pos(log, log->cnt++)] = *event; 424 return; 425 } 426 427 max_events = BPF_DIAG_EVENT_LOG_MAX_SIZE / sizeof(*events); 428 if (log->growth_failed || log->cap == max_events) 429 goto rotate; 430 431 cap = min(log->cap ? log->cap * 2 : 64, max_events); 432 events = kvrealloc(log->events, array_size(cap, sizeof(*events)), GFP_KERNEL_ACCOUNT); 433 if (!events) { 434 log->growth_failed = true; 435 goto rotate; 436 } 437 log->events = events; 438 log->cap = cap; 439 log->events[log->cnt++] = *event; 440 return; 441 442 rotate: 443 if (log->cap) { 444 log->events[log->head++] = *event; 445 if (log->head == log->cap) 446 log->head = 0; 447 } 448 log->first_seq++; 449 } 450 451 static void diag_print_wrapped_prefixed(struct bpf_verifier_env *env, const char *first_prefix, 452 const char *next_prefix, const char *text) 453 { 454 const char *prefix = first_prefix; 455 456 while (*text) { 457 const char *line = text; 458 int prefix_len = strlen(prefix); 459 int text_width = BPF_DIAG_TEXT_WIDTH - prefix_len; 460 int len = 0, last_space = -1; 461 462 if (text_width < 1) 463 text_width = 1; 464 465 while (line[len] && line[len] != '\n' && len < text_width) { 466 if (line[len] == ' ') 467 last_space = len; 468 len++; 469 } 470 471 if (line[len] && line[len] != '\n' && line[len] != ' ' && last_space > 0) 472 len = last_space; 473 474 diag_write(env, "%s%.*s\n", prefix, len, line); 475 476 text = line + len; 477 while (*text == ' ') 478 text++; 479 if (*text == '\n') 480 text++; 481 482 prefix = next_prefix; 483 } 484 } 485 486 const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id) 487 { 488 char *buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); 489 size_t len; 490 int ret; 491 492 if (!buf) 493 return ""; 494 495 buf[0] = '\0'; 496 ret = btf_type_name_to_buf(btf, type_id, buf, BPF_DIAG_FMT_BUF_SIZE); 497 if (ret < 0 || !buf[0]) { 498 scnprintf(buf, BPF_DIAG_FMT_BUF_SIZE, "BTF type ID %u", type_id); 499 return buf; 500 } 501 502 len = strlen(buf); 503 if (len && buf[len - 1] == '{') 504 buf[len - 1] = '\0'; 505 return buf; 506 } 507 508 static int diag_line_width(unsigned int line) 509 { 510 int width = 1; 511 512 while (line >= 10) { 513 line /= 10; 514 width++; 515 } 516 517 return width; 518 } 519 520 static int diag_line_indent(const char *line) 521 { 522 int indent = 0; 523 524 while (*line == ' ' || *line == '\t') { 525 if (*line == '\t') 526 indent = round_up(indent + 1, BPF_DIAG_TAB_WIDTH); 527 else 528 indent++; 529 line++; 530 } 531 532 return indent; 533 } 534 535 static void disasm_print(void *private_data, const char *fmt, ...) __printf(2, 3); 536 537 static void disasm_print(void *private_data, const char *fmt, ...) 538 { 539 struct disasm_ctx *ctx = private_data; 540 va_list args; 541 542 va_start(args, fmt); 543 seq_buf_vprintf(&ctx->seq, fmt, args); 544 va_end(args); 545 } 546 547 static const char *disasm_kfunc_name(void *private_data, const struct bpf_insn *insn) 548 { 549 struct disasm_ctx *ctx = private_data; 550 551 return bpf_disasm_kfunc_name(ctx->env, insn); 552 } 553 554 static void format_disasm_line(struct bpf_verifier_env *env, int insn_idx, 555 struct disasm_line *line) 556 { 557 struct disasm_ctx ctx = { .env = env }; 558 struct bpf_insn *insn; 559 const struct bpf_insn_cbs cbs = { 560 .cb_call = disasm_kfunc_name, 561 .cb_print = disasm_print, 562 .private_data = &ctx, 563 }; 564 565 line->idx = insn_idx; 566 line->valid = false; 567 seq_buf_init(&ctx.seq, line->text, sizeof(line->text)); 568 569 if (insn_idx < 0 || insn_idx >= env->prog->len) 570 return; 571 572 if (insn_idx > 0 && bpf_is_ldimm64(&env->prog->insnsi[insn_idx - 1])) 573 return; 574 575 insn = &env->prog->insnsi[insn_idx]; 576 if (bpf_is_ldimm64(insn) && insn_idx + 1 >= env->prog->len) 577 return; 578 579 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 580 seq_buf_str(&ctx.seq); 581 ctx.seq.len = strnlen(line->text, sizeof(line->text)); 582 while (ctx.seq.len && line->text[ctx.seq.len - 1] == '\n') 583 seq_buf_pop(&ctx.seq); 584 seq_buf_str(&ctx.seq); 585 586 line->valid = true; 587 } 588 589 static void diag_format_source_text(char *buf, size_t size, const char *line, int width) 590 { 591 int col = 0, len = 0; 592 593 if (!size) 594 return; 595 if (width <= 0) { 596 buf[0] = '\0'; 597 return; 598 } 599 600 line = line ?: "..."; 601 while (*line && col < width && len + 1 < size) { 602 if (*line == '\t') { 603 int next = round_up(col + 1, BPF_DIAG_TAB_WIDTH); 604 605 while (col < next && col < width && len + 1 < size) { 606 buf[len++] = ' '; 607 col++; 608 } 609 line++; 610 continue; 611 } 612 613 buf[len++] = *line++; 614 col++; 615 } 616 617 if (*line) { 618 int ellipsis_len = min(3, width); 619 620 while (len > 0 && col > width - ellipsis_len) { 621 len--; 622 col--; 623 } 624 while (ellipsis_len-- && len + 1 < size) 625 buf[len++] = '.'; 626 } 627 628 buf[len] = '\0'; 629 } 630 631 static void diag_format_source_lane(char *buf, size_t size, const char *source_prefix, 632 int source_line_width, int line_num, const char *line) 633 { 634 int len, text_width; 635 636 if (line_num <= 0) { 637 buf[0] = '\0'; 638 return; 639 } 640 641 len = scnprintf(buf, size, "%s%*d | ", source_prefix, source_line_width, line_num); 642 text_width = BPF_DIAG_SOURCE_LANE_WIDTH - len; 643 diag_format_source_text(buf + len, size - len, line, text_width); 644 } 645 646 static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, 647 const char *problem) 648 { 649 char first; 650 651 if (!bpf_diag_enabled(env)) 652 return; 653 654 category = category ?: "Verifier Error"; 655 problem = problem ?: ""; 656 657 if (!problem[0]) { 658 diag_write(env, "\nVerification failed: %s\n", category); 659 return; 660 } 661 662 first = toupper(problem[0]); 663 diag_write(env, "\nVerification failed: %s: %c%s\n", category, first, problem + 1); 664 } 665 666 static void diag_print_source_annotation(struct bpf_verifier_env *env, int line_width, int indent, 667 const char *label, const char *msg) 668 { 669 const char *first_prefix, *next_prefix, *text; 670 671 indent = min_t(int, indent, max_t(int, 0, BPF_DIAG_SOURCE_LANE_WIDTH - line_width - 8)); 672 text = bpf_diag_fmt(env, "%s: %s", label, msg); 673 first_prefix = bpf_diag_fmt(env, " %*s | %*s^-- ", line_width + 4, "", indent, ""); 674 next_prefix = bpf_diag_fmt(env, " %*s | %*s ", line_width + 4, "", indent, ""); 675 676 diag_print_wrapped_prefixed(env, first_prefix, next_prefix, text); 677 } 678 679 static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, 680 struct disasm_line *disasm_lines) 681 { 682 int insn_width = diag_line_width(env->prog->len ? env->prog->len - 1 : 0); 683 int i; 684 685 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { 686 int row = i - BPF_DIAG_CONTEXT; 687 688 format_disasm_line(env, insn_idx + row, &disasm_lines[i]); 689 } 690 691 diag_write(env, " Instruction context:\n"); 692 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { 693 struct disasm_line *line = &disasm_lines[i]; 694 695 if (line->valid) 696 diag_write(env, " %s%*d | %s\n", 697 line->idx == insn_idx ? ">>> " : " ", 698 insn_width, line->idx, line->text); 699 } 700 } 701 702 static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, 703 const char *fmt, ...) 704 { 705 struct bpf_diag_scratch *scratch; 706 struct bpf_linfo_source *source_lines; 707 struct disasm_line *disasm_lines; 708 struct bpf_linfo_source src = {}; 709 struct diag_fmt_mark mark; 710 const struct bpf_line_info *linfo; 711 const struct bpf_subprog_info *subprog; 712 struct btf *btf = env->prog->aux->btf; 713 char *source_lane; 714 const char *msg; 715 const char *func; 716 int start_line, end_line, width, indent, subprogno, linfo_start, linfo_end, i; 717 va_list args; 718 719 if (!bpf_diag_enabled(env)) 720 return; 721 if (!env->diag) 722 return; 723 724 mark = diag_fmt_save(env); 725 label = label ?: "note"; 726 scratch = &env->diag->scratch; 727 source_lines = scratch->source_lines; 728 disasm_lines = scratch->disasm_lines; 729 memset(source_lines, 0, sizeof(scratch->source_lines)); 730 memset(disasm_lines, 0, sizeof(scratch->disasm_lines)); 731 732 va_start(args, fmt); 733 msg = bpf_diag_vfmt(env, fmt, args); 734 va_end(args); 735 if (!*msg) 736 msg = "<failed to allocate diagnostic text>"; 737 738 linfo = bpf_find_linfo(env->prog, insn_idx); 739 if (btf && linfo) 740 bpf_get_linfo_source(btf, linfo, &src); 741 if (!src.file || !*src.file || !src.line || !*src.line) { 742 diag_write(env, " insn %u\n", insn_idx); 743 diag_print_source_annotation(env, 0, 0, label, msg); 744 diag_print_insn_context(env, insn_idx, disasm_lines); 745 goto out_restore; 746 } 747 748 subprog = bpf_find_containing_subprog(env, insn_idx); 749 subprogno = subprog ? subprog - env->subprog_info : -ENOENT; 750 func = subprogno >= 0 ? bpf_subprog_name(env, subprogno) : NULL; 751 if (func && *func) 752 diag_write(env, " %s @ %s:%d:%d\n", func, src.file, src.line_num, src.line_col); 753 else 754 diag_write(env, " %s:%d:%d\n", src.file, src.line_num, src.line_col); 755 756 start_line = src.line_num - BPF_DIAG_CONTEXT; 757 end_line = src.line_num + BPF_DIAG_CONTEXT; 758 width = diag_line_width(end_line); 759 indent = diag_line_indent(src.line); 760 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) 761 source_lines[i].line_num = start_line + i; 762 763 linfo = env->prog->aux->linfo; 764 linfo_start = subprog ? subprog->linfo_idx : 0; 765 linfo_end = subprogno >= 0 && subprogno + 1 < env->subprog_cnt ? 766 env->subprog_info[subprogno + 1].linfo_idx : env->prog->aux->nr_linfo; 767 for (i = linfo_start; i < linfo_end; i++) { 768 struct bpf_linfo_source line_src; 769 int idx; 770 771 bpf_get_linfo_source(btf, &linfo[i], &line_src); 772 if (line_src.file_name_off != src.file_name_off || 773 line_src.line_num < start_line || line_src.line_num > end_line || 774 !line_src.line || !*line_src.line) 775 continue; 776 777 idx = line_src.line_num - start_line; 778 if (!source_lines[idx].line) 779 source_lines[idx] = line_src; 780 } 781 782 diag_write(env, " Source context:\n"); 783 source_lane = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); 784 if (!source_lane) 785 goto out_restore; 786 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { 787 const char *source_prefix; 788 789 source_prefix = source_lines[i].line_num == src.line_num ? ">>> " : " "; 790 diag_format_source_lane(source_lane, BPF_DIAG_FMT_BUF_SIZE, source_prefix, width, 791 source_lines[i].line_num, source_lines[i].line); 792 diag_write(env, " %s\n", source_lane); 793 if (source_lines[i].line_num == src.line_num) 794 diag_print_source_annotation(env, width, indent, label, msg); 795 } 796 diag_print_insn_context(env, insn_idx, disasm_lines); 797 798 out_restore: 799 diag_fmt_restore(env, mark); 800 } 801 802 void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true) 803 { 804 struct bpf_diag_history_event event = { 805 .insn_idx = insn_idx, 806 .kind = BPF_DIAG_HISTORY_BRANCH, 807 .branch = { 808 .cond_true = cond_true, 809 }, 810 }; 811 812 diag_append_history(env, &event); 813 } 814 815 static void diag_snapshot_reg(struct bpf_diag_reg_snapshot *snapshot, 816 const struct bpf_reg_state *reg) 817 { 818 snapshot->type = reg->type; 819 if (type_is_map_ptr(reg->type)) 820 snapshot->map_ptr = reg->map_ptr; 821 if (base_type(reg->type) == PTR_TO_BTF_ID && reg->btf && reg->btf_id) { 822 snapshot->btf_id = reg->btf_id; 823 snapshot->btf = reg->btf; 824 } 825 snapshot->var_off = reg->var_off; 826 snapshot->r64 = reg->r64; 827 } 828 829 static bool diag_mod_insn_origin(struct bpf_verifier_env *env, u32 insn_idx, 830 const struct bpf_diag_mod_target *target, 831 struct bpf_diag_mod_target *origin) 832 { 833 const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 834 u8 class = BPF_CLASS(insn->code); 835 const struct bpf_func_state *state; 836 837 if (target->kind == BPF_DIAG_MOD_TARGET_REG && (class == BPF_ALU || class == BPF_ALU64) && 838 BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X) { 839 *origin = diag_reg_target(target->frame_id, target->frameno, insn->src_reg); 840 return true; 841 } 842 843 if ((target->kind != BPF_DIAG_MOD_TARGET_STACK_ARG && 844 target->kind != BPF_DIAG_MOD_TARGET_STACK_SLOT) || 845 class != BPF_STX) 846 return false; 847 848 state = env->cur_state->frame[env->cur_state->curframe]; 849 *origin = diag_reg_target(state->diag_frame_id, state->frameno, insn->src_reg); 850 return true; 851 } 852 853 static bool diag_mod_keeps_lineage(struct bpf_verifier_env *env, 854 const struct bpf_diag_history_event *event) 855 { 856 const struct bpf_insn *insn; 857 u8 class; 858 859 if (event->mod.reason != BPF_DIAG_MOD_WRITE || 860 event->mod.target.kind != BPF_DIAG_MOD_TARGET_REG) 861 return false; 862 863 insn = &env->prog->insnsi[event->insn_idx]; 864 class = BPF_CLASS(insn->code); 865 if (class != BPF_ALU && class != BPF_ALU64) 866 return false; 867 868 switch (BPF_OP(insn->code)) { 869 case BPF_ADD: 870 case BPF_SUB: 871 case BPF_MUL: 872 case BPF_OR: 873 case BPF_AND: 874 case BPF_LSH: 875 case BPF_RSH: 876 case BPF_ARSH: 877 case BPF_XOR: 878 case BPF_NEG: 879 case BPF_END: 880 return true; 881 default: 882 return false; 883 } 884 } 885 886 static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, 887 struct bpf_diag_mod_target target, 888 enum bpf_diag_mod_reason reason, 889 const struct bpf_reg_state *old_reg, 890 const struct bpf_reg_state *new_reg, 891 const struct bpf_diag_mod_target *origin) 892 { 893 struct bpf_diag_history_event event = { 894 .insn_idx = insn_idx, 895 .kind = BPF_DIAG_HISTORY_MOD, 896 .mod = { 897 .target = target, 898 .reason = reason, 899 }, 900 }; 901 902 if (old_reg) 903 diag_snapshot_reg(&event.mod.old, old_reg); 904 if (new_reg) 905 diag_snapshot_reg(&event.mod.new, new_reg); 906 if (origin) { 907 event.mod.origin = *origin; 908 event.mod.origin_valid = true; 909 } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { 910 event.mod.origin_valid = true; 911 } 912 if (old_reg && new_reg && 913 (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) && 914 !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) && 915 !event.mod.origin_valid && 916 diag_mod_keeps_lineage(env, &event)) 917 return; 918 919 diag_append_history(env, &event); 920 } 921 922 static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env, 923 const struct bpf_diag_mod_target *target) 924 { 925 struct bpf_verifier_state *vstate = env->cur_state; 926 struct bpf_func_state *state; 927 928 state = target->frameno <= vstate->curframe ? vstate->frame[target->frameno] : NULL; 929 930 if (!state) 931 return NULL; 932 if (state->diag_frame_id != target->frame_id) 933 return NULL; 934 935 switch (target->kind) { 936 case BPF_DIAG_MOD_TARGET_REG: 937 if (target->regno >= MAX_BPF_REG) 938 return NULL; 939 return &state->regs[target->regno]; 940 case BPF_DIAG_MOD_TARGET_STACK_ARG: 941 if (target->stack_arg >= state->out_stack_arg_cnt) 942 return NULL; 943 return &state->stack_arg_regs[target->stack_arg]; 944 case BPF_DIAG_MOD_TARGET_STACK_SLOT: 945 if (target->spi >= state->allocated_stack / BPF_REG_SIZE) 946 return NULL; 947 return &state->stack[target->spi].spilled_ptr; 948 default: 949 return NULL; 950 } 951 } 952 953 static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, 954 struct bpf_diag_mod_target *target) 955 { 956 struct bpf_verifier_state *vstate = env->cur_state; 957 unsigned long addr = (unsigned long)reg; 958 int frame; 959 960 for (frame = 0; frame <= vstate->curframe; frame++) { 961 struct bpf_func_state *state = vstate->frame[frame]; 962 unsigned long start, end; 963 u32 nslots = state->allocated_stack / BPF_REG_SIZE; 964 int spi; 965 966 start = (unsigned long)state->regs; 967 end = (unsigned long)(state->regs + MAX_BPF_REG); 968 if (addr >= start && addr < end) { 969 *target = diag_reg_target(state->diag_frame_id, state->frameno, 970 reg - state->regs); 971 return true; 972 } 973 974 start = (unsigned long)state->stack_arg_regs; 975 end = (unsigned long)(state->stack_arg_regs + state->out_stack_arg_cnt); 976 if (state->out_stack_arg_cnt && addr >= start && addr < end) { 977 *target = diag_stack_arg_target(state->diag_frame_id, state->frameno, 978 reg - state->stack_arg_regs); 979 return true; 980 } 981 982 start = (unsigned long)state->stack; 983 end = (unsigned long)(state->stack + nslots); 984 if (nslots && addr >= start && addr < end) { 985 spi = ((const char *)reg - (const char *)state->stack) / 986 sizeof(*state->stack); 987 *target = diag_stack_slot_target(state->diag_frame_id, state->frameno, spi); 988 return true; 989 } 990 } 991 return false; 992 } 993 994 void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, 995 const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason) 996 { 997 struct bpf_diag *diag = env->diag; 998 999 if (!diag) 1000 return; 1001 diag->mod.active = reg_to_target(env, reg, &diag->mod.target); 1002 if (!diag->mod.active) 1003 return; 1004 diag->mod.target_reg_snapshot = *reg; 1005 diag->mod.insn_idx = env->insn_idx; 1006 diag->mod.reason = reason; 1007 diag->mod.origin_valid = origin && reg_to_target(env, origin, &diag->mod.origin); 1008 } 1009 1010 void bpf_diag_mod_end(struct bpf_verifier_env *env) 1011 { 1012 struct bpf_diag *diag = env->diag; 1013 const struct bpf_reg_state *new_reg; 1014 1015 if (!diag || !diag->mod.active) 1016 return; 1017 diag->mod.active = false; 1018 /* 1019 * Resolve the target again because the enclosing function state's stack 1020 * may have been reallocated while the modification was in progress. 1021 */ 1022 new_reg = target_to_reg(env, &diag->mod.target); 1023 if (!new_reg) 1024 return; 1025 diag_record_mod(env, diag->mod.insn_idx, diag->mod.target, diag->mod.reason, 1026 &diag->mod.target_reg_snapshot, new_reg, 1027 diag->mod.origin_valid ? &diag->mod.origin : NULL); 1028 } 1029 1030 void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, 1031 enum bpf_diag_mod_reason reason) 1032 { 1033 struct bpf_diag_mod_target target; 1034 1035 if (!env->diag || reg->type == NOT_INIT || !reg_to_target(env, reg, &target)) 1036 return; 1037 diag_record_mod(env, env->insn_idx, target, reason, reg, NULL, NULL); 1038 } 1039 1040 void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, 1041 const struct bpf_func_state *state, s16 min_off, s16 max_off, 1042 enum bpf_diag_mod_reason reason) 1043 { 1044 diag_record_mod(env, env->insn_idx, 1045 diag_stack_range_target(state->diag_frame_id, state->frameno, min_off, max_off), 1046 reason, NULL, NULL, NULL); 1047 } 1048 1049 static void diag_record_ref(struct bpf_verifier_env *env, u32 insn_idx, u8 kind, u32 ref_id) 1050 { 1051 struct bpf_diag_history_event event = { 1052 .insn_idx = insn_idx, 1053 .kind = kind, 1054 .ref = { 1055 .ref_id = ref_id, 1056 }, 1057 }; 1058 1059 diag_append_history(env, &event); 1060 } 1061 1062 void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) 1063 { 1064 diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_ACQUIRE, ref_id); 1065 } 1066 1067 void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) 1068 { 1069 diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_RELEASE, ref_id); 1070 } 1071 1072 void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx, 1073 enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth) 1074 { 1075 /* 1076 * Keep leave events so context rendering can stop at a depth-zero exit 1077 * and show nested-region depth accurately for the active path. 1078 */ 1079 struct bpf_diag_history_event event = { 1080 .insn_idx = insn_idx, 1081 .kind = BPF_DIAG_HISTORY_CONTEXT, 1082 .ctx = { 1083 .kind = ctx_kind, 1084 .enter = enter, 1085 .depth = depth, 1086 }, 1087 }; 1088 1089 diag_append_history(env, &event); 1090 } 1091