1 // SPDX-License-Identifier: GPL-2.0-only 2 // Copyright (c) 2026 Meta Platforms, Inc. and affiliates. 3 4 #include <linux/bpf.h> 5 #include <linux/bpf_verifier.h> 6 #include <linux/btf.h> 7 #include <linux/ctype.h> 8 #include <linux/kernel.h> 9 #include <linux/list.h> 10 #include <linux/seq_buf.h> 11 #include <linux/slab.h> 12 #include <linux/stdarg.h> 13 #include <linux/string.h> 14 15 #include "disasm.h" 16 #include "diagnostics.h" 17 18 #define BPF_DIAG_TEXT_WIDTH 100 19 #define BPF_DIAG_CONTEXT 2 20 #define BPF_DIAG_CONTEXT_CNT (1 + BPF_DIAG_CONTEXT * 2) 21 #define BPF_DIAG_SOURCE_LANE_WIDTH 88 22 #define BPF_DIAG_TAB_WIDTH 8 23 #define BPF_DIAG_FMT_CHUNK_SIZE (PAGE_SIZE - sizeof(struct diag_fmt_chunk)) 24 #define BPF_DIAG_FMT_BUF_SIZE 256 25 #define BPF_DIAG_EVENT_LOG_MAX_SIZE (64U << 20) 26 #define DISASM_LINE_LEN 160 27 28 enum bpf_diag_mod_target_kind { 29 BPF_DIAG_MOD_TARGET_NONE, 30 BPF_DIAG_MOD_TARGET_REG, 31 BPF_DIAG_MOD_TARGET_STACK_ARG, 32 BPF_DIAG_MOD_TARGET_STACK_SLOT, 33 BPF_DIAG_MOD_TARGET_STACK_RANGE, 34 }; 35 36 struct bpf_diag_mod_target { 37 u32 frame_id; 38 union { 39 struct { 40 s16 min_off; 41 s16 max_off; 42 } range; 43 u16 spi; 44 u8 regno; 45 u8 stack_arg; 46 }; 47 u8 frameno; 48 u8 kind; 49 }; 50 51 static struct bpf_diag_mod_target diag_reg_target(u32 frame_id, u8 frameno, u8 regno) 52 { 53 return (struct bpf_diag_mod_target){ 54 .frame_id = frame_id, 55 .frameno = frameno, 56 .kind = BPF_DIAG_MOD_TARGET_REG, 57 .regno = regno, 58 }; 59 } 60 61 static struct bpf_diag_mod_target diag_stack_arg_target(u32 frame_id, u8 frameno, u8 slot) 62 { 63 return (struct bpf_diag_mod_target){ 64 .frame_id = frame_id, 65 .frameno = frameno, 66 .kind = BPF_DIAG_MOD_TARGET_STACK_ARG, 67 .stack_arg = slot, 68 }; 69 } 70 71 static struct bpf_diag_mod_target diag_stack_slot_target(u32 frame_id, u8 frameno, u16 spi) 72 { 73 return (struct bpf_diag_mod_target){ 74 .frame_id = frame_id, 75 .frameno = frameno, 76 .kind = BPF_DIAG_MOD_TARGET_STACK_SLOT, 77 .spi = spi, 78 }; 79 } 80 81 static struct bpf_diag_mod_target diag_stack_range_target(u32 frame_id, u8 frameno, 82 s16 min_off, s16 max_off) 83 { 84 return (struct bpf_diag_mod_target){ 85 .frame_id = frame_id, 86 .frameno = frameno, 87 .kind = BPF_DIAG_MOD_TARGET_STACK_RANGE, 88 .range.min_off = min_off, 89 .range.max_off = max_off, 90 }; 91 } 92 93 struct bpf_diag_reg_snapshot { 94 u32 type; 95 u32 btf_id; 96 const struct bpf_map *map_ptr; 97 const struct btf *btf; 98 struct tnum var_off; 99 struct cnum64 r64; 100 }; 101 102 enum bpf_diag_history_kind { 103 BPF_DIAG_HISTORY_BRANCH, 104 BPF_DIAG_HISTORY_MOD, 105 }; 106 107 struct bpf_diag_history_event { 108 u32 insn_idx : 24; 109 u32 kind : 8; 110 u8 in_lineage : 1; 111 union { 112 struct { 113 bool cond_true; 114 } branch; 115 struct { 116 struct bpf_diag_mod_target target; 117 struct bpf_diag_mod_target origin; 118 struct bpf_diag_reg_snapshot old, new; 119 u8 reason; 120 bool origin_valid; 121 } mod; 122 }; 123 }; 124 125 struct disasm_line { 126 char text[DISASM_LINE_LEN]; 127 int idx; 128 bool valid; 129 }; 130 131 struct disasm_ctx { 132 struct bpf_verifier_env *env; 133 struct seq_buf seq; 134 }; 135 136 struct diag_fmt_chunk { 137 struct list_head node; 138 struct seq_buf seq; 139 char data[]; 140 }; 141 142 struct diag_fmt_mark { 143 struct diag_fmt_chunk *chunk; 144 size_t len; 145 }; 146 147 struct bpf_diag_log { 148 struct bpf_diag_history_event *events; 149 /* Sequence number of the oldest retained event on the active path. */ 150 u64 first_seq; 151 u32 cnt; 152 u32 cap; 153 u32 head; 154 bool growth_failed; 155 }; 156 157 struct bpf_diag_scratch { 158 struct bpf_linfo_source source_lines[BPF_DIAG_CONTEXT_CNT]; 159 struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT]; 160 }; 161 162 struct bpf_diag_mod_scope { 163 struct bpf_reg_state target_reg_snapshot; 164 struct bpf_diag_mod_target target; 165 struct bpf_diag_mod_target origin; 166 enum bpf_diag_mod_reason reason; 167 u32 insn_idx; 168 bool active; 169 bool origin_valid; 170 }; 171 172 struct bpf_diag { 173 struct bpf_diag_log log; 174 struct bpf_diag_scratch scratch; 175 struct list_head fmt_chunks; 176 struct bpf_diag_mod_scope mod; 177 u32 frame_id_gen; 178 }; 179 180 bool bpf_diag_enabled(const struct bpf_verifier_env *env) 181 { 182 return env->log.level & BPF_LOG_LEVEL; 183 } 184 185 static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); 186 187 int bpf_diag_init(struct bpf_verifier_env *env) 188 { 189 if (!bpf_diag_enabled(env)) 190 return 0; 191 192 env->diag = kzalloc_obj(struct bpf_diag, GFP_KERNEL_ACCOUNT); 193 if (!env->diag) 194 return -ENOMEM; 195 196 INIT_LIST_HEAD(&env->diag->fmt_chunks); 197 return 0; 198 } 199 200 void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state) 201 { 202 if (env->diag) 203 state->diag_frame_id = ++env->diag->frame_id_gen; 204 } 205 206 static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size) 207 { 208 struct bpf_diag *diag = env->diag; 209 struct diag_fmt_chunk *chunk; 210 size_t capacity, available; 211 char *buf; 212 213 if (!diag || !size || size > INT_MAX) 214 return NULL; 215 216 if (!list_empty(&diag->fmt_chunks)) { 217 chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); 218 available = seq_buf_get_buf(&chunk->seq, &buf); 219 if (available >= size) 220 goto commit; 221 } 222 223 capacity = max_t(size_t, BPF_DIAG_FMT_CHUNK_SIZE, size); 224 chunk = kmalloc(struct_size(chunk, data, capacity), GFP_KERNEL_ACCOUNT); 225 if (!chunk) 226 return NULL; 227 228 seq_buf_init(&chunk->seq, chunk->data, capacity); 229 list_add_tail(&chunk->node, &diag->fmt_chunks); 230 available = seq_buf_get_buf(&chunk->seq, &buf); 231 if (WARN_ON_ONCE(available < size)) 232 return NULL; 233 234 commit: 235 seq_buf_commit(&chunk->seq, size); 236 return buf; 237 } 238 239 char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size) 240 { 241 char *buf; 242 243 buf = diag_fmt_alloc(env, size); 244 if (buf) 245 buf[0] = '\0'; 246 return buf; 247 } 248 249 const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) 250 { 251 va_list copy; 252 char *buf; 253 int len; 254 255 va_copy(copy, args); 256 len = vsnprintf(NULL, 0, fmt, copy); 257 va_end(copy); 258 if (len < 0 || len == INT_MAX) 259 return ""; 260 261 buf = diag_fmt_alloc(env, len + 1); 262 if (buf) 263 vsnprintf(buf, len + 1, fmt, args); 264 return buf ?: ""; 265 } 266 267 const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) 268 { 269 const char *buf; 270 va_list args; 271 272 va_start(args, fmt); 273 buf = bpf_diag_vfmt(env, fmt, args); 274 va_end(args); 275 return buf; 276 } 277 278 static struct diag_fmt_mark diag_fmt_save(struct bpf_verifier_env *env) 279 { 280 struct bpf_diag *diag = env->diag; 281 struct diag_fmt_mark mark = {}; 282 283 if (!diag || list_empty(&diag->fmt_chunks)) 284 return mark; 285 286 mark.chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); 287 mark.len = mark.chunk->seq.len; 288 return mark; 289 } 290 291 static void diag_fmt_restore(struct bpf_verifier_env *env, struct diag_fmt_mark mark) 292 { 293 struct bpf_diag *diag = env->diag; 294 struct diag_fmt_chunk *chunk; 295 296 if (!diag) 297 return; 298 299 while (!list_empty(&diag->fmt_chunks)) { 300 chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); 301 if (chunk == mark.chunk) 302 break; 303 list_del(&chunk->node); 304 kfree(chunk); 305 } 306 307 if (mark.chunk) { 308 mark.chunk->seq.len = mark.len; 309 seq_buf_str(&mark.chunk->seq); 310 } 311 } 312 313 void bpf_diag_free(struct bpf_verifier_env *env) 314 { 315 struct bpf_diag *diag = env->diag; 316 317 if (!diag) 318 return; 319 320 diag_fmt_restore(env, (struct diag_fmt_mark){}); 321 kvfree(diag->log.events); 322 kfree(diag); 323 env->diag = NULL; 324 } 325 326 static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) 327 { 328 va_list args; 329 330 if (!bpf_diag_enabled(env)) 331 return; 332 333 va_start(args, fmt); 334 bpf_verifier_vlog(&env->log, fmt, args); 335 va_end(args); 336 } 337 338 static u64 log_end(const struct bpf_diag_log *log) 339 { 340 return log->first_seq + log->cnt; 341 } 342 343 static u32 log_pos(const struct bpf_diag_log *log, u32 idx) 344 { 345 u32 pos = log->head + idx; 346 347 return pos < log->cap ? pos : pos - log->cap; 348 } 349 350 u64 bpf_diag_event_log_save(struct bpf_verifier_env *env) 351 { 352 struct bpf_diag *diag = env->diag; 353 354 return diag ? log_end(&diag->log) : 0; 355 } 356 357 void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos) 358 { 359 struct bpf_diag *diag = env->diag; 360 struct bpf_diag_log *log; 361 u64 end_seq; 362 363 if (!diag) 364 return; 365 366 log = &diag->log; 367 end_seq = log_end(log); 368 if (WARN_ON_ONCE(log_pos > end_seq)) 369 log_pos = end_seq; 370 371 /* 372 * A deep abandoned path may have rotated away the shared prefix. In 373 * that case, restart with an empty retained suffix and remember that 374 * every event before the restored mark is unavailable. 375 */ 376 if (log_pos <= log->first_seq) { 377 log->first_seq = log_pos; 378 log->head = 0; 379 log->cnt = 0; 380 return; 381 } 382 383 log->cnt = log_pos - log->first_seq; 384 } 385 386 static void diag_append_history(struct bpf_verifier_env *env, 387 const struct bpf_diag_history_event *event) 388 { 389 struct bpf_diag_history_event *events; 390 struct bpf_diag *diag = env->diag; 391 struct bpf_diag_log *log; 392 u32 cap, max_events; 393 394 if (!diag) 395 return; 396 log = &diag->log; 397 398 if (log->cnt < log->cap) { 399 log->events[log_pos(log, log->cnt++)] = *event; 400 return; 401 } 402 403 max_events = BPF_DIAG_EVENT_LOG_MAX_SIZE / sizeof(*events); 404 if (log->growth_failed || log->cap == max_events) 405 goto rotate; 406 407 cap = min(log->cap ? log->cap * 2 : 64, max_events); 408 events = kvrealloc(log->events, array_size(cap, sizeof(*events)), GFP_KERNEL_ACCOUNT); 409 if (!events) { 410 log->growth_failed = true; 411 goto rotate; 412 } 413 log->events = events; 414 log->cap = cap; 415 log->events[log->cnt++] = *event; 416 return; 417 418 rotate: 419 if (log->cap) { 420 log->events[log->head++] = *event; 421 if (log->head == log->cap) 422 log->head = 0; 423 } 424 log->first_seq++; 425 } 426 427 static void diag_print_wrapped_prefixed(struct bpf_verifier_env *env, const char *first_prefix, 428 const char *next_prefix, const char *text) 429 { 430 const char *prefix = first_prefix; 431 432 while (*text) { 433 const char *line = text; 434 int prefix_len = strlen(prefix); 435 int text_width = BPF_DIAG_TEXT_WIDTH - prefix_len; 436 int len = 0, last_space = -1; 437 438 if (text_width < 1) 439 text_width = 1; 440 441 while (line[len] && line[len] != '\n' && len < text_width) { 442 if (line[len] == ' ') 443 last_space = len; 444 len++; 445 } 446 447 if (line[len] && line[len] != '\n' && line[len] != ' ' && last_space > 0) 448 len = last_space; 449 450 diag_write(env, "%s%.*s\n", prefix, len, line); 451 452 text = line + len; 453 while (*text == ' ') 454 text++; 455 if (*text == '\n') 456 text++; 457 458 prefix = next_prefix; 459 } 460 } 461 462 const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id) 463 { 464 char *buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); 465 size_t len; 466 int ret; 467 468 if (!buf) 469 return ""; 470 471 buf[0] = '\0'; 472 ret = btf_type_name_to_buf(btf, type_id, buf, BPF_DIAG_FMT_BUF_SIZE); 473 if (ret < 0 || !buf[0]) { 474 scnprintf(buf, BPF_DIAG_FMT_BUF_SIZE, "BTF type ID %u", type_id); 475 return buf; 476 } 477 478 len = strlen(buf); 479 if (len && buf[len - 1] == '{') 480 buf[len - 1] = '\0'; 481 return buf; 482 } 483 484 static int diag_line_width(unsigned int line) 485 { 486 int width = 1; 487 488 while (line >= 10) { 489 line /= 10; 490 width++; 491 } 492 493 return width; 494 } 495 496 static int diag_line_indent(const char *line) 497 { 498 int indent = 0; 499 500 while (*line == ' ' || *line == '\t') { 501 if (*line == '\t') 502 indent = round_up(indent + 1, BPF_DIAG_TAB_WIDTH); 503 else 504 indent++; 505 line++; 506 } 507 508 return indent; 509 } 510 511 static void disasm_print(void *private_data, const char *fmt, ...) __printf(2, 3); 512 513 static void disasm_print(void *private_data, const char *fmt, ...) 514 { 515 struct disasm_ctx *ctx = private_data; 516 va_list args; 517 518 va_start(args, fmt); 519 seq_buf_vprintf(&ctx->seq, fmt, args); 520 va_end(args); 521 } 522 523 static const char *disasm_kfunc_name(void *private_data, const struct bpf_insn *insn) 524 { 525 struct disasm_ctx *ctx = private_data; 526 527 return bpf_disasm_kfunc_name(ctx->env, insn); 528 } 529 530 static void format_disasm_line(struct bpf_verifier_env *env, int insn_idx, 531 struct disasm_line *line) 532 { 533 struct disasm_ctx ctx = { .env = env }; 534 struct bpf_insn *insn; 535 const struct bpf_insn_cbs cbs = { 536 .cb_call = disasm_kfunc_name, 537 .cb_print = disasm_print, 538 .private_data = &ctx, 539 }; 540 541 line->idx = insn_idx; 542 line->valid = false; 543 seq_buf_init(&ctx.seq, line->text, sizeof(line->text)); 544 545 if (insn_idx < 0 || insn_idx >= env->prog->len) 546 return; 547 548 if (insn_idx > 0 && bpf_is_ldimm64(&env->prog->insnsi[insn_idx - 1])) 549 return; 550 551 insn = &env->prog->insnsi[insn_idx]; 552 if (bpf_is_ldimm64(insn) && insn_idx + 1 >= env->prog->len) 553 return; 554 555 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 556 seq_buf_str(&ctx.seq); 557 ctx.seq.len = strnlen(line->text, sizeof(line->text)); 558 while (ctx.seq.len && line->text[ctx.seq.len - 1] == '\n') 559 seq_buf_pop(&ctx.seq); 560 seq_buf_str(&ctx.seq); 561 562 line->valid = true; 563 } 564 565 static void diag_format_source_text(char *buf, size_t size, const char *line, int width) 566 { 567 int col = 0, len = 0; 568 569 if (!size) 570 return; 571 if (width <= 0) { 572 buf[0] = '\0'; 573 return; 574 } 575 576 line = line ?: "..."; 577 while (*line && col < width && len + 1 < size) { 578 if (*line == '\t') { 579 int next = round_up(col + 1, BPF_DIAG_TAB_WIDTH); 580 581 while (col < next && col < width && len + 1 < size) { 582 buf[len++] = ' '; 583 col++; 584 } 585 line++; 586 continue; 587 } 588 589 buf[len++] = *line++; 590 col++; 591 } 592 593 if (*line) { 594 int ellipsis_len = min(3, width); 595 596 while (len > 0 && col > width - ellipsis_len) { 597 len--; 598 col--; 599 } 600 while (ellipsis_len-- && len + 1 < size) 601 buf[len++] = '.'; 602 } 603 604 buf[len] = '\0'; 605 } 606 607 static void diag_format_source_lane(char *buf, size_t size, const char *source_prefix, 608 int source_line_width, int line_num, const char *line) 609 { 610 int len, text_width; 611 612 if (line_num <= 0) { 613 buf[0] = '\0'; 614 return; 615 } 616 617 len = scnprintf(buf, size, "%s%*d | ", source_prefix, source_line_width, line_num); 618 text_width = BPF_DIAG_SOURCE_LANE_WIDTH - len; 619 diag_format_source_text(buf + len, size - len, line, text_width); 620 } 621 622 static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, 623 const char *problem) 624 { 625 char first; 626 627 if (!bpf_diag_enabled(env)) 628 return; 629 630 category = category ?: "Verifier Error"; 631 problem = problem ?: ""; 632 633 if (!problem[0]) { 634 diag_write(env, "\nVerification failed: %s\n", category); 635 return; 636 } 637 638 first = toupper(problem[0]); 639 diag_write(env, "\nVerification failed: %s: %c%s\n", category, first, problem + 1); 640 } 641 642 static void diag_print_source_annotation(struct bpf_verifier_env *env, int line_width, int indent, 643 const char *label, const char *msg) 644 { 645 const char *first_prefix, *next_prefix, *text; 646 647 indent = min_t(int, indent, max_t(int, 0, BPF_DIAG_SOURCE_LANE_WIDTH - line_width - 8)); 648 text = bpf_diag_fmt(env, "%s: %s", label, msg); 649 first_prefix = bpf_diag_fmt(env, " %*s | %*s^-- ", line_width + 4, "", indent, ""); 650 next_prefix = bpf_diag_fmt(env, " %*s | %*s ", line_width + 4, "", indent, ""); 651 652 diag_print_wrapped_prefixed(env, first_prefix, next_prefix, text); 653 } 654 655 static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, 656 struct disasm_line *disasm_lines) 657 { 658 int insn_width = diag_line_width(env->prog->len ? env->prog->len - 1 : 0); 659 int i; 660 661 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { 662 int row = i - BPF_DIAG_CONTEXT; 663 664 format_disasm_line(env, insn_idx + row, &disasm_lines[i]); 665 } 666 667 diag_write(env, " Instruction context:\n"); 668 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { 669 struct disasm_line *line = &disasm_lines[i]; 670 671 if (line->valid) 672 diag_write(env, " %s%*d | %s\n", 673 line->idx == insn_idx ? ">>> " : " ", 674 insn_width, line->idx, line->text); 675 } 676 } 677 678 static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, 679 const char *fmt, ...) 680 { 681 struct bpf_diag_scratch *scratch; 682 struct bpf_linfo_source *source_lines; 683 struct disasm_line *disasm_lines; 684 struct bpf_linfo_source src = {}; 685 struct diag_fmt_mark mark; 686 const struct bpf_line_info *linfo; 687 const struct bpf_subprog_info *subprog; 688 struct btf *btf = env->prog->aux->btf; 689 char *source_lane; 690 const char *msg; 691 const char *func; 692 int start_line, end_line, width, indent, subprogno, linfo_start, linfo_end, i; 693 va_list args; 694 695 if (!bpf_diag_enabled(env)) 696 return; 697 if (!env->diag) 698 return; 699 700 mark = diag_fmt_save(env); 701 label = label ?: "note"; 702 scratch = &env->diag->scratch; 703 source_lines = scratch->source_lines; 704 disasm_lines = scratch->disasm_lines; 705 memset(source_lines, 0, sizeof(scratch->source_lines)); 706 memset(disasm_lines, 0, sizeof(scratch->disasm_lines)); 707 708 va_start(args, fmt); 709 msg = bpf_diag_vfmt(env, fmt, args); 710 va_end(args); 711 if (!*msg) 712 msg = "<failed to allocate diagnostic text>"; 713 714 linfo = bpf_find_linfo(env->prog, insn_idx); 715 if (btf && linfo) 716 bpf_get_linfo_source(btf, linfo, &src); 717 if (!src.file || !*src.file || !src.line || !*src.line) { 718 diag_write(env, " insn %u\n", insn_idx); 719 diag_print_source_annotation(env, 0, 0, label, msg); 720 diag_print_insn_context(env, insn_idx, disasm_lines); 721 goto out_restore; 722 } 723 724 subprog = bpf_find_containing_subprog(env, insn_idx); 725 subprogno = subprog ? subprog - env->subprog_info : -ENOENT; 726 func = subprogno >= 0 ? bpf_subprog_name(env, subprogno) : NULL; 727 if (func && *func) 728 diag_write(env, " %s @ %s:%d:%d\n", func, src.file, src.line_num, src.line_col); 729 else 730 diag_write(env, " %s:%d:%d\n", src.file, src.line_num, src.line_col); 731 732 start_line = src.line_num - BPF_DIAG_CONTEXT; 733 end_line = src.line_num + BPF_DIAG_CONTEXT; 734 width = diag_line_width(end_line); 735 indent = diag_line_indent(src.line); 736 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) 737 source_lines[i].line_num = start_line + i; 738 739 linfo = env->prog->aux->linfo; 740 linfo_start = subprog ? subprog->linfo_idx : 0; 741 linfo_end = subprogno >= 0 && subprogno + 1 < env->subprog_cnt ? 742 env->subprog_info[subprogno + 1].linfo_idx : env->prog->aux->nr_linfo; 743 for (i = linfo_start; i < linfo_end; i++) { 744 struct bpf_linfo_source line_src; 745 int idx; 746 747 bpf_get_linfo_source(btf, &linfo[i], &line_src); 748 if (line_src.file_name_off != src.file_name_off || 749 line_src.line_num < start_line || line_src.line_num > end_line || 750 !line_src.line || !*line_src.line) 751 continue; 752 753 idx = line_src.line_num - start_line; 754 if (!source_lines[idx].line) 755 source_lines[idx] = line_src; 756 } 757 758 diag_write(env, " Source context:\n"); 759 source_lane = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); 760 if (!source_lane) 761 goto out_restore; 762 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { 763 const char *source_prefix; 764 765 source_prefix = source_lines[i].line_num == src.line_num ? ">>> " : " "; 766 diag_format_source_lane(source_lane, BPF_DIAG_FMT_BUF_SIZE, source_prefix, width, 767 source_lines[i].line_num, source_lines[i].line); 768 diag_write(env, " %s\n", source_lane); 769 if (source_lines[i].line_num == src.line_num) 770 diag_print_source_annotation(env, width, indent, label, msg); 771 } 772 diag_print_insn_context(env, insn_idx, disasm_lines); 773 774 out_restore: 775 diag_fmt_restore(env, mark); 776 } 777 778 void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true) 779 { 780 struct bpf_diag_history_event event = { 781 .insn_idx = insn_idx, 782 .kind = BPF_DIAG_HISTORY_BRANCH, 783 .branch = { 784 .cond_true = cond_true, 785 }, 786 }; 787 788 diag_append_history(env, &event); 789 } 790 791 static void diag_snapshot_reg(struct bpf_diag_reg_snapshot *snapshot, 792 const struct bpf_reg_state *reg) 793 { 794 snapshot->type = reg->type; 795 if (type_is_map_ptr(reg->type)) 796 snapshot->map_ptr = reg->map_ptr; 797 if (base_type(reg->type) == PTR_TO_BTF_ID && reg->btf && reg->btf_id) { 798 snapshot->btf_id = reg->btf_id; 799 snapshot->btf = reg->btf; 800 } 801 snapshot->var_off = reg->var_off; 802 snapshot->r64 = reg->r64; 803 } 804 805 static bool diag_mod_insn_origin(struct bpf_verifier_env *env, u32 insn_idx, 806 const struct bpf_diag_mod_target *target, 807 struct bpf_diag_mod_target *origin) 808 { 809 const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 810 u8 class = BPF_CLASS(insn->code); 811 const struct bpf_func_state *state; 812 813 if (target->kind == BPF_DIAG_MOD_TARGET_REG && (class == BPF_ALU || class == BPF_ALU64) && 814 BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X) { 815 *origin = diag_reg_target(target->frame_id, target->frameno, insn->src_reg); 816 return true; 817 } 818 819 if ((target->kind != BPF_DIAG_MOD_TARGET_STACK_ARG && 820 target->kind != BPF_DIAG_MOD_TARGET_STACK_SLOT) || 821 class != BPF_STX) 822 return false; 823 824 state = env->cur_state->frame[env->cur_state->curframe]; 825 *origin = diag_reg_target(state->diag_frame_id, state->frameno, insn->src_reg); 826 return true; 827 } 828 829 static bool diag_mod_keeps_lineage(struct bpf_verifier_env *env, 830 const struct bpf_diag_history_event *event) 831 { 832 const struct bpf_insn *insn; 833 u8 class; 834 835 if (event->mod.reason != BPF_DIAG_MOD_WRITE || 836 event->mod.target.kind != BPF_DIAG_MOD_TARGET_REG) 837 return false; 838 839 insn = &env->prog->insnsi[event->insn_idx]; 840 class = BPF_CLASS(insn->code); 841 if (class != BPF_ALU && class != BPF_ALU64) 842 return false; 843 844 switch (BPF_OP(insn->code)) { 845 case BPF_ADD: 846 case BPF_SUB: 847 case BPF_MUL: 848 case BPF_OR: 849 case BPF_AND: 850 case BPF_LSH: 851 case BPF_RSH: 852 case BPF_ARSH: 853 case BPF_XOR: 854 case BPF_NEG: 855 case BPF_END: 856 return true; 857 default: 858 return false; 859 } 860 } 861 862 static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, 863 struct bpf_diag_mod_target target, 864 enum bpf_diag_mod_reason reason, 865 const struct bpf_reg_state *old_reg, 866 const struct bpf_reg_state *new_reg, 867 const struct bpf_diag_mod_target *origin) 868 { 869 struct bpf_diag_history_event event = { 870 .insn_idx = insn_idx, 871 .kind = BPF_DIAG_HISTORY_MOD, 872 .mod = { 873 .target = target, 874 .reason = reason, 875 }, 876 }; 877 878 if (old_reg) 879 diag_snapshot_reg(&event.mod.old, old_reg); 880 if (new_reg) 881 diag_snapshot_reg(&event.mod.new, new_reg); 882 if (origin) { 883 event.mod.origin = *origin; 884 event.mod.origin_valid = true; 885 } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { 886 event.mod.origin_valid = true; 887 } 888 if (old_reg && new_reg && 889 (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) && 890 !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) && 891 !event.mod.origin_valid && 892 diag_mod_keeps_lineage(env, &event)) 893 return; 894 895 diag_append_history(env, &event); 896 } 897 898 static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env, 899 const struct bpf_diag_mod_target *target) 900 { 901 struct bpf_verifier_state *vstate = env->cur_state; 902 struct bpf_func_state *state; 903 904 state = target->frameno <= vstate->curframe ? vstate->frame[target->frameno] : NULL; 905 906 if (!state) 907 return NULL; 908 if (state->diag_frame_id != target->frame_id) 909 return NULL; 910 911 switch (target->kind) { 912 case BPF_DIAG_MOD_TARGET_REG: 913 if (target->regno >= MAX_BPF_REG) 914 return NULL; 915 return &state->regs[target->regno]; 916 case BPF_DIAG_MOD_TARGET_STACK_ARG: 917 if (target->stack_arg >= state->out_stack_arg_cnt) 918 return NULL; 919 return &state->stack_arg_regs[target->stack_arg]; 920 case BPF_DIAG_MOD_TARGET_STACK_SLOT: 921 if (target->spi >= state->allocated_stack / BPF_REG_SIZE) 922 return NULL; 923 return &state->stack[target->spi].spilled_ptr; 924 default: 925 return NULL; 926 } 927 } 928 929 static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, 930 struct bpf_diag_mod_target *target) 931 { 932 struct bpf_verifier_state *vstate = env->cur_state; 933 unsigned long addr = (unsigned long)reg; 934 int frame; 935 936 for (frame = 0; frame <= vstate->curframe; frame++) { 937 struct bpf_func_state *state = vstate->frame[frame]; 938 unsigned long start, end; 939 u32 nslots = state->allocated_stack / BPF_REG_SIZE; 940 int spi; 941 942 start = (unsigned long)state->regs; 943 end = (unsigned long)(state->regs + MAX_BPF_REG); 944 if (addr >= start && addr < end) { 945 *target = diag_reg_target(state->diag_frame_id, state->frameno, 946 reg - state->regs); 947 return true; 948 } 949 950 start = (unsigned long)state->stack_arg_regs; 951 end = (unsigned long)(state->stack_arg_regs + state->out_stack_arg_cnt); 952 if (state->out_stack_arg_cnt && addr >= start && addr < end) { 953 *target = diag_stack_arg_target(state->diag_frame_id, state->frameno, 954 reg - state->stack_arg_regs); 955 return true; 956 } 957 958 start = (unsigned long)state->stack; 959 end = (unsigned long)(state->stack + nslots); 960 if (nslots && addr >= start && addr < end) { 961 spi = ((const char *)reg - (const char *)state->stack) / 962 sizeof(*state->stack); 963 *target = diag_stack_slot_target(state->diag_frame_id, state->frameno, spi); 964 return true; 965 } 966 } 967 return false; 968 } 969 970 void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, 971 const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason) 972 { 973 struct bpf_diag *diag = env->diag; 974 975 if (!diag) 976 return; 977 diag->mod.active = reg_to_target(env, reg, &diag->mod.target); 978 if (!diag->mod.active) 979 return; 980 diag->mod.target_reg_snapshot = *reg; 981 diag->mod.insn_idx = env->insn_idx; 982 diag->mod.reason = reason; 983 diag->mod.origin_valid = origin && reg_to_target(env, origin, &diag->mod.origin); 984 } 985 986 void bpf_diag_mod_end(struct bpf_verifier_env *env) 987 { 988 struct bpf_diag *diag = env->diag; 989 const struct bpf_reg_state *new_reg; 990 991 if (!diag || !diag->mod.active) 992 return; 993 diag->mod.active = false; 994 /* 995 * Resolve the target again because the enclosing function state's stack 996 * may have been reallocated while the modification was in progress. 997 */ 998 new_reg = target_to_reg(env, &diag->mod.target); 999 if (!new_reg) 1000 return; 1001 diag_record_mod(env, diag->mod.insn_idx, diag->mod.target, diag->mod.reason, 1002 &diag->mod.target_reg_snapshot, new_reg, 1003 diag->mod.origin_valid ? &diag->mod.origin : NULL); 1004 } 1005 1006 void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, 1007 enum bpf_diag_mod_reason reason) 1008 { 1009 struct bpf_diag_mod_target target; 1010 1011 if (!env->diag || reg->type == NOT_INIT || !reg_to_target(env, reg, &target)) 1012 return; 1013 diag_record_mod(env, env->insn_idx, target, reason, reg, NULL, NULL); 1014 } 1015 1016 void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, 1017 const struct bpf_func_state *state, s16 min_off, s16 max_off, 1018 enum bpf_diag_mod_reason reason) 1019 { 1020 diag_record_mod(env, env->insn_idx, 1021 diag_stack_range_target(state->diag_frame_id, state->frameno, min_off, max_off), 1022 reason, NULL, NULL, NULL); 1023 } 1024