1 // SPDX-License-Identifier: GPL-2.0-only 2 // Copyright (c) 2026 Meta Platforms, Inc. and affiliates. 3 4 #include <linux/bpf.h> 5 #include <linux/bpf_verifier.h> 6 #include <linux/btf.h> 7 #include <linux/ctype.h> 8 #include <linux/kernel.h> 9 #include <linux/list.h> 10 #include <linux/seq_buf.h> 11 #include <linux/slab.h> 12 #include <linux/stdarg.h> 13 #include <linux/string.h> 14 15 #include "disasm.h" 16 #include "diagnostics.h" 17 18 #define BPF_DIAG_TEXT_WIDTH 100 19 #define BPF_DIAG_CONTEXT 2 20 #define BPF_DIAG_CONTEXT_CNT (1 + BPF_DIAG_CONTEXT * 2) 21 #define BPF_DIAG_SOURCE_LANE_WIDTH 88 22 #define BPF_DIAG_TAB_WIDTH 8 23 #define BPF_DIAG_FMT_CHUNK_SIZE (PAGE_SIZE - sizeof(struct diag_fmt_chunk)) 24 #define BPF_DIAG_FMT_BUF_SIZE 256 25 #define BPF_DIAG_EVENT_LOG_MAX_SIZE (64U << 20) 26 #define DISASM_LINE_LEN 160 27 28 enum bpf_diag_mod_target_kind { 29 BPF_DIAG_MOD_TARGET_NONE, 30 BPF_DIAG_MOD_TARGET_REG, 31 BPF_DIAG_MOD_TARGET_STACK_ARG, 32 BPF_DIAG_MOD_TARGET_STACK_SLOT, 33 BPF_DIAG_MOD_TARGET_STACK_RANGE, 34 }; 35 36 struct bpf_diag_mod_target { 37 u32 frame_id; 38 union { 39 struct { 40 s16 min_off; 41 s16 max_off; 42 } range; 43 u16 spi; 44 u8 regno; 45 u8 stack_arg; 46 }; 47 u8 frameno; 48 u8 kind; 49 }; 50 51 static struct bpf_diag_mod_target diag_reg_target(u32 frame_id, u8 frameno, u8 regno) 52 { 53 return (struct bpf_diag_mod_target){ 54 .frame_id = frame_id, 55 .frameno = frameno, 56 .kind = BPF_DIAG_MOD_TARGET_REG, 57 .regno = regno, 58 }; 59 } 60 61 static struct bpf_diag_mod_target diag_stack_arg_target(u32 frame_id, u8 frameno, u8 slot) 62 { 63 return (struct bpf_diag_mod_target){ 64 .frame_id = frame_id, 65 .frameno = frameno, 66 .kind = BPF_DIAG_MOD_TARGET_STACK_ARG, 67 .stack_arg = slot, 68 }; 69 } 70 71 static struct bpf_diag_mod_target diag_stack_slot_target(u32 frame_id, u8 frameno, u16 spi) 72 { 73 return (struct bpf_diag_mod_target){ 74 .frame_id = frame_id, 75 .frameno = frameno, 76 .kind = BPF_DIAG_MOD_TARGET_STACK_SLOT, 77 .spi = spi, 78 }; 79 } 80 81 static struct bpf_diag_mod_target diag_stack_range_target(u32 frame_id, u8 frameno, 82 s16 min_off, s16 max_off) 83 { 84 return (struct bpf_diag_mod_target){ 85 .frame_id = frame_id, 86 .frameno = frameno, 87 .kind = BPF_DIAG_MOD_TARGET_STACK_RANGE, 88 .range.min_off = min_off, 89 .range.max_off = max_off, 90 }; 91 } 92 93 struct bpf_diag_reg_snapshot { 94 u32 type; 95 u32 btf_id; 96 const struct bpf_map *map_ptr; 97 const struct btf *btf; 98 struct tnum var_off; 99 struct cnum64 r64; 100 }; 101 102 enum bpf_diag_history_kind { 103 BPF_DIAG_HISTORY_BRANCH, 104 BPF_DIAG_HISTORY_MOD, 105 BPF_DIAG_HISTORY_REF_ACQUIRE, 106 BPF_DIAG_HISTORY_REF_RELEASE, 107 }; 108 109 struct bpf_diag_history_event { 110 u32 insn_idx : 24; 111 u32 kind : 8; 112 u8 in_lineage : 1; 113 union { 114 struct { 115 bool cond_true; 116 } branch; 117 struct { 118 struct bpf_diag_mod_target target; 119 struct bpf_diag_mod_target origin; 120 struct bpf_diag_reg_snapshot old, new; 121 u8 reason; 122 bool origin_valid; 123 } mod; 124 struct { 125 u32 ref_id; 126 } ref; 127 }; 128 }; 129 130 struct disasm_line { 131 char text[DISASM_LINE_LEN]; 132 int idx; 133 bool valid; 134 }; 135 136 struct disasm_ctx { 137 struct bpf_verifier_env *env; 138 struct seq_buf seq; 139 }; 140 141 struct diag_fmt_chunk { 142 struct list_head node; 143 struct seq_buf seq; 144 char data[]; 145 }; 146 147 struct diag_fmt_mark { 148 struct diag_fmt_chunk *chunk; 149 size_t len; 150 }; 151 152 struct bpf_diag_log { 153 struct bpf_diag_history_event *events; 154 /* Sequence number of the oldest retained event on the active path. */ 155 u64 first_seq; 156 u32 cnt; 157 u32 cap; 158 u32 head; 159 bool growth_failed; 160 }; 161 162 struct bpf_diag_scratch { 163 struct bpf_linfo_source source_lines[BPF_DIAG_CONTEXT_CNT]; 164 struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT]; 165 }; 166 167 struct bpf_diag_mod_scope { 168 struct bpf_reg_state target_reg_snapshot; 169 struct bpf_diag_mod_target target; 170 struct bpf_diag_mod_target origin; 171 enum bpf_diag_mod_reason reason; 172 u32 insn_idx; 173 bool active; 174 bool origin_valid; 175 }; 176 177 struct bpf_diag { 178 struct bpf_diag_log log; 179 struct bpf_diag_scratch scratch; 180 struct list_head fmt_chunks; 181 struct bpf_diag_mod_scope mod; 182 u32 frame_id_gen; 183 }; 184 185 bool bpf_diag_enabled(const struct bpf_verifier_env *env) 186 { 187 return env->log.level & BPF_LOG_LEVEL; 188 } 189 190 static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); 191 192 int bpf_diag_init(struct bpf_verifier_env *env) 193 { 194 if (!bpf_diag_enabled(env)) 195 return 0; 196 197 env->diag = kzalloc_obj(struct bpf_diag, GFP_KERNEL_ACCOUNT); 198 if (!env->diag) 199 return -ENOMEM; 200 201 INIT_LIST_HEAD(&env->diag->fmt_chunks); 202 return 0; 203 } 204 205 void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state) 206 { 207 if (env->diag) 208 state->diag_frame_id = ++env->diag->frame_id_gen; 209 } 210 211 static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size) 212 { 213 struct bpf_diag *diag = env->diag; 214 struct diag_fmt_chunk *chunk; 215 size_t capacity, available; 216 char *buf; 217 218 if (!diag || !size || size > INT_MAX) 219 return NULL; 220 221 if (!list_empty(&diag->fmt_chunks)) { 222 chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); 223 available = seq_buf_get_buf(&chunk->seq, &buf); 224 if (available >= size) 225 goto commit; 226 } 227 228 capacity = max_t(size_t, BPF_DIAG_FMT_CHUNK_SIZE, size); 229 chunk = kmalloc(struct_size(chunk, data, capacity), GFP_KERNEL_ACCOUNT); 230 if (!chunk) 231 return NULL; 232 233 seq_buf_init(&chunk->seq, chunk->data, capacity); 234 list_add_tail(&chunk->node, &diag->fmt_chunks); 235 available = seq_buf_get_buf(&chunk->seq, &buf); 236 if (WARN_ON_ONCE(available < size)) 237 return NULL; 238 239 commit: 240 seq_buf_commit(&chunk->seq, size); 241 return buf; 242 } 243 244 char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size) 245 { 246 char *buf; 247 248 buf = diag_fmt_alloc(env, size); 249 if (buf) 250 buf[0] = '\0'; 251 return buf; 252 } 253 254 const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) 255 { 256 va_list copy; 257 char *buf; 258 int len; 259 260 va_copy(copy, args); 261 len = vsnprintf(NULL, 0, fmt, copy); 262 va_end(copy); 263 if (len < 0 || len == INT_MAX) 264 return ""; 265 266 buf = diag_fmt_alloc(env, len + 1); 267 if (buf) 268 vsnprintf(buf, len + 1, fmt, args); 269 return buf ?: ""; 270 } 271 272 const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) 273 { 274 const char *buf; 275 va_list args; 276 277 va_start(args, fmt); 278 buf = bpf_diag_vfmt(env, fmt, args); 279 va_end(args); 280 return buf; 281 } 282 283 static struct diag_fmt_mark diag_fmt_save(struct bpf_verifier_env *env) 284 { 285 struct bpf_diag *diag = env->diag; 286 struct diag_fmt_mark mark = {}; 287 288 if (!diag || list_empty(&diag->fmt_chunks)) 289 return mark; 290 291 mark.chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); 292 mark.len = mark.chunk->seq.len; 293 return mark; 294 } 295 296 static void diag_fmt_restore(struct bpf_verifier_env *env, struct diag_fmt_mark mark) 297 { 298 struct bpf_diag *diag = env->diag; 299 struct diag_fmt_chunk *chunk; 300 301 if (!diag) 302 return; 303 304 while (!list_empty(&diag->fmt_chunks)) { 305 chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); 306 if (chunk == mark.chunk) 307 break; 308 list_del(&chunk->node); 309 kfree(chunk); 310 } 311 312 if (mark.chunk) { 313 mark.chunk->seq.len = mark.len; 314 seq_buf_str(&mark.chunk->seq); 315 } 316 } 317 318 void bpf_diag_free(struct bpf_verifier_env *env) 319 { 320 struct bpf_diag *diag = env->diag; 321 322 if (!diag) 323 return; 324 325 diag_fmt_restore(env, (struct diag_fmt_mark){}); 326 kvfree(diag->log.events); 327 kfree(diag); 328 env->diag = NULL; 329 } 330 331 static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) 332 { 333 va_list args; 334 335 if (!bpf_diag_enabled(env)) 336 return; 337 338 va_start(args, fmt); 339 bpf_verifier_vlog(&env->log, fmt, args); 340 va_end(args); 341 } 342 343 static u64 log_end(const struct bpf_diag_log *log) 344 { 345 return log->first_seq + log->cnt; 346 } 347 348 static u32 log_pos(const struct bpf_diag_log *log, u32 idx) 349 { 350 u32 pos = log->head + idx; 351 352 return pos < log->cap ? pos : pos - log->cap; 353 } 354 355 u64 bpf_diag_event_log_save(struct bpf_verifier_env *env) 356 { 357 struct bpf_diag *diag = env->diag; 358 359 return diag ? log_end(&diag->log) : 0; 360 } 361 362 void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos) 363 { 364 struct bpf_diag *diag = env->diag; 365 struct bpf_diag_log *log; 366 u64 end_seq; 367 368 if (!diag) 369 return; 370 371 log = &diag->log; 372 end_seq = log_end(log); 373 if (WARN_ON_ONCE(log_pos > end_seq)) 374 log_pos = end_seq; 375 376 /* 377 * A deep abandoned path may have rotated away the shared prefix. In 378 * that case, restart with an empty retained suffix and remember that 379 * every event before the restored mark is unavailable. 380 */ 381 if (log_pos <= log->first_seq) { 382 log->first_seq = log_pos; 383 log->head = 0; 384 log->cnt = 0; 385 return; 386 } 387 388 log->cnt = log_pos - log->first_seq; 389 } 390 391 static void diag_append_history(struct bpf_verifier_env *env, 392 const struct bpf_diag_history_event *event) 393 { 394 struct bpf_diag_history_event *events; 395 struct bpf_diag *diag = env->diag; 396 struct bpf_diag_log *log; 397 u32 cap, max_events; 398 399 if (!diag) 400 return; 401 log = &diag->log; 402 403 if (log->cnt < log->cap) { 404 log->events[log_pos(log, log->cnt++)] = *event; 405 return; 406 } 407 408 max_events = BPF_DIAG_EVENT_LOG_MAX_SIZE / sizeof(*events); 409 if (log->growth_failed || log->cap == max_events) 410 goto rotate; 411 412 cap = min(log->cap ? log->cap * 2 : 64, max_events); 413 events = kvrealloc(log->events, array_size(cap, sizeof(*events)), GFP_KERNEL_ACCOUNT); 414 if (!events) { 415 log->growth_failed = true; 416 goto rotate; 417 } 418 log->events = events; 419 log->cap = cap; 420 log->events[log->cnt++] = *event; 421 return; 422 423 rotate: 424 if (log->cap) { 425 log->events[log->head++] = *event; 426 if (log->head == log->cap) 427 log->head = 0; 428 } 429 log->first_seq++; 430 } 431 432 static void diag_print_wrapped_prefixed(struct bpf_verifier_env *env, const char *first_prefix, 433 const char *next_prefix, const char *text) 434 { 435 const char *prefix = first_prefix; 436 437 while (*text) { 438 const char *line = text; 439 int prefix_len = strlen(prefix); 440 int text_width = BPF_DIAG_TEXT_WIDTH - prefix_len; 441 int len = 0, last_space = -1; 442 443 if (text_width < 1) 444 text_width = 1; 445 446 while (line[len] && line[len] != '\n' && len < text_width) { 447 if (line[len] == ' ') 448 last_space = len; 449 len++; 450 } 451 452 if (line[len] && line[len] != '\n' && line[len] != ' ' && last_space > 0) 453 len = last_space; 454 455 diag_write(env, "%s%.*s\n", prefix, len, line); 456 457 text = line + len; 458 while (*text == ' ') 459 text++; 460 if (*text == '\n') 461 text++; 462 463 prefix = next_prefix; 464 } 465 } 466 467 const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id) 468 { 469 char *buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); 470 size_t len; 471 int ret; 472 473 if (!buf) 474 return ""; 475 476 buf[0] = '\0'; 477 ret = btf_type_name_to_buf(btf, type_id, buf, BPF_DIAG_FMT_BUF_SIZE); 478 if (ret < 0 || !buf[0]) { 479 scnprintf(buf, BPF_DIAG_FMT_BUF_SIZE, "BTF type ID %u", type_id); 480 return buf; 481 } 482 483 len = strlen(buf); 484 if (len && buf[len - 1] == '{') 485 buf[len - 1] = '\0'; 486 return buf; 487 } 488 489 static int diag_line_width(unsigned int line) 490 { 491 int width = 1; 492 493 while (line >= 10) { 494 line /= 10; 495 width++; 496 } 497 498 return width; 499 } 500 501 static int diag_line_indent(const char *line) 502 { 503 int indent = 0; 504 505 while (*line == ' ' || *line == '\t') { 506 if (*line == '\t') 507 indent = round_up(indent + 1, BPF_DIAG_TAB_WIDTH); 508 else 509 indent++; 510 line++; 511 } 512 513 return indent; 514 } 515 516 static void disasm_print(void *private_data, const char *fmt, ...) __printf(2, 3); 517 518 static void disasm_print(void *private_data, const char *fmt, ...) 519 { 520 struct disasm_ctx *ctx = private_data; 521 va_list args; 522 523 va_start(args, fmt); 524 seq_buf_vprintf(&ctx->seq, fmt, args); 525 va_end(args); 526 } 527 528 static const char *disasm_kfunc_name(void *private_data, const struct bpf_insn *insn) 529 { 530 struct disasm_ctx *ctx = private_data; 531 532 return bpf_disasm_kfunc_name(ctx->env, insn); 533 } 534 535 static void format_disasm_line(struct bpf_verifier_env *env, int insn_idx, 536 struct disasm_line *line) 537 { 538 struct disasm_ctx ctx = { .env = env }; 539 struct bpf_insn *insn; 540 const struct bpf_insn_cbs cbs = { 541 .cb_call = disasm_kfunc_name, 542 .cb_print = disasm_print, 543 .private_data = &ctx, 544 }; 545 546 line->idx = insn_idx; 547 line->valid = false; 548 seq_buf_init(&ctx.seq, line->text, sizeof(line->text)); 549 550 if (insn_idx < 0 || insn_idx >= env->prog->len) 551 return; 552 553 if (insn_idx > 0 && bpf_is_ldimm64(&env->prog->insnsi[insn_idx - 1])) 554 return; 555 556 insn = &env->prog->insnsi[insn_idx]; 557 if (bpf_is_ldimm64(insn) && insn_idx + 1 >= env->prog->len) 558 return; 559 560 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 561 seq_buf_str(&ctx.seq); 562 ctx.seq.len = strnlen(line->text, sizeof(line->text)); 563 while (ctx.seq.len && line->text[ctx.seq.len - 1] == '\n') 564 seq_buf_pop(&ctx.seq); 565 seq_buf_str(&ctx.seq); 566 567 line->valid = true; 568 } 569 570 static void diag_format_source_text(char *buf, size_t size, const char *line, int width) 571 { 572 int col = 0, len = 0; 573 574 if (!size) 575 return; 576 if (width <= 0) { 577 buf[0] = '\0'; 578 return; 579 } 580 581 line = line ?: "..."; 582 while (*line && col < width && len + 1 < size) { 583 if (*line == '\t') { 584 int next = round_up(col + 1, BPF_DIAG_TAB_WIDTH); 585 586 while (col < next && col < width && len + 1 < size) { 587 buf[len++] = ' '; 588 col++; 589 } 590 line++; 591 continue; 592 } 593 594 buf[len++] = *line++; 595 col++; 596 } 597 598 if (*line) { 599 int ellipsis_len = min(3, width); 600 601 while (len > 0 && col > width - ellipsis_len) { 602 len--; 603 col--; 604 } 605 while (ellipsis_len-- && len + 1 < size) 606 buf[len++] = '.'; 607 } 608 609 buf[len] = '\0'; 610 } 611 612 static void diag_format_source_lane(char *buf, size_t size, const char *source_prefix, 613 int source_line_width, int line_num, const char *line) 614 { 615 int len, text_width; 616 617 if (line_num <= 0) { 618 buf[0] = '\0'; 619 return; 620 } 621 622 len = scnprintf(buf, size, "%s%*d | ", source_prefix, source_line_width, line_num); 623 text_width = BPF_DIAG_SOURCE_LANE_WIDTH - len; 624 diag_format_source_text(buf + len, size - len, line, text_width); 625 } 626 627 static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, 628 const char *problem) 629 { 630 char first; 631 632 if (!bpf_diag_enabled(env)) 633 return; 634 635 category = category ?: "Verifier Error"; 636 problem = problem ?: ""; 637 638 if (!problem[0]) { 639 diag_write(env, "\nVerification failed: %s\n", category); 640 return; 641 } 642 643 first = toupper(problem[0]); 644 diag_write(env, "\nVerification failed: %s: %c%s\n", category, first, problem + 1); 645 } 646 647 static void diag_print_source_annotation(struct bpf_verifier_env *env, int line_width, int indent, 648 const char *label, const char *msg) 649 { 650 const char *first_prefix, *next_prefix, *text; 651 652 indent = min_t(int, indent, max_t(int, 0, BPF_DIAG_SOURCE_LANE_WIDTH - line_width - 8)); 653 text = bpf_diag_fmt(env, "%s: %s", label, msg); 654 first_prefix = bpf_diag_fmt(env, " %*s | %*s^-- ", line_width + 4, "", indent, ""); 655 next_prefix = bpf_diag_fmt(env, " %*s | %*s ", line_width + 4, "", indent, ""); 656 657 diag_print_wrapped_prefixed(env, first_prefix, next_prefix, text); 658 } 659 660 static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, 661 struct disasm_line *disasm_lines) 662 { 663 int insn_width = diag_line_width(env->prog->len ? env->prog->len - 1 : 0); 664 int i; 665 666 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { 667 int row = i - BPF_DIAG_CONTEXT; 668 669 format_disasm_line(env, insn_idx + row, &disasm_lines[i]); 670 } 671 672 diag_write(env, " Instruction context:\n"); 673 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { 674 struct disasm_line *line = &disasm_lines[i]; 675 676 if (line->valid) 677 diag_write(env, " %s%*d | %s\n", 678 line->idx == insn_idx ? ">>> " : " ", 679 insn_width, line->idx, line->text); 680 } 681 } 682 683 static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, 684 const char *fmt, ...) 685 { 686 struct bpf_diag_scratch *scratch; 687 struct bpf_linfo_source *source_lines; 688 struct disasm_line *disasm_lines; 689 struct bpf_linfo_source src = {}; 690 struct diag_fmt_mark mark; 691 const struct bpf_line_info *linfo; 692 const struct bpf_subprog_info *subprog; 693 struct btf *btf = env->prog->aux->btf; 694 char *source_lane; 695 const char *msg; 696 const char *func; 697 int start_line, end_line, width, indent, subprogno, linfo_start, linfo_end, i; 698 va_list args; 699 700 if (!bpf_diag_enabled(env)) 701 return; 702 if (!env->diag) 703 return; 704 705 mark = diag_fmt_save(env); 706 label = label ?: "note"; 707 scratch = &env->diag->scratch; 708 source_lines = scratch->source_lines; 709 disasm_lines = scratch->disasm_lines; 710 memset(source_lines, 0, sizeof(scratch->source_lines)); 711 memset(disasm_lines, 0, sizeof(scratch->disasm_lines)); 712 713 va_start(args, fmt); 714 msg = bpf_diag_vfmt(env, fmt, args); 715 va_end(args); 716 if (!*msg) 717 msg = "<failed to allocate diagnostic text>"; 718 719 linfo = bpf_find_linfo(env->prog, insn_idx); 720 if (btf && linfo) 721 bpf_get_linfo_source(btf, linfo, &src); 722 if (!src.file || !*src.file || !src.line || !*src.line) { 723 diag_write(env, " insn %u\n", insn_idx); 724 diag_print_source_annotation(env, 0, 0, label, msg); 725 diag_print_insn_context(env, insn_idx, disasm_lines); 726 goto out_restore; 727 } 728 729 subprog = bpf_find_containing_subprog(env, insn_idx); 730 subprogno = subprog ? subprog - env->subprog_info : -ENOENT; 731 func = subprogno >= 0 ? bpf_subprog_name(env, subprogno) : NULL; 732 if (func && *func) 733 diag_write(env, " %s @ %s:%d:%d\n", func, src.file, src.line_num, src.line_col); 734 else 735 diag_write(env, " %s:%d:%d\n", src.file, src.line_num, src.line_col); 736 737 start_line = src.line_num - BPF_DIAG_CONTEXT; 738 end_line = src.line_num + BPF_DIAG_CONTEXT; 739 width = diag_line_width(end_line); 740 indent = diag_line_indent(src.line); 741 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) 742 source_lines[i].line_num = start_line + i; 743 744 linfo = env->prog->aux->linfo; 745 linfo_start = subprog ? subprog->linfo_idx : 0; 746 linfo_end = subprogno >= 0 && subprogno + 1 < env->subprog_cnt ? 747 env->subprog_info[subprogno + 1].linfo_idx : env->prog->aux->nr_linfo; 748 for (i = linfo_start; i < linfo_end; i++) { 749 struct bpf_linfo_source line_src; 750 int idx; 751 752 bpf_get_linfo_source(btf, &linfo[i], &line_src); 753 if (line_src.file_name_off != src.file_name_off || 754 line_src.line_num < start_line || line_src.line_num > end_line || 755 !line_src.line || !*line_src.line) 756 continue; 757 758 idx = line_src.line_num - start_line; 759 if (!source_lines[idx].line) 760 source_lines[idx] = line_src; 761 } 762 763 diag_write(env, " Source context:\n"); 764 source_lane = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); 765 if (!source_lane) 766 goto out_restore; 767 for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { 768 const char *source_prefix; 769 770 source_prefix = source_lines[i].line_num == src.line_num ? ">>> " : " "; 771 diag_format_source_lane(source_lane, BPF_DIAG_FMT_BUF_SIZE, source_prefix, width, 772 source_lines[i].line_num, source_lines[i].line); 773 diag_write(env, " %s\n", source_lane); 774 if (source_lines[i].line_num == src.line_num) 775 diag_print_source_annotation(env, width, indent, label, msg); 776 } 777 diag_print_insn_context(env, insn_idx, disasm_lines); 778 779 out_restore: 780 diag_fmt_restore(env, mark); 781 } 782 783 void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true) 784 { 785 struct bpf_diag_history_event event = { 786 .insn_idx = insn_idx, 787 .kind = BPF_DIAG_HISTORY_BRANCH, 788 .branch = { 789 .cond_true = cond_true, 790 }, 791 }; 792 793 diag_append_history(env, &event); 794 } 795 796 static void diag_snapshot_reg(struct bpf_diag_reg_snapshot *snapshot, 797 const struct bpf_reg_state *reg) 798 { 799 snapshot->type = reg->type; 800 if (type_is_map_ptr(reg->type)) 801 snapshot->map_ptr = reg->map_ptr; 802 if (base_type(reg->type) == PTR_TO_BTF_ID && reg->btf && reg->btf_id) { 803 snapshot->btf_id = reg->btf_id; 804 snapshot->btf = reg->btf; 805 } 806 snapshot->var_off = reg->var_off; 807 snapshot->r64 = reg->r64; 808 } 809 810 static bool diag_mod_insn_origin(struct bpf_verifier_env *env, u32 insn_idx, 811 const struct bpf_diag_mod_target *target, 812 struct bpf_diag_mod_target *origin) 813 { 814 const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 815 u8 class = BPF_CLASS(insn->code); 816 const struct bpf_func_state *state; 817 818 if (target->kind == BPF_DIAG_MOD_TARGET_REG && (class == BPF_ALU || class == BPF_ALU64) && 819 BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X) { 820 *origin = diag_reg_target(target->frame_id, target->frameno, insn->src_reg); 821 return true; 822 } 823 824 if ((target->kind != BPF_DIAG_MOD_TARGET_STACK_ARG && 825 target->kind != BPF_DIAG_MOD_TARGET_STACK_SLOT) || 826 class != BPF_STX) 827 return false; 828 829 state = env->cur_state->frame[env->cur_state->curframe]; 830 *origin = diag_reg_target(state->diag_frame_id, state->frameno, insn->src_reg); 831 return true; 832 } 833 834 static bool diag_mod_keeps_lineage(struct bpf_verifier_env *env, 835 const struct bpf_diag_history_event *event) 836 { 837 const struct bpf_insn *insn; 838 u8 class; 839 840 if (event->mod.reason != BPF_DIAG_MOD_WRITE || 841 event->mod.target.kind != BPF_DIAG_MOD_TARGET_REG) 842 return false; 843 844 insn = &env->prog->insnsi[event->insn_idx]; 845 class = BPF_CLASS(insn->code); 846 if (class != BPF_ALU && class != BPF_ALU64) 847 return false; 848 849 switch (BPF_OP(insn->code)) { 850 case BPF_ADD: 851 case BPF_SUB: 852 case BPF_MUL: 853 case BPF_OR: 854 case BPF_AND: 855 case BPF_LSH: 856 case BPF_RSH: 857 case BPF_ARSH: 858 case BPF_XOR: 859 case BPF_NEG: 860 case BPF_END: 861 return true; 862 default: 863 return false; 864 } 865 } 866 867 static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, 868 struct bpf_diag_mod_target target, 869 enum bpf_diag_mod_reason reason, 870 const struct bpf_reg_state *old_reg, 871 const struct bpf_reg_state *new_reg, 872 const struct bpf_diag_mod_target *origin) 873 { 874 struct bpf_diag_history_event event = { 875 .insn_idx = insn_idx, 876 .kind = BPF_DIAG_HISTORY_MOD, 877 .mod = { 878 .target = target, 879 .reason = reason, 880 }, 881 }; 882 883 if (old_reg) 884 diag_snapshot_reg(&event.mod.old, old_reg); 885 if (new_reg) 886 diag_snapshot_reg(&event.mod.new, new_reg); 887 if (origin) { 888 event.mod.origin = *origin; 889 event.mod.origin_valid = true; 890 } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { 891 event.mod.origin_valid = true; 892 } 893 if (old_reg && new_reg && 894 (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) && 895 !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) && 896 !event.mod.origin_valid && 897 diag_mod_keeps_lineage(env, &event)) 898 return; 899 900 diag_append_history(env, &event); 901 } 902 903 static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env, 904 const struct bpf_diag_mod_target *target) 905 { 906 struct bpf_verifier_state *vstate = env->cur_state; 907 struct bpf_func_state *state; 908 909 state = target->frameno <= vstate->curframe ? vstate->frame[target->frameno] : NULL; 910 911 if (!state) 912 return NULL; 913 if (state->diag_frame_id != target->frame_id) 914 return NULL; 915 916 switch (target->kind) { 917 case BPF_DIAG_MOD_TARGET_REG: 918 if (target->regno >= MAX_BPF_REG) 919 return NULL; 920 return &state->regs[target->regno]; 921 case BPF_DIAG_MOD_TARGET_STACK_ARG: 922 if (target->stack_arg >= state->out_stack_arg_cnt) 923 return NULL; 924 return &state->stack_arg_regs[target->stack_arg]; 925 case BPF_DIAG_MOD_TARGET_STACK_SLOT: 926 if (target->spi >= state->allocated_stack / BPF_REG_SIZE) 927 return NULL; 928 return &state->stack[target->spi].spilled_ptr; 929 default: 930 return NULL; 931 } 932 } 933 934 static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, 935 struct bpf_diag_mod_target *target) 936 { 937 struct bpf_verifier_state *vstate = env->cur_state; 938 unsigned long addr = (unsigned long)reg; 939 int frame; 940 941 for (frame = 0; frame <= vstate->curframe; frame++) { 942 struct bpf_func_state *state = vstate->frame[frame]; 943 unsigned long start, end; 944 u32 nslots = state->allocated_stack / BPF_REG_SIZE; 945 int spi; 946 947 start = (unsigned long)state->regs; 948 end = (unsigned long)(state->regs + MAX_BPF_REG); 949 if (addr >= start && addr < end) { 950 *target = diag_reg_target(state->diag_frame_id, state->frameno, 951 reg - state->regs); 952 return true; 953 } 954 955 start = (unsigned long)state->stack_arg_regs; 956 end = (unsigned long)(state->stack_arg_regs + state->out_stack_arg_cnt); 957 if (state->out_stack_arg_cnt && addr >= start && addr < end) { 958 *target = diag_stack_arg_target(state->diag_frame_id, state->frameno, 959 reg - state->stack_arg_regs); 960 return true; 961 } 962 963 start = (unsigned long)state->stack; 964 end = (unsigned long)(state->stack + nslots); 965 if (nslots && addr >= start && addr < end) { 966 spi = ((const char *)reg - (const char *)state->stack) / 967 sizeof(*state->stack); 968 *target = diag_stack_slot_target(state->diag_frame_id, state->frameno, spi); 969 return true; 970 } 971 } 972 return false; 973 } 974 975 void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, 976 const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason) 977 { 978 struct bpf_diag *diag = env->diag; 979 980 if (!diag) 981 return; 982 diag->mod.active = reg_to_target(env, reg, &diag->mod.target); 983 if (!diag->mod.active) 984 return; 985 diag->mod.target_reg_snapshot = *reg; 986 diag->mod.insn_idx = env->insn_idx; 987 diag->mod.reason = reason; 988 diag->mod.origin_valid = origin && reg_to_target(env, origin, &diag->mod.origin); 989 } 990 991 void bpf_diag_mod_end(struct bpf_verifier_env *env) 992 { 993 struct bpf_diag *diag = env->diag; 994 const struct bpf_reg_state *new_reg; 995 996 if (!diag || !diag->mod.active) 997 return; 998 diag->mod.active = false; 999 /* 1000 * Resolve the target again because the enclosing function state's stack 1001 * may have been reallocated while the modification was in progress. 1002 */ 1003 new_reg = target_to_reg(env, &diag->mod.target); 1004 if (!new_reg) 1005 return; 1006 diag_record_mod(env, diag->mod.insn_idx, diag->mod.target, diag->mod.reason, 1007 &diag->mod.target_reg_snapshot, new_reg, 1008 diag->mod.origin_valid ? &diag->mod.origin : NULL); 1009 } 1010 1011 void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, 1012 enum bpf_diag_mod_reason reason) 1013 { 1014 struct bpf_diag_mod_target target; 1015 1016 if (!env->diag || reg->type == NOT_INIT || !reg_to_target(env, reg, &target)) 1017 return; 1018 diag_record_mod(env, env->insn_idx, target, reason, reg, NULL, NULL); 1019 } 1020 1021 void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, 1022 const struct bpf_func_state *state, s16 min_off, s16 max_off, 1023 enum bpf_diag_mod_reason reason) 1024 { 1025 diag_record_mod(env, env->insn_idx, 1026 diag_stack_range_target(state->diag_frame_id, state->frameno, min_off, max_off), 1027 reason, NULL, NULL, NULL); 1028 } 1029 1030 static void diag_record_ref(struct bpf_verifier_env *env, u32 insn_idx, u8 kind, u32 ref_id) 1031 { 1032 struct bpf_diag_history_event event = { 1033 .insn_idx = insn_idx, 1034 .kind = kind, 1035 .ref = { 1036 .ref_id = ref_id, 1037 }, 1038 }; 1039 1040 diag_append_history(env, &event); 1041 } 1042 1043 void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) 1044 { 1045 diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_ACQUIRE, ref_id); 1046 } 1047 1048 void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) 1049 { 1050 diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_RELEASE, ref_id); 1051 } 1052