1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/bpf.h> 4 #include <linux/btf.h> 5 #include <linux/bpf_verifier.h> 6 #include <linux/filter.h> 7 #include <linux/vmalloc.h> 8 #include <linux/perf_event.h> 9 #include <linux/sched/signal.h> 10 #include <net/xdp.h> 11 #include "disasm.h" 12 13 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) 14 15 /* 16 * Matches BPF_PROBE_ATOMIC too: bpf_convert_ctx_accesses() rewrites arena 17 * atomics before bpf_opt_subreg_zext_lo32_rnd_hi32() runs. 18 */ 19 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 20 { 21 return BPF_CLASS(insn->code) == BPF_STX && 22 (BPF_MODE(insn->code) == BPF_ATOMIC || 23 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && 24 insn->imm == BPF_CMPXCHG; 25 } 26 27 /* Returns true if 'insn' is an address space cast instruction translated as BPF_ALU op */ 28 static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *insn) 29 { 30 struct bpf_map *arena = (struct bpf_map *)prog->aux->arena; 31 32 if (insn->code != (BPF_ALU64 | BPF_MOV | BPF_X) || insn->off != BPF_ADDR_SPACE_CAST) 33 return false; 34 35 /* cast from as(1) to as(0) */ 36 if (insn->imm == 1) 37 return true; 38 39 /* cast from as(0) to as(1) */ 40 if (insn->imm == 1 << 16) 41 return arena && arena->map_flags & BPF_F_NO_USER_CONV; 42 43 /* non-BPF_F_NO_USER_CONV cast from as(0) to as(1) should be handled by JIT */ 44 return false; 45 } 46 47 /* Return the regno defined by the insn, or -1. */ 48 static int insn_def_regno(const struct bpf_insn *insn) 49 { 50 switch (BPF_CLASS(insn->code)) { 51 case BPF_JMP: 52 case BPF_JMP32: 53 case BPF_ST: 54 return -1; 55 case BPF_STX: 56 return bpf_atomic_load_reg(insn); 57 default: 58 return insn->dst_reg; 59 } 60 } 61 62 /* 63 * For use only in combination with insn_def_regno() >= 0. 64 * Returns TRUE if the destination register operates on 64-bit, 65 * otherwise return FALSE. 66 */ 67 static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn) 68 { 69 u8 class = BPF_CLASS(insn->code); 70 u8 mode = BPF_MODE(insn->code); 71 u8 size = BPF_SIZE(insn->code); 72 u8 op = BPF_OP(insn->code); 73 bool mode_mem; 74 75 /* subregister endiness swap */ 76 if ((class == BPF_ALU || class == BPF_ALU64) && op == BPF_END && insn->imm != 64) 77 return false; 78 79 /* w0 += 1 */ 80 if (class == BPF_ALU && op != BPF_END) 81 return false; 82 83 /* address space casts converted to BPF_ALU, see bpf_do_misc_fixups() */ 84 if (is_addr_space_cast32(prog, insn)) 85 return false; 86 87 /* non 64-bit, non signed extended loads */ 88 mode_mem = mode == BPF_MEM || mode == BPF_PROBE_MEM || mode == BPF_PROBE_MEM32; 89 if (class == BPF_LDX && mode_mem && size != BPF_DW) 90 return false; 91 92 /* atomics, see insn_def_regno() */ 93 if (class == BPF_STX && size != BPF_DW) 94 return false; 95 96 /* both LD_IND and LD_ABS return 32-bit data. */ 97 if (class == BPF_LD && (mode == BPF_IND || mode == BPF_ABS)) 98 return false; 99 100 /* Conservatively return true at default. */ 101 return true; 102 } 103 104 /* 105 * Return the 32-bit subregister defined by INSN, or -1 if INSN does not 106 * explicitly define a 32-bit value. 107 */ 108 int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) 109 { 110 int dst_reg = insn_def_regno(insn); 111 112 if (dst_reg < 0 || bpf_is_reg64(prog, insn)) 113 return -1; 114 115 return dst_reg; 116 } 117 118 const struct btf_func_model * 119 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 120 const struct bpf_insn *insn) 121 { 122 const struct bpf_kfunc_desc *res; 123 struct bpf_kfunc_desc_tab *tab; 124 125 tab = prog->aux->kfunc_tab; 126 if (insn->off < 0 || insn->off >= tab->nr_descs) 127 return NULL; 128 129 res = &tab->descs[insn->off]; 130 if (bpf_jit_supports_far_kfunc_call()) { 131 if (res->func_id != insn->imm) 132 return NULL; 133 } else if ((s32)BPF_CALL_IMM(res->addr) != insn->imm) { 134 return NULL; 135 } 136 137 return &res->func_model; 138 } 139 140 static int add_kfunc_in_insns(struct bpf_verifier_env *env, 141 struct bpf_insn *insn, int cnt) 142 { 143 int i, ret; 144 145 for (i = 0; i < cnt; i++, insn++) { 146 if (bpf_pseudo_kfunc_call(insn)) { 147 ret = bpf_add_kfunc_call(env, insn->imm, insn->off); 148 if (ret < 0) 149 return ret; 150 } 151 } 152 return 0; 153 } 154 155 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 156 static int get_callee_stack_depth(struct bpf_verifier_env *env, 157 const struct bpf_insn *insn, int idx) 158 { 159 int start = idx + insn->imm + 1, subprog; 160 161 subprog = bpf_find_subprog(env, start); 162 if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start)) 163 return -EFAULT; 164 return env->subprog_info[subprog].stack_depth; 165 } 166 #endif 167 168 enum bpf_patch_mode { 169 BPF_PATCH_KEEP_TARGET, /* targets of 'off' keep naming 'off', the first new insn */ 170 BPF_PATCH_MOVE_TARGET, /* targets of 'off' follow the original insn to 'off + len - 1' */ 171 }; 172 173 /* single env->prog->insni[off] instruction was replaced with the range 174 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 175 * [0, off) and [off, end) to new locations, so the patched range stays zero 176 */ 177 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 178 struct bpf_prog *new_prog, u32 off, u32 cnt, 179 struct bpf_insn *original_insn, 180 enum bpf_patch_mode mode) 181 { 182 struct bpf_insn_aux_data *data = env->insn_aux_data; 183 struct bpf_insn *insn = new_prog->insnsi; 184 u32 old_seen = data[off].seen; 185 u32 prog_len; 186 int i; 187 188 /* aux info at OFF always needs adjustment, no matter fast path 189 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 190 * original insn at old prog. 191 */ 192 data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; 193 194 if (cnt == 1) { 195 /* 196 * A non-memory accessing insn could have been replaced by a 197 * memory accessing insn, systematically mark it for non-stack 198 * access 199 */ 200 data[off].non_stack_access = bpf_is_mem_insn(insn + off); 201 return; 202 } 203 prog_len = new_prog->len; 204 env->insn_aux_data_len = prog_len; 205 206 memmove(data + off + cnt - 1, data + off, 207 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 208 memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1)); 209 for (i = off; i < off + cnt - 1; i++) { 210 /* Expand insni[off]'s seen count to the patched range. */ 211 data[i].seen = old_seen; 212 data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; 213 /* 214 * In BPF_PATCH_MOVE_TARGET mode the original insn is the last 215 * slot by construction. 216 */ 217 if (mode == BPF_PATCH_KEEP_TARGET && 218 !memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { 219 data[i].non_stack_access = 220 data[off + cnt - 1].non_stack_access; 221 data[off + cnt - 1].non_stack_access = false; 222 } else if (bpf_is_mem_insn(insn + i)) { 223 data[i].non_stack_access = true; 224 } 225 } 226 227 /* 228 * Last slot instruction could be a newly generated 229 * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access 230 * if it is not the original instruction, otherwise keep the 231 * original marking 232 */ 233 if (bpf_is_mem_insn(insn + off + cnt - 1) && 234 memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn))) 235 data[off + cnt - 1].non_stack_access = true; 236 237 /* 238 * The indirect_target flag of the original instruction was moved to the last of the 239 * new instructions by the above memmove and memset, but the indirect jump target is 240 * actually the first instruction, so move it back. This also matches with the behavior 241 * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new 242 * instruction. For BPF_PATCH_MOVE_TARGET the original instruction is the last one, 243 * so the flag already sits where needed. 244 */ 245 if (mode == BPF_PATCH_KEEP_TARGET && data[off + cnt - 1].indirect_target) { 246 data[off].indirect_target = 1; 247 data[off + cnt - 1].indirect_target = 0; 248 } 249 } 250 251 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 252 { 253 int i; 254 255 if (len == 1) 256 return; 257 /* NOTE: fake 'exit' subprog should be updated as well. */ 258 for (i = 0; i <= env->subprog_cnt; i++) { 259 if (env->subprog_info[i].start <= off) 260 continue; 261 env->subprog_info[i].start += len - 1; 262 } 263 } 264 265 static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 first, u32 len) 266 { 267 int i; 268 269 if (len == 1) 270 return; 271 272 for (i = 0; i < env->insn_array_map_cnt; i++) 273 bpf_insn_array_adjust(env->insn_array_maps[i], first, len); 274 } 275 276 static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) 277 { 278 int i; 279 280 for (i = 0; i < env->insn_array_map_cnt; i++) 281 bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len); 282 } 283 284 static void adjust_poke_descs(struct bpf_prog *prog, u32 first, u32 len) 285 { 286 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 287 int i, sz = prog->aux->size_poke_tab; 288 struct bpf_jit_poke_descriptor *desc; 289 290 for (i = 0; i < sz; i++) { 291 desc = &tab[i]; 292 if (desc->insn_idx < first) 293 continue; 294 desc->insn_idx += len - 1; 295 } 296 } 297 298 /* 299 * Some post-verification instruction rewriting passes require an 300 * O(prog->len) operation per instruction. Keep their shared primitives 301 * killable and preemptible. 302 */ 303 static bool bpf_rewrite_must_abort(void) 304 { 305 if (fatal_signal_pending(current)) 306 return true; 307 cond_resched(); 308 return false; 309 } 310 311 static struct bpf_prog *__bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 312 const struct bpf_insn *patch, u32 len, 313 enum bpf_patch_mode mode) 314 { 315 struct bpf_prog *new_prog; 316 struct bpf_insn_aux_data *new_data = NULL; 317 /* The first slot whose targets move. */ 318 u32 first = mode == BPF_PATCH_MOVE_TARGET ? off : off + 1; 319 struct bpf_insn original_insn; 320 321 if (bpf_rewrite_must_abort()) 322 return NULL; 323 324 if (len > 1) { 325 new_data = vrealloc(env->insn_aux_data, 326 array_size(env->prog->len + len - 1, 327 sizeof(struct bpf_insn_aux_data)), 328 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 329 if (!new_data) 330 return NULL; 331 332 env->insn_aux_data = new_data; 333 } 334 335 memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn)); 336 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 337 if (IS_ERR(new_prog)) { 338 if (PTR_ERR(new_prog) == -ERANGE) 339 verbose(env, 340 "insn %d cannot be patched due to 16-bit range\n", 341 env->insn_aux_data[off].orig_idx); 342 return NULL; 343 } 344 adjust_insn_aux_data(env, new_prog, off, len, &original_insn, mode); 345 adjust_subprog_starts(env, off, len); 346 adjust_insn_arrays(env, first, len); 347 bpf_adjust_func_ptrs(env, off, len); 348 adjust_poke_descs(new_prog, first, len); 349 return new_prog; 350 } 351 352 struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 353 const struct bpf_insn *patch, u32 len) 354 { 355 return __bpf_patch_insn_data(env, off, patch, len, BPF_PATCH_KEEP_TARGET); 356 } 357 358 /* 359 * insn was moved down by delta insns inside its own patch. Operands relative 360 * to the pc that point in front of the old position did not move with it. 361 */ 362 static int bpf_adj_moved_insn(struct bpf_insn *insn, u32 delta) 363 { 364 u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code); 365 s64 off = insn->imm, off_min = S32_MIN; 366 bool is_imm = true; 367 368 if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) { 369 /* subprog that started at the old position starts with the patch */ 370 if (off >= 0) 371 return 0; 372 } else if ((class == BPF_JMP || class == BPF_JMP32) && 373 op != BPF_CALL && op != BPF_EXIT) { 374 if (insn->code != (BPF_JMP32 | BPF_JA)) { 375 off = insn->off; 376 off_min = S16_MIN; 377 is_imm = false; 378 } 379 /* jump to itself stays */ 380 if (off >= -1) 381 return 0; 382 } else { 383 return 0; 384 } 385 386 off -= delta; 387 if (off < off_min) 388 return -ERANGE; 389 if (is_imm) 390 insn->imm = off; 391 else 392 insn->off = off; 393 return 0; 394 } 395 396 /* 397 * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the 398 * jump offset by 'delta'. 399 */ 400 static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta) 401 { 402 struct bpf_insn *insn = prog->insnsi; 403 u32 insn_cnt = prog->len, i; 404 s32 imm; 405 s16 off; 406 407 for (i = 0; i < insn_cnt; i++, insn++) { 408 u8 code = insn->code; 409 410 if (tgt_idx <= i && i < tgt_idx + delta) 411 continue; 412 413 if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) || 414 BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT) 415 continue; 416 417 if (insn->code == (BPF_JMP32 | BPF_JA)) { 418 if (i + 1 + insn->imm != tgt_idx) 419 continue; 420 if (check_add_overflow(insn->imm, delta, &imm)) 421 return -ERANGE; 422 insn->imm = imm; 423 } else { 424 if (i + 1 + insn->off != tgt_idx) 425 continue; 426 if (check_add_overflow(insn->off, delta, &off)) 427 return -ERANGE; 428 insn->off = off; 429 } 430 } 431 return 0; 432 } 433 434 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 435 u32 off, u32 cnt) 436 { 437 int i, j; 438 439 /* find first prog starting at or after off (first to remove) */ 440 for (i = 0; i < env->subprog_cnt; i++) 441 if (env->subprog_info[i].start >= off) 442 break; 443 /* find first prog starting at or after off + cnt (first to stay) */ 444 for (j = i; j < env->subprog_cnt; j++) 445 if (env->subprog_info[j].start >= off + cnt) 446 break; 447 /* if j doesn't start exactly at off + cnt, we are just removing 448 * the front of previous prog 449 */ 450 if (env->subprog_info[j].start != off + cnt) 451 j--; 452 453 if (j > i) { 454 struct bpf_prog_aux *aux = env->prog->aux; 455 int move; 456 457 /* move fake 'exit' subprog as well */ 458 move = env->subprog_cnt + 1 - j; 459 460 memmove(env->subprog_info + i, 461 env->subprog_info + j, 462 sizeof(*env->subprog_info) * move); 463 env->subprog_cnt -= j - i; 464 465 /* remove func_info and its aux */ 466 if (aux->func_info) { 467 move = aux->func_info_cnt - j; 468 469 memmove(aux->func_info + i, 470 aux->func_info + j, 471 sizeof(*aux->func_info) * move); 472 if (aux->func_info_aux) 473 memmove(aux->func_info_aux + i, 474 aux->func_info_aux + j, 475 sizeof(*aux->func_info_aux) * move); 476 aux->func_info_cnt -= j - i; 477 /* func_info->insn_off is set after all code rewrites, 478 * in adjust_btf_func() - no need to adjust 479 */ 480 } 481 /* The exception callback is live, so it cannot sit inside the removed range. */ 482 if (env->exception_callback_subprog >= j) 483 env->exception_callback_subprog -= j - i; 484 } else { 485 /* convert i from "first prog to remove" to "first to adjust" */ 486 if (env->subprog_info[i].start == off) 487 i++; 488 } 489 490 /* update fake 'exit' subprog as well */ 491 for (; i <= env->subprog_cnt; i++) 492 env->subprog_info[i].start -= cnt; 493 494 return 0; 495 } 496 497 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 498 u32 cnt) 499 { 500 struct bpf_prog *prog = env->prog; 501 u32 i, l_off, l_cnt, nr_linfo; 502 struct bpf_line_info *linfo; 503 504 nr_linfo = prog->aux->nr_linfo; 505 if (!nr_linfo) 506 return 0; 507 508 linfo = prog->aux->linfo; 509 510 /* find first line info to remove, count lines to be removed */ 511 for (i = 0; i < nr_linfo; i++) 512 if (linfo[i].insn_off >= off) 513 break; 514 515 l_off = i; 516 l_cnt = 0; 517 for (; i < nr_linfo; i++) 518 if (linfo[i].insn_off < off + cnt) 519 l_cnt++; 520 else 521 break; 522 523 /* First live insn doesn't match first live linfo, it needs to "inherit" 524 * last removed linfo. prog is already modified, so prog->len == off 525 * means no live instructions after (tail of the program was removed). 526 */ 527 if (prog->len != off && l_cnt && 528 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 529 l_cnt--; 530 linfo[--i].insn_off = off + cnt; 531 } 532 533 /* remove the line info which refer to the removed instructions */ 534 if (l_cnt) { 535 memmove(linfo + l_off, linfo + i, 536 sizeof(*linfo) * (nr_linfo - i)); 537 538 prog->aux->nr_linfo -= l_cnt; 539 nr_linfo = prog->aux->nr_linfo; 540 } 541 542 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 543 for (i = l_off; i < nr_linfo; i++) 544 linfo[i].insn_off -= cnt; 545 546 /* fix up all subprogs (incl. 'exit') which start >= off */ 547 for (i = 0; i <= env->subprog_cnt; i++) 548 if (env->subprog_info[i].linfo_idx > l_off) { 549 /* program may have started in the removed region but 550 * may not be fully removed 551 */ 552 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 553 env->subprog_info[i].linfo_idx -= l_cnt; 554 else 555 env->subprog_info[i].linfo_idx = l_off; 556 } 557 558 return 0; 559 } 560 561 /* Clean up dynamically allocated fields of aux data for instructions [start, ...] */ 562 void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) 563 { 564 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 565 int end = start + len; 566 int i; 567 568 for (i = start; i < end; i++) { 569 if (aux_data[i].loop) { 570 kvfree(aux_data[i].loop->exits); 571 kvfree(aux_data[i].loop); 572 aux_data[i].loop = NULL; 573 } 574 } 575 } 576 577 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 578 { 579 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 580 unsigned int orig_prog_len = env->prog->len; 581 int err; 582 583 if (bpf_rewrite_must_abort()) 584 return -EINTR; 585 586 if (bpf_prog_is_offloaded(env->prog->aux)) 587 bpf_prog_offload_remove_insns(env, off, cnt); 588 589 bpf_clear_insn_aux_data(env, off, cnt); 590 591 err = bpf_remove_insns(env->prog, off, cnt); 592 if (err) 593 return err; 594 595 /* before subprogs are adjusted, since it looks at them */ 596 bpf_adjust_func_ptrs_after_remove(env, off, cnt); 597 598 err = adjust_subprog_starts_after_remove(env, off, cnt); 599 if (err) 600 return err; 601 602 err = bpf_adj_linfo_after_remove(env, off, cnt); 603 if (err) 604 return err; 605 606 adjust_insn_arrays_after_remove(env, off, cnt); 607 608 memmove(aux_data + off, aux_data + off + cnt, 609 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 610 env->insn_aux_data_len -= cnt; 611 612 return 0; 613 } 614 615 static const struct bpf_insn NOP = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 616 static const struct bpf_insn MAY_GOTO_0 = BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0, 0); 617 618 bool bpf_insn_is_cond_jump(u8 code) 619 { 620 u8 op; 621 622 op = BPF_OP(code); 623 if (BPF_CLASS(code) == BPF_JMP32) 624 return op != BPF_JA; 625 626 if (BPF_CLASS(code) != BPF_JMP) 627 return false; 628 629 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 630 } 631 632 void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 633 { 634 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 635 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 636 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 637 struct bpf_insn *insn = env->prog->insnsi; 638 const int insn_cnt = env->prog->len; 639 int i; 640 641 for (i = 0; i < insn_cnt; i++, insn++) { 642 if (!bpf_insn_is_cond_jump(insn->code)) 643 continue; 644 645 if (bpf_is_unwind_goto(insn) && aux_data[i + 1 + insn->off].seen) { 646 /* 647 * The callee never exits. Its unwind_exit needs 648 * unwind_goto all the same, and the cpu predicts that it 649 * returns to what follows. Don't let the removal of dead 650 * code put something else there. 651 */ 652 if (!aux_data[i + 1].seen) { 653 memcpy(insn + 1, &trap, sizeof(trap)); 654 aux_data[i + 1].seen = env->pass_cnt; 655 } 656 continue; 657 } 658 659 if (!aux_data[i + 1].seen) 660 ja.off = insn->off; 661 else if (!aux_data[i + 1 + insn->off].seen) 662 ja.off = 0; 663 else 664 continue; 665 666 if (bpf_prog_is_offloaded(env->prog->aux)) 667 bpf_prog_offload_replace_insn(env, i, &ja); 668 669 memcpy(insn, &ja, sizeof(ja)); 670 } 671 } 672 673 /* 674 * The address of a function might be taken by the code that is not dead, 675 * while the function itself is dead code, since nothing calls it. Keep the first 676 * instruction of such function, so that the address remains valid and differs 677 * from the addresses of other functions. Make it a trap like 678 * sanitize_dead_code() does. 679 */ 680 static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env) 681 { 682 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 683 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 684 struct bpf_insn *insn = env->prog->insnsi; 685 int insn_cnt = env->prog->len; 686 int i, t; 687 688 for (i = 0; i < insn_cnt; i++) { 689 if (!aux_data[i].seen || !bpf_pseudo_func(insn + i)) 690 continue; 691 t = i + insn[i].imm + 1; 692 if (aux_data[t].seen) 693 continue; 694 memcpy(insn + t, &trap, sizeof(trap)); 695 aux_data[t].zext_dst = false; 696 aux_data[t].seen = env->pass_cnt; 697 } 698 } 699 700 int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) 701 { 702 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 703 int insn_cnt = env->prog->len; 704 int i, err; 705 706 keep_funcs_with_addr_taken(env); 707 708 for (i = 0; i < insn_cnt; i++) { 709 int j; 710 711 j = 0; 712 while (i + j < insn_cnt && !aux_data[i + j].seen) 713 j++; 714 if (!j) 715 continue; 716 717 err = verifier_remove_insns(env, i, j); 718 if (err) 719 return err; 720 insn_cnt = env->prog->len; 721 } 722 723 return 0; 724 } 725 726 int bpf_opt_remove_nops(struct bpf_verifier_env *env) 727 { 728 struct bpf_insn_aux_data *aux = env->insn_aux_data; 729 struct bpf_insn *insn = env->prog->insnsi; 730 int insn_cnt = env->prog->len; 731 bool is_may_goto_0, is_ja; 732 int i, err; 733 734 for (i = 0; i < insn_cnt; i++) { 735 is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0)); 736 is_ja = !memcmp(&insn[i], &NOP, sizeof(NOP)); 737 738 if (!is_may_goto_0 && !is_ja) 739 continue; 740 if (aux[i].indirect_target) 741 continue; 742 743 err = verifier_remove_insns(env, i, 1); 744 if (err) 745 return err; 746 insn_cnt--; 747 /* Go back one insn to catch may_goto +1; may_goto +0 sequence */ 748 i -= (is_may_goto_0 && i > 0) ? 2 : 1; 749 } 750 751 return 0; 752 } 753 754 int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 755 const union bpf_attr *attr) 756 { 757 struct bpf_insn *patch; 758 /* use env->insn_buf as two independent buffers */ 759 struct bpf_insn *zext_patch = env->insn_buf; 760 struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2]; 761 struct bpf_insn_aux_data *aux = env->insn_aux_data; 762 int i, patch_len, delta = 0, len = env->prog->len; 763 struct bpf_insn *insns = env->prog->insnsi; 764 struct bpf_prog *new_prog; 765 bool rnd_hi32; 766 767 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 768 zext_patch[1] = BPF_ZEXT_REG(0); 769 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 770 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 771 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 772 for (i = 0; i < len; i++) { 773 int adj_idx = i + delta; 774 struct bpf_insn insn; 775 int load_reg; 776 777 insn = insns[adj_idx]; 778 load_reg = insn_def_regno(&insn); 779 if (!aux[adj_idx].zext_dst) { 780 u8 code, class; 781 u32 imm_rnd; 782 783 if (!rnd_hi32) 784 continue; 785 786 code = insn.code; 787 class = BPF_CLASS(code); 788 if (load_reg == -1) 789 continue; 790 791 if (bpf_is_reg64(env->prog, &insn)) { 792 if (class == BPF_LD && 793 BPF_MODE(code) == BPF_IMM) 794 i++; 795 continue; 796 } 797 798 /* ctx load could be transformed into wider load. */ 799 if (class == BPF_LDX && 800 aux[adj_idx].ptr_type == PTR_TO_CTX) 801 continue; 802 803 imm_rnd = get_random_u32(); 804 rnd_hi32_patch[0] = insn; 805 rnd_hi32_patch[1].imm = imm_rnd; 806 rnd_hi32_patch[3].dst_reg = load_reg; 807 patch = rnd_hi32_patch; 808 patch_len = 4; 809 goto apply_patch_buffer; 810 } 811 812 /* Add in an zero-extend instruction if a) the JIT has requested 813 * it or b) it's a CMPXCHG. 814 * 815 * The latter is because: BPF_CMPXCHG always loads a value into 816 * R0, therefore always zero-extends. However some archs' 817 * equivalent instruction only does this load when the 818 * comparison is successful. This detail of CMPXCHG is 819 * orthogonal to the general zero-extension behaviour of the 820 * CPU, so it's treated independently of bpf_jit_needs_zext. 821 */ 822 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 823 continue; 824 825 /* Zero-extension is done by the caller. */ 826 if (bpf_pseudo_kfunc_call(&insn)) 827 continue; 828 829 if (verifier_bug_if(load_reg == -1, env, 830 "zext_dst is set, but no reg is defined")) 831 return -EFAULT; 832 833 zext_patch[0] = insn; 834 zext_patch[1].dst_reg = load_reg; 835 zext_patch[1].src_reg = load_reg; 836 patch = zext_patch; 837 patch_len = 2; 838 apply_patch_buffer: 839 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 840 if (!new_prog) 841 return -ENOMEM; 842 env->prog = new_prog; 843 insns = new_prog->insnsi; 844 aux = env->insn_aux_data; 845 delta += patch_len - 1; 846 } 847 848 return 0; 849 } 850 851 /* 852 * The prologue takes the context from R1, and it can clobber R2-R5. An 853 * extension gets the arguments of the function that it replaces. 854 */ 855 static bool ext_prog_takes_prologue(const struct bpf_verifier_env *env) 856 { 857 const struct bpf_subprog_info *sub = &env->subprog_info[0]; 858 int i, cnt = min_t(int, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); 859 860 if (!cnt || sub->args[0].arg_type != ARG_PTR_TO_CTX) 861 return false; 862 863 for (i = 1; i < cnt; i++) 864 if (sub->args[i].arg_type != ARG_SCALAR) 865 return false; 866 867 return true; 868 } 869 870 /* convert load instructions that access fields of a context type into a 871 * sequence of instructions that access fields of the underlying structure: 872 * struct __sk_buff -> struct sk_buff 873 * struct bpf_sock_ops -> struct sock 874 */ 875 /* 876 * A load through an __arena pointer member of a kernel struct reads the kernel 877 * address of arena memory. Append the conversion to the arena pointer form, 878 * (u32)(kaddr - kern_vm_start), that the struct_ops trampoline applies to 879 * __arena arguments. A __nullable member keeps NULL as NULL, tested on the 880 * full kernel address, and so does a probed load, which reads as 0 when it 881 * faults. Returns the number of insns in @buf, 0 when the insn loads no such 882 * member, or an error when it does so on some paths only. 883 */ 884 static int arena_member_fixup(struct bpf_verifier_env *env, int idx, struct bpf_insn *insn, 885 struct bpf_insn *buf) 886 { 887 struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx]; 888 int cnt = 0; 889 u32 base; 890 891 if (!aux->arena_member) 892 return 0; 893 if (aux->non_arena_load) { 894 verbose(env, "an insn loads an arena pointer member on one path and something else on another\n"); 895 return -EINVAL; 896 } 897 898 base = bpf_arena_get_kern_vm_start(env->prog->aux->arena); 899 buf[cnt++] = *insn; 900 if (aux->arena_member_nullable || BPF_MODE(insn->code) == BPF_PROBE_MEM) 901 buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, insn->dst_reg, 0, 1); 902 buf[cnt++] = BPF_ALU32_IMM(BPF_SUB, insn->dst_reg, (s32)base); 903 return cnt; 904 } 905 906 int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) 907 { 908 struct bpf_subprog_info *subprogs = env->subprog_info; 909 const struct bpf_verifier_ops *ops = env->ops; 910 int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0; 911 const int insn_cnt = env->prog->len; 912 struct bpf_insn *epilogue_buf = env->epilogue_buf; 913 struct bpf_insn *insn_buf = env->insn_buf; 914 struct bpf_insn *insn; 915 u32 target_size, size_default, off; 916 struct bpf_prog *new_prog; 917 enum bpf_access_type type; 918 bool is_narrower_load; 919 int epilogue_idx = 0; 920 921 if (ops->gen_epilogue) { 922 epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog, 923 -(subprogs[0].stack_depth + 8)); 924 if (epilogue_cnt >= INSN_BUF_SIZE) { 925 verifier_bug(env, "epilogue is too long"); 926 return -EFAULT; 927 } else if (epilogue_cnt) { 928 /* Save the ARG_PTR_TO_CTX for the epilogue to use */ 929 cnt = 0; 930 subprogs[0].stack_depth += 8; 931 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, 932 -subprogs[0].stack_depth); 933 insn_buf[cnt++] = env->prog->insnsi[0]; 934 new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, 935 BPF_PATCH_MOVE_TARGET); 936 if (!new_prog) 937 return -ENOMEM; 938 env->prog = new_prog; 939 delta += cnt - 1; 940 941 ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1); 942 if (ret < 0) 943 return ret; 944 } 945 } 946 947 if (ops->gen_prologue || env->seen_direct_write) { 948 if (!ops->gen_prologue) { 949 verifier_bug(env, "gen_prologue is null"); 950 return -EFAULT; 951 } 952 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 953 env->prog); 954 if (cnt >= INSN_BUF_SIZE) { 955 verifier_bug(env, "prologue is too long"); 956 return -EFAULT; 957 } else if (cnt) { 958 if (env->prog->type == BPF_PROG_TYPE_EXT && 959 !ext_prog_takes_prologue(env)) { 960 verbose(env, 961 "Extension writing packet data must take ctx and scalars only\n"); 962 return -EINVAL; 963 } 964 if (env->prog->type == BPF_PROG_TYPE_EXT && 965 bpf_check_ext_prologue(&env->log, 966 env->prog->aux->dst_prog, 967 env->prog->aux->attach_btf_id)) 968 return -EINVAL; 969 env->prog->aux->has_prologue = true; 970 new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, 971 BPF_PATCH_MOVE_TARGET); 972 if (!new_prog) 973 return -ENOMEM; 974 975 env->prog = new_prog; 976 delta += cnt - 1; 977 978 ret = add_kfunc_in_insns(env, insn_buf, cnt - 1); 979 if (ret < 0) 980 return ret; 981 } 982 } 983 984 if (delta) 985 WARN_ON(adjust_jmp_off(env->prog, 0, delta)); 986 987 if (bpf_prog_is_offloaded(env->prog->aux)) 988 return 0; 989 990 insn = env->prog->insnsi + delta; 991 992 for (i = 0; i < insn_cnt; i++, insn++) { 993 bpf_convert_ctx_access_t convert_ctx_access; 994 enum bpf_reg_type ptr_type; 995 u8 mode; 996 997 if (env->insn_aux_data[i + delta].nospec) { 998 WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state); 999 struct bpf_insn *patch = insn_buf; 1000 1001 *patch++ = BPF_ST_NOSPEC(); 1002 *patch = *insn; 1003 ret = bpf_adj_moved_insn(patch++, 1); 1004 if (ret) { 1005 verbose(env, "insn %d cannot be patched due to 16-bit range\n", 1006 env->insn_aux_data[i + delta].orig_idx); 1007 return ret; 1008 } 1009 cnt = patch - insn_buf; 1010 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1011 if (!new_prog) 1012 return -ENOMEM; 1013 1014 delta += cnt - 1; 1015 env->prog = new_prog; 1016 insn = new_prog->insnsi + i + delta; 1017 /* This can not be easily merged with the 1018 * nospec_result-case, because an insn may require a 1019 * nospec before and after itself. Therefore also do not 1020 * 'continue' here but potentially apply further 1021 * patching to insn. *insn should equal patch[1] now. 1022 */ 1023 } 1024 1025 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 1026 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 1027 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 1028 insn->code == (BPF_LDX | BPF_MEM | BPF_DW) || 1029 insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) || 1030 insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) || 1031 insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) { 1032 type = BPF_READ; 1033 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 1034 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 1035 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 1036 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 1037 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 1038 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 1039 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 1040 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 1041 type = BPF_WRITE; 1042 } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) || 1043 insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) || 1044 insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) || 1045 insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) && 1046 env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) { 1047 insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); 1048 env->prog->aux->num_exentries++; 1049 continue; 1050 } else if (insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && 1051 insn->imm == BPF_LOAD_ACQ && 1052 env->insn_aux_data[i + delta].arena_member) { 1053 /* an acquire load of an __arena member is rewritten like a plain load */ 1054 cnt = arena_member_fixup(env, i + delta, insn, insn_buf); 1055 if (cnt < 0) 1056 return cnt; 1057 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1058 if (!new_prog) 1059 return -ENOMEM; 1060 1061 delta += cnt - 1; 1062 env->prog = new_prog; 1063 insn = new_prog->insnsi + i + delta; 1064 continue; 1065 } else if (insn->code == (BPF_JMP | BPF_EXIT) && 1066 epilogue_cnt && 1067 i + delta < subprogs[1].start) { 1068 /* Generate epilogue for the main prog */ 1069 if (epilogue_idx) { 1070 /* jump back to the earlier generated epilogue */ 1071 insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1); 1072 cnt = 1; 1073 } else { 1074 memcpy(insn_buf, epilogue_buf, 1075 epilogue_cnt * sizeof(*epilogue_buf)); 1076 cnt = epilogue_cnt; 1077 /* epilogue_idx cannot be 0. It must have at 1078 * least one ctx ptr saving insn before the 1079 * epilogue. 1080 */ 1081 epilogue_idx = i + delta; 1082 } 1083 goto patch_insn_buf; 1084 } else { 1085 continue; 1086 } 1087 1088 if (type == BPF_WRITE && 1089 env->insn_aux_data[i + delta].nospec_result) { 1090 /* nospec_result is only used to mitigate Spectre v4 and 1091 * to limit verification-time for Spectre v1. 1092 */ 1093 struct bpf_insn *patch = insn_buf; 1094 1095 *patch++ = *insn; 1096 *patch++ = BPF_ST_NOSPEC(); 1097 cnt = patch - insn_buf; 1098 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1099 if (!new_prog) 1100 return -ENOMEM; 1101 1102 delta += cnt - 1; 1103 env->prog = new_prog; 1104 insn = new_prog->insnsi + i + delta; 1105 continue; 1106 } 1107 1108 ptr_type = env->insn_aux_data[i + delta].ptr_type; 1109 switch ((int)ptr_type) { 1110 case PTR_TO_CTX: 1111 if (!ops->convert_ctx_access) 1112 continue; 1113 convert_ctx_access = ops->convert_ctx_access; 1114 break; 1115 case PTR_TO_SOCKET: 1116 case PTR_TO_SOCK_COMMON: 1117 convert_ctx_access = bpf_sock_convert_ctx_access; 1118 break; 1119 case PTR_TO_TCP_SOCK: 1120 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 1121 break; 1122 case PTR_TO_XDP_SOCK: 1123 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 1124 break; 1125 case PTR_TO_ARENA: 1126 if (BPF_MODE(insn->code) == BPF_MEMSX) { 1127 if (!bpf_jit_supports_insn(insn, true)) { 1128 verbose(env, "sign extending loads from arena are not supported yet\n"); 1129 return -EOPNOTSUPP; 1130 } 1131 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code); 1132 } else { 1133 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code); 1134 } 1135 env->prog->aux->num_exentries++; 1136 continue; 1137 default: 1138 /* 1139 * A pointer which may fault on a dereference must not 1140 * be loaded from without fault protection, hence turn 1141 * the BPF_LDX into a BPF_PROBE_MEM one so that a bad 1142 * address is handled rather than panicking the kernel. 1143 * A store through one is rejected earlier, there is no 1144 * probed counterpart to rewrite it into. 1145 */ 1146 if (bpf_is_ptr_to_mem_or_btf_id(ptr_type) && 1147 bpf_may_fault_on_deref(ptr_type) && 1148 type == BPF_READ) { 1149 if (BPF_MODE(insn->code) == BPF_MEM) 1150 insn->code = BPF_LDX | BPF_PROBE_MEM | 1151 BPF_SIZE(insn->code); 1152 else 1153 insn->code = BPF_LDX | BPF_PROBE_MEMSX | 1154 BPF_SIZE(insn->code); 1155 env->prog->aux->num_exentries++; 1156 } else if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, 1157 "access to a fault prone pointer is not rewritten as a probed one")) { 1158 return -EFAULT; 1159 } 1160 1161 cnt = arena_member_fixup(env, i + delta, insn, insn_buf); 1162 if (cnt < 0) 1163 return cnt; 1164 if (!cnt) 1165 continue; 1166 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1167 if (!new_prog) 1168 return -ENOMEM; 1169 1170 delta += cnt - 1; 1171 env->prog = new_prog; 1172 insn = new_prog->insnsi + i + delta; 1173 continue; 1174 } 1175 1176 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 1177 size = BPF_LDST_BYTES(insn); 1178 mode = BPF_MODE(insn->code); 1179 1180 /* If the read access is a narrower load of the field, 1181 * convert to a 4/8-byte load, to minimum program type specific 1182 * convert_ctx_access changes. If conversion is successful, 1183 * we will apply proper mask to the result. 1184 */ 1185 is_narrower_load = size < ctx_field_size; 1186 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 1187 off = insn->off; 1188 if (is_narrower_load) { 1189 u8 size_code; 1190 1191 if (type == BPF_WRITE) { 1192 verifier_bug(env, "narrow ctx access misconfigured"); 1193 return -EFAULT; 1194 } 1195 1196 size_code = BPF_H; 1197 if (ctx_field_size == 4) 1198 size_code = BPF_W; 1199 else if (ctx_field_size == 8) 1200 size_code = BPF_DW; 1201 1202 insn->off = off & ~(size_default - 1); 1203 insn->code = BPF_LDX | BPF_MEM | size_code; 1204 } 1205 1206 target_size = 0; 1207 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 1208 &target_size); 1209 if (cnt == 0 || cnt >= INSN_BUF_SIZE || 1210 (ctx_field_size && !target_size)) { 1211 verifier_bug(env, "error during ctx access conversion (%d)", cnt); 1212 return -EFAULT; 1213 } 1214 1215 if (is_narrower_load && size < target_size) { 1216 u8 shift = bpf_ctx_narrow_access_offset( 1217 off, size, size_default) * 8; 1218 if (shift && cnt + 1 >= INSN_BUF_SIZE) { 1219 verifier_bug(env, "narrow ctx load misconfigured"); 1220 return -EFAULT; 1221 } 1222 if (ctx_field_size <= 4) { 1223 if (shift) 1224 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 1225 insn->dst_reg, 1226 shift); 1227 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1228 (1 << size * 8) - 1); 1229 } else { 1230 if (shift) 1231 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 1232 insn->dst_reg, 1233 shift); 1234 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1235 (1ULL << size * 8) - 1); 1236 } 1237 } 1238 if (mode == BPF_MEMSX) 1239 insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, 1240 insn->dst_reg, insn->dst_reg, 1241 size * 8, 0); 1242 1243 patch_insn_buf: 1244 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1245 if (!new_prog) 1246 return -ENOMEM; 1247 1248 delta += cnt - 1; 1249 1250 /* keep walking new program and skip insns we just inserted */ 1251 env->prog = new_prog; 1252 insn = new_prog->insnsi + i + delta; 1253 } 1254 1255 return 0; 1256 } 1257 1258 static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env) 1259 { 1260 u32 *starts = NULL; 1261 1262 starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT); 1263 if (starts) { 1264 for (int i = 0; i < env->subprog_cnt; i++) 1265 starts[i] = env->subprog_info[i].start; 1266 } 1267 return starts; 1268 } 1269 1270 static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts) 1271 { 1272 for (int i = 0; i < env->subprog_cnt; i++) 1273 env->subprog_info[i].start = orig_starts[i]; 1274 /* restore the start of fake 'exit' subprog as well */ 1275 env->subprog_info[env->subprog_cnt].start = env->prog->len; 1276 } 1277 1278 /* 1279 * Replace the offsets of functions with their addresses in the maps of 1280 * the program, see resolve_func_ptrs() in verifier.c. The program must be 1281 * the only user of such map. From now on no other program can use it, see 1282 * bpf_map_claim(). 1283 */ 1284 static int resolve_func_ptrs(struct bpf_verifier_env *env, struct bpf_prog *prog, 1285 struct bpf_prog **func) 1286 { 1287 unsigned long me = (unsigned long)prog->aux; 1288 struct bpf_func_ptr *ptr; 1289 int i, err, subprog; 1290 u64 addr, new, *slot; 1291 1292 for (i = 0; i < env->func_ptr_cnt; i++) { 1293 ptr = &env->func_ptrs[i]; 1294 1295 /* pointers are sorted by map */ 1296 if ((!i || ptr->map != ptr[-1].map) && 1297 cmpxchg(&ptr->map->user, me, me | BPF_MAP_USER_PATCHED) != me) { 1298 verbose(env, "map '%s' is used by another program\n", ptr->map->name); 1299 return -EBUSY; 1300 } 1301 1302 /* it's the address of the value of the map whatever the offset is */ 1303 err = ptr->map->ops->map_direct_value_addr(ptr->map, &addr, 0); 1304 if (verifier_bug_if(err, env, "no value of map '%s'", ptr->map->name)) 1305 return -EFAULT; 1306 slot = (u64 *)(unsigned long)(addr + ptr->map_off); 1307 1308 /* 1309 * The function that is dead code is removed and the pointer is 1310 * NULL. The program may read it, but callx of it is dead code 1311 * too, see check_func_callx(). 1312 */ 1313 new = 0; 1314 if (ptr->xlated_off != BPF_FUNC_PTR_DELETED) { 1315 subprog = bpf_find_subprog(env, ptr->xlated_off); 1316 if (verifier_bug_if(!func || subprog <= 0, env, "no function at insn %u", 1317 ptr->xlated_off)) 1318 return -EFAULT; 1319 new = (unsigned long)func[subprog]->bpf_func; 1320 } 1321 1322 /* the map is frozen and nothing else uses it */ 1323 if (verifier_bug_if(*slot != (u64)ptr->orig_off * sizeof(struct bpf_insn), env, 1324 "map '%s' offset %u changed", ptr->map->name, ptr->map_off)) 1325 return -EFAULT; 1326 WRITE_ONCE(*slot, new); 1327 } 1328 return 0; 1329 } 1330 1331 static int jit_subprogs(struct bpf_verifier_env *env) 1332 { 1333 struct bpf_prog *prog = env->prog, **func, *tmp; 1334 int i, j, subprog_start, subprog_end = 0, len, subprog; 1335 struct bpf_map *map_ptr; 1336 struct bpf_insn *insn; 1337 void *old_bpf_func; 1338 int err, num_exentries; 1339 1340 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1341 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 1342 continue; 1343 1344 /* Upon error here we cannot fall back to interpreter but 1345 * need a hard reject of the program. Thus -EFAULT is 1346 * propagated in any case. 1347 */ 1348 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1349 if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d", 1350 i + insn->imm + 1)) 1351 return -EFAULT; 1352 /* temporarily remember subprog id inside insn instead of 1353 * aux_data, since next loop will split up all insns into funcs 1354 */ 1355 insn->off = subprog; 1356 /* remember original imm in case JIT fails and fallback 1357 * to interpreter will be needed 1358 */ 1359 env->insn_aux_data[i].call_imm = insn->imm; 1360 /* point imm to __bpf_call_base+1 from JITs point of view */ 1361 insn->imm = 1; 1362 if (bpf_pseudo_func(insn)) { 1363 #if defined(MODULES_VADDR) 1364 u64 addr = MODULES_VADDR; 1365 #else 1366 u64 addr = VMALLOC_START; 1367 #endif 1368 /* jit (e.g. x86_64) may emit fewer instructions 1369 * if it learns a u32 imm is the same as a u64 imm. 1370 * Set close enough to possible prog address. 1371 */ 1372 insn[0].imm = (u32)addr; 1373 insn[1].imm = addr >> 32; 1374 } 1375 } 1376 1377 err = bpf_prog_alloc_jited_linfo(prog); 1378 if (err) 1379 goto out_undo_insn; 1380 1381 err = -ENOMEM; 1382 func = kzalloc_objs(prog, env->subprog_cnt); 1383 if (!func) 1384 goto out_undo_insn; 1385 1386 for (i = 0; i < env->subprog_cnt; i++) { 1387 subprog_start = subprog_end; 1388 subprog_end = env->subprog_info[i + 1].start; 1389 1390 len = subprog_end - subprog_start; 1391 /* bpf_prog_run() doesn't call subprogs directly, 1392 * hence main prog stats include the runtime of subprogs. 1393 * subprogs don't have IDs and not reachable via prog_get_next_id 1394 * func[i]->stats will never be accessed and stays NULL 1395 */ 1396 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 1397 if (!func[i]) 1398 goto out_free; 1399 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 1400 len * sizeof(struct bpf_insn)); 1401 func[i]->type = prog->type; 1402 func[i]->len = len; 1403 if (bpf_prog_calc_tag(func[i])) 1404 goto out_free; 1405 func[i]->is_func = 1; 1406 func[i]->sleepable = prog->sleepable; 1407 func[i]->blinded = prog->blinded; 1408 func[i]->aux->func_idx = i; 1409 /* Below members will be freed only at prog->aux */ 1410 func[i]->aux->btf = prog->aux->btf; 1411 func[i]->aux->subprog_start = subprog_start; 1412 func[i]->aux->func_info = prog->aux->func_info; 1413 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 1414 func[i]->aux->poke_tab = prog->aux->poke_tab; 1415 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 1416 func[i]->aux->main_prog_aux = prog->aux; 1417 1418 for (j = 0; j < prog->aux->size_poke_tab; j++) { 1419 struct bpf_jit_poke_descriptor *poke; 1420 1421 poke = &prog->aux->poke_tab[j]; 1422 if (poke->insn_idx < subprog_end && 1423 poke->insn_idx >= subprog_start) 1424 poke->aux = func[i]->aux; 1425 } 1426 1427 func[i]->aux->name[0] = 'F'; 1428 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 1429 if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) 1430 func[i]->aux->jits_use_priv_stack = true; 1431 1432 func[i]->jit_requested = 1; 1433 func[i]->blinding_requested = prog->blinding_requested; 1434 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 1435 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 1436 func[i]->aux->linfo = prog->aux->linfo; 1437 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 1438 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 1439 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 1440 func[i]->aux->arena = prog->aux->arena; 1441 func[i]->aux->uses_arena_insns = bpf_prog_has_arena_insns(func[i]); 1442 func[i]->aux->used_maps = env->used_maps; 1443 func[i]->aux->used_map_cnt = env->used_map_cnt; 1444 num_exentries = 0; 1445 insn = func[i]->insnsi; 1446 for (j = 0; j < func[i]->len; j++, insn++) { 1447 if (BPF_CLASS(insn->code) == BPF_LDX && 1448 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 1449 BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 1450 BPF_MODE(insn->code) == BPF_PROBE_MEM32SX || 1451 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) 1452 num_exentries++; 1453 if ((BPF_CLASS(insn->code) == BPF_STX || 1454 BPF_CLASS(insn->code) == BPF_ST) && 1455 BPF_MODE(insn->code) == BPF_PROBE_MEM32) 1456 num_exentries++; 1457 if (BPF_CLASS(insn->code) == BPF_STX && 1458 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) 1459 num_exentries++; 1460 if (bpf_is_unwind_goto(insn)) 1461 num_exentries++; 1462 } 1463 func[i]->aux->num_exentries = num_exentries; 1464 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 1465 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; 1466 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; 1467 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; 1468 func[i]->aux->can_unwind = env->subprog_info[i].can_unwind; 1469 func[i]->aux->token = prog->aux->token; 1470 if (!i) 1471 func[i]->aux->exception_boundary = env->seen_exception; 1472 func[i] = bpf_int_jit_compile(env, func[i]); 1473 if (!func[i]->jited) { 1474 err = -ENOTSUPP; 1475 goto out_free; 1476 } 1477 cond_resched(); 1478 } 1479 1480 /* at this point all bpf functions were successfully JITed 1481 * now populate all bpf_calls with correct addresses and 1482 * run last pass of JIT 1483 */ 1484 for (i = 0; i < env->subprog_cnt; i++) { 1485 insn = func[i]->insnsi; 1486 for (j = 0; j < func[i]->len; j++, insn++) { 1487 if (bpf_pseudo_func(insn)) { 1488 subprog = insn->off; 1489 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 1490 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 1491 continue; 1492 } 1493 if (!bpf_pseudo_call(insn)) 1494 continue; 1495 subprog = insn->off; 1496 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 1497 } 1498 1499 /* we use the aux data to keep a list of the start addresses 1500 * of the JITed images for each function in the program 1501 * 1502 * for some architectures, such as powerpc64, the imm field 1503 * might not be large enough to hold the offset of the start 1504 * address of the callee's JITed image from __bpf_call_base 1505 * 1506 * in such cases, we can lookup the start address of a callee 1507 * by using its subprog id, available from the off field of 1508 * the call instruction, as an index for this list 1509 */ 1510 func[i]->aux->func = func; 1511 func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1512 func[i]->aux->real_func_cnt = env->subprog_cnt; 1513 } 1514 for (i = 0; i < env->subprog_cnt; i++) { 1515 old_bpf_func = func[i]->bpf_func; 1516 tmp = bpf_int_jit_compile(env, func[i]); 1517 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 1518 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 1519 err = -ENOTSUPP; 1520 goto out_free; 1521 } 1522 cond_resched(); 1523 } 1524 1525 /* the addresses of all functions are final */ 1526 err = resolve_func_ptrs(env, prog, func); 1527 if (err) 1528 goto out_free; 1529 1530 /* 1531 * Cleanup func[i]->aux fields which aren't required 1532 * or can become invalid in future 1533 */ 1534 for (i = 0; i < env->subprog_cnt; i++) { 1535 func[i]->aux->used_maps = NULL; 1536 func[i]->aux->used_map_cnt = 0; 1537 } 1538 1539 /* finally lock prog and jit images for all functions and 1540 * populate kallsysm. Begin at the first subprogram, since 1541 * bpf_prog_load will add the kallsyms for the main program. 1542 */ 1543 for (i = 1; i < env->subprog_cnt; i++) { 1544 err = bpf_prog_lock_ro(func[i]); 1545 if (err) 1546 goto out_free; 1547 } 1548 1549 for (i = 1; i < env->subprog_cnt; i++) 1550 bpf_prog_kallsyms_add(func[i]); 1551 1552 /* Last step: make now unused interpreter insns from main 1553 * prog consistent for later dump requests, so they can 1554 * later look the same as if they were interpreted only. 1555 */ 1556 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1557 if (bpf_pseudo_func(insn)) { 1558 insn[0].imm = env->insn_aux_data[i].call_imm; 1559 insn[1].imm = insn->off; 1560 insn->off = 0; 1561 continue; 1562 } 1563 if (!bpf_pseudo_call(insn)) 1564 continue; 1565 insn->imm = env->insn_aux_data[i].call_imm; 1566 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1567 insn->off = subprog; 1568 } 1569 1570 prog->jited = 1; 1571 prog->bpf_func = func[0]->bpf_func; 1572 prog->jited_len = func[0]->jited_len; 1573 prog->aux->extable = func[0]->aux->extable; 1574 prog->aux->num_exentries = func[0]->aux->num_exentries; 1575 prog->aux->func = func; 1576 prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1577 prog->aux->real_func_cnt = env->subprog_cnt; 1578 prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; 1579 prog->aux->exception_boundary = func[0]->aux->exception_boundary; 1580 prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust; 1581 bpf_prog_jit_attempt_done(prog); 1582 return 0; 1583 out_free: 1584 /* We failed JIT'ing, so at this point we need to unregister poke 1585 * descriptors from subprogs, so that kernel is not attempting to 1586 * patch it anymore as we're freeing the subprog JIT memory. 1587 */ 1588 for (i = 0; i < prog->aux->size_poke_tab; i++) { 1589 map_ptr = prog->aux->poke_tab[i].tail_call.map; 1590 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 1591 } 1592 /* At this point we're guaranteed that poke descriptors are not 1593 * live anymore. We can just unlink its descriptor table as it's 1594 * released with the main prog. 1595 */ 1596 for (i = 0; i < env->subprog_cnt; i++) { 1597 if (!func[i]) 1598 continue; 1599 func[i]->aux->poke_tab = NULL; 1600 bpf_jit_free(func[i]); 1601 } 1602 kfree(func); 1603 out_undo_insn: 1604 bpf_prog_jit_attempt_done(prog); 1605 return err; 1606 } 1607 1608 int bpf_jit_subprogs(struct bpf_verifier_env *env) 1609 { 1610 int err, i; 1611 bool blinded = false; 1612 struct bpf_insn *insn; 1613 struct bpf_prog *prog, *orig_prog; 1614 u32 *orig_subprog_starts; 1615 1616 /* all functions that the maps point to are removed as dead code */ 1617 if (env->subprog_cnt <= 1) 1618 return resolve_func_ptrs(env, env->prog, NULL); 1619 1620 prog = orig_prog = env->prog; 1621 if (bpf_prog_need_blind(prog)) { 1622 orig_subprog_starts = bpf_dup_subprog_starts(env); 1623 if (!orig_subprog_starts) { 1624 err = -ENOMEM; 1625 goto out_cleanup; 1626 } 1627 prog = bpf_jit_blind_constants(env, prog); 1628 if (IS_ERR(prog)) { 1629 err = PTR_ERR(prog); 1630 prog = orig_prog; 1631 goto out_restore; 1632 } 1633 blinded = true; 1634 } 1635 1636 err = jit_subprogs(env); 1637 if (err) 1638 goto out_jit_err; 1639 1640 if (blinded) { 1641 bpf_jit_prog_release_other(prog, orig_prog); 1642 kvfree(orig_subprog_starts); 1643 } 1644 1645 return 0; 1646 1647 out_jit_err: 1648 if (blinded) { 1649 bpf_jit_prog_release_other(orig_prog, prog); 1650 /* roll back to the clean original prog */ 1651 prog = env->prog = orig_prog; 1652 goto out_restore; 1653 } else { 1654 if (err != -EFAULT) { 1655 /* 1656 * We will fall back to interpreter mode when err is not -EFAULT, before 1657 * that, insn->off and insn->imm should be restored to their original 1658 * values since they were modified by jit_subprogs. 1659 */ 1660 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1661 if (!bpf_pseudo_call(insn)) 1662 continue; 1663 insn->off = 0; 1664 insn->imm = env->insn_aux_data[i].call_imm; 1665 } 1666 } 1667 goto out_cleanup; 1668 } 1669 1670 out_restore: 1671 bpf_restore_subprog_starts(env, orig_subprog_starts); 1672 kvfree(orig_subprog_starts); 1673 out_cleanup: 1674 /* cleanup main prog to be interpreted */ 1675 prog->jit_requested = 0; 1676 prog->blinding_requested = 0; 1677 return err; 1678 } 1679 1680 int bpf_fixup_call_args(struct bpf_verifier_env *env) 1681 { 1682 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1683 struct bpf_prog *prog = env->prog; 1684 struct bpf_insn *insn = prog->insnsi; 1685 int depth; 1686 #endif 1687 int i, err = 0; 1688 1689 for (i = 0; i < env->subprog_cnt; i++) { 1690 struct bpf_subprog_info *subprog = &env->subprog_info[i]; 1691 u16 outgoing = subprog->stack_arg_cnt - bpf_in_stack_arg_cnt(subprog); 1692 1693 if (subprog->max_out_stack_arg_cnt > outgoing) { 1694 verbose(env, 1695 "func#%d writes %u stack arg slots, but calls only require %u\n", 1696 i, subprog->max_out_stack_arg_cnt, outgoing); 1697 return -EINVAL; 1698 } 1699 } 1700 1701 if (env->prog->jit_requested && 1702 !bpf_prog_is_offloaded(env->prog->aux)) { 1703 err = bpf_jit_subprogs(env); 1704 if (err == 0) 1705 return 0; 1706 if (err == -EFAULT || err == -EINTR) 1707 return err; 1708 } 1709 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1710 if (prog->jit_required) { 1711 verbose(env, "program requires BPF JIT compiler but it is not available\n"); 1712 return -EINVAL; 1713 } 1714 for (i = 0; i < env->subprog_cnt; i++) { 1715 if (bpf_in_stack_arg_cnt(&env->subprog_info[i])) { 1716 verbose(env, "stack args are not supported in non-JITed programs\n"); 1717 return -EINVAL; 1718 } 1719 } 1720 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 1721 /* When JIT fails the progs with bpf2bpf calls and tail_calls 1722 * have to be rejected, since interpreter doesn't support them yet. 1723 */ 1724 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 1725 return -EINVAL; 1726 } 1727 for (i = 0; i < prog->len; i++, insn++) { 1728 if (bpf_pseudo_func(insn)) { 1729 /* When JIT fails the progs with callback calls 1730 * have to be rejected, since interpreter doesn't support them yet. 1731 */ 1732 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 1733 return -EINVAL; 1734 } 1735 1736 if (!bpf_pseudo_call(insn)) 1737 continue; 1738 depth = get_callee_stack_depth(env, insn, i); 1739 if (depth < 0) 1740 return depth; 1741 err = bpf_patch_call_args(insn, depth); 1742 if (err) { 1743 verbose(env, "stack depth %d exceeds interpreter stack depth limit\n", 1744 depth); 1745 return err; 1746 } 1747 } 1748 err = 0; 1749 #endif 1750 return err; 1751 } 1752 1753 /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */ 1754 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) 1755 { 1756 struct bpf_subprog_info *info = env->subprog_info; 1757 int cnt = env->subprog_cnt; 1758 struct bpf_prog *prog; 1759 1760 /* We only reserve one slot for hidden subprogs in subprog_info. */ 1761 if (env->hidden_subprog_cnt) { 1762 verifier_bug(env, "only one hidden subprog supported"); 1763 return -EFAULT; 1764 } 1765 /* We're not patching any existing instruction, just appending the new 1766 * ones for the hidden subprog. Hence all of the adjustment operations 1767 * in bpf_patch_insn_data are no-ops. 1768 */ 1769 prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); 1770 if (!prog) 1771 return -ENOMEM; 1772 env->prog = prog; 1773 info[cnt + 1].start = info[cnt].start; 1774 info[cnt].start = prog->len - len + 1; 1775 /* The patch ends with the exit. */ 1776 info[cnt].exit_idx = prog->len - 1; 1777 env->subprog_cnt++; 1778 env->hidden_subprog_cnt++; 1779 return 0; 1780 } 1781 1782 /* 1783 * Expand may_goto: load the count from the stack, jump to the target of 1784 * may_goto when it is zero, then the tail that updates the count. 1785 * Use gotol when the target is too far for 16-bit offset of a conditional jump. 1786 */ 1787 static int may_goto_expand(struct bpf_insn *insn_buf, int off, int stack_off, 1788 const struct bpf_insn *tail, int tail_cnt) 1789 { 1790 int cnt = 0; 1791 1792 /* Forward jump has to step over the tail */ 1793 off = off >= 0 ? off + tail_cnt : off - 1; 1794 1795 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); 1796 if (off == (s16)off) { 1797 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off); 1798 } else { 1799 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1); 1800 insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1); 1801 } 1802 memcpy(insn_buf + cnt, tail, tail_cnt * sizeof(*tail)); 1803 return cnt + tail_cnt; 1804 } 1805 1806 /* Do various post-verification rewrites in a single program pass. 1807 * These rewrites simplify JIT and interpreter implementations. 1808 */ 1809 int bpf_do_misc_fixups(struct bpf_verifier_env *env) 1810 { 1811 struct bpf_prog *prog = env->prog; 1812 enum bpf_attach_type eatype = prog->expected_attach_type; 1813 enum bpf_prog_type prog_type = resolve_prog_type(prog); 1814 struct bpf_insn *insn = prog->insnsi; 1815 const struct bpf_func_proto *fn; 1816 const int insn_cnt = prog->len; 1817 const struct bpf_map_ops *ops; 1818 struct bpf_insn_aux_data *aux; 1819 struct bpf_insn *insn_buf = env->insn_buf; 1820 struct bpf_prog *new_prog; 1821 struct bpf_map *map_ptr; 1822 int i, ret, cnt, delta = 0, cur_subprog = 0; 1823 struct bpf_subprog_info *subprogs = env->subprog_info; 1824 u16 stack_depth = subprogs[cur_subprog].stack_depth; 1825 u16 stack_depth_extra = 0; 1826 1827 if (env->seen_exception && !env->exception_callback_subprog) { 1828 struct bpf_insn *patch = insn_buf; 1829 1830 *patch++ = env->prog->insnsi[insn_cnt - 1]; 1831 *patch++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 1832 *patch++ = BPF_EXIT_INSN(); 1833 ret = add_hidden_subprog(env, insn_buf, patch - insn_buf); 1834 if (ret < 0) 1835 return ret; 1836 prog = env->prog; 1837 insn = prog->insnsi; 1838 1839 env->exception_callback_subprog = env->subprog_cnt - 1; 1840 /* Don't update insn_cnt, as add_hidden_subprog always appends insns */ 1841 bpf_mark_subprog_exc_cb(env, env->exception_callback_subprog); 1842 } 1843 1844 for (i = 0; i < insn_cnt;) { 1845 if (is_addr_space_cast32(env->prog, insn)) { 1846 /* convert to 32-bit mov that clears upper 32-bit */ 1847 insn->code = BPF_ALU | BPF_MOV | BPF_X; 1848 /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ 1849 insn->off = 0; 1850 insn->imm = 0; 1851 goto next_insn; 1852 } 1853 1854 if (env->insn_aux_data[i + delta].needs_zext) 1855 /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ 1856 insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); 1857 1858 /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ 1859 if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || 1860 insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || 1861 insn->code == (BPF_ALU | BPF_MOD | BPF_K) || 1862 insn->code == (BPF_ALU | BPF_DIV | BPF_K)) && 1863 insn->off == 1 && insn->imm == -1) { 1864 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1865 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1866 struct bpf_insn *patch = insn_buf; 1867 1868 if (isdiv) 1869 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1870 BPF_NEG | BPF_K, insn->dst_reg, 1871 0, 0, 0); 1872 else 1873 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1874 1875 cnt = patch - insn_buf; 1876 1877 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1878 if (!new_prog) 1879 return -ENOMEM; 1880 1881 delta += cnt - 1; 1882 env->prog = prog = new_prog; 1883 insn = new_prog->insnsi + i + delta; 1884 goto next_insn; 1885 } 1886 1887 /* Make divide-by-zero and divide-by-minus-one exceptions impossible. */ 1888 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 1889 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 1890 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 1891 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 1892 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1893 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1894 bool is_sdiv = isdiv && insn->off == 1; 1895 bool is_smod = !isdiv && insn->off == 1; 1896 struct bpf_insn *patch = insn_buf; 1897 1898 if (is_sdiv) { 1899 /* [R,W]x sdiv 0 -> 0 1900 * LLONG_MIN sdiv -1 -> LLONG_MIN 1901 * INT_MIN sdiv -1 -> INT_MIN 1902 */ 1903 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1904 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1905 BPF_ADD | BPF_K, BPF_REG_AX, 1906 0, 0, 1); 1907 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1908 BPF_JGT | BPF_K, BPF_REG_AX, 1909 0, 4, 1); 1910 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1911 BPF_JEQ | BPF_K, BPF_REG_AX, 1912 0, 1, 0); 1913 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1914 BPF_MOV | BPF_K, insn->dst_reg, 1915 0, 0, 0); 1916 /* BPF_NEG(LLONG_MIN) == -LLONG_MIN == LLONG_MIN */ 1917 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1918 BPF_NEG | BPF_K, insn->dst_reg, 1919 0, 0, 0); 1920 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1921 *patch++ = *insn; 1922 cnt = patch - insn_buf; 1923 } else if (is_smod) { 1924 /* [R,W]x mod 0 -> [R,W]x */ 1925 /* [R,W]x mod -1 -> 0 */ 1926 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1927 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1928 BPF_ADD | BPF_K, BPF_REG_AX, 1929 0, 0, 1); 1930 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1931 BPF_JGT | BPF_K, BPF_REG_AX, 1932 0, 3, 1); 1933 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1934 BPF_JEQ | BPF_K, BPF_REG_AX, 1935 0, 3 + (is64 ? 0 : 1), 1); 1936 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1937 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1938 *patch++ = *insn; 1939 1940 if (!is64) { 1941 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1942 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 1943 } 1944 cnt = patch - insn_buf; 1945 } else if (isdiv) { 1946 /* [R,W]x div 0 -> 0 */ 1947 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1948 BPF_JNE | BPF_K, insn->src_reg, 1949 0, 2, 0); 1950 *patch++ = BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg); 1951 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1952 *patch++ = *insn; 1953 cnt = patch - insn_buf; 1954 } else { 1955 /* [R,W]x mod 0 -> [R,W]x */ 1956 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1957 BPF_JEQ | BPF_K, insn->src_reg, 1958 0, 1 + (is64 ? 0 : 1), 0); 1959 *patch++ = *insn; 1960 1961 if (!is64) { 1962 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1963 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 1964 } 1965 cnt = patch - insn_buf; 1966 } 1967 1968 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1969 if (!new_prog) 1970 return -ENOMEM; 1971 1972 delta += cnt - 1; 1973 env->prog = prog = new_prog; 1974 insn = new_prog->insnsi + i + delta; 1975 goto next_insn; 1976 } 1977 1978 /* Make it impossible to de-reference a userspace address */ 1979 if (BPF_CLASS(insn->code) == BPF_LDX && 1980 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 1981 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) { 1982 struct bpf_insn *patch = insn_buf; 1983 u64 uaddress_limit = bpf_arch_uaddress_limit(); 1984 1985 if (!uaddress_limit) 1986 goto next_insn; 1987 1988 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1989 if (insn->off) 1990 *patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off); 1991 *patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32); 1992 *patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2); 1993 *patch++ = *insn; 1994 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1995 *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0); 1996 1997 cnt = patch - insn_buf; 1998 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1999 if (!new_prog) 2000 return -ENOMEM; 2001 2002 delta += cnt - 1; 2003 env->prog = prog = new_prog; 2004 insn = new_prog->insnsi + i + delta; 2005 goto next_insn; 2006 } 2007 2008 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 2009 if (BPF_CLASS(insn->code) == BPF_LD && 2010 (BPF_MODE(insn->code) == BPF_ABS || 2011 BPF_MODE(insn->code) == BPF_IND)) { 2012 cnt = env->ops->gen_ld_abs(insn, insn_buf); 2013 if (cnt == 0 || cnt >= INSN_BUF_SIZE) { 2014 verifier_bug(env, "%d insns generated for ld_abs", cnt); 2015 return -EFAULT; 2016 } 2017 2018 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2019 if (!new_prog) 2020 return -ENOMEM; 2021 2022 delta += cnt - 1; 2023 env->prog = prog = new_prog; 2024 insn = new_prog->insnsi + i + delta; 2025 goto next_insn; 2026 } 2027 2028 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 2029 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 2030 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 2031 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 2032 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 2033 struct bpf_insn *patch = insn_buf; 2034 bool issrc, isneg, isimm; 2035 u32 off_reg; 2036 2037 aux = &env->insn_aux_data[i + delta]; 2038 if (!aux->alu_state || 2039 aux->alu_state == BPF_ALU_NON_POINTER) 2040 goto next_insn; 2041 2042 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 2043 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 2044 BPF_ALU_SANITIZE_SRC; 2045 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 2046 2047 off_reg = issrc ? insn->src_reg : insn->dst_reg; 2048 if (isimm) { 2049 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 2050 } else { 2051 if (isneg) 2052 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 2053 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 2054 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 2055 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 2056 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 2057 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 2058 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 2059 } 2060 if (!issrc) 2061 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 2062 insn->src_reg = BPF_REG_AX; 2063 if (isneg) 2064 insn->code = insn->code == code_add ? 2065 code_sub : code_add; 2066 *patch++ = *insn; 2067 if (issrc && isneg && !isimm) 2068 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 2069 cnt = patch - insn_buf; 2070 2071 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2072 if (!new_prog) 2073 return -ENOMEM; 2074 2075 delta += cnt - 1; 2076 env->prog = prog = new_prog; 2077 insn = new_prog->insnsi + i + delta; 2078 goto next_insn; 2079 } 2080 2081 if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { 2082 int stack_off_cnt = -stack_depth - 16; 2083 /* 2084 * AX is used as an argument to pass in stack_off_cnt 2085 * (to add to r10/fp), and also as the return value of 2086 * the call to arch_bpf_timed_may_goto. 2087 */ 2088 struct bpf_insn tail[] = { 2089 BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), 2090 BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2), 2091 BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt), 2092 BPF_EMIT_CALL(arch_bpf_timed_may_goto), 2093 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt), 2094 }; 2095 2096 /* 2097 * Two 8 byte slots, depth-16 stores the count, and 2098 * depth-8 stores the start timestamp of the loop. 2099 * 2100 * The starting value of count is BPF_MAX_TIMED_LOOPS 2101 * (0xffff). Every iteration loads it and subs it by 1, 2102 * until the value becomes 0 in AX (thus, 1 in stack), 2103 * after which we call arch_bpf_timed_may_goto, which 2104 * either sets AX to 0xffff to keep looping, or to 0 2105 * upon timeout. AX is then stored into the stack. In 2106 * the next iteration, we either see 0 and break out, or 2107 * continue iterating until the next time value is 0 2108 * after subtraction, rinse and repeat. 2109 */ 2110 stack_depth_extra = 16; 2111 cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt, 2112 tail, ARRAY_SIZE(tail)); 2113 2114 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2115 if (!new_prog) 2116 return -ENOMEM; 2117 2118 delta += cnt - 1; 2119 env->prog = prog = new_prog; 2120 insn = new_prog->insnsi + i + delta; 2121 goto next_insn; 2122 } else if (bpf_is_may_goto_insn(insn)) { 2123 int stack_off = -stack_depth - 8; 2124 struct bpf_insn tail[] = { 2125 BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), 2126 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off), 2127 }; 2128 2129 stack_depth_extra = 8; 2130 cnt = may_goto_expand(insn_buf, insn->off, stack_off, 2131 tail, ARRAY_SIZE(tail)); 2132 2133 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2134 if (!new_prog) 2135 return -ENOMEM; 2136 2137 delta += cnt - 1; 2138 env->prog = prog = new_prog; 2139 insn = new_prog->insnsi + i + delta; 2140 goto next_insn; 2141 } 2142 2143 if (bpf_jit_supports_percpu_insn() && 2144 insn->code == (BPF_LD | BPF_IMM | BPF_DW) && 2145 (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 2146 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) { 2147 struct bpf_map *map; 2148 2149 aux = &env->insn_aux_data[i + delta]; 2150 map = env->used_maps[aux->map_index]; 2151 if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) 2152 goto next_insn; 2153 2154 prog->jit_required = true; 2155 2156 /* 2157 * We are *skipping* first half of ld_imm64 insn 2158 * with 'i++;', patching over second half of it 2159 * with that same half + mov64_percpu_reg insn. 2160 * All because bpf_patch_insn_data() can only 2161 * replace one 8-byte insn, which does not work 2162 * well for ld_imm64 insn. 2163 */ 2164 2165 insn_buf[0] = insn[1]; 2166 insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg); 2167 cnt = 2; 2168 2169 i++; 2170 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2171 if (!new_prog) 2172 return -ENOMEM; 2173 2174 delta += cnt - 1; 2175 env->prog = prog = new_prog; 2176 insn = new_prog->insnsi + i + delta; 2177 goto next_insn; 2178 } 2179 2180 if (insn->code != (BPF_JMP | BPF_CALL)) 2181 goto next_insn; 2182 if (insn->src_reg == BPF_PSEUDO_CALL) 2183 goto next_insn; 2184 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 2185 ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); 2186 if (ret) 2187 return ret; 2188 if (cnt == 0) 2189 goto next_insn; 2190 2191 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2192 if (!new_prog) 2193 return -ENOMEM; 2194 2195 delta += cnt - 1; 2196 env->prog = prog = new_prog; 2197 insn = new_prog->insnsi + i + delta; 2198 goto next_insn; 2199 } 2200 2201 /* Skip inlining the helper call if the JIT does it. */ 2202 if (bpf_jit_inlines_helper_call(insn->imm)) { 2203 prog->jit_required = 1; 2204 goto next_insn; 2205 } 2206 2207 if (insn->imm == BPF_FUNC_get_route_realm) 2208 prog->dst_needed = 1; 2209 if (insn->imm == BPF_FUNC_get_prandom_u32) 2210 bpf_user_rnd_init_once(); 2211 if (insn->imm == BPF_FUNC_override_return) 2212 prog->kprobe_override = 1; 2213 if (insn->imm == BPF_FUNC_tail_call) { 2214 /* If we tail call into other programs, we 2215 * cannot make any assumptions since they can 2216 * be replaced dynamically during runtime in 2217 * the program array. 2218 */ 2219 prog->cb_access = 1; 2220 if (!bpf_allow_tail_call_in_subprogs(env)) 2221 prog->aux->stack_depth = MAX_BPF_STACK; 2222 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 2223 prog->aux->max_rdonly_access = U32_MAX; 2224 prog->aux->max_rdwr_access = U32_MAX; 2225 if (prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) 2226 prog->aux->max_tp_access = U32_MAX; 2227 2228 /* mark bpf_tail_call as different opcode to avoid 2229 * conditional branch in the interpreter for every normal 2230 * call and to prevent accidental JITing by JIT compiler 2231 * that doesn't support bpf_tail_call yet 2232 */ 2233 insn->imm = 0; 2234 insn->code = BPF_JMP | BPF_TAIL_CALL; 2235 2236 aux = &env->insn_aux_data[i + delta]; 2237 if (env->bpf_capable && !prog->blinding_requested && 2238 prog->jit_requested && 2239 !bpf_map_key_poisoned(aux) && 2240 !bpf_map_ptr_poisoned(aux) && 2241 !bpf_map_ptr_unpriv(aux)) { 2242 struct bpf_jit_poke_descriptor desc = { 2243 .reason = BPF_POKE_REASON_TAIL_CALL, 2244 .tail_call.map = aux->map_ptr_state.map_ptr, 2245 .tail_call.key = bpf_map_key_immediate(aux), 2246 .insn_idx = i + delta, 2247 }; 2248 2249 ret = bpf_jit_add_poke_descriptor(prog, &desc); 2250 if (ret < 0) { 2251 verbose(env, "adding tail call poke descriptor failed\n"); 2252 return ret; 2253 } 2254 2255 insn->imm = ret + 1; 2256 goto next_insn; 2257 } 2258 2259 if (!bpf_map_ptr_unpriv(aux)) 2260 goto next_insn; 2261 2262 /* instead of changing every JIT dealing with tail_call 2263 * emit two extra insns: 2264 * if (index >= max_entries) goto out; 2265 * index &= array->index_mask; 2266 * to avoid out-of-bounds cpu speculation 2267 */ 2268 if (bpf_map_ptr_poisoned(aux)) { 2269 verbose(env, "tail_call abusing map_ptr\n"); 2270 return -EINVAL; 2271 } 2272 2273 map_ptr = aux->map_ptr_state.map_ptr; 2274 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 2275 map_ptr->max_entries, 2); 2276 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 2277 container_of(map_ptr, 2278 struct bpf_array, 2279 map)->index_mask); 2280 insn_buf[2] = *insn; 2281 cnt = 3; 2282 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2283 if (!new_prog) 2284 return -ENOMEM; 2285 2286 delta += cnt - 1; 2287 env->prog = prog = new_prog; 2288 insn = new_prog->insnsi + i + delta; 2289 goto next_insn; 2290 } 2291 2292 aux = &env->insn_aux_data[i + delta]; 2293 if (aux->arg_prog) { 2294 /* The verifier will process callback_fn as many times as necessary 2295 * with different maps and the register states prepared by 2296 * set_timer_callback_state will be accurate. 2297 * 2298 * The following use case is valid: 2299 * map1 is shared by prog1, prog2, prog3. 2300 * prog1 calls bpf_timer_init for some map1 elements 2301 * prog2 calls bpf_timer_set_callback for some map1 elements. 2302 * Those that were not bpf_timer_init-ed will return -EINVAL. 2303 * prog3 calls bpf_timer_start for some map1 elements. 2304 * Those that were not both bpf_timer_init-ed and 2305 * bpf_timer_set_callback-ed will return -EINVAL. 2306 */ 2307 struct bpf_insn ld_addrs[2] = { 2308 BPF_LD_IMM64(aux->arg_prog, (long)prog->aux), 2309 }; 2310 2311 insn_buf[0] = ld_addrs[0]; 2312 insn_buf[1] = ld_addrs[1]; 2313 insn_buf[2] = *insn; 2314 cnt = 3; 2315 2316 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2317 if (!new_prog) 2318 return -ENOMEM; 2319 2320 delta += cnt - 1; 2321 env->prog = prog = new_prog; 2322 insn = new_prog->insnsi + i + delta; 2323 goto patch_call_imm; 2324 } 2325 2326 /* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */ 2327 if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) { 2328 /* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data, 2329 * bpf_mem_alloc() returns a ptr to the percpu data ptr. 2330 */ 2331 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 2332 insn_buf[1] = *insn; 2333 cnt = 2; 2334 2335 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2336 if (!new_prog) 2337 return -ENOMEM; 2338 2339 delta += cnt - 1; 2340 env->prog = prog = new_prog; 2341 insn = new_prog->insnsi + i + delta; 2342 goto patch_call_imm; 2343 } 2344 2345 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 2346 * and other inlining handlers are currently limited to 64 bit 2347 * only. 2348 */ 2349 if (prog->jit_requested && BITS_PER_LONG == 64 && 2350 (insn->imm == BPF_FUNC_map_lookup_elem || 2351 insn->imm == BPF_FUNC_map_update_elem || 2352 insn->imm == BPF_FUNC_map_delete_elem || 2353 insn->imm == BPF_FUNC_map_push_elem || 2354 insn->imm == BPF_FUNC_map_pop_elem || 2355 insn->imm == BPF_FUNC_map_peek_elem || 2356 insn->imm == BPF_FUNC_redirect_map || 2357 insn->imm == BPF_FUNC_for_each_map_elem || 2358 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { 2359 aux = &env->insn_aux_data[i + delta]; 2360 if (bpf_map_ptr_poisoned(aux)) 2361 goto patch_call_imm; 2362 2363 map_ptr = aux->map_ptr_state.map_ptr; 2364 ops = map_ptr->ops; 2365 if (insn->imm == BPF_FUNC_map_lookup_elem && 2366 ops->map_gen_lookup) { 2367 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 2368 if (cnt == -EOPNOTSUPP) 2369 goto patch_map_ops_generic; 2370 if (cnt <= 0 || cnt >= INSN_BUF_SIZE) { 2371 verifier_bug(env, "%d insns generated for map lookup", cnt); 2372 return -EFAULT; 2373 } 2374 2375 if (bpf_map_is_percpu_map(map_ptr->map_type)) 2376 prog->jit_required = true; 2377 2378 new_prog = bpf_patch_insn_data(env, i + delta, 2379 insn_buf, cnt); 2380 if (!new_prog) 2381 return -ENOMEM; 2382 2383 delta += cnt - 1; 2384 env->prog = prog = new_prog; 2385 insn = new_prog->insnsi + i + delta; 2386 goto next_insn; 2387 } 2388 2389 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 2390 (void *(*)(struct bpf_map *map, void *key))NULL)); 2391 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 2392 (long (*)(struct bpf_map *map, void *key))NULL)); 2393 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 2394 (long (*)(struct bpf_map *map, void *key, void *value, 2395 u64 flags))NULL)); 2396 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 2397 (long (*)(struct bpf_map *map, void *value, 2398 u64 flags))NULL)); 2399 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 2400 (long (*)(struct bpf_map *map, void *value))NULL)); 2401 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 2402 (long (*)(struct bpf_map *map, void *value))NULL)); 2403 BUILD_BUG_ON(!__same_type(ops->map_redirect, 2404 (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); 2405 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 2406 (long (*)(struct bpf_map *map, 2407 bpf_callback_t callback_fn, 2408 void *callback_ctx, 2409 u64 flags))NULL)); 2410 BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, 2411 (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL)); 2412 2413 patch_map_ops_generic: 2414 switch (insn->imm) { 2415 case BPF_FUNC_map_lookup_elem: 2416 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 2417 goto next_insn; 2418 case BPF_FUNC_map_update_elem: 2419 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 2420 goto next_insn; 2421 case BPF_FUNC_map_delete_elem: 2422 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 2423 goto next_insn; 2424 case BPF_FUNC_map_push_elem: 2425 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 2426 goto next_insn; 2427 case BPF_FUNC_map_pop_elem: 2428 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 2429 goto next_insn; 2430 case BPF_FUNC_map_peek_elem: 2431 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 2432 goto next_insn; 2433 case BPF_FUNC_redirect_map: 2434 insn->imm = BPF_CALL_IMM(ops->map_redirect); 2435 goto next_insn; 2436 case BPF_FUNC_for_each_map_elem: 2437 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 2438 goto next_insn; 2439 case BPF_FUNC_map_lookup_percpu_elem: 2440 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); 2441 goto next_insn; 2442 } 2443 2444 goto patch_call_imm; 2445 } 2446 2447 /* Implement bpf_jiffies64 inline. */ 2448 if (prog->jit_requested && BITS_PER_LONG == 64 && 2449 insn->imm == BPF_FUNC_jiffies64) { 2450 struct bpf_insn ld_jiffies_addr[2] = { 2451 BPF_LD_IMM64(BPF_REG_0, 2452 (unsigned long)&jiffies), 2453 }; 2454 2455 insn_buf[0] = ld_jiffies_addr[0]; 2456 insn_buf[1] = ld_jiffies_addr[1]; 2457 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 2458 BPF_REG_0, 0); 2459 cnt = 3; 2460 2461 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 2462 cnt); 2463 if (!new_prog) 2464 return -ENOMEM; 2465 2466 delta += cnt - 1; 2467 env->prog = prog = new_prog; 2468 insn = new_prog->insnsi + i + delta; 2469 goto next_insn; 2470 } 2471 2472 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML) 2473 /* Implement bpf_get_smp_processor_id() inline. */ 2474 if (insn->imm == BPF_FUNC_get_smp_processor_id && 2475 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2476 /* BPF_FUNC_get_smp_processor_id inlining is an 2477 * optimization, so if cpu_number is ever 2478 * changed in some incompatible and hard to support 2479 * way, it's fine to back out this inlining logic 2480 */ 2481 #ifdef CONFIG_SMP 2482 prog->jit_required = true; 2483 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); 2484 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2485 insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); 2486 cnt = 3; 2487 #else 2488 insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); 2489 cnt = 1; 2490 #endif 2491 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2492 if (!new_prog) 2493 return -ENOMEM; 2494 2495 delta += cnt - 1; 2496 env->prog = prog = new_prog; 2497 insn = new_prog->insnsi + i + delta; 2498 goto next_insn; 2499 } 2500 2501 /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ 2502 if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && 2503 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2504 prog->jit_required = true; 2505 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); 2506 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2507 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); 2508 cnt = 3; 2509 2510 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2511 if (!new_prog) 2512 return -ENOMEM; 2513 2514 delta += cnt - 1; 2515 env->prog = prog = new_prog; 2516 insn = new_prog->insnsi + i + delta; 2517 goto next_insn; 2518 } 2519 #endif 2520 /* Implement bpf_get_func_arg inline. */ 2521 if (prog_type == BPF_PROG_TYPE_TRACING && 2522 insn->imm == BPF_FUNC_get_func_arg) { 2523 if (eatype == BPF_TRACE_RAW_TP) { 2524 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2525 2526 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2527 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2528 cnt = 1; 2529 } else { 2530 /* Load nr_args from ctx - 8 */ 2531 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2532 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2533 cnt = 2; 2534 } 2535 insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 2536 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_LSH, BPF_REG_2, 3); 2537 insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 2538 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 2539 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2540 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 0); 2541 insn_buf[cnt++] = BPF_JMP_A(1); 2542 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2543 2544 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2545 if (!new_prog) 2546 return -ENOMEM; 2547 2548 delta += cnt - 1; 2549 env->prog = prog = new_prog; 2550 insn = new_prog->insnsi + i + delta; 2551 goto next_insn; 2552 } 2553 2554 /* Implement bpf_get_func_ret inline. */ 2555 if (prog_type == BPF_PROG_TYPE_TRACING && 2556 insn->imm == BPF_FUNC_get_func_ret) { 2557 if (eatype == BPF_TRACE_FEXIT || 2558 eatype == BPF_TRACE_FSESSION || 2559 eatype == BPF_TRACE_FEXIT_MULTI || 2560 eatype == BPF_TRACE_FSESSION_MULTI || 2561 eatype == BPF_MODIFY_RETURN) { 2562 /* Load nr_args from ctx - 8 */ 2563 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2564 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2565 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 2566 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 2567 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2568 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 2569 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 2570 cnt = 7; 2571 } else { 2572 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 2573 cnt = 1; 2574 } 2575 2576 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2577 if (!new_prog) 2578 return -ENOMEM; 2579 2580 delta += cnt - 1; 2581 env->prog = prog = new_prog; 2582 insn = new_prog->insnsi + i + delta; 2583 goto next_insn; 2584 } 2585 2586 /* Implement get_func_arg_cnt inline. */ 2587 if (prog_type == BPF_PROG_TYPE_TRACING && 2588 insn->imm == BPF_FUNC_get_func_arg_cnt) { 2589 if (eatype == BPF_TRACE_RAW_TP) { 2590 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2591 2592 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2593 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2594 cnt = 1; 2595 } else { 2596 /* Load nr_args from ctx - 8 */ 2597 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2598 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2599 cnt = 2; 2600 } 2601 2602 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2603 if (!new_prog) 2604 return -ENOMEM; 2605 2606 delta += cnt - 1; 2607 env->prog = prog = new_prog; 2608 insn = new_prog->insnsi + i + delta; 2609 goto next_insn; 2610 } 2611 2612 /* Implement bpf_get_func_ip inline. */ 2613 if (prog_type == BPF_PROG_TYPE_TRACING && 2614 insn->imm == BPF_FUNC_get_func_ip) { 2615 /* Load IP address from ctx - 16 */ 2616 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 2617 2618 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 2619 if (!new_prog) 2620 return -ENOMEM; 2621 2622 env->prog = prog = new_prog; 2623 insn = new_prog->insnsi + i + delta; 2624 goto next_insn; 2625 } 2626 2627 /* Implement bpf_get_branch_snapshot inline. */ 2628 if (IS_ENABLED(CONFIG_PERF_EVENTS) && 2629 prog->jit_requested && BITS_PER_LONG == 64 && 2630 insn->imm == BPF_FUNC_get_branch_snapshot) { 2631 /* We are dealing with the following func protos: 2632 * u64 bpf_get_branch_snapshot(void *buf, u32 size, u64 flags); 2633 * int perf_snapshot_branch_stack(struct perf_branch_entry *entries, u32 cnt); 2634 */ 2635 const u32 br_entry_size = sizeof(struct perf_branch_entry); 2636 2637 /* struct perf_branch_entry is part of UAPI and is 2638 * used as an array element, so extremely unlikely to 2639 * ever grow or shrink 2640 */ 2641 BUILD_BUG_ON(br_entry_size != 24); 2642 2643 /* if (unlikely(flags)) return -EINVAL */ 2644 insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7); 2645 2646 /* Transform size (bytes) into number of entries (cnt = size / 24). 2647 * But to avoid expensive division instruction, we implement 2648 * divide-by-3 through multiplication, followed by further 2649 * division by 8 through 3-bit right shift. 2650 * Refer to book "Hacker's Delight, 2nd ed." by Henry S. Warren, Jr., 2651 * p. 227, chapter "Unsigned Division by 3" for details and proofs. 2652 * 2653 * N / 3 <=> M * N / 2^33, where M = (2^33 + 1) / 3 = 0xaaaaaaab. 2654 */ 2655 insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab); 2656 insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0); 2657 insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36); 2658 2659 /* call perf_snapshot_branch_stack implementation */ 2660 insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack)); 2661 /* if (entry_cnt == 0) return -ENOENT */ 2662 insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4); 2663 /* return entry_cnt * sizeof(struct perf_branch_entry) */ 2664 insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size); 2665 insn_buf[7] = BPF_JMP_A(3); 2666 /* return -EINVAL; */ 2667 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2668 insn_buf[9] = BPF_JMP_A(1); 2669 /* return -ENOENT; */ 2670 insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT); 2671 cnt = 11; 2672 2673 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2674 if (!new_prog) 2675 return -ENOMEM; 2676 2677 delta += cnt - 1; 2678 env->prog = prog = new_prog; 2679 insn = new_prog->insnsi + i + delta; 2680 goto next_insn; 2681 } 2682 2683 /* Implement bpf_kptr_xchg inline */ 2684 if (prog->jit_requested && BITS_PER_LONG == 64 && 2685 insn->imm == BPF_FUNC_kptr_xchg && 2686 bpf_jit_supports_ptr_xchg()) { 2687 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); 2688 insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); 2689 cnt = 2; 2690 2691 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2692 if (!new_prog) 2693 return -ENOMEM; 2694 2695 delta += cnt - 1; 2696 env->prog = prog = new_prog; 2697 insn = new_prog->insnsi + i + delta; 2698 goto next_insn; 2699 } 2700 patch_call_imm: 2701 fn = env->ops->get_func_proto(insn->imm, env->prog); 2702 /* all functions that have prototype and verifier allowed 2703 * programs to call them, must be real in-kernel functions 2704 */ 2705 if (!fn->func) { 2706 verifier_bug(env, 2707 "not inlined functions %s#%d is missing func", 2708 func_id_name(insn->imm), insn->imm); 2709 return -EFAULT; 2710 } 2711 insn->imm = BPF_CALL_IMM(fn->func); 2712 next_insn: 2713 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2714 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2715 subprogs[cur_subprog].stack_extra = stack_depth_extra; 2716 2717 stack_depth = subprogs[cur_subprog].stack_depth; 2718 if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) { 2719 verbose(env, "stack size %d(extra %d) is too large\n", 2720 stack_depth, stack_depth_extra); 2721 return -EINVAL; 2722 } 2723 cur_subprog++; 2724 stack_depth = subprogs[cur_subprog].stack_depth; 2725 stack_depth_extra = 0; 2726 } 2727 i++; 2728 insn++; 2729 } 2730 2731 env->prog->aux->stack_depth = subprogs[0].stack_depth; 2732 for (i = 0; i < env->subprog_cnt; i++) { 2733 int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1; 2734 int subprog_start = subprogs[i].start; 2735 int stack_slots = subprogs[i].stack_extra / 8; 2736 int slots = delta, cnt = 0; 2737 2738 if (!stack_slots) 2739 continue; 2740 /* We need two slots in case timed may_goto is supported. */ 2741 if (stack_slots > slots) { 2742 verifier_bug(env, "stack_slots supports may_goto only"); 2743 return -EFAULT; 2744 } 2745 2746 stack_depth = subprogs[i].stack_depth; 2747 if (bpf_jit_supports_timed_may_goto()) { 2748 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2749 BPF_MAX_TIMED_LOOPS); 2750 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0); 2751 } else { 2752 /* Add ST insn to subprog prologue to init extra stack */ 2753 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2754 BPF_MAX_LOOPS); 2755 } 2756 /* Copy first actual insn to preserve it */ 2757 insn_buf[cnt] = env->prog->insnsi[subprog_start]; 2758 ret = bpf_adj_moved_insn(&insn_buf[cnt], cnt); 2759 if (ret) { 2760 verbose(env, "insn %d cannot be patched due to 16-bit range\n", 2761 env->insn_aux_data[subprog_start].orig_idx); 2762 return ret; 2763 } 2764 cnt++; 2765 2766 new_prog = __bpf_patch_insn_data(env, subprog_start, insn_buf, cnt, 2767 BPF_PATCH_MOVE_TARGET); 2768 if (!new_prog) 2769 return -ENOMEM; 2770 env->prog = prog = new_prog; 2771 /* 2772 * If may_goto is a first insn of a prog there could be a jmp 2773 * insn that points to it, hence adjust all such jmps to point 2774 * to insn after BPF_ST that inits may_goto count. 2775 * Adjustment will succeed because bpf_patch_insn_data() didn't fail. 2776 */ 2777 WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta)); 2778 } 2779 2780 /* Since poke tab is now finalized, publish aux to tracker. */ 2781 for (i = 0; i < prog->aux->size_poke_tab; i++) { 2782 map_ptr = prog->aux->poke_tab[i].tail_call.map; 2783 if (!map_ptr->ops->map_poke_track || 2784 !map_ptr->ops->map_poke_untrack || 2785 !map_ptr->ops->map_poke_run) { 2786 verifier_bug(env, "poke tab is misconfigured"); 2787 return -EFAULT; 2788 } 2789 2790 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 2791 if (ret < 0) { 2792 verbose(env, "tracking tail call prog failed\n"); 2793 return ret; 2794 } 2795 } 2796 2797 return 0; 2798 } 2799 2800 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 2801 int position, 2802 s32 stack_base, 2803 u32 callback_subprogno, 2804 u32 *total_cnt) 2805 { 2806 s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; 2807 s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; 2808 s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; 2809 int reg_loop_max = BPF_REG_6; 2810 int reg_loop_cnt = BPF_REG_7; 2811 int reg_loop_ctx = BPF_REG_8; 2812 2813 struct bpf_insn *insn_buf = env->insn_buf; 2814 struct bpf_prog *new_prog; 2815 u32 callback_start; 2816 u32 call_insn_offset; 2817 s32 callback_offset; 2818 u32 cnt = 0; 2819 2820 /* This represents an inlined version of bpf_iter.c:bpf_loop, 2821 * be careful to modify this code in sync. 2822 */ 2823 2824 /* Return error and jump to the end of the patch if 2825 * expected number of iterations is too big. 2826 */ 2827 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2); 2828 insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG); 2829 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16); 2830 /* spill R6, R7, R8 to use these as loop vars */ 2831 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset); 2832 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset); 2833 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset); 2834 /* initialize loop vars */ 2835 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1); 2836 insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0); 2837 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3); 2838 /* loop header, 2839 * if reg_loop_cnt >= reg_loop_max skip the loop body 2840 */ 2841 insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5); 2842 /* callback call, 2843 * correct callback offset would be set after patching 2844 */ 2845 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt); 2846 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx); 2847 insn_buf[cnt++] = BPF_CALL_REL(0); 2848 /* increment loop counter */ 2849 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1); 2850 /* jump to loop header if callback returned 0 */ 2851 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6); 2852 /* return value of bpf_loop, 2853 * set R0 to the number of iterations 2854 */ 2855 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt); 2856 /* restore original values of R6, R7, R8 */ 2857 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset); 2858 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset); 2859 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset); 2860 2861 *total_cnt = cnt; 2862 new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); 2863 if (!new_prog) 2864 return new_prog; 2865 2866 /* callback start is known only after patching */ 2867 callback_start = env->subprog_info[callback_subprogno].start; 2868 /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ 2869 call_insn_offset = position + 12; 2870 callback_offset = callback_start - call_insn_offset - 1; 2871 new_prog->insnsi[call_insn_offset].imm = callback_offset; 2872 2873 return new_prog; 2874 } 2875 2876 static bool is_bpf_loop_call(struct bpf_insn *insn) 2877 { 2878 return insn->code == (BPF_JMP | BPF_CALL) && 2879 insn->src_reg == 0 && 2880 insn->imm == BPF_FUNC_loop; 2881 } 2882 2883 /* For all sub-programs in the program (including main) check 2884 * insn_aux_data to see if there are bpf_loop calls that require 2885 * inlining. If such calls are found the calls are replaced with a 2886 * sequence of instructions produced by `inline_bpf_loop` function and 2887 * subprog stack_depth is increased by the size of 3 registers. 2888 * This stack space is used to spill values of the R6, R7, R8. These 2889 * registers are used to store the loop bound, counter and context 2890 * variables. 2891 */ 2892 int bpf_optimize_bpf_loop(struct bpf_verifier_env *env) 2893 { 2894 struct bpf_subprog_info *subprogs = env->subprog_info; 2895 int i, cur_subprog = 0, cnt, delta = 0; 2896 struct bpf_insn *insn = env->prog->insnsi; 2897 int insn_cnt = env->prog->len; 2898 u16 stack_depth = subprogs[cur_subprog].stack_depth; 2899 u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2900 u16 stack_depth_extra = 0; 2901 2902 for (i = 0; i < insn_cnt; i++, insn++) { 2903 struct bpf_loop_inline_state *inline_state = 2904 &env->insn_aux_data[i + delta].loop_inline_state; 2905 2906 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { 2907 struct bpf_prog *new_prog; 2908 2909 stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; 2910 new_prog = inline_bpf_loop(env, 2911 i + delta, 2912 -(stack_depth + stack_depth_extra), 2913 inline_state->callback_subprogno, 2914 &cnt); 2915 if (!new_prog) 2916 return -ENOMEM; 2917 2918 delta += cnt - 1; 2919 env->prog = new_prog; 2920 insn = new_prog->insnsi + i + delta; 2921 } 2922 2923 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2924 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2925 cur_subprog++; 2926 stack_depth = subprogs[cur_subprog].stack_depth; 2927 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2928 stack_depth_extra = 0; 2929 } 2930 } 2931 2932 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 2933 2934 return 0; 2935 } 2936 2937 /* Remove unnecessary spill/fill pairs, members of fastcall pattern, 2938 * adjust subprograms stack depth when possible. 2939 */ 2940 int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) 2941 { 2942 struct bpf_subprog_info *subprog = env->subprog_info; 2943 struct bpf_insn_aux_data *aux = env->insn_aux_data; 2944 struct bpf_insn *insn = env->prog->insnsi; 2945 int insn_cnt = env->prog->len; 2946 u32 spills_num; 2947 bool modified = false; 2948 int i, j; 2949 2950 for (i = 0; i < insn_cnt; i++, insn++) { 2951 if (aux[i].fastcall_spills_num > 0) { 2952 spills_num = aux[i].fastcall_spills_num; 2953 /* NOPs would be removed by opt_remove_nops() */ 2954 for (j = 1; j <= spills_num; ++j) { 2955 *(insn - j) = NOP; 2956 *(insn + j) = NOP; 2957 } 2958 modified = true; 2959 } 2960 if ((subprog + 1)->start == i + 1) { 2961 if (modified && !subprog->keep_fastcall_stack) 2962 subprog->stack_depth = -subprog->fastcall_stack_off; 2963 subprog++; 2964 modified = false; 2965 } 2966 } 2967 2968 return 0; 2969 } 2970