1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/bpf.h> 4 #include <linux/btf.h> 5 #include <linux/bpf_verifier.h> 6 #include <linux/filter.h> 7 #include <linux/vmalloc.h> 8 #include <linux/perf_event.h> 9 #include <linux/sched/signal.h> 10 #include <net/xdp.h> 11 #include "disasm.h" 12 13 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) 14 15 /* 16 * Matches BPF_PROBE_ATOMIC too: bpf_convert_ctx_accesses() rewrites arena 17 * atomics before bpf_opt_subreg_zext_lo32_rnd_hi32() runs. 18 */ 19 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 20 { 21 return BPF_CLASS(insn->code) == BPF_STX && 22 (BPF_MODE(insn->code) == BPF_ATOMIC || 23 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && 24 insn->imm == BPF_CMPXCHG; 25 } 26 27 /* Returns true if 'insn' is an address space cast instruction translated as BPF_ALU op */ 28 static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *insn) 29 { 30 struct bpf_map *arena = (struct bpf_map *)prog->aux->arena; 31 32 if (insn->code != (BPF_ALU64 | BPF_MOV | BPF_X) || insn->off != BPF_ADDR_SPACE_CAST) 33 return false; 34 35 /* cast from as(1) to as(0) */ 36 if (insn->imm == 1) 37 return true; 38 39 /* cast from as(0) to as(1) */ 40 if (insn->imm == 1 << 16) 41 return arena && arena->map_flags & BPF_F_NO_USER_CONV; 42 43 /* non-BPF_F_NO_USER_CONV cast from as(0) to as(1) should be handled by JIT */ 44 return false; 45 } 46 47 /* 48 * The insn accesses arena through a number. JITs add the base of arena 49 * to the register as it is, so the access goes through the low 32 bits of 50 * the number in BPF_REG_AX. Registers of the program are not changed: the 51 * number may be compared or stored later, and on another path the register 52 * may be PTR_TO_ARENA. 53 * 54 * Constant blinding leaves insns that use BPF_REG_AX alone, so the immediate 55 * of st through a number is not blinded. 56 */ 57 static int arena_scalar_access(const struct bpf_insn *insn, struct bpf_insn *buf) 58 { 59 bool load = BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn); 60 struct bpf_insn *patch = buf; 61 62 *patch++ = BPF_MOV32_REG(BPF_REG_AX, load ? insn->src_reg : insn->dst_reg); 63 *patch = *insn; 64 if (load) 65 patch->src_reg = BPF_REG_AX; 66 else 67 patch->dst_reg = BPF_REG_AX; 68 patch++; 69 return patch - buf; 70 } 71 72 /* Return the regno defined by the insn, or -1. */ 73 static int insn_def_regno(const struct bpf_insn *insn) 74 { 75 switch (BPF_CLASS(insn->code)) { 76 case BPF_JMP: 77 case BPF_JMP32: 78 case BPF_ST: 79 return -1; 80 case BPF_STX: 81 return bpf_atomic_load_reg(insn); 82 default: 83 return insn->dst_reg; 84 } 85 } 86 87 /* 88 * For use only in combination with insn_def_regno() >= 0. 89 * Returns TRUE if the destination register operates on 64-bit, 90 * otherwise return FALSE. 91 */ 92 static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn) 93 { 94 u8 class = BPF_CLASS(insn->code); 95 u8 mode = BPF_MODE(insn->code); 96 u8 size = BPF_SIZE(insn->code); 97 u8 op = BPF_OP(insn->code); 98 bool mode_mem; 99 100 /* subregister endiness swap */ 101 if ((class == BPF_ALU || class == BPF_ALU64) && op == BPF_END && insn->imm != 64) 102 return false; 103 104 /* w0 += 1 */ 105 if (class == BPF_ALU && op != BPF_END) 106 return false; 107 108 /* address space casts converted to BPF_ALU, see bpf_do_misc_fixups() */ 109 if (is_addr_space_cast32(prog, insn)) 110 return false; 111 112 /* non 64-bit, non signed extended loads */ 113 mode_mem = mode == BPF_MEM || mode == BPF_PROBE_MEM || mode == BPF_PROBE_MEM32; 114 if (class == BPF_LDX && mode_mem && size != BPF_DW) 115 return false; 116 117 /* atomics, see insn_def_regno() */ 118 if (class == BPF_STX && size != BPF_DW) 119 return false; 120 121 /* both LD_IND and LD_ABS return 32-bit data. */ 122 if (class == BPF_LD && (mode == BPF_IND || mode == BPF_ABS)) 123 return false; 124 125 /* Conservatively return true at default. */ 126 return true; 127 } 128 129 /* 130 * Return the 32-bit subregister defined by INSN, or -1 if INSN does not 131 * explicitly define a 32-bit value. 132 */ 133 int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) 134 { 135 int dst_reg = insn_def_regno(insn); 136 137 if (dst_reg < 0 || bpf_is_reg64(prog, insn)) 138 return -1; 139 140 return dst_reg; 141 } 142 143 const struct btf_func_model * 144 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 145 const struct bpf_insn *insn) 146 { 147 const struct bpf_kfunc_desc *res; 148 struct bpf_kfunc_desc_tab *tab; 149 150 tab = prog->aux->kfunc_tab; 151 if (insn->off < 0 || insn->off >= tab->nr_descs) 152 return NULL; 153 154 res = &tab->descs[insn->off]; 155 if (bpf_jit_supports_far_kfunc_call()) { 156 if (res->func_id != insn->imm) 157 return NULL; 158 } else if ((s32)BPF_CALL_IMM(res->addr) != insn->imm) { 159 return NULL; 160 } 161 162 return &res->func_model; 163 } 164 165 static int add_kfunc_in_insns(struct bpf_verifier_env *env, 166 struct bpf_insn *insn, int cnt) 167 { 168 int i, ret; 169 170 for (i = 0; i < cnt; i++, insn++) { 171 if (bpf_pseudo_kfunc_call(insn)) { 172 ret = bpf_add_kfunc_call(env, insn->imm, insn->off); 173 if (ret < 0) 174 return ret; 175 } 176 } 177 return 0; 178 } 179 180 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 181 static int get_callee_stack_depth(struct bpf_verifier_env *env, 182 const struct bpf_insn *insn, int idx) 183 { 184 int start = idx + insn->imm + 1, subprog; 185 186 subprog = bpf_find_subprog(env, start); 187 if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start)) 188 return -EFAULT; 189 return env->subprog_info[subprog].stack_depth; 190 } 191 #endif 192 193 enum bpf_patch_mode { 194 BPF_PATCH_KEEP_TARGET, /* targets of 'off' keep naming 'off', the first new insn */ 195 BPF_PATCH_MOVE_TARGET, /* targets of 'off' follow the original insn to 'off + len - 1' */ 196 }; 197 198 /* single env->prog->insni[off] instruction was replaced with the range 199 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 200 * [0, off) and [off, end) to new locations, so the patched range stays zero 201 */ 202 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 203 struct bpf_prog *new_prog, u32 off, u32 cnt, 204 struct bpf_insn *original_insn, 205 enum bpf_patch_mode mode) 206 { 207 struct bpf_insn_aux_data *data = env->insn_aux_data; 208 struct bpf_insn *insn = new_prog->insnsi; 209 u32 old_seen = data[off].seen; 210 u32 prog_len; 211 int i; 212 213 /* aux info at OFF always needs adjustment, no matter fast path 214 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 215 * original insn at old prog. 216 */ 217 data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; 218 219 if (cnt == 1) { 220 /* 221 * A non-memory accessing insn could have been replaced by a 222 * memory accessing insn, systematically mark it for non-stack 223 * access 224 */ 225 data[off].non_stack_access = bpf_is_mem_insn(insn + off); 226 return; 227 } 228 prog_len = new_prog->len; 229 env->insn_aux_data_len = prog_len; 230 231 memmove(data + off + cnt - 1, data + off, 232 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 233 memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1)); 234 for (i = off; i < off + cnt - 1; i++) { 235 /* Expand insni[off]'s seen count to the patched range. */ 236 data[i].seen = old_seen; 237 data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; 238 /* 239 * In BPF_PATCH_MOVE_TARGET mode the original insn is the last 240 * slot by construction. 241 */ 242 if (mode == BPF_PATCH_KEEP_TARGET && 243 !memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { 244 data[i].non_stack_access = 245 data[off + cnt - 1].non_stack_access; 246 data[off + cnt - 1].non_stack_access = false; 247 } else if (bpf_is_mem_insn(insn + i)) { 248 data[i].non_stack_access = true; 249 } 250 } 251 252 /* 253 * Last slot instruction could be a newly generated 254 * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access 255 * if it is not the original instruction, otherwise keep the 256 * original marking 257 */ 258 if (bpf_is_mem_insn(insn + off + cnt - 1) && 259 memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn))) 260 data[off + cnt - 1].non_stack_access = true; 261 262 /* 263 * The indirect_target flag of the original instruction was moved to the last of the 264 * new instructions by the above memmove and memset, but the indirect jump target is 265 * actually the first instruction, so move it back. This also matches with the behavior 266 * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new 267 * instruction. For BPF_PATCH_MOVE_TARGET the original instruction is the last one, 268 * so the flag already sits where needed. 269 */ 270 if (mode == BPF_PATCH_KEEP_TARGET && data[off + cnt - 1].indirect_target) { 271 data[off].indirect_target = 1; 272 data[off + cnt - 1].indirect_target = 0; 273 } 274 } 275 276 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 277 { 278 int i; 279 280 if (len == 1) 281 return; 282 /* NOTE: fake 'exit' subprog should be updated as well. */ 283 for (i = 0; i <= env->subprog_cnt; i++) { 284 if (env->subprog_info[i].start <= off) 285 continue; 286 env->subprog_info[i].start += len - 1; 287 } 288 } 289 290 static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 first, u32 len) 291 { 292 int i; 293 294 if (len == 1) 295 return; 296 297 for (i = 0; i < env->insn_array_map_cnt; i++) 298 bpf_insn_array_adjust(env->insn_array_maps[i], first, len); 299 } 300 301 static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) 302 { 303 int i; 304 305 for (i = 0; i < env->insn_array_map_cnt; i++) 306 bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len); 307 } 308 309 static void adjust_poke_descs(struct bpf_prog *prog, u32 first, u32 len) 310 { 311 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 312 int i, sz = prog->aux->size_poke_tab; 313 struct bpf_jit_poke_descriptor *desc; 314 315 for (i = 0; i < sz; i++) { 316 desc = &tab[i]; 317 if (desc->insn_idx < first) 318 continue; 319 desc->insn_idx += len - 1; 320 } 321 } 322 323 /* 324 * Some post-verification instruction rewriting passes require an 325 * O(prog->len) operation per instruction. Keep their shared primitives 326 * killable and preemptible. 327 */ 328 static bool bpf_rewrite_must_abort(void) 329 { 330 if (fatal_signal_pending(current)) 331 return true; 332 cond_resched(); 333 return false; 334 } 335 336 static struct bpf_prog *__bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 337 const struct bpf_insn *patch, u32 len, 338 enum bpf_patch_mode mode) 339 { 340 struct bpf_prog *new_prog; 341 struct bpf_insn_aux_data *new_data = NULL; 342 /* The first slot whose targets move. */ 343 u32 first = mode == BPF_PATCH_MOVE_TARGET ? off : off + 1; 344 struct bpf_insn original_insn; 345 346 if (bpf_rewrite_must_abort()) 347 return NULL; 348 349 if (len > 1) { 350 new_data = vrealloc(env->insn_aux_data, 351 array_size(env->prog->len + len - 1, 352 sizeof(struct bpf_insn_aux_data)), 353 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 354 if (!new_data) 355 return NULL; 356 357 env->insn_aux_data = new_data; 358 } 359 360 memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn)); 361 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 362 if (IS_ERR(new_prog)) { 363 if (PTR_ERR(new_prog) == -ERANGE) 364 verbose(env, 365 "insn %d cannot be patched due to 16-bit range\n", 366 env->insn_aux_data[off].orig_idx); 367 return NULL; 368 } 369 adjust_insn_aux_data(env, new_prog, off, len, &original_insn, mode); 370 adjust_subprog_starts(env, off, len); 371 adjust_insn_arrays(env, first, len); 372 bpf_adjust_func_ptrs(env, off, len); 373 adjust_poke_descs(new_prog, first, len); 374 return new_prog; 375 } 376 377 struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 378 const struct bpf_insn *patch, u32 len) 379 { 380 return __bpf_patch_insn_data(env, off, patch, len, BPF_PATCH_KEEP_TARGET); 381 } 382 383 /* 384 * insn was moved down by delta insns inside its own patch. Operands relative 385 * to the pc that point in front of the old position did not move with it. 386 */ 387 static int bpf_adj_moved_insn(struct bpf_insn *insn, u32 delta) 388 { 389 u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code); 390 s64 off = insn->imm, off_min = S32_MIN; 391 bool is_imm = true; 392 393 if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) { 394 /* subprog that started at the old position starts with the patch */ 395 if (off >= 0) 396 return 0; 397 } else if ((class == BPF_JMP || class == BPF_JMP32) && 398 op != BPF_CALL && op != BPF_EXIT) { 399 if (insn->code != (BPF_JMP32 | BPF_JA)) { 400 off = insn->off; 401 off_min = S16_MIN; 402 is_imm = false; 403 } 404 /* jump to itself stays */ 405 if (off >= -1) 406 return 0; 407 } else { 408 return 0; 409 } 410 411 off -= delta; 412 if (off < off_min) 413 return -ERANGE; 414 if (is_imm) 415 insn->imm = off; 416 else 417 insn->off = off; 418 return 0; 419 } 420 421 /* 422 * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the 423 * jump offset by 'delta'. 424 */ 425 static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta) 426 { 427 struct bpf_insn *insn = prog->insnsi; 428 u32 insn_cnt = prog->len, i; 429 s32 imm; 430 s16 off; 431 432 for (i = 0; i < insn_cnt; i++, insn++) { 433 u8 code = insn->code; 434 435 if (tgt_idx <= i && i < tgt_idx + delta) 436 continue; 437 438 if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) || 439 BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT) 440 continue; 441 442 if (insn->code == (BPF_JMP32 | BPF_JA)) { 443 if (i + 1 + insn->imm != tgt_idx) 444 continue; 445 if (check_add_overflow(insn->imm, delta, &imm)) 446 return -ERANGE; 447 insn->imm = imm; 448 } else { 449 if (i + 1 + insn->off != tgt_idx) 450 continue; 451 if (check_add_overflow(insn->off, delta, &off)) 452 return -ERANGE; 453 insn->off = off; 454 } 455 } 456 return 0; 457 } 458 459 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 460 u32 off, u32 cnt) 461 { 462 int i, j; 463 464 /* find first prog starting at or after off (first to remove) */ 465 for (i = 0; i < env->subprog_cnt; i++) 466 if (env->subprog_info[i].start >= off) 467 break; 468 /* find first prog starting at or after off + cnt (first to stay) */ 469 for (j = i; j < env->subprog_cnt; j++) 470 if (env->subprog_info[j].start >= off + cnt) 471 break; 472 /* if j doesn't start exactly at off + cnt, we are just removing 473 * the front of previous prog 474 */ 475 if (env->subprog_info[j].start != off + cnt) 476 j--; 477 478 if (j > i) { 479 struct bpf_prog_aux *aux = env->prog->aux; 480 int move; 481 482 /* move fake 'exit' subprog as well */ 483 move = env->subprog_cnt + 1 - j; 484 485 memmove(env->subprog_info + i, 486 env->subprog_info + j, 487 sizeof(*env->subprog_info) * move); 488 env->subprog_cnt -= j - i; 489 490 /* remove func_info and its aux */ 491 if (aux->func_info) { 492 move = aux->func_info_cnt - j; 493 494 memmove(aux->func_info + i, 495 aux->func_info + j, 496 sizeof(*aux->func_info) * move); 497 if (aux->func_info_aux) 498 memmove(aux->func_info_aux + i, 499 aux->func_info_aux + j, 500 sizeof(*aux->func_info_aux) * move); 501 aux->func_info_cnt -= j - i; 502 /* func_info->insn_off is set after all code rewrites, 503 * in adjust_btf_func() - no need to adjust 504 */ 505 } 506 /* The exception callback is live, so it cannot sit inside the removed range. */ 507 if (env->exception_callback_subprog >= j) 508 env->exception_callback_subprog -= j - i; 509 } else { 510 /* convert i from "first prog to remove" to "first to adjust" */ 511 if (env->subprog_info[i].start == off) 512 i++; 513 } 514 515 /* update fake 'exit' subprog as well */ 516 for (; i <= env->subprog_cnt; i++) 517 env->subprog_info[i].start -= cnt; 518 519 return 0; 520 } 521 522 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 523 u32 cnt) 524 { 525 struct bpf_prog *prog = env->prog; 526 u32 i, l_off, l_cnt, nr_linfo; 527 struct bpf_line_info *linfo; 528 529 nr_linfo = prog->aux->nr_linfo; 530 if (!nr_linfo) 531 return 0; 532 533 linfo = prog->aux->linfo; 534 535 /* find first line info to remove, count lines to be removed */ 536 for (i = 0; i < nr_linfo; i++) 537 if (linfo[i].insn_off >= off) 538 break; 539 540 l_off = i; 541 l_cnt = 0; 542 for (; i < nr_linfo; i++) 543 if (linfo[i].insn_off < off + cnt) 544 l_cnt++; 545 else 546 break; 547 548 /* First live insn doesn't match first live linfo, it needs to "inherit" 549 * last removed linfo. prog is already modified, so prog->len == off 550 * means no live instructions after (tail of the program was removed). 551 */ 552 if (prog->len != off && l_cnt && 553 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 554 l_cnt--; 555 linfo[--i].insn_off = off + cnt; 556 } 557 558 /* remove the line info which refer to the removed instructions */ 559 if (l_cnt) { 560 memmove(linfo + l_off, linfo + i, 561 sizeof(*linfo) * (nr_linfo - i)); 562 563 prog->aux->nr_linfo -= l_cnt; 564 nr_linfo = prog->aux->nr_linfo; 565 } 566 567 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 568 for (i = l_off; i < nr_linfo; i++) 569 linfo[i].insn_off -= cnt; 570 571 /* fix up all subprogs (incl. 'exit') which start >= off */ 572 for (i = 0; i <= env->subprog_cnt; i++) 573 if (env->subprog_info[i].linfo_idx > l_off) { 574 /* program may have started in the removed region but 575 * may not be fully removed 576 */ 577 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 578 env->subprog_info[i].linfo_idx -= l_cnt; 579 else 580 env->subprog_info[i].linfo_idx = l_off; 581 } 582 583 return 0; 584 } 585 586 /* Clean up dynamically allocated fields of aux data for instructions [start, ...] */ 587 void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) 588 { 589 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 590 int end = start + len; 591 int i; 592 593 for (i = start; i < end; i++) { 594 if (aux_data[i].loop) { 595 kvfree(aux_data[i].loop->exits); 596 kvfree(aux_data[i].loop); 597 aux_data[i].loop = NULL; 598 } 599 } 600 } 601 602 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 603 { 604 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 605 unsigned int orig_prog_len = env->prog->len; 606 int err; 607 608 if (bpf_rewrite_must_abort()) 609 return -EINTR; 610 611 if (bpf_prog_is_offloaded(env->prog->aux)) 612 bpf_prog_offload_remove_insns(env, off, cnt); 613 614 bpf_clear_insn_aux_data(env, off, cnt); 615 616 err = bpf_remove_insns(env->prog, off, cnt); 617 if (err) 618 return err; 619 620 /* before subprogs are adjusted, since it looks at them */ 621 bpf_adjust_func_ptrs_after_remove(env, off, cnt); 622 623 err = adjust_subprog_starts_after_remove(env, off, cnt); 624 if (err) 625 return err; 626 627 err = bpf_adj_linfo_after_remove(env, off, cnt); 628 if (err) 629 return err; 630 631 adjust_insn_arrays_after_remove(env, off, cnt); 632 633 memmove(aux_data + off, aux_data + off + cnt, 634 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 635 env->insn_aux_data_len -= cnt; 636 637 return 0; 638 } 639 640 static const struct bpf_insn NOP = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 641 static const struct bpf_insn MAY_GOTO_0 = BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0, 0); 642 643 bool bpf_insn_is_cond_jump(u8 code) 644 { 645 u8 op; 646 647 op = BPF_OP(code); 648 if (BPF_CLASS(code) == BPF_JMP32) 649 return op != BPF_JA; 650 651 if (BPF_CLASS(code) != BPF_JMP) 652 return false; 653 654 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 655 } 656 657 void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 658 { 659 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 660 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 661 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 662 struct bpf_insn *insn = env->prog->insnsi; 663 const int insn_cnt = env->prog->len; 664 int i; 665 666 for (i = 0; i < insn_cnt; i++, insn++) { 667 if (!bpf_insn_is_cond_jump(insn->code)) 668 continue; 669 670 if (bpf_is_unwind_goto(insn) && aux_data[i + 1 + insn->off].seen) { 671 /* 672 * The callee never exits. Its unwind_exit needs 673 * unwind_goto all the same, and the cpu predicts that it 674 * returns to what follows. Don't let the removal of dead 675 * code put something else there. 676 */ 677 if (!aux_data[i + 1].seen) { 678 memcpy(insn + 1, &trap, sizeof(trap)); 679 aux_data[i + 1].seen = env->pass_cnt; 680 } 681 continue; 682 } 683 684 if (!aux_data[i + 1].seen) 685 ja.off = insn->off; 686 else if (!aux_data[i + 1 + insn->off].seen) 687 ja.off = 0; 688 else 689 continue; 690 691 if (bpf_prog_is_offloaded(env->prog->aux)) 692 bpf_prog_offload_replace_insn(env, i, &ja); 693 694 memcpy(insn, &ja, sizeof(ja)); 695 } 696 } 697 698 /* 699 * The address of a function might be taken by the code that is not dead, 700 * while the function itself is dead code, since nothing calls it. Keep the first 701 * instruction of such function, so that the address remains valid and differs 702 * from the addresses of other functions. Make it a trap like 703 * sanitize_dead_code() does. 704 */ 705 static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env) 706 { 707 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 708 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 709 struct bpf_insn *insn = env->prog->insnsi; 710 int insn_cnt = env->prog->len; 711 int i, t; 712 713 for (i = 0; i < insn_cnt; i++) { 714 if (!aux_data[i].seen || !bpf_pseudo_func(insn + i)) 715 continue; 716 t = i + insn[i].imm + 1; 717 if (aux_data[t].seen) 718 continue; 719 memcpy(insn + t, &trap, sizeof(trap)); 720 aux_data[t].zext_dst = false; 721 aux_data[t].seen = env->pass_cnt; 722 } 723 } 724 725 int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) 726 { 727 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 728 int insn_cnt = env->prog->len; 729 int i, err; 730 731 keep_funcs_with_addr_taken(env); 732 733 for (i = 0; i < insn_cnt; i++) { 734 int j; 735 736 j = 0; 737 while (i + j < insn_cnt && !aux_data[i + j].seen) 738 j++; 739 if (!j) 740 continue; 741 742 err = verifier_remove_insns(env, i, j); 743 if (err) 744 return err; 745 insn_cnt = env->prog->len; 746 } 747 748 return 0; 749 } 750 751 int bpf_opt_remove_nops(struct bpf_verifier_env *env) 752 { 753 struct bpf_insn_aux_data *aux = env->insn_aux_data; 754 struct bpf_insn *insn = env->prog->insnsi; 755 int insn_cnt = env->prog->len; 756 bool is_may_goto_0, is_ja; 757 int i, err; 758 759 for (i = 0; i < insn_cnt; i++) { 760 is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0)); 761 is_ja = !memcmp(&insn[i], &NOP, sizeof(NOP)); 762 763 if (!is_may_goto_0 && !is_ja) 764 continue; 765 if (aux[i].indirect_target) 766 continue; 767 768 err = verifier_remove_insns(env, i, 1); 769 if (err) 770 return err; 771 insn_cnt--; 772 /* Go back one insn to catch may_goto +1; may_goto +0 sequence */ 773 i -= (is_may_goto_0 && i > 0) ? 2 : 1; 774 } 775 776 return 0; 777 } 778 779 int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 780 const union bpf_attr *attr) 781 { 782 struct bpf_insn *patch; 783 /* use env->insn_buf as two independent buffers */ 784 struct bpf_insn *zext_patch = env->insn_buf; 785 struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2]; 786 struct bpf_insn_aux_data *aux = env->insn_aux_data; 787 int i, patch_len, delta = 0, len = env->prog->len; 788 struct bpf_insn *insns = env->prog->insnsi; 789 struct bpf_prog *new_prog; 790 bool rnd_hi32; 791 792 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 793 zext_patch[1] = BPF_ZEXT_REG(0); 794 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 795 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 796 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 797 for (i = 0; i < len; i++) { 798 int adj_idx = i + delta; 799 struct bpf_insn insn; 800 int load_reg; 801 802 insn = insns[adj_idx]; 803 load_reg = insn_def_regno(&insn); 804 if (!aux[adj_idx].zext_dst) { 805 u8 code, class; 806 u32 imm_rnd; 807 808 if (!rnd_hi32) 809 continue; 810 811 code = insn.code; 812 class = BPF_CLASS(code); 813 if (load_reg == -1) 814 continue; 815 816 if (bpf_is_reg64(env->prog, &insn)) { 817 if (class == BPF_LD && 818 BPF_MODE(code) == BPF_IMM) 819 i++; 820 continue; 821 } 822 823 /* ctx load could be transformed into wider load. */ 824 if (class == BPF_LDX && 825 aux[adj_idx].ptr_type == PTR_TO_CTX) 826 continue; 827 828 imm_rnd = get_random_u32(); 829 rnd_hi32_patch[0] = insn; 830 rnd_hi32_patch[1].imm = imm_rnd; 831 rnd_hi32_patch[3].dst_reg = load_reg; 832 patch = rnd_hi32_patch; 833 patch_len = 4; 834 goto apply_patch_buffer; 835 } 836 837 /* Add in an zero-extend instruction if a) the JIT has requested 838 * it or b) it's a CMPXCHG. 839 * 840 * The latter is because: BPF_CMPXCHG always loads a value into 841 * R0, therefore always zero-extends. However some archs' 842 * equivalent instruction only does this load when the 843 * comparison is successful. This detail of CMPXCHG is 844 * orthogonal to the general zero-extension behaviour of the 845 * CPU, so it's treated independently of bpf_jit_needs_zext. 846 */ 847 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 848 continue; 849 850 /* Zero-extension is done by the caller. */ 851 if (bpf_pseudo_kfunc_call(&insn)) 852 continue; 853 854 if (verifier_bug_if(load_reg == -1, env, 855 "zext_dst is set, but no reg is defined")) 856 return -EFAULT; 857 858 zext_patch[0] = insn; 859 zext_patch[1].dst_reg = load_reg; 860 zext_patch[1].src_reg = load_reg; 861 patch = zext_patch; 862 patch_len = 2; 863 apply_patch_buffer: 864 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 865 if (!new_prog) 866 return -ENOMEM; 867 env->prog = new_prog; 868 insns = new_prog->insnsi; 869 aux = env->insn_aux_data; 870 delta += patch_len - 1; 871 } 872 873 return 0; 874 } 875 876 /* 877 * The prologue takes the context from R1, and it can clobber R2-R5. An 878 * extension gets the arguments of the function that it replaces. 879 */ 880 static bool ext_prog_takes_prologue(const struct bpf_verifier_env *env) 881 { 882 const struct bpf_subprog_info *sub = &env->subprog_info[0]; 883 int i, cnt = min_t(int, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); 884 885 if (!cnt || sub->args[0].arg_type != ARG_PTR_TO_CTX) 886 return false; 887 888 for (i = 1; i < cnt; i++) 889 if (sub->args[i].arg_type != ARG_SCALAR) 890 return false; 891 892 return true; 893 } 894 895 /* convert load instructions that access fields of a context type into a 896 * sequence of instructions that access fields of the underlying structure: 897 * struct __sk_buff -> struct sk_buff 898 * struct bpf_sock_ops -> struct sock 899 */ 900 /* 901 * A load through an __arena pointer member of a kernel struct reads the kernel 902 * address of arena memory. Append the conversion to the arena pointer form, 903 * (u32)(kaddr - kern_vm_start), that the struct_ops trampoline applies to 904 * __arena arguments. A __nullable member keeps NULL as NULL, tested on the 905 * full kernel address, and so does a probed load, which reads as 0 when it 906 * faults. Returns the number of insns in @buf, 0 when the insn loads no such 907 * member, or an error when it does so on some paths only. 908 */ 909 static int arena_member_fixup(struct bpf_verifier_env *env, int idx, struct bpf_insn *insn, 910 struct bpf_insn *buf) 911 { 912 struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx]; 913 int cnt = 0; 914 u32 base; 915 916 if (!aux->arena_member) 917 return 0; 918 if (aux->non_arena_load) { 919 verbose(env, "an insn loads an arena pointer member on one path and something else on another\n"); 920 return -EINVAL; 921 } 922 923 base = bpf_arena_get_kern_vm_start(env->prog->aux->arena); 924 buf[cnt++] = *insn; 925 if (aux->arena_member_nullable || BPF_MODE(insn->code) == BPF_PROBE_MEM) 926 buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, insn->dst_reg, 0, 1); 927 buf[cnt++] = BPF_ALU32_IMM(BPF_SUB, insn->dst_reg, (s32)base); 928 return cnt; 929 } 930 931 int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) 932 { 933 struct bpf_subprog_info *subprogs = env->subprog_info; 934 const struct bpf_verifier_ops *ops = env->ops; 935 int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0; 936 const int insn_cnt = env->prog->len; 937 struct bpf_insn *epilogue_buf = env->epilogue_buf; 938 struct bpf_insn *insn_buf = env->insn_buf; 939 struct bpf_insn *insn; 940 u32 target_size, size_default, off; 941 struct bpf_prog *new_prog; 942 enum bpf_access_type type; 943 bool is_narrower_load; 944 int epilogue_idx = 0; 945 946 if (ops->gen_epilogue) { 947 epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog, 948 -(subprogs[0].stack_depth + 8)); 949 if (epilogue_cnt >= INSN_BUF_SIZE) { 950 verifier_bug(env, "epilogue is too long"); 951 return -EFAULT; 952 } else if (epilogue_cnt) { 953 /* Save the ARG_PTR_TO_CTX for the epilogue to use */ 954 cnt = 0; 955 subprogs[0].stack_depth += 8; 956 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, 957 -subprogs[0].stack_depth); 958 insn_buf[cnt++] = env->prog->insnsi[0]; 959 new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, 960 BPF_PATCH_MOVE_TARGET); 961 if (!new_prog) 962 return -ENOMEM; 963 env->prog = new_prog; 964 delta += cnt - 1; 965 966 ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1); 967 if (ret < 0) 968 return ret; 969 } 970 } 971 972 if (ops->gen_prologue || env->seen_direct_write) { 973 if (!ops->gen_prologue) { 974 verifier_bug(env, "gen_prologue is null"); 975 return -EFAULT; 976 } 977 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 978 env->prog); 979 if (cnt >= INSN_BUF_SIZE) { 980 verifier_bug(env, "prologue is too long"); 981 return -EFAULT; 982 } else if (cnt) { 983 if (env->prog->type == BPF_PROG_TYPE_EXT && 984 !ext_prog_takes_prologue(env)) { 985 verbose(env, 986 "Extension writing packet data must take ctx and scalars only\n"); 987 return -EINVAL; 988 } 989 if (env->prog->type == BPF_PROG_TYPE_EXT && 990 bpf_check_ext_prologue(&env->log, 991 env->prog->aux->dst_prog, 992 env->prog->aux->attach_btf_id)) 993 return -EINVAL; 994 env->prog->aux->has_prologue = true; 995 new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, 996 BPF_PATCH_MOVE_TARGET); 997 if (!new_prog) 998 return -ENOMEM; 999 1000 env->prog = new_prog; 1001 delta += cnt - 1; 1002 1003 ret = add_kfunc_in_insns(env, insn_buf, cnt - 1); 1004 if (ret < 0) 1005 return ret; 1006 } 1007 } 1008 1009 if (delta) 1010 WARN_ON(adjust_jmp_off(env->prog, 0, delta)); 1011 1012 if (bpf_prog_is_offloaded(env->prog->aux)) 1013 return 0; 1014 1015 insn = env->prog->insnsi + delta; 1016 1017 for (i = 0; i < insn_cnt; i++, insn++) { 1018 bpf_convert_ctx_access_t convert_ctx_access; 1019 enum bpf_reg_type ptr_type; 1020 u8 mode; 1021 1022 if (env->insn_aux_data[i + delta].nospec) { 1023 WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state); 1024 struct bpf_insn *patch = insn_buf; 1025 1026 *patch++ = BPF_ST_NOSPEC(); 1027 *patch = *insn; 1028 ret = bpf_adj_moved_insn(patch++, 1); 1029 if (ret) { 1030 verbose(env, "insn %d cannot be patched due to 16-bit range\n", 1031 env->insn_aux_data[i + delta].orig_idx); 1032 return ret; 1033 } 1034 cnt = patch - insn_buf; 1035 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1036 if (!new_prog) 1037 return -ENOMEM; 1038 1039 delta += cnt - 1; 1040 env->prog = new_prog; 1041 insn = new_prog->insnsi + i + delta; 1042 /* This can not be easily merged with the 1043 * nospec_result-case, because an insn may require a 1044 * nospec before and after itself. Therefore also do not 1045 * 'continue' here but potentially apply further 1046 * patching to insn. *insn should equal patch[1] now. 1047 */ 1048 } 1049 1050 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 1051 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 1052 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 1053 insn->code == (BPF_LDX | BPF_MEM | BPF_DW) || 1054 insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) || 1055 insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) || 1056 insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) { 1057 type = BPF_READ; 1058 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 1059 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 1060 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 1061 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 1062 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 1063 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 1064 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 1065 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 1066 type = BPF_WRITE; 1067 } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) || 1068 insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) || 1069 insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) || 1070 insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) && 1071 env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) { 1072 insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); 1073 env->prog->aux->num_exentries++; 1074 continue; 1075 } else if (insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && 1076 insn->imm == BPF_LOAD_ACQ && 1077 env->insn_aux_data[i + delta].arena_member) { 1078 /* an acquire load of an __arena member is rewritten like a plain load */ 1079 cnt = arena_member_fixup(env, i + delta, insn, insn_buf); 1080 if (cnt < 0) 1081 return cnt; 1082 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1083 if (!new_prog) 1084 return -ENOMEM; 1085 1086 delta += cnt - 1; 1087 env->prog = new_prog; 1088 insn = new_prog->insnsi + i + delta; 1089 continue; 1090 } else if (insn->code == (BPF_JMP | BPF_EXIT) && 1091 epilogue_cnt && 1092 i + delta < subprogs[1].start) { 1093 /* Generate epilogue for the main prog */ 1094 if (epilogue_idx) { 1095 /* jump back to the earlier generated epilogue */ 1096 insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1); 1097 cnt = 1; 1098 } else { 1099 memcpy(insn_buf, epilogue_buf, 1100 epilogue_cnt * sizeof(*epilogue_buf)); 1101 cnt = epilogue_cnt; 1102 /* epilogue_idx cannot be 0. It must have at 1103 * least one ctx ptr saving insn before the 1104 * epilogue. 1105 */ 1106 epilogue_idx = i + delta; 1107 } 1108 goto patch_insn_buf; 1109 } else { 1110 continue; 1111 } 1112 1113 if (type == BPF_WRITE && 1114 env->insn_aux_data[i + delta].nospec_result) { 1115 /* nospec_result is only used to mitigate Spectre v4 and 1116 * to limit verification-time for Spectre v1. 1117 */ 1118 struct bpf_insn *patch = insn_buf; 1119 1120 *patch++ = *insn; 1121 *patch++ = BPF_ST_NOSPEC(); 1122 cnt = patch - insn_buf; 1123 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1124 if (!new_prog) 1125 return -ENOMEM; 1126 1127 delta += cnt - 1; 1128 env->prog = new_prog; 1129 insn = new_prog->insnsi + i + delta; 1130 continue; 1131 } 1132 1133 ptr_type = env->insn_aux_data[i + delta].ptr_type; 1134 switch ((int)ptr_type) { 1135 case PTR_TO_CTX: 1136 if (!ops->convert_ctx_access) 1137 continue; 1138 convert_ctx_access = ops->convert_ctx_access; 1139 break; 1140 case PTR_TO_SOCKET: 1141 case PTR_TO_SOCK_COMMON: 1142 convert_ctx_access = bpf_sock_convert_ctx_access; 1143 break; 1144 case PTR_TO_TCP_SOCK: 1145 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 1146 break; 1147 case PTR_TO_XDP_SOCK: 1148 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 1149 break; 1150 case PTR_TO_ARENA: 1151 if (BPF_MODE(insn->code) == BPF_MEMSX) { 1152 if (!bpf_jit_supports_insn(insn, true)) { 1153 verbose(env, "sign extending loads from arena are not supported yet\n"); 1154 return -EOPNOTSUPP; 1155 } 1156 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code); 1157 } else { 1158 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code); 1159 } 1160 env->prog->aux->num_exentries++; 1161 continue; 1162 default: 1163 /* 1164 * A pointer which may fault on a dereference must not 1165 * be loaded from without fault protection, hence turn 1166 * the BPF_LDX into a BPF_PROBE_MEM one so that a bad 1167 * address is handled rather than panicking the kernel. 1168 * A store through one is rejected earlier, there is no 1169 * probed counterpart to rewrite it into. 1170 */ 1171 if (bpf_is_ptr_to_mem_or_btf_id(ptr_type) && 1172 bpf_may_fault_on_deref(ptr_type) && 1173 type == BPF_READ) { 1174 if (BPF_MODE(insn->code) == BPF_MEM) 1175 insn->code = BPF_LDX | BPF_PROBE_MEM | 1176 BPF_SIZE(insn->code); 1177 else 1178 insn->code = BPF_LDX | BPF_PROBE_MEMSX | 1179 BPF_SIZE(insn->code); 1180 env->prog->aux->num_exentries++; 1181 } else if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, 1182 "access to a fault prone pointer is not rewritten as a probed one")) { 1183 return -EFAULT; 1184 } 1185 1186 cnt = arena_member_fixup(env, i + delta, insn, insn_buf); 1187 if (cnt < 0) 1188 return cnt; 1189 if (!cnt) 1190 continue; 1191 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1192 if (!new_prog) 1193 return -ENOMEM; 1194 1195 delta += cnt - 1; 1196 env->prog = new_prog; 1197 insn = new_prog->insnsi + i + delta; 1198 continue; 1199 } 1200 1201 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 1202 size = BPF_LDST_BYTES(insn); 1203 mode = BPF_MODE(insn->code); 1204 1205 /* If the read access is a narrower load of the field, 1206 * convert to a 4/8-byte load, to minimum program type specific 1207 * convert_ctx_access changes. If conversion is successful, 1208 * we will apply proper mask to the result. 1209 */ 1210 is_narrower_load = size < ctx_field_size; 1211 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 1212 off = insn->off; 1213 if (is_narrower_load) { 1214 u8 size_code; 1215 1216 if (type == BPF_WRITE) { 1217 verifier_bug(env, "narrow ctx access misconfigured"); 1218 return -EFAULT; 1219 } 1220 1221 size_code = BPF_H; 1222 if (ctx_field_size == 4) 1223 size_code = BPF_W; 1224 else if (ctx_field_size == 8) 1225 size_code = BPF_DW; 1226 1227 insn->off = off & ~(size_default - 1); 1228 insn->code = BPF_LDX | BPF_MEM | size_code; 1229 } 1230 1231 target_size = 0; 1232 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 1233 &target_size); 1234 if (cnt == 0 || cnt >= INSN_BUF_SIZE || 1235 (ctx_field_size && !target_size)) { 1236 verifier_bug(env, "error during ctx access conversion (%d)", cnt); 1237 return -EFAULT; 1238 } 1239 1240 if (is_narrower_load && size < target_size) { 1241 u8 shift = bpf_ctx_narrow_access_offset( 1242 off, size, size_default) * 8; 1243 if (shift && cnt + 1 >= INSN_BUF_SIZE) { 1244 verifier_bug(env, "narrow ctx load misconfigured"); 1245 return -EFAULT; 1246 } 1247 if (ctx_field_size <= 4) { 1248 if (shift) 1249 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 1250 insn->dst_reg, 1251 shift); 1252 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1253 (1 << size * 8) - 1); 1254 } else { 1255 if (shift) 1256 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 1257 insn->dst_reg, 1258 shift); 1259 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1260 (1ULL << size * 8) - 1); 1261 } 1262 } 1263 if (mode == BPF_MEMSX) 1264 insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, 1265 insn->dst_reg, insn->dst_reg, 1266 size * 8, 0); 1267 1268 patch_insn_buf: 1269 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1270 if (!new_prog) 1271 return -ENOMEM; 1272 1273 delta += cnt - 1; 1274 1275 /* keep walking new program and skip insns we just inserted */ 1276 env->prog = new_prog; 1277 insn = new_prog->insnsi + i + delta; 1278 } 1279 1280 return 0; 1281 } 1282 1283 static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env) 1284 { 1285 u32 *starts = NULL; 1286 1287 starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT); 1288 if (starts) { 1289 for (int i = 0; i < env->subprog_cnt; i++) 1290 starts[i] = env->subprog_info[i].start; 1291 } 1292 return starts; 1293 } 1294 1295 static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts) 1296 { 1297 for (int i = 0; i < env->subprog_cnt; i++) 1298 env->subprog_info[i].start = orig_starts[i]; 1299 /* restore the start of fake 'exit' subprog as well */ 1300 env->subprog_info[env->subprog_cnt].start = env->prog->len; 1301 } 1302 1303 /* 1304 * Replace the offsets of functions with their addresses in the maps of 1305 * the program, see resolve_func_ptrs() in verifier.c. The program must be 1306 * the only user of such map. From now on no other program can use it, see 1307 * bpf_map_claim(). 1308 */ 1309 static int resolve_func_ptrs(struct bpf_verifier_env *env, struct bpf_prog *prog, 1310 struct bpf_prog **func) 1311 { 1312 unsigned long me = (unsigned long)prog->aux; 1313 struct bpf_func_ptr *ptr; 1314 int i, err, subprog; 1315 u64 addr, new, *slot; 1316 1317 for (i = 0; i < env->func_ptr_cnt; i++) { 1318 ptr = &env->func_ptrs[i]; 1319 1320 /* pointers are sorted by map */ 1321 if ((!i || ptr->map != ptr[-1].map) && 1322 cmpxchg(&ptr->map->user, me, me | BPF_MAP_USER_PATCHED) != me) { 1323 verbose(env, "map '%s' is used by another program\n", ptr->map->name); 1324 return -EBUSY; 1325 } 1326 1327 /* it's the address of the value of the map whatever the offset is */ 1328 err = ptr->map->ops->map_direct_value_addr(ptr->map, &addr, 0); 1329 if (verifier_bug_if(err, env, "no value of map '%s'", ptr->map->name)) 1330 return -EFAULT; 1331 slot = (u64 *)(unsigned long)(addr + ptr->map_off); 1332 1333 /* 1334 * The function that is dead code is removed and the pointer is 1335 * NULL. The program may read it, but callx of it is dead code 1336 * too, see check_func_callx(). 1337 */ 1338 new = 0; 1339 if (ptr->xlated_off != BPF_FUNC_PTR_DELETED) { 1340 subprog = bpf_find_subprog(env, ptr->xlated_off); 1341 if (verifier_bug_if(!func || subprog <= 0, env, "no function at insn %u", 1342 ptr->xlated_off)) 1343 return -EFAULT; 1344 new = (unsigned long)func[subprog]->bpf_func; 1345 } 1346 1347 /* the map is frozen and nothing else uses it */ 1348 if (verifier_bug_if(*slot != (u64)ptr->orig_off * sizeof(struct bpf_insn), env, 1349 "map '%s' offset %u changed", ptr->map->name, ptr->map_off)) 1350 return -EFAULT; 1351 WRITE_ONCE(*slot, new); 1352 } 1353 return 0; 1354 } 1355 1356 static int jit_subprogs(struct bpf_verifier_env *env) 1357 { 1358 struct bpf_prog *prog = env->prog, **func, *tmp; 1359 int i, j, subprog_start, subprog_end = 0, len, subprog; 1360 struct bpf_map *map_ptr; 1361 struct bpf_insn *insn; 1362 void *old_bpf_func; 1363 int err, num_exentries; 1364 1365 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1366 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 1367 continue; 1368 1369 /* Upon error here we cannot fall back to interpreter but 1370 * need a hard reject of the program. Thus -EFAULT is 1371 * propagated in any case. 1372 */ 1373 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1374 if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d", 1375 i + insn->imm + 1)) 1376 return -EFAULT; 1377 /* temporarily remember subprog id inside insn instead of 1378 * aux_data, since next loop will split up all insns into funcs 1379 */ 1380 insn->off = subprog; 1381 /* remember original imm in case JIT fails and fallback 1382 * to interpreter will be needed 1383 */ 1384 env->insn_aux_data[i].call_imm = insn->imm; 1385 /* point imm to __bpf_call_base+1 from JITs point of view */ 1386 insn->imm = 1; 1387 if (bpf_pseudo_func(insn)) { 1388 #if defined(MODULES_VADDR) 1389 u64 addr = MODULES_VADDR; 1390 #else 1391 u64 addr = VMALLOC_START; 1392 #endif 1393 /* jit (e.g. x86_64) may emit fewer instructions 1394 * if it learns a u32 imm is the same as a u64 imm. 1395 * Set close enough to possible prog address. 1396 */ 1397 insn[0].imm = (u32)addr; 1398 insn[1].imm = addr >> 32; 1399 } 1400 } 1401 1402 err = bpf_prog_alloc_jited_linfo(prog); 1403 if (err) 1404 goto out_undo_insn; 1405 1406 err = -ENOMEM; 1407 func = kzalloc_objs(prog, env->subprog_cnt); 1408 if (!func) 1409 goto out_undo_insn; 1410 1411 for (i = 0; i < env->subprog_cnt; i++) { 1412 subprog_start = subprog_end; 1413 subprog_end = env->subprog_info[i + 1].start; 1414 1415 len = subprog_end - subprog_start; 1416 /* bpf_prog_run() doesn't call subprogs directly, 1417 * hence main prog stats include the runtime of subprogs. 1418 * subprogs don't have IDs and not reachable via prog_get_next_id 1419 * func[i]->stats will never be accessed and stays NULL 1420 */ 1421 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 1422 if (!func[i]) 1423 goto out_free; 1424 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 1425 len * sizeof(struct bpf_insn)); 1426 func[i]->type = prog->type; 1427 func[i]->len = len; 1428 if (bpf_prog_calc_tag(func[i])) 1429 goto out_free; 1430 func[i]->is_func = 1; 1431 func[i]->sleepable = prog->sleepable; 1432 func[i]->blinded = prog->blinded; 1433 func[i]->aux->func_idx = i; 1434 /* Below members will be freed only at prog->aux */ 1435 func[i]->aux->btf = prog->aux->btf; 1436 func[i]->aux->subprog_start = subprog_start; 1437 func[i]->aux->func_info = prog->aux->func_info; 1438 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 1439 func[i]->aux->poke_tab = prog->aux->poke_tab; 1440 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 1441 func[i]->aux->main_prog_aux = prog->aux; 1442 1443 for (j = 0; j < prog->aux->size_poke_tab; j++) { 1444 struct bpf_jit_poke_descriptor *poke; 1445 1446 poke = &prog->aux->poke_tab[j]; 1447 if (poke->insn_idx < subprog_end && 1448 poke->insn_idx >= subprog_start) 1449 poke->aux = func[i]->aux; 1450 } 1451 1452 func[i]->aux->name[0] = 'F'; 1453 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 1454 if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) 1455 func[i]->aux->jits_use_priv_stack = true; 1456 1457 func[i]->jit_requested = 1; 1458 func[i]->blinding_requested = prog->blinding_requested; 1459 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 1460 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 1461 func[i]->aux->linfo = prog->aux->linfo; 1462 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 1463 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 1464 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 1465 func[i]->aux->arena = prog->aux->arena; 1466 func[i]->aux->uses_arena_insns = bpf_prog_has_arena_insns(func[i]); 1467 func[i]->aux->used_maps = env->used_maps; 1468 func[i]->aux->used_map_cnt = env->used_map_cnt; 1469 num_exentries = 0; 1470 insn = func[i]->insnsi; 1471 for (j = 0; j < func[i]->len; j++, insn++) { 1472 if (BPF_CLASS(insn->code) == BPF_LDX && 1473 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 1474 BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 1475 BPF_MODE(insn->code) == BPF_PROBE_MEM32SX || 1476 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) 1477 num_exentries++; 1478 if ((BPF_CLASS(insn->code) == BPF_STX || 1479 BPF_CLASS(insn->code) == BPF_ST) && 1480 BPF_MODE(insn->code) == BPF_PROBE_MEM32) 1481 num_exentries++; 1482 if (BPF_CLASS(insn->code) == BPF_STX && 1483 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) 1484 num_exentries++; 1485 if (bpf_is_unwind_goto(insn)) 1486 num_exentries++; 1487 } 1488 func[i]->aux->num_exentries = num_exentries; 1489 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 1490 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; 1491 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; 1492 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; 1493 func[i]->aux->can_unwind = env->subprog_info[i].can_unwind; 1494 func[i]->aux->token = prog->aux->token; 1495 if (!i) 1496 func[i]->aux->exception_boundary = env->seen_exception; 1497 func[i] = bpf_int_jit_compile(env, func[i]); 1498 if (!func[i]->jited) { 1499 err = -ENOTSUPP; 1500 goto out_free; 1501 } 1502 cond_resched(); 1503 } 1504 1505 /* at this point all bpf functions were successfully JITed 1506 * now populate all bpf_calls with correct addresses and 1507 * run last pass of JIT 1508 */ 1509 for (i = 0; i < env->subprog_cnt; i++) { 1510 insn = func[i]->insnsi; 1511 for (j = 0; j < func[i]->len; j++, insn++) { 1512 if (bpf_pseudo_func(insn)) { 1513 subprog = insn->off; 1514 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 1515 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 1516 continue; 1517 } 1518 if (!bpf_pseudo_call(insn)) 1519 continue; 1520 subprog = insn->off; 1521 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 1522 } 1523 1524 /* we use the aux data to keep a list of the start addresses 1525 * of the JITed images for each function in the program 1526 * 1527 * for some architectures, such as powerpc64, the imm field 1528 * might not be large enough to hold the offset of the start 1529 * address of the callee's JITed image from __bpf_call_base 1530 * 1531 * in such cases, we can lookup the start address of a callee 1532 * by using its subprog id, available from the off field of 1533 * the call instruction, as an index for this list 1534 */ 1535 func[i]->aux->func = func; 1536 func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1537 func[i]->aux->real_func_cnt = env->subprog_cnt; 1538 } 1539 for (i = 0; i < env->subprog_cnt; i++) { 1540 old_bpf_func = func[i]->bpf_func; 1541 tmp = bpf_int_jit_compile(env, func[i]); 1542 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 1543 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 1544 err = -ENOTSUPP; 1545 goto out_free; 1546 } 1547 cond_resched(); 1548 } 1549 1550 /* the addresses of all functions are final */ 1551 err = resolve_func_ptrs(env, prog, func); 1552 if (err) 1553 goto out_free; 1554 1555 /* 1556 * Cleanup func[i]->aux fields which aren't required 1557 * or can become invalid in future 1558 */ 1559 for (i = 0; i < env->subprog_cnt; i++) { 1560 func[i]->aux->used_maps = NULL; 1561 func[i]->aux->used_map_cnt = 0; 1562 } 1563 1564 /* finally lock prog and jit images for all functions and 1565 * populate kallsysm. Begin at the first subprogram, since 1566 * bpf_prog_load will add the kallsyms for the main program. 1567 */ 1568 for (i = 1; i < env->subprog_cnt; i++) { 1569 err = bpf_prog_lock_ro(func[i]); 1570 if (err) 1571 goto out_free; 1572 } 1573 1574 for (i = 1; i < env->subprog_cnt; i++) 1575 bpf_prog_kallsyms_add(func[i]); 1576 1577 /* Last step: make now unused interpreter insns from main 1578 * prog consistent for later dump requests, so they can 1579 * later look the same as if they were interpreted only. 1580 */ 1581 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1582 if (bpf_pseudo_func(insn)) { 1583 insn[0].imm = env->insn_aux_data[i].call_imm; 1584 insn[1].imm = insn->off; 1585 insn->off = 0; 1586 continue; 1587 } 1588 if (!bpf_pseudo_call(insn)) 1589 continue; 1590 insn->imm = env->insn_aux_data[i].call_imm; 1591 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1592 insn->off = subprog; 1593 } 1594 1595 prog->jited = 1; 1596 prog->bpf_func = func[0]->bpf_func; 1597 prog->jited_len = func[0]->jited_len; 1598 prog->aux->extable = func[0]->aux->extable; 1599 prog->aux->num_exentries = func[0]->aux->num_exentries; 1600 prog->aux->func = func; 1601 prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1602 prog->aux->real_func_cnt = env->subprog_cnt; 1603 prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; 1604 prog->aux->exception_boundary = func[0]->aux->exception_boundary; 1605 prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust; 1606 bpf_prog_jit_attempt_done(prog); 1607 return 0; 1608 out_free: 1609 /* We failed JIT'ing, so at this point we need to unregister poke 1610 * descriptors from subprogs, so that kernel is not attempting to 1611 * patch it anymore as we're freeing the subprog JIT memory. 1612 */ 1613 for (i = 0; i < prog->aux->size_poke_tab; i++) { 1614 map_ptr = prog->aux->poke_tab[i].tail_call.map; 1615 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 1616 } 1617 /* At this point we're guaranteed that poke descriptors are not 1618 * live anymore. We can just unlink its descriptor table as it's 1619 * released with the main prog. 1620 */ 1621 for (i = 0; i < env->subprog_cnt; i++) { 1622 if (!func[i]) 1623 continue; 1624 func[i]->aux->poke_tab = NULL; 1625 bpf_jit_free(func[i]); 1626 } 1627 kfree(func); 1628 out_undo_insn: 1629 bpf_prog_jit_attempt_done(prog); 1630 return err; 1631 } 1632 1633 int bpf_jit_subprogs(struct bpf_verifier_env *env) 1634 { 1635 int err, i; 1636 bool blinded = false; 1637 struct bpf_insn *insn; 1638 struct bpf_prog *prog, *orig_prog; 1639 u32 *orig_subprog_starts; 1640 1641 /* all functions that the maps point to are removed as dead code */ 1642 if (env->subprog_cnt <= 1) 1643 return resolve_func_ptrs(env, env->prog, NULL); 1644 1645 prog = orig_prog = env->prog; 1646 if (bpf_prog_need_blind(prog)) { 1647 orig_subprog_starts = bpf_dup_subprog_starts(env); 1648 if (!orig_subprog_starts) { 1649 err = -ENOMEM; 1650 goto out_cleanup; 1651 } 1652 prog = bpf_jit_blind_constants(env, prog); 1653 if (IS_ERR(prog)) { 1654 err = PTR_ERR(prog); 1655 prog = orig_prog; 1656 goto out_restore; 1657 } 1658 blinded = true; 1659 } 1660 1661 err = jit_subprogs(env); 1662 if (err) 1663 goto out_jit_err; 1664 1665 if (blinded) { 1666 bpf_jit_prog_release_other(prog, orig_prog); 1667 kvfree(orig_subprog_starts); 1668 } 1669 1670 return 0; 1671 1672 out_jit_err: 1673 if (blinded) { 1674 bpf_jit_prog_release_other(orig_prog, prog); 1675 /* roll back to the clean original prog */ 1676 prog = env->prog = orig_prog; 1677 goto out_restore; 1678 } else { 1679 if (err != -EFAULT) { 1680 /* 1681 * We will fall back to interpreter mode when err is not -EFAULT, before 1682 * that, insn->off and insn->imm should be restored to their original 1683 * values since they were modified by jit_subprogs. 1684 */ 1685 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1686 if (!bpf_pseudo_call(insn)) 1687 continue; 1688 insn->off = 0; 1689 insn->imm = env->insn_aux_data[i].call_imm; 1690 } 1691 } 1692 goto out_cleanup; 1693 } 1694 1695 out_restore: 1696 bpf_restore_subprog_starts(env, orig_subprog_starts); 1697 kvfree(orig_subprog_starts); 1698 out_cleanup: 1699 /* cleanup main prog to be interpreted */ 1700 prog->jit_requested = 0; 1701 prog->blinding_requested = 0; 1702 return err; 1703 } 1704 1705 int bpf_fixup_call_args(struct bpf_verifier_env *env) 1706 { 1707 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1708 struct bpf_prog *prog = env->prog; 1709 struct bpf_insn *insn = prog->insnsi; 1710 int depth; 1711 #endif 1712 int i, err = 0; 1713 1714 for (i = 0; i < env->subprog_cnt; i++) { 1715 struct bpf_subprog_info *subprog = &env->subprog_info[i]; 1716 u16 outgoing = subprog->stack_arg_cnt - bpf_in_stack_arg_cnt(subprog); 1717 1718 if (subprog->max_out_stack_arg_cnt > outgoing) { 1719 verbose(env, 1720 "func#%d writes %u stack arg slots, but calls only require %u\n", 1721 i, subprog->max_out_stack_arg_cnt, outgoing); 1722 return -EINVAL; 1723 } 1724 } 1725 1726 if (env->prog->jit_requested && 1727 !bpf_prog_is_offloaded(env->prog->aux)) { 1728 err = bpf_jit_subprogs(env); 1729 if (err == 0) 1730 return 0; 1731 if (err == -EFAULT || err == -EINTR) 1732 return err; 1733 } 1734 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1735 if (prog->jit_required) { 1736 verbose(env, "program requires BPF JIT compiler but it is not available\n"); 1737 return -EINVAL; 1738 } 1739 for (i = 0; i < env->subprog_cnt; i++) { 1740 if (bpf_in_stack_arg_cnt(&env->subprog_info[i])) { 1741 verbose(env, "stack args are not supported in non-JITed programs\n"); 1742 return -EINVAL; 1743 } 1744 } 1745 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 1746 /* When JIT fails the progs with bpf2bpf calls and tail_calls 1747 * have to be rejected, since interpreter doesn't support them yet. 1748 */ 1749 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 1750 return -EINVAL; 1751 } 1752 for (i = 0; i < prog->len; i++, insn++) { 1753 if (bpf_pseudo_func(insn)) { 1754 /* When JIT fails the progs with callback calls 1755 * have to be rejected, since interpreter doesn't support them yet. 1756 */ 1757 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 1758 return -EINVAL; 1759 } 1760 1761 if (!bpf_pseudo_call(insn)) 1762 continue; 1763 depth = get_callee_stack_depth(env, insn, i); 1764 if (depth < 0) 1765 return depth; 1766 err = bpf_patch_call_args(insn, depth); 1767 if (err) { 1768 verbose(env, "stack depth %d exceeds interpreter stack depth limit\n", 1769 depth); 1770 return err; 1771 } 1772 } 1773 err = 0; 1774 #endif 1775 return err; 1776 } 1777 1778 /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */ 1779 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) 1780 { 1781 struct bpf_subprog_info *info = env->subprog_info; 1782 int cnt = env->subprog_cnt; 1783 struct bpf_prog *prog; 1784 1785 /* We only reserve one slot for hidden subprogs in subprog_info. */ 1786 if (env->hidden_subprog_cnt) { 1787 verifier_bug(env, "only one hidden subprog supported"); 1788 return -EFAULT; 1789 } 1790 /* We're not patching any existing instruction, just appending the new 1791 * ones for the hidden subprog. Hence all of the adjustment operations 1792 * in bpf_patch_insn_data are no-ops. 1793 */ 1794 prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); 1795 if (!prog) 1796 return -ENOMEM; 1797 env->prog = prog; 1798 info[cnt + 1].start = info[cnt].start; 1799 info[cnt].start = prog->len - len + 1; 1800 /* The patch ends with the exit. */ 1801 info[cnt].exit_idx = prog->len - 1; 1802 env->subprog_cnt++; 1803 env->hidden_subprog_cnt++; 1804 return 0; 1805 } 1806 1807 /* 1808 * Expand may_goto: load the count from the stack, jump to the target of 1809 * may_goto when it is zero, then the tail that updates the count. 1810 * Use gotol when the target is too far for 16-bit offset of a conditional jump. 1811 */ 1812 static int may_goto_expand(struct bpf_insn *insn_buf, int off, int stack_off, 1813 const struct bpf_insn *tail, int tail_cnt) 1814 { 1815 int cnt = 0; 1816 1817 /* Forward jump has to step over the tail */ 1818 off = off >= 0 ? off + tail_cnt : off - 1; 1819 1820 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); 1821 if (off == (s16)off) { 1822 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off); 1823 } else { 1824 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1); 1825 insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1); 1826 } 1827 memcpy(insn_buf + cnt, tail, tail_cnt * sizeof(*tail)); 1828 return cnt + tail_cnt; 1829 } 1830 1831 /* Do various post-verification rewrites in a single program pass. 1832 * These rewrites simplify JIT and interpreter implementations. 1833 */ 1834 int bpf_do_misc_fixups(struct bpf_verifier_env *env) 1835 { 1836 struct bpf_prog *prog = env->prog; 1837 enum bpf_attach_type eatype = prog->expected_attach_type; 1838 enum bpf_prog_type prog_type = resolve_prog_type(prog); 1839 struct bpf_insn *insn = prog->insnsi; 1840 const struct bpf_func_proto *fn; 1841 const int insn_cnt = prog->len; 1842 const struct bpf_map_ops *ops; 1843 struct bpf_insn_aux_data *aux; 1844 struct bpf_insn *insn_buf = env->insn_buf; 1845 struct bpf_prog *new_prog; 1846 struct bpf_map *map_ptr; 1847 int i, ret, cnt, delta = 0, cur_subprog = 0; 1848 struct bpf_subprog_info *subprogs = env->subprog_info; 1849 u16 stack_depth = subprogs[cur_subprog].stack_depth; 1850 u16 stack_depth_extra = 0; 1851 1852 if (env->seen_exception && !env->exception_callback_subprog) { 1853 struct bpf_insn *patch = insn_buf; 1854 1855 *patch++ = env->prog->insnsi[insn_cnt - 1]; 1856 *patch++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 1857 *patch++ = BPF_EXIT_INSN(); 1858 ret = add_hidden_subprog(env, insn_buf, patch - insn_buf); 1859 if (ret < 0) 1860 return ret; 1861 prog = env->prog; 1862 insn = prog->insnsi; 1863 1864 env->exception_callback_subprog = env->subprog_cnt - 1; 1865 /* Don't update insn_cnt, as add_hidden_subprog always appends insns */ 1866 bpf_mark_subprog_exc_cb(env, env->exception_callback_subprog); 1867 } 1868 1869 for (i = 0; i < insn_cnt;) { 1870 if (is_addr_space_cast32(env->prog, insn)) { 1871 /* convert to 32-bit mov that clears upper 32-bit */ 1872 insn->code = BPF_ALU | BPF_MOV | BPF_X; 1873 /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ 1874 insn->off = 0; 1875 insn->imm = 0; 1876 goto next_insn; 1877 } 1878 1879 if (env->insn_aux_data[i + delta].needs_zext) 1880 /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ 1881 insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); 1882 1883 if (env->insn_aux_data[i + delta].arena_scalar) { 1884 cnt = arena_scalar_access(insn, insn_buf); 1885 1886 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1887 if (!new_prog) 1888 return -ENOMEM; 1889 1890 delta += cnt - 1; 1891 prog = new_prog; 1892 env->prog = prog; 1893 insn = prog->insnsi + i + delta; 1894 goto next_insn; 1895 } 1896 1897 /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ 1898 if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || 1899 insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || 1900 insn->code == (BPF_ALU | BPF_MOD | BPF_K) || 1901 insn->code == (BPF_ALU | BPF_DIV | BPF_K)) && 1902 insn->off == 1 && insn->imm == -1) { 1903 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1904 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1905 struct bpf_insn *patch = insn_buf; 1906 1907 if (isdiv) 1908 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1909 BPF_NEG | BPF_K, insn->dst_reg, 1910 0, 0, 0); 1911 else 1912 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1913 1914 cnt = patch - insn_buf; 1915 1916 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1917 if (!new_prog) 1918 return -ENOMEM; 1919 1920 delta += cnt - 1; 1921 env->prog = prog = new_prog; 1922 insn = new_prog->insnsi + i + delta; 1923 goto next_insn; 1924 } 1925 1926 /* Make divide-by-zero and divide-by-minus-one exceptions impossible. */ 1927 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 1928 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 1929 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 1930 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 1931 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1932 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1933 bool is_sdiv = isdiv && insn->off == 1; 1934 bool is_smod = !isdiv && insn->off == 1; 1935 struct bpf_insn *patch = insn_buf; 1936 1937 if (is_sdiv) { 1938 /* [R,W]x sdiv 0 -> 0 1939 * LLONG_MIN sdiv -1 -> LLONG_MIN 1940 * INT_MIN sdiv -1 -> INT_MIN 1941 */ 1942 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1943 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1944 BPF_ADD | BPF_K, BPF_REG_AX, 1945 0, 0, 1); 1946 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1947 BPF_JGT | BPF_K, BPF_REG_AX, 1948 0, 4, 1); 1949 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1950 BPF_JEQ | BPF_K, BPF_REG_AX, 1951 0, 1, 0); 1952 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1953 BPF_MOV | BPF_K, insn->dst_reg, 1954 0, 0, 0); 1955 /* BPF_NEG(LLONG_MIN) == -LLONG_MIN == LLONG_MIN */ 1956 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1957 BPF_NEG | BPF_K, insn->dst_reg, 1958 0, 0, 0); 1959 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1960 *patch++ = *insn; 1961 cnt = patch - insn_buf; 1962 } else if (is_smod) { 1963 /* [R,W]x mod 0 -> [R,W]x */ 1964 /* [R,W]x mod -1 -> 0 */ 1965 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1966 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1967 BPF_ADD | BPF_K, BPF_REG_AX, 1968 0, 0, 1); 1969 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1970 BPF_JGT | BPF_K, BPF_REG_AX, 1971 0, 3, 1); 1972 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1973 BPF_JEQ | BPF_K, BPF_REG_AX, 1974 0, 3 + (is64 ? 0 : 1), 1); 1975 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1976 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1977 *patch++ = *insn; 1978 1979 if (!is64) { 1980 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1981 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 1982 } 1983 cnt = patch - insn_buf; 1984 } else if (isdiv) { 1985 /* [R,W]x div 0 -> 0 */ 1986 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1987 BPF_JNE | BPF_K, insn->src_reg, 1988 0, 2, 0); 1989 *patch++ = BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg); 1990 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1991 *patch++ = *insn; 1992 cnt = patch - insn_buf; 1993 } else { 1994 /* [R,W]x mod 0 -> [R,W]x */ 1995 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1996 BPF_JEQ | BPF_K, insn->src_reg, 1997 0, 1 + (is64 ? 0 : 1), 0); 1998 *patch++ = *insn; 1999 2000 if (!is64) { 2001 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 2002 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 2003 } 2004 cnt = patch - insn_buf; 2005 } 2006 2007 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2008 if (!new_prog) 2009 return -ENOMEM; 2010 2011 delta += cnt - 1; 2012 env->prog = prog = new_prog; 2013 insn = new_prog->insnsi + i + delta; 2014 goto next_insn; 2015 } 2016 2017 /* Make it impossible to de-reference a userspace address */ 2018 if (BPF_CLASS(insn->code) == BPF_LDX && 2019 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 2020 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) { 2021 struct bpf_insn *patch = insn_buf; 2022 u64 uaddress_limit = bpf_arch_uaddress_limit(); 2023 2024 if (!uaddress_limit) 2025 goto next_insn; 2026 2027 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 2028 if (insn->off) 2029 *patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off); 2030 *patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32); 2031 *patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2); 2032 *patch++ = *insn; 2033 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 2034 *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0); 2035 2036 cnt = patch - insn_buf; 2037 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2038 if (!new_prog) 2039 return -ENOMEM; 2040 2041 delta += cnt - 1; 2042 env->prog = prog = new_prog; 2043 insn = new_prog->insnsi + i + delta; 2044 goto next_insn; 2045 } 2046 2047 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 2048 if (BPF_CLASS(insn->code) == BPF_LD && 2049 (BPF_MODE(insn->code) == BPF_ABS || 2050 BPF_MODE(insn->code) == BPF_IND)) { 2051 cnt = env->ops->gen_ld_abs(insn, insn_buf); 2052 if (cnt == 0 || cnt >= INSN_BUF_SIZE) { 2053 verifier_bug(env, "%d insns generated for ld_abs", cnt); 2054 return -EFAULT; 2055 } 2056 2057 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2058 if (!new_prog) 2059 return -ENOMEM; 2060 2061 delta += cnt - 1; 2062 env->prog = prog = new_prog; 2063 insn = new_prog->insnsi + i + delta; 2064 goto next_insn; 2065 } 2066 2067 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 2068 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 2069 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 2070 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 2071 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 2072 struct bpf_insn *patch = insn_buf; 2073 bool issrc, isneg, isimm; 2074 u32 off_reg; 2075 2076 aux = &env->insn_aux_data[i + delta]; 2077 if (!aux->alu_state || 2078 aux->alu_state == BPF_ALU_NON_POINTER) 2079 goto next_insn; 2080 2081 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 2082 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 2083 BPF_ALU_SANITIZE_SRC; 2084 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 2085 2086 off_reg = issrc ? insn->src_reg : insn->dst_reg; 2087 if (isimm) { 2088 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 2089 } else { 2090 if (isneg) 2091 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 2092 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 2093 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 2094 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 2095 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 2096 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 2097 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 2098 } 2099 if (!issrc) 2100 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 2101 insn->src_reg = BPF_REG_AX; 2102 if (isneg) 2103 insn->code = insn->code == code_add ? 2104 code_sub : code_add; 2105 *patch++ = *insn; 2106 if (issrc && isneg && !isimm) 2107 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 2108 cnt = patch - insn_buf; 2109 2110 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2111 if (!new_prog) 2112 return -ENOMEM; 2113 2114 delta += cnt - 1; 2115 env->prog = prog = new_prog; 2116 insn = new_prog->insnsi + i + delta; 2117 goto next_insn; 2118 } 2119 2120 if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { 2121 int stack_off_cnt = -stack_depth - 16; 2122 /* 2123 * AX is used as an argument to pass in stack_off_cnt 2124 * (to add to r10/fp), and also as the return value of 2125 * the call to arch_bpf_timed_may_goto. 2126 */ 2127 struct bpf_insn tail[] = { 2128 BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), 2129 BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2), 2130 BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt), 2131 BPF_EMIT_CALL(arch_bpf_timed_may_goto), 2132 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt), 2133 }; 2134 2135 /* 2136 * Two 8 byte slots, depth-16 stores the count, and 2137 * depth-8 stores the start timestamp of the loop. 2138 * 2139 * The starting value of count is BPF_MAX_TIMED_LOOPS 2140 * (0xffff). Every iteration loads it and subs it by 1, 2141 * until the value becomes 0 in AX (thus, 1 in stack), 2142 * after which we call arch_bpf_timed_may_goto, which 2143 * either sets AX to 0xffff to keep looping, or to 0 2144 * upon timeout. AX is then stored into the stack. In 2145 * the next iteration, we either see 0 and break out, or 2146 * continue iterating until the next time value is 0 2147 * after subtraction, rinse and repeat. 2148 */ 2149 stack_depth_extra = 16; 2150 cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt, 2151 tail, ARRAY_SIZE(tail)); 2152 2153 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2154 if (!new_prog) 2155 return -ENOMEM; 2156 2157 delta += cnt - 1; 2158 env->prog = prog = new_prog; 2159 insn = new_prog->insnsi + i + delta; 2160 goto next_insn; 2161 } else if (bpf_is_may_goto_insn(insn)) { 2162 int stack_off = -stack_depth - 8; 2163 struct bpf_insn tail[] = { 2164 BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), 2165 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off), 2166 }; 2167 2168 stack_depth_extra = 8; 2169 cnt = may_goto_expand(insn_buf, insn->off, stack_off, 2170 tail, ARRAY_SIZE(tail)); 2171 2172 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2173 if (!new_prog) 2174 return -ENOMEM; 2175 2176 delta += cnt - 1; 2177 env->prog = prog = new_prog; 2178 insn = new_prog->insnsi + i + delta; 2179 goto next_insn; 2180 } 2181 2182 if (bpf_jit_supports_percpu_insn() && 2183 insn->code == (BPF_LD | BPF_IMM | BPF_DW) && 2184 (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 2185 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) { 2186 struct bpf_map *map; 2187 2188 aux = &env->insn_aux_data[i + delta]; 2189 map = env->used_maps[aux->map_index]; 2190 if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) 2191 goto next_insn; 2192 2193 prog->jit_required = true; 2194 2195 /* 2196 * We are *skipping* first half of ld_imm64 insn 2197 * with 'i++;', patching over second half of it 2198 * with that same half + mov64_percpu_reg insn. 2199 * All because bpf_patch_insn_data() can only 2200 * replace one 8-byte insn, which does not work 2201 * well for ld_imm64 insn. 2202 */ 2203 2204 insn_buf[0] = insn[1]; 2205 insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg); 2206 cnt = 2; 2207 2208 i++; 2209 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2210 if (!new_prog) 2211 return -ENOMEM; 2212 2213 delta += cnt - 1; 2214 env->prog = prog = new_prog; 2215 insn = new_prog->insnsi + i + delta; 2216 goto next_insn; 2217 } 2218 2219 if (insn->code != (BPF_JMP | BPF_CALL)) 2220 goto next_insn; 2221 if (insn->src_reg == BPF_PSEUDO_CALL) 2222 goto next_insn; 2223 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 2224 ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); 2225 if (ret) 2226 return ret; 2227 if (cnt == 0) 2228 goto next_insn; 2229 2230 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2231 if (!new_prog) 2232 return -ENOMEM; 2233 2234 delta += cnt - 1; 2235 env->prog = prog = new_prog; 2236 insn = new_prog->insnsi + i + delta; 2237 goto next_insn; 2238 } 2239 2240 /* Skip inlining the helper call if the JIT does it. */ 2241 if (bpf_jit_inlines_helper_call(insn->imm)) { 2242 prog->jit_required = 1; 2243 goto next_insn; 2244 } 2245 2246 if (insn->imm == BPF_FUNC_get_route_realm) 2247 prog->dst_needed = 1; 2248 if (insn->imm == BPF_FUNC_get_prandom_u32) 2249 bpf_user_rnd_init_once(); 2250 if (insn->imm == BPF_FUNC_override_return) 2251 prog->kprobe_override = 1; 2252 if (insn->imm == BPF_FUNC_tail_call) { 2253 /* If we tail call into other programs, we 2254 * cannot make any assumptions since they can 2255 * be replaced dynamically during runtime in 2256 * the program array. 2257 */ 2258 prog->cb_access = 1; 2259 if (!bpf_allow_tail_call_in_subprogs(env)) 2260 prog->aux->stack_depth = MAX_BPF_STACK; 2261 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 2262 prog->aux->max_rdonly_access = U32_MAX; 2263 prog->aux->max_rdwr_access = U32_MAX; 2264 if (prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) 2265 prog->aux->max_tp_access = U32_MAX; 2266 2267 /* mark bpf_tail_call as different opcode to avoid 2268 * conditional branch in the interpreter for every normal 2269 * call and to prevent accidental JITing by JIT compiler 2270 * that doesn't support bpf_tail_call yet 2271 */ 2272 insn->imm = 0; 2273 insn->code = BPF_JMP | BPF_TAIL_CALL; 2274 2275 aux = &env->insn_aux_data[i + delta]; 2276 if (env->bpf_capable && !prog->blinding_requested && 2277 prog->jit_requested && 2278 !bpf_map_key_poisoned(aux) && 2279 !bpf_map_ptr_poisoned(aux) && 2280 !bpf_map_ptr_unpriv(aux)) { 2281 struct bpf_jit_poke_descriptor desc = { 2282 .reason = BPF_POKE_REASON_TAIL_CALL, 2283 .tail_call.map = aux->map_ptr_state.map_ptr, 2284 .tail_call.key = bpf_map_key_immediate(aux), 2285 .insn_idx = i + delta, 2286 }; 2287 2288 ret = bpf_jit_add_poke_descriptor(prog, &desc); 2289 if (ret < 0) { 2290 verbose(env, "adding tail call poke descriptor failed\n"); 2291 return ret; 2292 } 2293 2294 insn->imm = ret + 1; 2295 goto next_insn; 2296 } 2297 2298 if (!bpf_map_ptr_unpriv(aux)) 2299 goto next_insn; 2300 2301 /* instead of changing every JIT dealing with tail_call 2302 * emit two extra insns: 2303 * if (index >= max_entries) goto out; 2304 * index &= array->index_mask; 2305 * to avoid out-of-bounds cpu speculation 2306 */ 2307 if (bpf_map_ptr_poisoned(aux)) { 2308 verbose(env, "tail_call abusing map_ptr\n"); 2309 return -EINVAL; 2310 } 2311 2312 map_ptr = aux->map_ptr_state.map_ptr; 2313 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 2314 map_ptr->max_entries, 2); 2315 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 2316 container_of(map_ptr, 2317 struct bpf_array, 2318 map)->index_mask); 2319 insn_buf[2] = *insn; 2320 cnt = 3; 2321 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2322 if (!new_prog) 2323 return -ENOMEM; 2324 2325 delta += cnt - 1; 2326 env->prog = prog = new_prog; 2327 insn = new_prog->insnsi + i + delta; 2328 goto next_insn; 2329 } 2330 2331 aux = &env->insn_aux_data[i + delta]; 2332 if (aux->arg_prog) { 2333 /* The verifier will process callback_fn as many times as necessary 2334 * with different maps and the register states prepared by 2335 * set_timer_callback_state will be accurate. 2336 * 2337 * The following use case is valid: 2338 * map1 is shared by prog1, prog2, prog3. 2339 * prog1 calls bpf_timer_init for some map1 elements 2340 * prog2 calls bpf_timer_set_callback for some map1 elements. 2341 * Those that were not bpf_timer_init-ed will return -EINVAL. 2342 * prog3 calls bpf_timer_start for some map1 elements. 2343 * Those that were not both bpf_timer_init-ed and 2344 * bpf_timer_set_callback-ed will return -EINVAL. 2345 */ 2346 struct bpf_insn ld_addrs[2] = { 2347 BPF_LD_IMM64(aux->arg_prog, (long)prog->aux), 2348 }; 2349 2350 insn_buf[0] = ld_addrs[0]; 2351 insn_buf[1] = ld_addrs[1]; 2352 insn_buf[2] = *insn; 2353 cnt = 3; 2354 2355 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2356 if (!new_prog) 2357 return -ENOMEM; 2358 2359 delta += cnt - 1; 2360 env->prog = prog = new_prog; 2361 insn = new_prog->insnsi + i + delta; 2362 goto patch_call_imm; 2363 } 2364 2365 /* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */ 2366 if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) { 2367 /* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data, 2368 * bpf_mem_alloc() returns a ptr to the percpu data ptr. 2369 */ 2370 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 2371 insn_buf[1] = *insn; 2372 cnt = 2; 2373 2374 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2375 if (!new_prog) 2376 return -ENOMEM; 2377 2378 delta += cnt - 1; 2379 env->prog = prog = new_prog; 2380 insn = new_prog->insnsi + i + delta; 2381 goto patch_call_imm; 2382 } 2383 2384 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 2385 * and other inlining handlers are currently limited to 64 bit 2386 * only. 2387 */ 2388 if (prog->jit_requested && BITS_PER_LONG == 64 && 2389 (insn->imm == BPF_FUNC_map_lookup_elem || 2390 insn->imm == BPF_FUNC_map_update_elem || 2391 insn->imm == BPF_FUNC_map_delete_elem || 2392 insn->imm == BPF_FUNC_map_push_elem || 2393 insn->imm == BPF_FUNC_map_pop_elem || 2394 insn->imm == BPF_FUNC_map_peek_elem || 2395 insn->imm == BPF_FUNC_redirect_map || 2396 insn->imm == BPF_FUNC_for_each_map_elem || 2397 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { 2398 aux = &env->insn_aux_data[i + delta]; 2399 if (bpf_map_ptr_poisoned(aux)) 2400 goto patch_call_imm; 2401 2402 map_ptr = aux->map_ptr_state.map_ptr; 2403 ops = map_ptr->ops; 2404 if (insn->imm == BPF_FUNC_map_lookup_elem && 2405 ops->map_gen_lookup) { 2406 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 2407 if (cnt == -EOPNOTSUPP) 2408 goto patch_map_ops_generic; 2409 if (cnt <= 0 || cnt >= INSN_BUF_SIZE) { 2410 verifier_bug(env, "%d insns generated for map lookup", cnt); 2411 return -EFAULT; 2412 } 2413 2414 if (bpf_map_is_percpu_map(map_ptr->map_type)) 2415 prog->jit_required = true; 2416 2417 new_prog = bpf_patch_insn_data(env, i + delta, 2418 insn_buf, cnt); 2419 if (!new_prog) 2420 return -ENOMEM; 2421 2422 delta += cnt - 1; 2423 env->prog = prog = new_prog; 2424 insn = new_prog->insnsi + i + delta; 2425 goto next_insn; 2426 } 2427 2428 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 2429 (void *(*)(struct bpf_map *map, void *key))NULL)); 2430 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 2431 (long (*)(struct bpf_map *map, void *key))NULL)); 2432 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 2433 (long (*)(struct bpf_map *map, void *key, void *value, 2434 u64 flags))NULL)); 2435 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 2436 (long (*)(struct bpf_map *map, void *value, 2437 u64 flags))NULL)); 2438 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 2439 (long (*)(struct bpf_map *map, void *value))NULL)); 2440 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 2441 (long (*)(struct bpf_map *map, void *value))NULL)); 2442 BUILD_BUG_ON(!__same_type(ops->map_redirect, 2443 (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); 2444 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 2445 (long (*)(struct bpf_map *map, 2446 bpf_callback_t callback_fn, 2447 void *callback_ctx, 2448 u64 flags))NULL)); 2449 BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, 2450 (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL)); 2451 2452 patch_map_ops_generic: 2453 switch (insn->imm) { 2454 case BPF_FUNC_map_lookup_elem: 2455 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 2456 goto next_insn; 2457 case BPF_FUNC_map_update_elem: 2458 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 2459 goto next_insn; 2460 case BPF_FUNC_map_delete_elem: 2461 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 2462 goto next_insn; 2463 case BPF_FUNC_map_push_elem: 2464 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 2465 goto next_insn; 2466 case BPF_FUNC_map_pop_elem: 2467 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 2468 goto next_insn; 2469 case BPF_FUNC_map_peek_elem: 2470 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 2471 goto next_insn; 2472 case BPF_FUNC_redirect_map: 2473 insn->imm = BPF_CALL_IMM(ops->map_redirect); 2474 goto next_insn; 2475 case BPF_FUNC_for_each_map_elem: 2476 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 2477 goto next_insn; 2478 case BPF_FUNC_map_lookup_percpu_elem: 2479 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); 2480 goto next_insn; 2481 } 2482 2483 goto patch_call_imm; 2484 } 2485 2486 /* Implement bpf_jiffies64 inline. */ 2487 if (prog->jit_requested && BITS_PER_LONG == 64 && 2488 insn->imm == BPF_FUNC_jiffies64) { 2489 struct bpf_insn ld_jiffies_addr[2] = { 2490 BPF_LD_IMM64(BPF_REG_0, 2491 (unsigned long)&jiffies), 2492 }; 2493 2494 insn_buf[0] = ld_jiffies_addr[0]; 2495 insn_buf[1] = ld_jiffies_addr[1]; 2496 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 2497 BPF_REG_0, 0); 2498 cnt = 3; 2499 2500 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 2501 cnt); 2502 if (!new_prog) 2503 return -ENOMEM; 2504 2505 delta += cnt - 1; 2506 env->prog = prog = new_prog; 2507 insn = new_prog->insnsi + i + delta; 2508 goto next_insn; 2509 } 2510 2511 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML) 2512 /* Implement bpf_get_smp_processor_id() inline. */ 2513 if (insn->imm == BPF_FUNC_get_smp_processor_id && 2514 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2515 /* BPF_FUNC_get_smp_processor_id inlining is an 2516 * optimization, so if cpu_number is ever 2517 * changed in some incompatible and hard to support 2518 * way, it's fine to back out this inlining logic 2519 */ 2520 #ifdef CONFIG_SMP 2521 prog->jit_required = true; 2522 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); 2523 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2524 insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); 2525 cnt = 3; 2526 #else 2527 insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); 2528 cnt = 1; 2529 #endif 2530 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2531 if (!new_prog) 2532 return -ENOMEM; 2533 2534 delta += cnt - 1; 2535 env->prog = prog = new_prog; 2536 insn = new_prog->insnsi + i + delta; 2537 goto next_insn; 2538 } 2539 2540 /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ 2541 if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && 2542 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2543 prog->jit_required = true; 2544 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); 2545 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2546 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); 2547 cnt = 3; 2548 2549 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2550 if (!new_prog) 2551 return -ENOMEM; 2552 2553 delta += cnt - 1; 2554 env->prog = prog = new_prog; 2555 insn = new_prog->insnsi + i + delta; 2556 goto next_insn; 2557 } 2558 #endif 2559 /* Implement bpf_get_func_arg inline. */ 2560 if (prog_type == BPF_PROG_TYPE_TRACING && 2561 insn->imm == BPF_FUNC_get_func_arg) { 2562 if (eatype == BPF_TRACE_RAW_TP) { 2563 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2564 2565 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2566 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2567 cnt = 1; 2568 } else { 2569 /* Load nr_args from ctx - 8 */ 2570 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2571 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2572 cnt = 2; 2573 } 2574 insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 2575 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_LSH, BPF_REG_2, 3); 2576 insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 2577 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 2578 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2579 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 0); 2580 insn_buf[cnt++] = BPF_JMP_A(1); 2581 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2582 2583 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2584 if (!new_prog) 2585 return -ENOMEM; 2586 2587 delta += cnt - 1; 2588 env->prog = prog = new_prog; 2589 insn = new_prog->insnsi + i + delta; 2590 goto next_insn; 2591 } 2592 2593 /* Implement bpf_get_func_ret inline. */ 2594 if (prog_type == BPF_PROG_TYPE_TRACING && 2595 insn->imm == BPF_FUNC_get_func_ret) { 2596 if (eatype == BPF_TRACE_FEXIT || 2597 eatype == BPF_TRACE_FSESSION || 2598 eatype == BPF_TRACE_FEXIT_MULTI || 2599 eatype == BPF_TRACE_FSESSION_MULTI || 2600 eatype == BPF_MODIFY_RETURN) { 2601 /* Load nr_args from ctx - 8 */ 2602 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2603 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2604 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 2605 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 2606 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2607 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 2608 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 2609 cnt = 7; 2610 } else { 2611 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 2612 cnt = 1; 2613 } 2614 2615 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2616 if (!new_prog) 2617 return -ENOMEM; 2618 2619 delta += cnt - 1; 2620 env->prog = prog = new_prog; 2621 insn = new_prog->insnsi + i + delta; 2622 goto next_insn; 2623 } 2624 2625 /* Implement get_func_arg_cnt inline. */ 2626 if (prog_type == BPF_PROG_TYPE_TRACING && 2627 insn->imm == BPF_FUNC_get_func_arg_cnt) { 2628 if (eatype == BPF_TRACE_RAW_TP) { 2629 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2630 2631 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2632 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2633 cnt = 1; 2634 } else { 2635 /* Load nr_args from ctx - 8 */ 2636 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2637 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2638 cnt = 2; 2639 } 2640 2641 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2642 if (!new_prog) 2643 return -ENOMEM; 2644 2645 delta += cnt - 1; 2646 env->prog = prog = new_prog; 2647 insn = new_prog->insnsi + i + delta; 2648 goto next_insn; 2649 } 2650 2651 /* Implement bpf_get_func_ip inline. */ 2652 if (prog_type == BPF_PROG_TYPE_TRACING && 2653 insn->imm == BPF_FUNC_get_func_ip) { 2654 /* Load IP address from ctx - 16 */ 2655 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 2656 2657 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 2658 if (!new_prog) 2659 return -ENOMEM; 2660 2661 env->prog = prog = new_prog; 2662 insn = new_prog->insnsi + i + delta; 2663 goto next_insn; 2664 } 2665 2666 /* Implement bpf_get_branch_snapshot inline. */ 2667 if (IS_ENABLED(CONFIG_PERF_EVENTS) && 2668 prog->jit_requested && BITS_PER_LONG == 64 && 2669 insn->imm == BPF_FUNC_get_branch_snapshot) { 2670 /* We are dealing with the following func protos: 2671 * u64 bpf_get_branch_snapshot(void *buf, u32 size, u64 flags); 2672 * int perf_snapshot_branch_stack(struct perf_branch_entry *entries, u32 cnt); 2673 */ 2674 const u32 br_entry_size = sizeof(struct perf_branch_entry); 2675 2676 /* struct perf_branch_entry is part of UAPI and is 2677 * used as an array element, so extremely unlikely to 2678 * ever grow or shrink 2679 */ 2680 BUILD_BUG_ON(br_entry_size != 24); 2681 2682 /* if (unlikely(flags)) return -EINVAL */ 2683 insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7); 2684 2685 /* Transform size (bytes) into number of entries (cnt = size / 24). 2686 * But to avoid expensive division instruction, we implement 2687 * divide-by-3 through multiplication, followed by further 2688 * division by 8 through 3-bit right shift. 2689 * Refer to book "Hacker's Delight, 2nd ed." by Henry S. Warren, Jr., 2690 * p. 227, chapter "Unsigned Division by 3" for details and proofs. 2691 * 2692 * N / 3 <=> M * N / 2^33, where M = (2^33 + 1) / 3 = 0xaaaaaaab. 2693 */ 2694 insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab); 2695 insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0); 2696 insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36); 2697 2698 /* call perf_snapshot_branch_stack implementation */ 2699 insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack)); 2700 /* if (entry_cnt == 0) return -ENOENT */ 2701 insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4); 2702 /* return entry_cnt * sizeof(struct perf_branch_entry) */ 2703 insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size); 2704 insn_buf[7] = BPF_JMP_A(3); 2705 /* return -EINVAL; */ 2706 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2707 insn_buf[9] = BPF_JMP_A(1); 2708 /* return -ENOENT; */ 2709 insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT); 2710 cnt = 11; 2711 2712 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2713 if (!new_prog) 2714 return -ENOMEM; 2715 2716 delta += cnt - 1; 2717 env->prog = prog = new_prog; 2718 insn = new_prog->insnsi + i + delta; 2719 goto next_insn; 2720 } 2721 2722 /* Implement bpf_kptr_xchg inline */ 2723 if (prog->jit_requested && BITS_PER_LONG == 64 && 2724 insn->imm == BPF_FUNC_kptr_xchg && 2725 bpf_jit_supports_ptr_xchg()) { 2726 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); 2727 insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); 2728 cnt = 2; 2729 2730 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2731 if (!new_prog) 2732 return -ENOMEM; 2733 2734 delta += cnt - 1; 2735 env->prog = prog = new_prog; 2736 insn = new_prog->insnsi + i + delta; 2737 goto next_insn; 2738 } 2739 patch_call_imm: 2740 fn = env->ops->get_func_proto(insn->imm, env->prog); 2741 /* all functions that have prototype and verifier allowed 2742 * programs to call them, must be real in-kernel functions 2743 */ 2744 if (!fn->func) { 2745 verifier_bug(env, 2746 "not inlined functions %s#%d is missing func", 2747 func_id_name(insn->imm), insn->imm); 2748 return -EFAULT; 2749 } 2750 insn->imm = BPF_CALL_IMM(fn->func); 2751 next_insn: 2752 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2753 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2754 subprogs[cur_subprog].stack_extra = stack_depth_extra; 2755 2756 stack_depth = subprogs[cur_subprog].stack_depth; 2757 if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) { 2758 verbose(env, "stack size %d(extra %d) is too large\n", 2759 stack_depth, stack_depth_extra); 2760 return -EINVAL; 2761 } 2762 cur_subprog++; 2763 stack_depth = subprogs[cur_subprog].stack_depth; 2764 stack_depth_extra = 0; 2765 } 2766 i++; 2767 insn++; 2768 } 2769 2770 env->prog->aux->stack_depth = subprogs[0].stack_depth; 2771 for (i = 0; i < env->subprog_cnt; i++) { 2772 int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1; 2773 int subprog_start = subprogs[i].start; 2774 int stack_slots = subprogs[i].stack_extra / 8; 2775 int slots = delta, cnt = 0; 2776 2777 if (!stack_slots) 2778 continue; 2779 /* We need two slots in case timed may_goto is supported. */ 2780 if (stack_slots > slots) { 2781 verifier_bug(env, "stack_slots supports may_goto only"); 2782 return -EFAULT; 2783 } 2784 2785 stack_depth = subprogs[i].stack_depth; 2786 if (bpf_jit_supports_timed_may_goto()) { 2787 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2788 BPF_MAX_TIMED_LOOPS); 2789 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0); 2790 } else { 2791 /* Add ST insn to subprog prologue to init extra stack */ 2792 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2793 BPF_MAX_LOOPS); 2794 } 2795 /* Copy first actual insn to preserve it */ 2796 insn_buf[cnt] = env->prog->insnsi[subprog_start]; 2797 ret = bpf_adj_moved_insn(&insn_buf[cnt], cnt); 2798 if (ret) { 2799 verbose(env, "insn %d cannot be patched due to 16-bit range\n", 2800 env->insn_aux_data[subprog_start].orig_idx); 2801 return ret; 2802 } 2803 cnt++; 2804 2805 new_prog = __bpf_patch_insn_data(env, subprog_start, insn_buf, cnt, 2806 BPF_PATCH_MOVE_TARGET); 2807 if (!new_prog) 2808 return -ENOMEM; 2809 env->prog = prog = new_prog; 2810 /* 2811 * If may_goto is a first insn of a prog there could be a jmp 2812 * insn that points to it, hence adjust all such jmps to point 2813 * to insn after BPF_ST that inits may_goto count. 2814 * Adjustment will succeed because bpf_patch_insn_data() didn't fail. 2815 */ 2816 WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta)); 2817 } 2818 2819 /* Since poke tab is now finalized, publish aux to tracker. */ 2820 for (i = 0; i < prog->aux->size_poke_tab; i++) { 2821 map_ptr = prog->aux->poke_tab[i].tail_call.map; 2822 if (!map_ptr->ops->map_poke_track || 2823 !map_ptr->ops->map_poke_untrack || 2824 !map_ptr->ops->map_poke_run) { 2825 verifier_bug(env, "poke tab is misconfigured"); 2826 return -EFAULT; 2827 } 2828 2829 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 2830 if (ret < 0) { 2831 verbose(env, "tracking tail call prog failed\n"); 2832 return ret; 2833 } 2834 } 2835 2836 return 0; 2837 } 2838 2839 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 2840 int position, 2841 s32 stack_base, 2842 u32 callback_subprogno, 2843 u32 *total_cnt) 2844 { 2845 s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; 2846 s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; 2847 s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; 2848 int reg_loop_max = BPF_REG_6; 2849 int reg_loop_cnt = BPF_REG_7; 2850 int reg_loop_ctx = BPF_REG_8; 2851 2852 struct bpf_insn *insn_buf = env->insn_buf; 2853 struct bpf_prog *new_prog; 2854 u32 callback_start; 2855 u32 call_insn_offset; 2856 s32 callback_offset; 2857 u32 cnt = 0; 2858 2859 /* This represents an inlined version of bpf_iter.c:bpf_loop, 2860 * be careful to modify this code in sync. 2861 */ 2862 2863 /* Return error and jump to the end of the patch if 2864 * expected number of iterations is too big. 2865 */ 2866 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2); 2867 insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG); 2868 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16); 2869 /* spill R6, R7, R8 to use these as loop vars */ 2870 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset); 2871 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset); 2872 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset); 2873 /* initialize loop vars */ 2874 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1); 2875 insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0); 2876 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3); 2877 /* loop header, 2878 * if reg_loop_cnt >= reg_loop_max skip the loop body 2879 */ 2880 insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5); 2881 /* callback call, 2882 * correct callback offset would be set after patching 2883 */ 2884 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt); 2885 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx); 2886 insn_buf[cnt++] = BPF_CALL_REL(0); 2887 /* increment loop counter */ 2888 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1); 2889 /* jump to loop header if callback returned 0 */ 2890 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6); 2891 /* return value of bpf_loop, 2892 * set R0 to the number of iterations 2893 */ 2894 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt); 2895 /* restore original values of R6, R7, R8 */ 2896 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset); 2897 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset); 2898 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset); 2899 2900 *total_cnt = cnt; 2901 new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); 2902 if (!new_prog) 2903 return new_prog; 2904 2905 /* callback start is known only after patching */ 2906 callback_start = env->subprog_info[callback_subprogno].start; 2907 /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ 2908 call_insn_offset = position + 12; 2909 callback_offset = callback_start - call_insn_offset - 1; 2910 new_prog->insnsi[call_insn_offset].imm = callback_offset; 2911 2912 return new_prog; 2913 } 2914 2915 static bool is_bpf_loop_call(struct bpf_insn *insn) 2916 { 2917 return insn->code == (BPF_JMP | BPF_CALL) && 2918 insn->src_reg == 0 && 2919 insn->imm == BPF_FUNC_loop; 2920 } 2921 2922 /* For all sub-programs in the program (including main) check 2923 * insn_aux_data to see if there are bpf_loop calls that require 2924 * inlining. If such calls are found the calls are replaced with a 2925 * sequence of instructions produced by `inline_bpf_loop` function and 2926 * subprog stack_depth is increased by the size of 3 registers. 2927 * This stack space is used to spill values of the R6, R7, R8. These 2928 * registers are used to store the loop bound, counter and context 2929 * variables. 2930 */ 2931 int bpf_optimize_bpf_loop(struct bpf_verifier_env *env) 2932 { 2933 struct bpf_subprog_info *subprogs = env->subprog_info; 2934 int i, cur_subprog = 0, cnt, delta = 0; 2935 struct bpf_insn *insn = env->prog->insnsi; 2936 int insn_cnt = env->prog->len; 2937 u16 stack_depth = subprogs[cur_subprog].stack_depth; 2938 u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2939 u16 stack_depth_extra = 0; 2940 2941 for (i = 0; i < insn_cnt; i++, insn++) { 2942 struct bpf_loop_inline_state *inline_state = 2943 &env->insn_aux_data[i + delta].loop_inline_state; 2944 2945 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { 2946 struct bpf_prog *new_prog; 2947 2948 stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; 2949 new_prog = inline_bpf_loop(env, 2950 i + delta, 2951 -(stack_depth + stack_depth_extra), 2952 inline_state->callback_subprogno, 2953 &cnt); 2954 if (!new_prog) 2955 return -ENOMEM; 2956 2957 delta += cnt - 1; 2958 env->prog = new_prog; 2959 insn = new_prog->insnsi + i + delta; 2960 } 2961 2962 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2963 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2964 cur_subprog++; 2965 stack_depth = subprogs[cur_subprog].stack_depth; 2966 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2967 stack_depth_extra = 0; 2968 } 2969 } 2970 2971 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 2972 2973 return 0; 2974 } 2975 2976 /* Remove unnecessary spill/fill pairs, members of fastcall pattern, 2977 * adjust subprograms stack depth when possible. 2978 */ 2979 int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) 2980 { 2981 struct bpf_subprog_info *subprog = env->subprog_info; 2982 struct bpf_insn_aux_data *aux = env->insn_aux_data; 2983 struct bpf_insn *insn = env->prog->insnsi; 2984 int insn_cnt = env->prog->len; 2985 u32 spills_num; 2986 bool modified = false; 2987 int i, j; 2988 2989 for (i = 0; i < insn_cnt; i++, insn++) { 2990 if (aux[i].fastcall_spills_num > 0) { 2991 spills_num = aux[i].fastcall_spills_num; 2992 /* NOPs would be removed by opt_remove_nops() */ 2993 for (j = 1; j <= spills_num; ++j) { 2994 *(insn - j) = NOP; 2995 *(insn + j) = NOP; 2996 } 2997 modified = true; 2998 } 2999 if ((subprog + 1)->start == i + 1) { 3000 if (modified && !subprog->keep_fastcall_stack) 3001 subprog->stack_depth = -subprog->fastcall_stack_off; 3002 subprog++; 3003 modified = false; 3004 } 3005 } 3006 3007 return 0; 3008 } 3009