1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/bpf.h> 4 #include <linux/btf.h> 5 #include <linux/bpf_verifier.h> 6 #include <linux/filter.h> 7 #include <linux/vmalloc.h> 8 #include <linux/perf_event.h> 9 #include <linux/sched/signal.h> 10 #include <net/xdp.h> 11 #include "disasm.h" 12 13 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) 14 15 /* 16 * Matches BPF_PROBE_ATOMIC too: bpf_convert_ctx_accesses() rewrites arena 17 * atomics before bpf_opt_subreg_zext_lo32_rnd_hi32() runs. 18 */ 19 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 20 { 21 return BPF_CLASS(insn->code) == BPF_STX && 22 (BPF_MODE(insn->code) == BPF_ATOMIC || 23 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && 24 insn->imm == BPF_CMPXCHG; 25 } 26 27 /* Returns true if 'insn' is an address space cast instruction translated as BPF_ALU op */ 28 static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *insn) 29 { 30 struct bpf_map *arena = (struct bpf_map *)prog->aux->arena; 31 32 if (insn->code != (BPF_ALU64 | BPF_MOV | BPF_X) || insn->off != BPF_ADDR_SPACE_CAST) 33 return false; 34 35 /* cast from as(1) to as(0) */ 36 if (insn->imm == 1) 37 return true; 38 39 /* cast from as(0) to as(1) */ 40 if (insn->imm == 1 << 16) 41 return arena && arena->map_flags & BPF_F_NO_USER_CONV; 42 43 /* non-BPF_F_NO_USER_CONV cast from as(0) to as(1) should be handled by JIT */ 44 return false; 45 } 46 47 /* 48 * The insn accesses arena through a number. JITs add the base of arena 49 * to the register as it is, so the access goes through the low 32 bits of 50 * the number in BPF_REG_AX. Registers of the program are not changed: the 51 * number may be compared or stored later, and on another path the register 52 * may be PTR_TO_ARENA. 53 * 54 * Constant blinding leaves insns that use BPF_REG_AX alone, so the immediate 55 * of st through a number is not blinded. 56 */ 57 static int arena_scalar_access(const struct bpf_insn *insn, struct bpf_insn *buf) 58 { 59 bool load = BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn); 60 struct bpf_insn *patch = buf; 61 62 *patch++ = BPF_MOV32_REG(BPF_REG_AX, load ? insn->src_reg : insn->dst_reg); 63 *patch = *insn; 64 if (load) 65 patch->src_reg = BPF_REG_AX; 66 else 67 patch->dst_reg = BPF_REG_AX; 68 patch++; 69 return patch - buf; 70 } 71 72 /* Return the regno defined by the insn, or -1. */ 73 static int insn_def_regno(const struct bpf_insn *insn) 74 { 75 switch (BPF_CLASS(insn->code)) { 76 case BPF_JMP: 77 case BPF_JMP32: 78 case BPF_ST: 79 return -1; 80 case BPF_STX: 81 return bpf_atomic_load_reg(insn); 82 default: 83 return insn->dst_reg; 84 } 85 } 86 87 /* 88 * For use only in combination with insn_def_regno() >= 0. 89 * Returns TRUE if the destination register operates on 64-bit, 90 * otherwise return FALSE. 91 */ 92 static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn) 93 { 94 u8 class = BPF_CLASS(insn->code); 95 u8 mode = BPF_MODE(insn->code); 96 u8 size = BPF_SIZE(insn->code); 97 u8 op = BPF_OP(insn->code); 98 bool mode_mem; 99 100 /* subregister endiness swap */ 101 if ((class == BPF_ALU || class == BPF_ALU64) && op == BPF_END && insn->imm != 64) 102 return false; 103 104 /* w0 += 1 */ 105 if (class == BPF_ALU && op != BPF_END) 106 return false; 107 108 /* address space casts converted to BPF_ALU, see bpf_do_misc_fixups() */ 109 if (is_addr_space_cast32(prog, insn)) 110 return false; 111 112 /* non 64-bit, non signed extended loads */ 113 mode_mem = mode == BPF_MEM || mode == BPF_PROBE_MEM || mode == BPF_PROBE_MEM32; 114 if (class == BPF_LDX && mode_mem && size != BPF_DW) 115 return false; 116 117 /* atomics, see insn_def_regno() */ 118 if (class == BPF_STX && size != BPF_DW) 119 return false; 120 121 /* both LD_IND and LD_ABS return 32-bit data. */ 122 if (class == BPF_LD && (mode == BPF_IND || mode == BPF_ABS)) 123 return false; 124 125 /* Conservatively return true at default. */ 126 return true; 127 } 128 129 /* 130 * Return the 32-bit subregister defined by INSN, or -1 if INSN does not 131 * explicitly define a 32-bit value. 132 */ 133 int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) 134 { 135 int dst_reg = insn_def_regno(insn); 136 137 if (dst_reg < 0 || bpf_is_reg64(prog, insn)) 138 return -1; 139 140 return dst_reg; 141 } 142 143 const struct btf_func_model * 144 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 145 const struct bpf_insn *insn) 146 { 147 const struct bpf_kfunc_desc *res; 148 struct bpf_kfunc_desc_tab *tab; 149 150 tab = prog->aux->kfunc_tab; 151 if (insn->off < 0 || insn->off >= tab->nr_descs) 152 return NULL; 153 154 res = &tab->descs[insn->off]; 155 if (bpf_jit_supports_far_kfunc_call()) { 156 if (res->func_id != insn->imm) 157 return NULL; 158 } else if ((s32)BPF_CALL_IMM(res->addr) != insn->imm) { 159 return NULL; 160 } 161 162 return &res->func_model; 163 } 164 165 static int add_kfunc_in_insns(struct bpf_verifier_env *env, 166 struct bpf_insn *insn, int cnt) 167 { 168 int i, ret; 169 170 for (i = 0; i < cnt; i++, insn++) { 171 if (bpf_pseudo_kfunc_call(insn)) { 172 ret = bpf_add_kfunc_call(env, insn->imm, insn->off); 173 if (ret < 0) 174 return ret; 175 } 176 } 177 return 0; 178 } 179 180 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 181 static int get_callee_stack_depth(struct bpf_verifier_env *env, 182 const struct bpf_insn *insn, int idx) 183 { 184 int start = idx + insn->imm + 1, subprog; 185 186 subprog = bpf_find_subprog(env, start); 187 if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start)) 188 return -EFAULT; 189 return env->subprog_info[subprog].stack_depth; 190 } 191 #endif 192 193 enum bpf_patch_mode { 194 BPF_PATCH_KEEP_TARGET, /* targets of 'off' keep naming 'off', the first new insn */ 195 BPF_PATCH_MOVE_TARGET, /* targets of 'off' follow the original insn to 'off + len - 1' */ 196 }; 197 198 /* single env->prog->insni[off] instruction was replaced with the range 199 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 200 * [0, off) and [off, end) to new locations, so the patched range stays zero 201 */ 202 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 203 struct bpf_prog *new_prog, u32 off, u32 cnt, 204 struct bpf_insn *original_insn, 205 enum bpf_patch_mode mode) 206 { 207 struct bpf_insn_aux_data *data = env->insn_aux_data; 208 struct bpf_insn *insn = new_prog->insnsi; 209 u32 old_seen = data[off].seen; 210 u32 prog_len; 211 int i; 212 213 /* aux info at OFF always needs adjustment, no matter fast path 214 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 215 * original insn at old prog. 216 */ 217 data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; 218 219 if (cnt == 1) { 220 /* 221 * A non-memory accessing insn could have been replaced by a 222 * memory accessing insn, systematically mark it for non-stack 223 * access 224 */ 225 data[off].non_stack_access = bpf_is_mem_insn(insn + off); 226 return; 227 } 228 prog_len = new_prog->len; 229 env->insn_aux_data_len = prog_len; 230 231 memmove(data + off + cnt - 1, data + off, 232 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 233 memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1)); 234 for (i = off; i < off + cnt - 1; i++) { 235 /* Expand insni[off]'s seen count to the patched range. */ 236 data[i].seen = old_seen; 237 data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; 238 /* 239 * In BPF_PATCH_MOVE_TARGET mode the original insn is the last 240 * slot by construction. 241 */ 242 if (mode == BPF_PATCH_KEEP_TARGET && 243 !memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { 244 data[i].non_stack_access = 245 data[off + cnt - 1].non_stack_access; 246 data[off + cnt - 1].non_stack_access = false; 247 } else if (bpf_is_mem_insn(insn + i)) { 248 data[i].non_stack_access = true; 249 } 250 } 251 252 /* 253 * Last slot instruction could be a newly generated 254 * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access 255 * if it is not the original instruction, otherwise keep the 256 * original marking 257 */ 258 if (bpf_is_mem_insn(insn + off + cnt - 1) && 259 memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn))) 260 data[off + cnt - 1].non_stack_access = true; 261 262 /* 263 * The indirect_target flag of the original instruction was moved to the last of the 264 * new instructions by the above memmove and memset, but the indirect jump target is 265 * actually the first instruction, so move it back. This also matches with the behavior 266 * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new 267 * instruction. For BPF_PATCH_MOVE_TARGET the original instruction is the last one, 268 * so the flag already sits where needed. 269 */ 270 if (mode == BPF_PATCH_KEEP_TARGET && data[off + cnt - 1].indirect_target) { 271 data[off].indirect_target = 1; 272 data[off + cnt - 1].indirect_target = 0; 273 } 274 } 275 276 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 277 { 278 int i; 279 280 if (len == 1) 281 return; 282 /* NOTE: fake 'exit' subprog should be updated as well. */ 283 for (i = 0; i <= env->subprog_cnt; i++) { 284 if (env->subprog_info[i].start <= off) 285 continue; 286 env->subprog_info[i].start += len - 1; 287 } 288 } 289 290 static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 first, u32 len) 291 { 292 int i; 293 294 if (len == 1) 295 return; 296 297 for (i = 0; i < env->insn_array_map_cnt; i++) 298 bpf_insn_array_adjust(env->insn_array_maps[i], first, len); 299 } 300 301 static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) 302 { 303 int i; 304 305 for (i = 0; i < env->insn_array_map_cnt; i++) 306 bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len); 307 } 308 309 static void adjust_poke_descs(struct bpf_prog *prog, u32 first, u32 len) 310 { 311 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 312 int i, sz = prog->aux->size_poke_tab; 313 struct bpf_jit_poke_descriptor *desc; 314 315 for (i = 0; i < sz; i++) { 316 desc = &tab[i]; 317 if (desc->insn_idx < first) 318 continue; 319 desc->insn_idx += len - 1; 320 } 321 } 322 323 /* 324 * Some post-verification instruction rewriting passes require an 325 * O(prog->len) operation per instruction. Keep their shared primitives 326 * killable and preemptible. 327 */ 328 static bool bpf_rewrite_must_abort(void) 329 { 330 if (fatal_signal_pending(current)) 331 return true; 332 cond_resched(); 333 return false; 334 } 335 336 static struct bpf_prog *__bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 337 const struct bpf_insn *patch, u32 len, 338 enum bpf_patch_mode mode) 339 { 340 struct bpf_prog *new_prog; 341 struct bpf_insn_aux_data *new_data = NULL; 342 /* The first slot whose targets move. */ 343 u32 first = mode == BPF_PATCH_MOVE_TARGET ? off : off + 1; 344 struct bpf_insn original_insn; 345 346 if (bpf_rewrite_must_abort()) 347 return NULL; 348 349 if (len > 1) { 350 new_data = vrealloc(env->insn_aux_data, 351 array_size(env->prog->len + len - 1, 352 sizeof(struct bpf_insn_aux_data)), 353 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 354 if (!new_data) 355 return NULL; 356 357 env->insn_aux_data = new_data; 358 } 359 360 memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn)); 361 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 362 if (IS_ERR(new_prog)) { 363 if (PTR_ERR(new_prog) == -ERANGE) 364 verbose(env, 365 "insn %d cannot be patched due to 16-bit range\n", 366 env->insn_aux_data[off].orig_idx); 367 return NULL; 368 } 369 adjust_insn_aux_data(env, new_prog, off, len, &original_insn, mode); 370 adjust_subprog_starts(env, off, len); 371 adjust_insn_arrays(env, first, len); 372 bpf_adjust_func_ptrs(env, off, len); 373 adjust_poke_descs(new_prog, first, len); 374 return new_prog; 375 } 376 377 struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 378 const struct bpf_insn *patch, u32 len) 379 { 380 return __bpf_patch_insn_data(env, off, patch, len, BPF_PATCH_KEEP_TARGET); 381 } 382 383 /* 384 * insn was moved down by delta insns inside its own patch. Operands relative 385 * to the pc that point in front of the old position did not move with it. 386 */ 387 static int bpf_adj_moved_insn(struct bpf_insn *insn, u32 delta) 388 { 389 u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code); 390 s64 off = insn->imm, off_min = S32_MIN; 391 bool is_imm = true; 392 393 if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) { 394 /* subprog that started at the old position starts with the patch */ 395 if (off >= 0) 396 return 0; 397 } else if ((class == BPF_JMP || class == BPF_JMP32) && 398 op != BPF_CALL && op != BPF_EXIT) { 399 if (insn->code != (BPF_JMP32 | BPF_JA)) { 400 off = insn->off; 401 off_min = S16_MIN; 402 is_imm = false; 403 } 404 /* jump to itself stays */ 405 if (off >= -1) 406 return 0; 407 } else { 408 return 0; 409 } 410 411 off -= delta; 412 if (off < off_min) 413 return -ERANGE; 414 if (is_imm) 415 insn->imm = off; 416 else 417 insn->off = off; 418 return 0; 419 } 420 421 /* 422 * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the 423 * jump offset by 'delta'. 424 */ 425 static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta) 426 { 427 struct bpf_insn *insn = prog->insnsi; 428 u32 insn_cnt = prog->len, i; 429 s32 imm; 430 s16 off; 431 432 for (i = 0; i < insn_cnt; i++, insn++) { 433 u8 code = insn->code; 434 435 if (tgt_idx <= i && i < tgt_idx + delta) 436 continue; 437 438 if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) || 439 BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT) 440 continue; 441 442 if (insn->code == (BPF_JMP32 | BPF_JA)) { 443 if (i + 1 + insn->imm != tgt_idx) 444 continue; 445 if (check_add_overflow(insn->imm, delta, &imm)) 446 return -ERANGE; 447 insn->imm = imm; 448 } else { 449 if (i + 1 + insn->off != tgt_idx) 450 continue; 451 if (check_add_overflow(insn->off, delta, &off)) 452 return -ERANGE; 453 insn->off = off; 454 } 455 } 456 return 0; 457 } 458 459 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 460 u32 off, u32 cnt) 461 { 462 int i, j; 463 464 /* find first prog starting at or after off (first to remove) */ 465 for (i = 0; i < env->subprog_cnt; i++) 466 if (env->subprog_info[i].start >= off) 467 break; 468 /* find first prog starting at or after off + cnt (first to stay) */ 469 for (j = i; j < env->subprog_cnt; j++) 470 if (env->subprog_info[j].start >= off + cnt) 471 break; 472 /* if j doesn't start exactly at off + cnt, we are just removing 473 * the front of previous prog 474 */ 475 if (env->subprog_info[j].start != off + cnt) 476 j--; 477 478 if (j > i) { 479 struct bpf_prog_aux *aux = env->prog->aux; 480 int move, k; 481 482 /* The removed subprogs own their jump tables. */ 483 for (k = i; k < j; k++) 484 kvfree(env->subprog_info[k].jt); 485 486 /* move fake 'exit' subprog as well */ 487 move = env->subprog_cnt + 1 - j; 488 489 memmove(env->subprog_info + i, 490 env->subprog_info + j, 491 sizeof(*env->subprog_info) * move); 492 /* The vacated tail still holds copies of the moved entries. */ 493 memset(env->subprog_info + i + move, 0, 494 sizeof(*env->subprog_info) * (j - i)); 495 env->subprog_cnt -= j - i; 496 497 /* remove func_info and its aux */ 498 if (aux->func_info) { 499 move = aux->func_info_cnt - j; 500 501 memmove(aux->func_info + i, 502 aux->func_info + j, 503 sizeof(*aux->func_info) * move); 504 if (aux->func_info_aux) 505 memmove(aux->func_info_aux + i, 506 aux->func_info_aux + j, 507 sizeof(*aux->func_info_aux) * move); 508 aux->func_info_cnt -= j - i; 509 /* func_info->insn_off is set after all code rewrites, 510 * in adjust_btf_func() - no need to adjust 511 */ 512 } 513 /* The exception callback is live, so it cannot sit inside the removed range. */ 514 if (env->exception_callback_subprog >= j) 515 env->exception_callback_subprog -= j - i; 516 } else { 517 /* convert i from "first prog to remove" to "first to adjust" */ 518 if (env->subprog_info[i].start == off) 519 i++; 520 } 521 522 /* update fake 'exit' subprog as well */ 523 for (; i <= env->subprog_cnt; i++) 524 env->subprog_info[i].start -= cnt; 525 526 return 0; 527 } 528 529 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 530 u32 cnt) 531 { 532 struct bpf_prog *prog = env->prog; 533 u32 i, l_off, l_cnt, nr_linfo; 534 struct bpf_line_info *linfo; 535 536 nr_linfo = prog->aux->nr_linfo; 537 if (!nr_linfo) 538 return 0; 539 540 linfo = prog->aux->linfo; 541 542 /* find first line info to remove, count lines to be removed */ 543 for (i = 0; i < nr_linfo; i++) 544 if (linfo[i].insn_off >= off) 545 break; 546 547 l_off = i; 548 l_cnt = 0; 549 for (; i < nr_linfo; i++) 550 if (linfo[i].insn_off < off + cnt) 551 l_cnt++; 552 else 553 break; 554 555 /* First live insn doesn't match first live linfo, it needs to "inherit" 556 * last removed linfo. prog is already modified, so prog->len == off 557 * means no live instructions after (tail of the program was removed). 558 */ 559 if (prog->len != off && l_cnt && 560 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 561 l_cnt--; 562 linfo[--i].insn_off = off + cnt; 563 } 564 565 /* remove the line info which refer to the removed instructions */ 566 if (l_cnt) { 567 memmove(linfo + l_off, linfo + i, 568 sizeof(*linfo) * (nr_linfo - i)); 569 570 prog->aux->nr_linfo -= l_cnt; 571 nr_linfo = prog->aux->nr_linfo; 572 } 573 574 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 575 for (i = l_off; i < nr_linfo; i++) 576 linfo[i].insn_off -= cnt; 577 578 /* fix up all subprogs (incl. 'exit') which start >= off */ 579 for (i = 0; i <= env->subprog_cnt; i++) 580 if (env->subprog_info[i].linfo_idx > l_off) { 581 /* program may have started in the removed region but 582 * may not be fully removed 583 */ 584 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 585 env->subprog_info[i].linfo_idx -= l_cnt; 586 else 587 env->subprog_info[i].linfo_idx = l_off; 588 } 589 590 return 0; 591 } 592 593 /* Clean up dynamically allocated fields of aux data for instructions [start, ...] */ 594 void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) 595 { 596 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 597 int end = start + len; 598 int i; 599 600 for (i = start; i < end; i++) { 601 if (aux_data[i].loop) { 602 kvfree(aux_data[i].loop->exits); 603 kvfree(aux_data[i].loop); 604 aux_data[i].loop = NULL; 605 } 606 } 607 } 608 609 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 610 { 611 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 612 unsigned int orig_prog_len = env->prog->len; 613 int err; 614 615 if (bpf_rewrite_must_abort()) 616 return -EINTR; 617 618 if (bpf_prog_is_offloaded(env->prog->aux)) 619 bpf_prog_offload_remove_insns(env, off, cnt); 620 621 bpf_clear_insn_aux_data(env, off, cnt); 622 623 err = bpf_remove_insns(env->prog, off, cnt); 624 if (err) 625 return err; 626 627 /* before subprogs are adjusted, since it looks at them */ 628 bpf_adjust_func_ptrs_after_remove(env, off, cnt); 629 630 err = adjust_subprog_starts_after_remove(env, off, cnt); 631 if (err) 632 return err; 633 634 err = bpf_adj_linfo_after_remove(env, off, cnt); 635 if (err) 636 return err; 637 638 adjust_insn_arrays_after_remove(env, off, cnt); 639 640 memmove(aux_data + off, aux_data + off + cnt, 641 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 642 env->insn_aux_data_len -= cnt; 643 644 return 0; 645 } 646 647 static const struct bpf_insn NOP = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 648 static const struct bpf_insn MAY_GOTO_0 = BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0, 0); 649 650 bool bpf_insn_is_cond_jump(u8 code) 651 { 652 u8 op; 653 654 op = BPF_OP(code); 655 if (BPF_CLASS(code) == BPF_JMP32) 656 return op != BPF_JA; 657 658 if (BPF_CLASS(code) != BPF_JMP) 659 return false; 660 661 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 662 } 663 664 void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 665 { 666 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 667 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 668 struct bpf_insn *insn = env->prog->insnsi; 669 const int insn_cnt = env->prog->len; 670 int i; 671 672 for (i = 0; i < insn_cnt; i++, insn++) { 673 if (!bpf_insn_is_cond_jump(insn->code)) 674 continue; 675 676 if (!aux_data[i + 1].seen) 677 ja.off = insn->off; 678 else if (!aux_data[i + 1 + insn->off].seen) 679 ja.off = 0; 680 else 681 continue; 682 683 if (bpf_prog_is_offloaded(env->prog->aux)) 684 bpf_prog_offload_replace_insn(env, i, &ja); 685 686 memcpy(insn, &ja, sizeof(ja)); 687 } 688 } 689 690 /* 691 * The address of a function might be taken by the code that is not dead, 692 * while the function itself is dead code, since nothing calls it. Keep the first 693 * instruction of such function, so that the address remains valid and differs 694 * from the addresses of other functions. Make it a trap like 695 * sanitize_dead_code() does. 696 */ 697 static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env) 698 { 699 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 700 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 701 struct bpf_insn *insn = env->prog->insnsi; 702 int insn_cnt = env->prog->len; 703 int i, t; 704 705 for (i = 0; i < insn_cnt; i++) { 706 if (!aux_data[i].seen || !bpf_pseudo_func(insn + i)) 707 continue; 708 t = i + insn[i].imm + 1; 709 if (aux_data[t].seen) 710 continue; 711 memcpy(insn + t, &trap, sizeof(trap)); 712 aux_data[t].zext_dst = false; 713 aux_data[t].seen = env->pass_cnt; 714 } 715 } 716 717 int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) 718 { 719 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 720 int insn_cnt = env->prog->len; 721 int i, err; 722 723 keep_funcs_with_addr_taken(env); 724 725 for (i = 0; i < insn_cnt; i++) { 726 int j; 727 728 j = 0; 729 while (i + j < insn_cnt && !aux_data[i + j].seen) 730 j++; 731 if (!j) 732 continue; 733 734 err = verifier_remove_insns(env, i, j); 735 if (err) 736 return err; 737 insn_cnt = env->prog->len; 738 } 739 740 return 0; 741 } 742 743 int bpf_opt_remove_nops(struct bpf_verifier_env *env) 744 { 745 struct bpf_insn_aux_data *aux = env->insn_aux_data; 746 struct bpf_insn *insn = env->prog->insnsi; 747 int insn_cnt = env->prog->len; 748 bool is_may_goto_0, is_ja; 749 int i, err; 750 751 for (i = 0; i < insn_cnt; i++) { 752 is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0)); 753 is_ja = !memcmp(&insn[i], &NOP, sizeof(NOP)); 754 755 if (!is_may_goto_0 && !is_ja) 756 continue; 757 if (aux[i].indirect_target) 758 continue; 759 760 err = verifier_remove_insns(env, i, 1); 761 if (err) 762 return err; 763 insn_cnt--; 764 /* Go back one insn to catch may_goto +1; may_goto +0 sequence */ 765 i -= (is_may_goto_0 && i > 0) ? 2 : 1; 766 } 767 768 return 0; 769 } 770 771 int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 772 const union bpf_attr *attr) 773 { 774 struct bpf_insn *patch; 775 /* use env->insn_buf as two independent buffers */ 776 struct bpf_insn *zext_patch = env->insn_buf; 777 struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2]; 778 struct bpf_insn_aux_data *aux = env->insn_aux_data; 779 int i, patch_len, delta = 0, len = env->prog->len; 780 struct bpf_insn *insns = env->prog->insnsi; 781 struct bpf_prog *new_prog; 782 bool rnd_hi32; 783 784 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 785 zext_patch[1] = BPF_ZEXT_REG(0); 786 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 787 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 788 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 789 for (i = 0; i < len; i++) { 790 int adj_idx = i + delta; 791 struct bpf_insn insn; 792 int load_reg; 793 794 insn = insns[adj_idx]; 795 load_reg = insn_def_regno(&insn); 796 if (!aux[adj_idx].zext_dst) { 797 u8 code, class; 798 u32 imm_rnd; 799 800 if (!rnd_hi32) 801 continue; 802 803 code = insn.code; 804 class = BPF_CLASS(code); 805 if (load_reg == -1) 806 continue; 807 808 if (bpf_is_reg64(env->prog, &insn)) { 809 if (class == BPF_LD && 810 BPF_MODE(code) == BPF_IMM) 811 i++; 812 continue; 813 } 814 815 /* ctx load could be transformed into wider load. */ 816 if (class == BPF_LDX && 817 aux[adj_idx].ptr_type == PTR_TO_CTX) 818 continue; 819 820 imm_rnd = get_random_u32(); 821 rnd_hi32_patch[0] = insn; 822 rnd_hi32_patch[1].imm = imm_rnd; 823 rnd_hi32_patch[3].dst_reg = load_reg; 824 patch = rnd_hi32_patch; 825 patch_len = 4; 826 goto apply_patch_buffer; 827 } 828 829 /* Add in an zero-extend instruction if a) the JIT has requested 830 * it or b) it's a CMPXCHG. 831 * 832 * The latter is because: BPF_CMPXCHG always loads a value into 833 * R0, therefore always zero-extends. However some archs' 834 * equivalent instruction only does this load when the 835 * comparison is successful. This detail of CMPXCHG is 836 * orthogonal to the general zero-extension behaviour of the 837 * CPU, so it's treated independently of bpf_jit_needs_zext. 838 */ 839 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 840 continue; 841 842 /* Zero-extension is done by the caller. */ 843 if (bpf_pseudo_kfunc_call(&insn)) 844 continue; 845 846 if (verifier_bug_if(load_reg == -1, env, 847 "zext_dst is set, but no reg is defined")) 848 return -EFAULT; 849 850 zext_patch[0] = insn; 851 zext_patch[1].dst_reg = load_reg; 852 zext_patch[1].src_reg = load_reg; 853 patch = zext_patch; 854 patch_len = 2; 855 apply_patch_buffer: 856 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 857 if (!new_prog) 858 return -ENOMEM; 859 env->prog = new_prog; 860 insns = new_prog->insnsi; 861 aux = env->insn_aux_data; 862 delta += patch_len - 1; 863 } 864 865 return 0; 866 } 867 868 /* convert load instructions that access fields of a context type into a 869 * sequence of instructions that access fields of the underlying structure: 870 * struct __sk_buff -> struct sk_buff 871 * struct bpf_sock_ops -> struct sock 872 */ 873 int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) 874 { 875 struct bpf_subprog_info *subprogs = env->subprog_info; 876 const struct bpf_verifier_ops *ops = env->ops; 877 int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0; 878 const int insn_cnt = env->prog->len; 879 struct bpf_insn *epilogue_buf = env->epilogue_buf; 880 struct bpf_insn *insn_buf = env->insn_buf; 881 struct bpf_insn *insn; 882 u32 target_size, size_default, off; 883 struct bpf_prog *new_prog; 884 enum bpf_access_type type; 885 bool is_narrower_load; 886 int epilogue_idx = 0; 887 888 if (ops->gen_epilogue) { 889 epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog, 890 -(subprogs[0].stack_depth + 8)); 891 if (epilogue_cnt >= INSN_BUF_SIZE) { 892 verifier_bug(env, "epilogue is too long"); 893 return -EFAULT; 894 } else if (epilogue_cnt) { 895 /* Save the ARG_PTR_TO_CTX for the epilogue to use */ 896 cnt = 0; 897 subprogs[0].stack_depth += 8; 898 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, 899 -subprogs[0].stack_depth); 900 insn_buf[cnt++] = env->prog->insnsi[0]; 901 new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, 902 BPF_PATCH_MOVE_TARGET); 903 if (!new_prog) 904 return -ENOMEM; 905 env->prog = new_prog; 906 delta += cnt - 1; 907 908 ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1); 909 if (ret < 0) 910 return ret; 911 } 912 } 913 914 if (ops->gen_prologue || env->seen_direct_write) { 915 if (!ops->gen_prologue) { 916 verifier_bug(env, "gen_prologue is null"); 917 return -EFAULT; 918 } 919 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 920 env->prog); 921 if (cnt >= INSN_BUF_SIZE) { 922 verifier_bug(env, "prologue is too long"); 923 return -EFAULT; 924 } else if (cnt) { 925 new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, 926 BPF_PATCH_MOVE_TARGET); 927 if (!new_prog) 928 return -ENOMEM; 929 930 env->prog = new_prog; 931 delta += cnt - 1; 932 933 ret = add_kfunc_in_insns(env, insn_buf, cnt - 1); 934 if (ret < 0) 935 return ret; 936 } 937 } 938 939 if (delta) 940 WARN_ON(adjust_jmp_off(env->prog, 0, delta)); 941 942 if (bpf_prog_is_offloaded(env->prog->aux)) 943 return 0; 944 945 insn = env->prog->insnsi + delta; 946 947 for (i = 0; i < insn_cnt; i++, insn++) { 948 bpf_convert_ctx_access_t convert_ctx_access; 949 enum bpf_reg_type ptr_type; 950 u8 mode; 951 952 if (env->insn_aux_data[i + delta].nospec) { 953 WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state); 954 struct bpf_insn *patch = insn_buf; 955 956 *patch++ = BPF_ST_NOSPEC(); 957 *patch = *insn; 958 ret = bpf_adj_moved_insn(patch++, 1); 959 if (ret) { 960 verbose(env, "insn %d cannot be patched due to 16-bit range\n", 961 env->insn_aux_data[i + delta].orig_idx); 962 return ret; 963 } 964 cnt = patch - insn_buf; 965 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 966 if (!new_prog) 967 return -ENOMEM; 968 969 delta += cnt - 1; 970 env->prog = new_prog; 971 insn = new_prog->insnsi + i + delta; 972 /* This can not be easily merged with the 973 * nospec_result-case, because an insn may require a 974 * nospec before and after itself. Therefore also do not 975 * 'continue' here but potentially apply further 976 * patching to insn. *insn should equal patch[1] now. 977 */ 978 } 979 980 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 981 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 982 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 983 insn->code == (BPF_LDX | BPF_MEM | BPF_DW) || 984 insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) || 985 insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) || 986 insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) { 987 type = BPF_READ; 988 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 989 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 990 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 991 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 992 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 993 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 994 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 995 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 996 type = BPF_WRITE; 997 } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) || 998 insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) || 999 insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) || 1000 insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) && 1001 env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) { 1002 insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); 1003 env->prog->aux->num_exentries++; 1004 continue; 1005 } else if (insn->code == (BPF_JMP | BPF_EXIT) && 1006 epilogue_cnt && 1007 i + delta < subprogs[1].start) { 1008 /* Generate epilogue for the main prog */ 1009 if (epilogue_idx) { 1010 /* jump back to the earlier generated epilogue */ 1011 insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1); 1012 cnt = 1; 1013 } else { 1014 memcpy(insn_buf, epilogue_buf, 1015 epilogue_cnt * sizeof(*epilogue_buf)); 1016 cnt = epilogue_cnt; 1017 /* epilogue_idx cannot be 0. It must have at 1018 * least one ctx ptr saving insn before the 1019 * epilogue. 1020 */ 1021 epilogue_idx = i + delta; 1022 } 1023 goto patch_insn_buf; 1024 } else { 1025 continue; 1026 } 1027 1028 if (type == BPF_WRITE && 1029 env->insn_aux_data[i + delta].nospec_result) { 1030 /* nospec_result is only used to mitigate Spectre v4 and 1031 * to limit verification-time for Spectre v1. 1032 */ 1033 struct bpf_insn *patch = insn_buf; 1034 1035 *patch++ = *insn; 1036 *patch++ = BPF_ST_NOSPEC(); 1037 cnt = patch - insn_buf; 1038 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1039 if (!new_prog) 1040 return -ENOMEM; 1041 1042 delta += cnt - 1; 1043 env->prog = new_prog; 1044 insn = new_prog->insnsi + i + delta; 1045 continue; 1046 } 1047 1048 ptr_type = env->insn_aux_data[i + delta].ptr_type; 1049 switch ((int)ptr_type) { 1050 case PTR_TO_CTX: 1051 if (!ops->convert_ctx_access) 1052 continue; 1053 convert_ctx_access = ops->convert_ctx_access; 1054 break; 1055 case PTR_TO_SOCKET: 1056 case PTR_TO_SOCK_COMMON: 1057 convert_ctx_access = bpf_sock_convert_ctx_access; 1058 break; 1059 case PTR_TO_TCP_SOCK: 1060 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 1061 break; 1062 case PTR_TO_XDP_SOCK: 1063 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 1064 break; 1065 case PTR_TO_ARENA: 1066 if (BPF_MODE(insn->code) == BPF_MEMSX) { 1067 if (!bpf_jit_supports_insn(insn, true)) { 1068 verbose(env, "sign extending loads from arena are not supported yet\n"); 1069 return -EOPNOTSUPP; 1070 } 1071 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code); 1072 } else { 1073 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code); 1074 } 1075 env->prog->aux->num_exentries++; 1076 continue; 1077 default: 1078 /* 1079 * A pointer which may fault on a dereference must not 1080 * be loaded from without fault protection, hence turn 1081 * the BPF_LDX into a BPF_PROBE_MEM one so that a bad 1082 * address is handled rather than panicking the kernel. 1083 * A store through one is rejected earlier, there is no 1084 * probed counterpart to rewrite it into. 1085 */ 1086 if (bpf_is_ptr_to_mem_or_btf_id(ptr_type) && 1087 bpf_may_fault_on_deref(ptr_type) && 1088 type == BPF_READ) { 1089 if (BPF_MODE(insn->code) == BPF_MEM) 1090 insn->code = BPF_LDX | BPF_PROBE_MEM | 1091 BPF_SIZE(insn->code); 1092 else 1093 insn->code = BPF_LDX | BPF_PROBE_MEMSX | 1094 BPF_SIZE(insn->code); 1095 env->prog->aux->num_exentries++; 1096 continue; 1097 } 1098 if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, 1099 "access to a fault prone pointer is not rewritten as a probed one")) 1100 return -EFAULT; 1101 continue; 1102 } 1103 1104 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 1105 size = BPF_LDST_BYTES(insn); 1106 mode = BPF_MODE(insn->code); 1107 1108 /* If the read access is a narrower load of the field, 1109 * convert to a 4/8-byte load, to minimum program type specific 1110 * convert_ctx_access changes. If conversion is successful, 1111 * we will apply proper mask to the result. 1112 */ 1113 is_narrower_load = size < ctx_field_size; 1114 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 1115 off = insn->off; 1116 if (is_narrower_load) { 1117 u8 size_code; 1118 1119 if (type == BPF_WRITE) { 1120 verifier_bug(env, "narrow ctx access misconfigured"); 1121 return -EFAULT; 1122 } 1123 1124 size_code = BPF_H; 1125 if (ctx_field_size == 4) 1126 size_code = BPF_W; 1127 else if (ctx_field_size == 8) 1128 size_code = BPF_DW; 1129 1130 insn->off = off & ~(size_default - 1); 1131 insn->code = BPF_LDX | BPF_MEM | size_code; 1132 } 1133 1134 target_size = 0; 1135 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 1136 &target_size); 1137 if (cnt == 0 || cnt >= INSN_BUF_SIZE || 1138 (ctx_field_size && !target_size)) { 1139 verifier_bug(env, "error during ctx access conversion (%d)", cnt); 1140 return -EFAULT; 1141 } 1142 1143 if (is_narrower_load && size < target_size) { 1144 u8 shift = bpf_ctx_narrow_access_offset( 1145 off, size, size_default) * 8; 1146 if (shift && cnt + 1 >= INSN_BUF_SIZE) { 1147 verifier_bug(env, "narrow ctx load misconfigured"); 1148 return -EFAULT; 1149 } 1150 if (ctx_field_size <= 4) { 1151 if (shift) 1152 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 1153 insn->dst_reg, 1154 shift); 1155 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1156 (1 << size * 8) - 1); 1157 } else { 1158 if (shift) 1159 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 1160 insn->dst_reg, 1161 shift); 1162 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1163 (1ULL << size * 8) - 1); 1164 } 1165 } 1166 if (mode == BPF_MEMSX) 1167 insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, 1168 insn->dst_reg, insn->dst_reg, 1169 size * 8, 0); 1170 1171 patch_insn_buf: 1172 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1173 if (!new_prog) 1174 return -ENOMEM; 1175 1176 delta += cnt - 1; 1177 1178 /* keep walking new program and skip insns we just inserted */ 1179 env->prog = new_prog; 1180 insn = new_prog->insnsi + i + delta; 1181 } 1182 1183 return 0; 1184 } 1185 1186 static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env) 1187 { 1188 u32 *starts = NULL; 1189 1190 starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT); 1191 if (starts) { 1192 for (int i = 0; i < env->subprog_cnt; i++) 1193 starts[i] = env->subprog_info[i].start; 1194 } 1195 return starts; 1196 } 1197 1198 static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts) 1199 { 1200 for (int i = 0; i < env->subprog_cnt; i++) 1201 env->subprog_info[i].start = orig_starts[i]; 1202 /* restore the start of fake 'exit' subprog as well */ 1203 env->subprog_info[env->subprog_cnt].start = env->prog->len; 1204 } 1205 1206 /* 1207 * Replace the offsets of functions with their addresses in the maps of 1208 * the program, see resolve_func_ptrs() in verifier.c. The program must be 1209 * the only user of such map. From now on no other program can use it, see 1210 * bpf_map_claim(). 1211 */ 1212 static int resolve_func_ptrs(struct bpf_verifier_env *env, struct bpf_prog *prog, 1213 struct bpf_prog **func) 1214 { 1215 unsigned long me = (unsigned long)prog->aux; 1216 struct bpf_func_ptr *ptr; 1217 int i, err, subprog; 1218 u64 addr, new, *slot; 1219 1220 for (i = 0; i < env->func_ptr_cnt; i++) { 1221 ptr = &env->func_ptrs[i]; 1222 1223 /* pointers are sorted by map */ 1224 if ((!i || ptr->map != ptr[-1].map) && 1225 cmpxchg(&ptr->map->user, me, me | BPF_MAP_USER_PATCHED) != me) { 1226 verbose(env, "map '%s' is used by another program\n", ptr->map->name); 1227 return -EBUSY; 1228 } 1229 1230 /* it's the address of the value of the map whatever the offset is */ 1231 err = ptr->map->ops->map_direct_value_addr(ptr->map, &addr, 0); 1232 if (verifier_bug_if(err, env, "no value of map '%s'", ptr->map->name)) 1233 return -EFAULT; 1234 slot = (u64 *)(unsigned long)(addr + ptr->map_off); 1235 1236 /* 1237 * The function that is dead code is removed and the pointer is 1238 * NULL. The program may read it, but callx of it is dead code 1239 * too, see check_func_callx(). 1240 */ 1241 new = 0; 1242 if (ptr->xlated_off != BPF_FUNC_PTR_DELETED) { 1243 subprog = bpf_find_subprog(env, ptr->xlated_off); 1244 if (verifier_bug_if(!func || subprog <= 0, env, "no function at insn %u", 1245 ptr->xlated_off)) 1246 return -EFAULT; 1247 new = (unsigned long)func[subprog]->bpf_func; 1248 } 1249 1250 /* the map is frozen and nothing else uses it */ 1251 if (verifier_bug_if(*slot != (u64)ptr->orig_off * sizeof(struct bpf_insn), env, 1252 "map '%s' offset %u changed", ptr->map->name, ptr->map_off)) 1253 return -EFAULT; 1254 WRITE_ONCE(*slot, new); 1255 } 1256 return 0; 1257 } 1258 1259 static int jit_subprogs(struct bpf_verifier_env *env) 1260 { 1261 struct bpf_prog *prog = env->prog, **func, *tmp; 1262 int i, j, subprog_start, subprog_end = 0, len, subprog; 1263 struct bpf_map *map_ptr; 1264 struct bpf_insn *insn; 1265 void *old_bpf_func; 1266 int err, num_exentries; 1267 1268 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1269 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 1270 continue; 1271 1272 /* Upon error here we cannot fall back to interpreter but 1273 * need a hard reject of the program. Thus -EFAULT is 1274 * propagated in any case. 1275 */ 1276 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1277 if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d", 1278 i + insn->imm + 1)) 1279 return -EFAULT; 1280 /* temporarily remember subprog id inside insn instead of 1281 * aux_data, since next loop will split up all insns into funcs 1282 */ 1283 insn->off = subprog; 1284 /* remember original imm in case JIT fails and fallback 1285 * to interpreter will be needed 1286 */ 1287 env->insn_aux_data[i].call_imm = insn->imm; 1288 /* point imm to __bpf_call_base+1 from JITs point of view */ 1289 insn->imm = 1; 1290 if (bpf_pseudo_func(insn)) { 1291 #if defined(MODULES_VADDR) 1292 u64 addr = MODULES_VADDR; 1293 #else 1294 u64 addr = VMALLOC_START; 1295 #endif 1296 /* jit (e.g. x86_64) may emit fewer instructions 1297 * if it learns a u32 imm is the same as a u64 imm. 1298 * Set close enough to possible prog address. 1299 */ 1300 insn[0].imm = (u32)addr; 1301 insn[1].imm = addr >> 32; 1302 } 1303 } 1304 1305 err = bpf_prog_alloc_jited_linfo(prog); 1306 if (err) 1307 goto out_undo_insn; 1308 1309 err = -ENOMEM; 1310 func = kzalloc_objs(prog, env->subprog_cnt); 1311 if (!func) 1312 goto out_undo_insn; 1313 1314 for (i = 0; i < env->subprog_cnt; i++) { 1315 subprog_start = subprog_end; 1316 subprog_end = env->subprog_info[i + 1].start; 1317 1318 len = subprog_end - subprog_start; 1319 /* bpf_prog_run() doesn't call subprogs directly, 1320 * hence main prog stats include the runtime of subprogs. 1321 * subprogs don't have IDs and not reachable via prog_get_next_id 1322 * func[i]->stats will never be accessed and stays NULL 1323 */ 1324 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 1325 if (!func[i]) 1326 goto out_free; 1327 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 1328 len * sizeof(struct bpf_insn)); 1329 func[i]->type = prog->type; 1330 func[i]->len = len; 1331 if (bpf_prog_calc_tag(func[i])) 1332 goto out_free; 1333 func[i]->is_func = 1; 1334 func[i]->sleepable = prog->sleepable; 1335 func[i]->blinded = prog->blinded; 1336 func[i]->aux->func_idx = i; 1337 /* Below members will be freed only at prog->aux */ 1338 func[i]->aux->btf = prog->aux->btf; 1339 func[i]->aux->subprog_start = subprog_start; 1340 func[i]->aux->func_info = prog->aux->func_info; 1341 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 1342 func[i]->aux->poke_tab = prog->aux->poke_tab; 1343 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 1344 func[i]->aux->main_prog_aux = prog->aux; 1345 1346 for (j = 0; j < prog->aux->size_poke_tab; j++) { 1347 struct bpf_jit_poke_descriptor *poke; 1348 1349 poke = &prog->aux->poke_tab[j]; 1350 if (poke->insn_idx < subprog_end && 1351 poke->insn_idx >= subprog_start) 1352 poke->aux = func[i]->aux; 1353 } 1354 1355 func[i]->aux->name[0] = 'F'; 1356 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 1357 if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) 1358 func[i]->aux->jits_use_priv_stack = true; 1359 1360 func[i]->jit_requested = 1; 1361 func[i]->blinding_requested = prog->blinding_requested; 1362 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 1363 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 1364 func[i]->aux->linfo = prog->aux->linfo; 1365 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 1366 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 1367 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 1368 func[i]->aux->arena = prog->aux->arena; 1369 func[i]->aux->used_maps = env->used_maps; 1370 func[i]->aux->used_map_cnt = env->used_map_cnt; 1371 num_exentries = 0; 1372 insn = func[i]->insnsi; 1373 for (j = 0; j < func[i]->len; j++, insn++) { 1374 if (BPF_CLASS(insn->code) == BPF_LDX && 1375 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 1376 BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 1377 BPF_MODE(insn->code) == BPF_PROBE_MEM32SX || 1378 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) 1379 num_exentries++; 1380 if ((BPF_CLASS(insn->code) == BPF_STX || 1381 BPF_CLASS(insn->code) == BPF_ST) && 1382 BPF_MODE(insn->code) == BPF_PROBE_MEM32) 1383 num_exentries++; 1384 if (BPF_CLASS(insn->code) == BPF_STX && 1385 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) 1386 num_exentries++; 1387 } 1388 func[i]->aux->num_exentries = num_exentries; 1389 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 1390 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; 1391 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; 1392 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; 1393 func[i]->aux->token = prog->aux->token; 1394 if (!i) 1395 func[i]->aux->exception_boundary = env->seen_exception; 1396 func[i] = bpf_int_jit_compile(env, func[i]); 1397 if (!func[i]->jited) { 1398 err = -ENOTSUPP; 1399 goto out_free; 1400 } 1401 cond_resched(); 1402 } 1403 1404 /* at this point all bpf functions were successfully JITed 1405 * now populate all bpf_calls with correct addresses and 1406 * run last pass of JIT 1407 */ 1408 for (i = 0; i < env->subprog_cnt; i++) { 1409 insn = func[i]->insnsi; 1410 for (j = 0; j < func[i]->len; j++, insn++) { 1411 if (bpf_pseudo_func(insn)) { 1412 subprog = insn->off; 1413 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 1414 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 1415 continue; 1416 } 1417 if (!bpf_pseudo_call(insn)) 1418 continue; 1419 subprog = insn->off; 1420 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 1421 } 1422 1423 /* we use the aux data to keep a list of the start addresses 1424 * of the JITed images for each function in the program 1425 * 1426 * for some architectures, such as powerpc64, the imm field 1427 * might not be large enough to hold the offset of the start 1428 * address of the callee's JITed image from __bpf_call_base 1429 * 1430 * in such cases, we can lookup the start address of a callee 1431 * by using its subprog id, available from the off field of 1432 * the call instruction, as an index for this list 1433 */ 1434 func[i]->aux->func = func; 1435 func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1436 func[i]->aux->real_func_cnt = env->subprog_cnt; 1437 } 1438 for (i = 0; i < env->subprog_cnt; i++) { 1439 old_bpf_func = func[i]->bpf_func; 1440 tmp = bpf_int_jit_compile(env, func[i]); 1441 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 1442 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 1443 err = -ENOTSUPP; 1444 goto out_free; 1445 } 1446 cond_resched(); 1447 } 1448 1449 /* the addresses of all functions are final */ 1450 err = resolve_func_ptrs(env, prog, func); 1451 if (err) 1452 goto out_free; 1453 1454 /* 1455 * Cleanup func[i]->aux fields which aren't required 1456 * or can become invalid in future 1457 */ 1458 for (i = 0; i < env->subprog_cnt; i++) { 1459 func[i]->aux->used_maps = NULL; 1460 func[i]->aux->used_map_cnt = 0; 1461 } 1462 1463 /* finally lock prog and jit images for all functions and 1464 * populate kallsysm. Begin at the first subprogram, since 1465 * bpf_prog_load will add the kallsyms for the main program. 1466 */ 1467 for (i = 1; i < env->subprog_cnt; i++) { 1468 err = bpf_prog_lock_ro(func[i]); 1469 if (err) 1470 goto out_free; 1471 } 1472 1473 for (i = 1; i < env->subprog_cnt; i++) 1474 bpf_prog_kallsyms_add(func[i]); 1475 1476 /* Last step: make now unused interpreter insns from main 1477 * prog consistent for later dump requests, so they can 1478 * later look the same as if they were interpreted only. 1479 */ 1480 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1481 if (bpf_pseudo_func(insn)) { 1482 insn[0].imm = env->insn_aux_data[i].call_imm; 1483 insn[1].imm = insn->off; 1484 insn->off = 0; 1485 continue; 1486 } 1487 if (!bpf_pseudo_call(insn)) 1488 continue; 1489 insn->imm = env->insn_aux_data[i].call_imm; 1490 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1491 insn->off = subprog; 1492 } 1493 1494 prog->jited = 1; 1495 prog->bpf_func = func[0]->bpf_func; 1496 prog->jited_len = func[0]->jited_len; 1497 prog->aux->extable = func[0]->aux->extable; 1498 prog->aux->num_exentries = func[0]->aux->num_exentries; 1499 prog->aux->func = func; 1500 prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1501 prog->aux->real_func_cnt = env->subprog_cnt; 1502 prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; 1503 prog->aux->exception_boundary = func[0]->aux->exception_boundary; 1504 prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust; 1505 bpf_prog_jit_attempt_done(prog); 1506 return 0; 1507 out_free: 1508 /* We failed JIT'ing, so at this point we need to unregister poke 1509 * descriptors from subprogs, so that kernel is not attempting to 1510 * patch it anymore as we're freeing the subprog JIT memory. 1511 */ 1512 for (i = 0; i < prog->aux->size_poke_tab; i++) { 1513 map_ptr = prog->aux->poke_tab[i].tail_call.map; 1514 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 1515 } 1516 /* At this point we're guaranteed that poke descriptors are not 1517 * live anymore. We can just unlink its descriptor table as it's 1518 * released with the main prog. 1519 */ 1520 for (i = 0; i < env->subprog_cnt; i++) { 1521 if (!func[i]) 1522 continue; 1523 func[i]->aux->poke_tab = NULL; 1524 bpf_jit_free(func[i]); 1525 } 1526 kfree(func); 1527 out_undo_insn: 1528 bpf_prog_jit_attempt_done(prog); 1529 return err; 1530 } 1531 1532 int bpf_jit_subprogs(struct bpf_verifier_env *env) 1533 { 1534 int err, i; 1535 bool blinded = false; 1536 struct bpf_insn *insn; 1537 struct bpf_prog *prog, *orig_prog; 1538 u32 *orig_subprog_starts; 1539 1540 /* all functions that the maps point to are removed as dead code */ 1541 if (env->subprog_cnt <= 1) 1542 return resolve_func_ptrs(env, env->prog, NULL); 1543 1544 prog = orig_prog = env->prog; 1545 if (bpf_prog_need_blind(prog)) { 1546 orig_subprog_starts = bpf_dup_subprog_starts(env); 1547 if (!orig_subprog_starts) { 1548 err = -ENOMEM; 1549 goto out_cleanup; 1550 } 1551 prog = bpf_jit_blind_constants(env, prog); 1552 if (IS_ERR(prog)) { 1553 err = PTR_ERR(prog); 1554 prog = orig_prog; 1555 goto out_restore; 1556 } 1557 blinded = true; 1558 } 1559 1560 err = jit_subprogs(env); 1561 if (err) 1562 goto out_jit_err; 1563 1564 if (blinded) { 1565 bpf_jit_prog_release_other(prog, orig_prog); 1566 kvfree(orig_subprog_starts); 1567 } 1568 1569 return 0; 1570 1571 out_jit_err: 1572 if (blinded) { 1573 bpf_jit_prog_release_other(orig_prog, prog); 1574 /* roll back to the clean original prog */ 1575 prog = env->prog = orig_prog; 1576 goto out_restore; 1577 } else { 1578 if (err != -EFAULT) { 1579 /* 1580 * We will fall back to interpreter mode when err is not -EFAULT, before 1581 * that, insn->off and insn->imm should be restored to their original 1582 * values since they were modified by jit_subprogs. 1583 */ 1584 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1585 if (!bpf_pseudo_call(insn)) 1586 continue; 1587 insn->off = 0; 1588 insn->imm = env->insn_aux_data[i].call_imm; 1589 } 1590 } 1591 goto out_cleanup; 1592 } 1593 1594 out_restore: 1595 bpf_restore_subprog_starts(env, orig_subprog_starts); 1596 kvfree(orig_subprog_starts); 1597 out_cleanup: 1598 /* cleanup main prog to be interpreted */ 1599 prog->jit_requested = 0; 1600 prog->blinding_requested = 0; 1601 return err; 1602 } 1603 1604 int bpf_fixup_call_args(struct bpf_verifier_env *env) 1605 { 1606 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1607 struct bpf_prog *prog = env->prog; 1608 struct bpf_insn *insn = prog->insnsi; 1609 int depth; 1610 #endif 1611 int i, err = 0; 1612 1613 for (i = 0; i < env->subprog_cnt; i++) { 1614 struct bpf_subprog_info *subprog = &env->subprog_info[i]; 1615 u16 outgoing = subprog->stack_arg_cnt - bpf_in_stack_arg_cnt(subprog); 1616 1617 if (subprog->max_out_stack_arg_cnt > outgoing) { 1618 verbose(env, 1619 "func#%d writes %u stack arg slots, but calls only require %u\n", 1620 i, subprog->max_out_stack_arg_cnt, outgoing); 1621 return -EINVAL; 1622 } 1623 } 1624 1625 if (env->prog->jit_requested && 1626 !bpf_prog_is_offloaded(env->prog->aux)) { 1627 err = bpf_jit_subprogs(env); 1628 if (err == 0) 1629 return 0; 1630 if (err == -EFAULT || err == -EINTR) 1631 return err; 1632 } 1633 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1634 if (prog->jit_required) { 1635 verbose(env, "program requires BPF JIT compiler but it is not available\n"); 1636 return -EINVAL; 1637 } 1638 for (i = 0; i < env->subprog_cnt; i++) { 1639 if (bpf_in_stack_arg_cnt(&env->subprog_info[i])) { 1640 verbose(env, "stack args are not supported in non-JITed programs\n"); 1641 return -EINVAL; 1642 } 1643 } 1644 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 1645 /* When JIT fails the progs with bpf2bpf calls and tail_calls 1646 * have to be rejected, since interpreter doesn't support them yet. 1647 */ 1648 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 1649 return -EINVAL; 1650 } 1651 for (i = 0; i < prog->len; i++, insn++) { 1652 if (bpf_pseudo_func(insn)) { 1653 /* When JIT fails the progs with callback calls 1654 * have to be rejected, since interpreter doesn't support them yet. 1655 */ 1656 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 1657 return -EINVAL; 1658 } 1659 1660 if (!bpf_pseudo_call(insn)) 1661 continue; 1662 depth = get_callee_stack_depth(env, insn, i); 1663 if (depth < 0) 1664 return depth; 1665 err = bpf_patch_call_args(insn, depth); 1666 if (err) { 1667 verbose(env, "stack depth %d exceeds interpreter stack depth limit\n", 1668 depth); 1669 return err; 1670 } 1671 } 1672 err = 0; 1673 #endif 1674 return err; 1675 } 1676 1677 /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */ 1678 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) 1679 { 1680 struct bpf_subprog_info *info = env->subprog_info; 1681 int cnt = env->subprog_cnt; 1682 struct bpf_prog *prog; 1683 1684 /* We only reserve one slot for hidden subprogs in subprog_info. */ 1685 if (env->hidden_subprog_cnt) { 1686 verifier_bug(env, "only one hidden subprog supported"); 1687 return -EFAULT; 1688 } 1689 /* We're not patching any existing instruction, just appending the new 1690 * ones for the hidden subprog. Hence all of the adjustment operations 1691 * in bpf_patch_insn_data are no-ops. 1692 */ 1693 prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); 1694 if (!prog) 1695 return -ENOMEM; 1696 env->prog = prog; 1697 info[cnt + 1].start = info[cnt].start; 1698 info[cnt].start = prog->len - len + 1; 1699 /* The patch ends with the exit. */ 1700 info[cnt].exit_idx = prog->len - 1; 1701 env->subprog_cnt++; 1702 env->hidden_subprog_cnt++; 1703 return 0; 1704 } 1705 1706 /* 1707 * Expand may_goto: load the count from the stack, jump to the target of 1708 * may_goto when it is zero, then the tail that updates the count. 1709 * Use gotol when the target is too far for 16-bit offset of a conditional jump. 1710 */ 1711 static int may_goto_expand(struct bpf_insn *insn_buf, int off, int stack_off, 1712 const struct bpf_insn *tail, int tail_cnt) 1713 { 1714 int cnt = 0; 1715 1716 /* Forward jump has to step over the tail */ 1717 off = off >= 0 ? off + tail_cnt : off - 1; 1718 1719 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); 1720 if (off == (s16)off) { 1721 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off); 1722 } else { 1723 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1); 1724 insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1); 1725 } 1726 memcpy(insn_buf + cnt, tail, tail_cnt * sizeof(*tail)); 1727 return cnt + tail_cnt; 1728 } 1729 1730 /* Do various post-verification rewrites in a single program pass. 1731 * These rewrites simplify JIT and interpreter implementations. 1732 */ 1733 int bpf_do_misc_fixups(struct bpf_verifier_env *env) 1734 { 1735 struct bpf_prog *prog = env->prog; 1736 enum bpf_attach_type eatype = prog->expected_attach_type; 1737 enum bpf_prog_type prog_type = resolve_prog_type(prog); 1738 struct bpf_insn *insn = prog->insnsi; 1739 const struct bpf_func_proto *fn; 1740 const int insn_cnt = prog->len; 1741 const struct bpf_map_ops *ops; 1742 struct bpf_insn_aux_data *aux; 1743 struct bpf_insn *insn_buf = env->insn_buf; 1744 struct bpf_prog *new_prog; 1745 struct bpf_map *map_ptr; 1746 int i, ret, cnt, delta = 0, cur_subprog = 0; 1747 struct bpf_subprog_info *subprogs = env->subprog_info; 1748 u16 stack_depth = subprogs[cur_subprog].stack_depth; 1749 u16 stack_depth_extra = 0; 1750 1751 if (env->seen_exception && !env->exception_callback_subprog) { 1752 struct bpf_insn *patch = insn_buf; 1753 1754 *patch++ = env->prog->insnsi[insn_cnt - 1]; 1755 *patch++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 1756 *patch++ = BPF_EXIT_INSN(); 1757 ret = add_hidden_subprog(env, insn_buf, patch - insn_buf); 1758 if (ret < 0) 1759 return ret; 1760 prog = env->prog; 1761 insn = prog->insnsi; 1762 1763 env->exception_callback_subprog = env->subprog_cnt - 1; 1764 /* Don't update insn_cnt, as add_hidden_subprog always appends insns */ 1765 bpf_mark_subprog_exc_cb(env, env->exception_callback_subprog); 1766 } 1767 1768 for (i = 0; i < insn_cnt;) { 1769 if (is_addr_space_cast32(env->prog, insn)) { 1770 /* convert to 32-bit mov that clears upper 32-bit */ 1771 insn->code = BPF_ALU | BPF_MOV | BPF_X; 1772 /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ 1773 insn->off = 0; 1774 insn->imm = 0; 1775 goto next_insn; 1776 } 1777 1778 if (env->insn_aux_data[i + delta].needs_zext) 1779 /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ 1780 insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); 1781 1782 if (env->insn_aux_data[i + delta].arena_scalar) { 1783 cnt = arena_scalar_access(insn, insn_buf); 1784 1785 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1786 if (!new_prog) 1787 return -ENOMEM; 1788 1789 delta += cnt - 1; 1790 prog = new_prog; 1791 env->prog = prog; 1792 insn = prog->insnsi + i + delta; 1793 goto next_insn; 1794 } 1795 1796 /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ 1797 if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || 1798 insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || 1799 insn->code == (BPF_ALU | BPF_MOD | BPF_K) || 1800 insn->code == (BPF_ALU | BPF_DIV | BPF_K)) && 1801 insn->off == 1 && insn->imm == -1) { 1802 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1803 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1804 struct bpf_insn *patch = insn_buf; 1805 1806 if (isdiv) 1807 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1808 BPF_NEG | BPF_K, insn->dst_reg, 1809 0, 0, 0); 1810 else 1811 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1812 1813 cnt = patch - insn_buf; 1814 1815 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1816 if (!new_prog) 1817 return -ENOMEM; 1818 1819 delta += cnt - 1; 1820 env->prog = prog = new_prog; 1821 insn = new_prog->insnsi + i + delta; 1822 goto next_insn; 1823 } 1824 1825 /* Make divide-by-zero and divide-by-minus-one exceptions impossible. */ 1826 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 1827 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 1828 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 1829 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 1830 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1831 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1832 bool is_sdiv = isdiv && insn->off == 1; 1833 bool is_smod = !isdiv && insn->off == 1; 1834 struct bpf_insn *patch = insn_buf; 1835 1836 if (is_sdiv) { 1837 /* [R,W]x sdiv 0 -> 0 1838 * LLONG_MIN sdiv -1 -> LLONG_MIN 1839 * INT_MIN sdiv -1 -> INT_MIN 1840 */ 1841 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1842 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1843 BPF_ADD | BPF_K, BPF_REG_AX, 1844 0, 0, 1); 1845 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1846 BPF_JGT | BPF_K, BPF_REG_AX, 1847 0, 4, 1); 1848 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1849 BPF_JEQ | BPF_K, BPF_REG_AX, 1850 0, 1, 0); 1851 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1852 BPF_MOV | BPF_K, insn->dst_reg, 1853 0, 0, 0); 1854 /* BPF_NEG(LLONG_MIN) == -LLONG_MIN == LLONG_MIN */ 1855 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1856 BPF_NEG | BPF_K, insn->dst_reg, 1857 0, 0, 0); 1858 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1859 *patch++ = *insn; 1860 cnt = patch - insn_buf; 1861 } else if (is_smod) { 1862 /* [R,W]x mod 0 -> [R,W]x */ 1863 /* [R,W]x mod -1 -> 0 */ 1864 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1865 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1866 BPF_ADD | BPF_K, BPF_REG_AX, 1867 0, 0, 1); 1868 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1869 BPF_JGT | BPF_K, BPF_REG_AX, 1870 0, 3, 1); 1871 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1872 BPF_JEQ | BPF_K, BPF_REG_AX, 1873 0, 3 + (is64 ? 0 : 1), 1); 1874 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1875 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1876 *patch++ = *insn; 1877 1878 if (!is64) { 1879 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1880 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 1881 } 1882 cnt = patch - insn_buf; 1883 } else if (isdiv) { 1884 /* [R,W]x div 0 -> 0 */ 1885 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1886 BPF_JNE | BPF_K, insn->src_reg, 1887 0, 2, 0); 1888 *patch++ = BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg); 1889 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1890 *patch++ = *insn; 1891 cnt = patch - insn_buf; 1892 } else { 1893 /* [R,W]x mod 0 -> [R,W]x */ 1894 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1895 BPF_JEQ | BPF_K, insn->src_reg, 1896 0, 1 + (is64 ? 0 : 1), 0); 1897 *patch++ = *insn; 1898 1899 if (!is64) { 1900 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1901 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 1902 } 1903 cnt = patch - insn_buf; 1904 } 1905 1906 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1907 if (!new_prog) 1908 return -ENOMEM; 1909 1910 delta += cnt - 1; 1911 env->prog = prog = new_prog; 1912 insn = new_prog->insnsi + i + delta; 1913 goto next_insn; 1914 } 1915 1916 /* Make it impossible to de-reference a userspace address */ 1917 if (BPF_CLASS(insn->code) == BPF_LDX && 1918 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 1919 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) { 1920 struct bpf_insn *patch = insn_buf; 1921 u64 uaddress_limit = bpf_arch_uaddress_limit(); 1922 1923 if (!uaddress_limit) 1924 goto next_insn; 1925 1926 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1927 if (insn->off) 1928 *patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off); 1929 *patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32); 1930 *patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2); 1931 *patch++ = *insn; 1932 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1933 *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0); 1934 1935 cnt = patch - insn_buf; 1936 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1937 if (!new_prog) 1938 return -ENOMEM; 1939 1940 delta += cnt - 1; 1941 env->prog = prog = new_prog; 1942 insn = new_prog->insnsi + i + delta; 1943 goto next_insn; 1944 } 1945 1946 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 1947 if (BPF_CLASS(insn->code) == BPF_LD && 1948 (BPF_MODE(insn->code) == BPF_ABS || 1949 BPF_MODE(insn->code) == BPF_IND)) { 1950 cnt = env->ops->gen_ld_abs(insn, insn_buf); 1951 if (cnt == 0 || cnt >= INSN_BUF_SIZE) { 1952 verifier_bug(env, "%d insns generated for ld_abs", cnt); 1953 return -EFAULT; 1954 } 1955 1956 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1957 if (!new_prog) 1958 return -ENOMEM; 1959 1960 delta += cnt - 1; 1961 env->prog = prog = new_prog; 1962 insn = new_prog->insnsi + i + delta; 1963 goto next_insn; 1964 } 1965 1966 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 1967 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 1968 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 1969 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 1970 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 1971 struct bpf_insn *patch = insn_buf; 1972 bool issrc, isneg, isimm; 1973 u32 off_reg; 1974 1975 aux = &env->insn_aux_data[i + delta]; 1976 if (!aux->alu_state || 1977 aux->alu_state == BPF_ALU_NON_POINTER) 1978 goto next_insn; 1979 1980 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 1981 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 1982 BPF_ALU_SANITIZE_SRC; 1983 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 1984 1985 off_reg = issrc ? insn->src_reg : insn->dst_reg; 1986 if (isimm) { 1987 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 1988 } else { 1989 if (isneg) 1990 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 1991 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 1992 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 1993 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 1994 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 1995 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 1996 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 1997 } 1998 if (!issrc) 1999 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 2000 insn->src_reg = BPF_REG_AX; 2001 if (isneg) 2002 insn->code = insn->code == code_add ? 2003 code_sub : code_add; 2004 *patch++ = *insn; 2005 if (issrc && isneg && !isimm) 2006 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 2007 cnt = patch - insn_buf; 2008 2009 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2010 if (!new_prog) 2011 return -ENOMEM; 2012 2013 delta += cnt - 1; 2014 env->prog = prog = new_prog; 2015 insn = new_prog->insnsi + i + delta; 2016 goto next_insn; 2017 } 2018 2019 if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { 2020 int stack_off_cnt = -stack_depth - 16; 2021 /* 2022 * AX is used as an argument to pass in stack_off_cnt 2023 * (to add to r10/fp), and also as the return value of 2024 * the call to arch_bpf_timed_may_goto. 2025 */ 2026 struct bpf_insn tail[] = { 2027 BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), 2028 BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2), 2029 BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt), 2030 BPF_EMIT_CALL(arch_bpf_timed_may_goto), 2031 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt), 2032 }; 2033 2034 /* 2035 * Two 8 byte slots, depth-16 stores the count, and 2036 * depth-8 stores the start timestamp of the loop. 2037 * 2038 * The starting value of count is BPF_MAX_TIMED_LOOPS 2039 * (0xffff). Every iteration loads it and subs it by 1, 2040 * until the value becomes 0 in AX (thus, 1 in stack), 2041 * after which we call arch_bpf_timed_may_goto, which 2042 * either sets AX to 0xffff to keep looping, or to 0 2043 * upon timeout. AX is then stored into the stack. In 2044 * the next iteration, we either see 0 and break out, or 2045 * continue iterating until the next time value is 0 2046 * after subtraction, rinse and repeat. 2047 */ 2048 stack_depth_extra = 16; 2049 cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt, 2050 tail, ARRAY_SIZE(tail)); 2051 2052 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2053 if (!new_prog) 2054 return -ENOMEM; 2055 2056 delta += cnt - 1; 2057 env->prog = prog = new_prog; 2058 insn = new_prog->insnsi + i + delta; 2059 goto next_insn; 2060 } else if (bpf_is_may_goto_insn(insn)) { 2061 int stack_off = -stack_depth - 8; 2062 struct bpf_insn tail[] = { 2063 BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), 2064 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off), 2065 }; 2066 2067 stack_depth_extra = 8; 2068 cnt = may_goto_expand(insn_buf, insn->off, stack_off, 2069 tail, ARRAY_SIZE(tail)); 2070 2071 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2072 if (!new_prog) 2073 return -ENOMEM; 2074 2075 delta += cnt - 1; 2076 env->prog = prog = new_prog; 2077 insn = new_prog->insnsi + i + delta; 2078 goto next_insn; 2079 } 2080 2081 if (bpf_jit_supports_percpu_insn() && 2082 insn->code == (BPF_LD | BPF_IMM | BPF_DW) && 2083 (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 2084 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) { 2085 struct bpf_map *map; 2086 2087 aux = &env->insn_aux_data[i + delta]; 2088 map = env->used_maps[aux->map_index]; 2089 if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) 2090 goto next_insn; 2091 2092 prog->jit_required = true; 2093 2094 /* 2095 * We are *skipping* first half of ld_imm64 insn 2096 * with 'i++;', patching over second half of it 2097 * with that same half + mov64_percpu_reg insn. 2098 * All because bpf_patch_insn_data() can only 2099 * replace one 8-byte insn, which does not work 2100 * well for ld_imm64 insn. 2101 */ 2102 2103 insn_buf[0] = insn[1]; 2104 insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg); 2105 cnt = 2; 2106 2107 i++; 2108 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2109 if (!new_prog) 2110 return -ENOMEM; 2111 2112 delta += cnt - 1; 2113 env->prog = prog = new_prog; 2114 insn = new_prog->insnsi + i + delta; 2115 goto next_insn; 2116 } 2117 2118 if (insn->code != (BPF_JMP | BPF_CALL)) 2119 goto next_insn; 2120 if (insn->src_reg == BPF_PSEUDO_CALL) 2121 goto next_insn; 2122 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 2123 ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); 2124 if (ret) 2125 return ret; 2126 if (cnt == 0) 2127 goto next_insn; 2128 2129 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2130 if (!new_prog) 2131 return -ENOMEM; 2132 2133 delta += cnt - 1; 2134 env->prog = prog = new_prog; 2135 insn = new_prog->insnsi + i + delta; 2136 goto next_insn; 2137 } 2138 2139 /* Skip inlining the helper call if the JIT does it. */ 2140 if (bpf_jit_inlines_helper_call(insn->imm)) { 2141 prog->jit_required = 1; 2142 goto next_insn; 2143 } 2144 2145 if (insn->imm == BPF_FUNC_get_route_realm) 2146 prog->dst_needed = 1; 2147 if (insn->imm == BPF_FUNC_get_prandom_u32) 2148 bpf_user_rnd_init_once(); 2149 if (insn->imm == BPF_FUNC_override_return) 2150 prog->kprobe_override = 1; 2151 if (insn->imm == BPF_FUNC_tail_call) { 2152 /* If we tail call into other programs, we 2153 * cannot make any assumptions since they can 2154 * be replaced dynamically during runtime in 2155 * the program array. 2156 */ 2157 prog->cb_access = 1; 2158 if (!bpf_allow_tail_call_in_subprogs(env)) 2159 prog->aux->stack_depth = MAX_BPF_STACK; 2160 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 2161 2162 /* mark bpf_tail_call as different opcode to avoid 2163 * conditional branch in the interpreter for every normal 2164 * call and to prevent accidental JITing by JIT compiler 2165 * that doesn't support bpf_tail_call yet 2166 */ 2167 insn->imm = 0; 2168 insn->code = BPF_JMP | BPF_TAIL_CALL; 2169 2170 aux = &env->insn_aux_data[i + delta]; 2171 if (env->bpf_capable && !prog->blinding_requested && 2172 prog->jit_requested && 2173 !bpf_map_key_poisoned(aux) && 2174 !bpf_map_ptr_poisoned(aux) && 2175 !bpf_map_ptr_unpriv(aux)) { 2176 struct bpf_jit_poke_descriptor desc = { 2177 .reason = BPF_POKE_REASON_TAIL_CALL, 2178 .tail_call.map = aux->map_ptr_state.map_ptr, 2179 .tail_call.key = bpf_map_key_immediate(aux), 2180 .insn_idx = i + delta, 2181 }; 2182 2183 ret = bpf_jit_add_poke_descriptor(prog, &desc); 2184 if (ret < 0) { 2185 verbose(env, "adding tail call poke descriptor failed\n"); 2186 return ret; 2187 } 2188 2189 insn->imm = ret + 1; 2190 goto next_insn; 2191 } 2192 2193 if (!bpf_map_ptr_unpriv(aux)) 2194 goto next_insn; 2195 2196 /* instead of changing every JIT dealing with tail_call 2197 * emit two extra insns: 2198 * if (index >= max_entries) goto out; 2199 * index &= array->index_mask; 2200 * to avoid out-of-bounds cpu speculation 2201 */ 2202 if (bpf_map_ptr_poisoned(aux)) { 2203 verbose(env, "tail_call abusing map_ptr\n"); 2204 return -EINVAL; 2205 } 2206 2207 map_ptr = aux->map_ptr_state.map_ptr; 2208 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 2209 map_ptr->max_entries, 2); 2210 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 2211 container_of(map_ptr, 2212 struct bpf_array, 2213 map)->index_mask); 2214 insn_buf[2] = *insn; 2215 cnt = 3; 2216 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2217 if (!new_prog) 2218 return -ENOMEM; 2219 2220 delta += cnt - 1; 2221 env->prog = prog = new_prog; 2222 insn = new_prog->insnsi + i + delta; 2223 goto next_insn; 2224 } 2225 2226 aux = &env->insn_aux_data[i + delta]; 2227 if (aux->arg_prog) { 2228 /* The verifier will process callback_fn as many times as necessary 2229 * with different maps and the register states prepared by 2230 * set_timer_callback_state will be accurate. 2231 * 2232 * The following use case is valid: 2233 * map1 is shared by prog1, prog2, prog3. 2234 * prog1 calls bpf_timer_init for some map1 elements 2235 * prog2 calls bpf_timer_set_callback for some map1 elements. 2236 * Those that were not bpf_timer_init-ed will return -EINVAL. 2237 * prog3 calls bpf_timer_start for some map1 elements. 2238 * Those that were not both bpf_timer_init-ed and 2239 * bpf_timer_set_callback-ed will return -EINVAL. 2240 */ 2241 struct bpf_insn ld_addrs[2] = { 2242 BPF_LD_IMM64(aux->arg_prog, (long)prog->aux), 2243 }; 2244 2245 insn_buf[0] = ld_addrs[0]; 2246 insn_buf[1] = ld_addrs[1]; 2247 insn_buf[2] = *insn; 2248 cnt = 3; 2249 2250 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2251 if (!new_prog) 2252 return -ENOMEM; 2253 2254 delta += cnt - 1; 2255 env->prog = prog = new_prog; 2256 insn = new_prog->insnsi + i + delta; 2257 goto patch_call_imm; 2258 } 2259 2260 /* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */ 2261 if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) { 2262 /* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data, 2263 * bpf_mem_alloc() returns a ptr to the percpu data ptr. 2264 */ 2265 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 2266 insn_buf[1] = *insn; 2267 cnt = 2; 2268 2269 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2270 if (!new_prog) 2271 return -ENOMEM; 2272 2273 delta += cnt - 1; 2274 env->prog = prog = new_prog; 2275 insn = new_prog->insnsi + i + delta; 2276 goto patch_call_imm; 2277 } 2278 2279 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 2280 * and other inlining handlers are currently limited to 64 bit 2281 * only. 2282 */ 2283 if (prog->jit_requested && BITS_PER_LONG == 64 && 2284 (insn->imm == BPF_FUNC_map_lookup_elem || 2285 insn->imm == BPF_FUNC_map_update_elem || 2286 insn->imm == BPF_FUNC_map_delete_elem || 2287 insn->imm == BPF_FUNC_map_push_elem || 2288 insn->imm == BPF_FUNC_map_pop_elem || 2289 insn->imm == BPF_FUNC_map_peek_elem || 2290 insn->imm == BPF_FUNC_redirect_map || 2291 insn->imm == BPF_FUNC_for_each_map_elem || 2292 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { 2293 aux = &env->insn_aux_data[i + delta]; 2294 if (bpf_map_ptr_poisoned(aux)) 2295 goto patch_call_imm; 2296 2297 map_ptr = aux->map_ptr_state.map_ptr; 2298 ops = map_ptr->ops; 2299 if (insn->imm == BPF_FUNC_map_lookup_elem && 2300 ops->map_gen_lookup) { 2301 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 2302 if (cnt == -EOPNOTSUPP) 2303 goto patch_map_ops_generic; 2304 if (cnt <= 0 || cnt >= INSN_BUF_SIZE) { 2305 verifier_bug(env, "%d insns generated for map lookup", cnt); 2306 return -EFAULT; 2307 } 2308 2309 if (bpf_map_is_percpu_map(map_ptr->map_type)) 2310 prog->jit_required = true; 2311 2312 new_prog = bpf_patch_insn_data(env, i + delta, 2313 insn_buf, cnt); 2314 if (!new_prog) 2315 return -ENOMEM; 2316 2317 delta += cnt - 1; 2318 env->prog = prog = new_prog; 2319 insn = new_prog->insnsi + i + delta; 2320 goto next_insn; 2321 } 2322 2323 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 2324 (void *(*)(struct bpf_map *map, void *key))NULL)); 2325 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 2326 (long (*)(struct bpf_map *map, void *key))NULL)); 2327 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 2328 (long (*)(struct bpf_map *map, void *key, void *value, 2329 u64 flags))NULL)); 2330 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 2331 (long (*)(struct bpf_map *map, void *value, 2332 u64 flags))NULL)); 2333 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 2334 (long (*)(struct bpf_map *map, void *value))NULL)); 2335 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 2336 (long (*)(struct bpf_map *map, void *value))NULL)); 2337 BUILD_BUG_ON(!__same_type(ops->map_redirect, 2338 (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); 2339 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 2340 (long (*)(struct bpf_map *map, 2341 bpf_callback_t callback_fn, 2342 void *callback_ctx, 2343 u64 flags))NULL)); 2344 BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, 2345 (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL)); 2346 2347 patch_map_ops_generic: 2348 switch (insn->imm) { 2349 case BPF_FUNC_map_lookup_elem: 2350 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 2351 goto next_insn; 2352 case BPF_FUNC_map_update_elem: 2353 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 2354 goto next_insn; 2355 case BPF_FUNC_map_delete_elem: 2356 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 2357 goto next_insn; 2358 case BPF_FUNC_map_push_elem: 2359 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 2360 goto next_insn; 2361 case BPF_FUNC_map_pop_elem: 2362 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 2363 goto next_insn; 2364 case BPF_FUNC_map_peek_elem: 2365 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 2366 goto next_insn; 2367 case BPF_FUNC_redirect_map: 2368 insn->imm = BPF_CALL_IMM(ops->map_redirect); 2369 goto next_insn; 2370 case BPF_FUNC_for_each_map_elem: 2371 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 2372 goto next_insn; 2373 case BPF_FUNC_map_lookup_percpu_elem: 2374 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); 2375 goto next_insn; 2376 } 2377 2378 goto patch_call_imm; 2379 } 2380 2381 /* Implement bpf_jiffies64 inline. */ 2382 if (prog->jit_requested && BITS_PER_LONG == 64 && 2383 insn->imm == BPF_FUNC_jiffies64) { 2384 struct bpf_insn ld_jiffies_addr[2] = { 2385 BPF_LD_IMM64(BPF_REG_0, 2386 (unsigned long)&jiffies), 2387 }; 2388 2389 insn_buf[0] = ld_jiffies_addr[0]; 2390 insn_buf[1] = ld_jiffies_addr[1]; 2391 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 2392 BPF_REG_0, 0); 2393 cnt = 3; 2394 2395 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 2396 cnt); 2397 if (!new_prog) 2398 return -ENOMEM; 2399 2400 delta += cnt - 1; 2401 env->prog = prog = new_prog; 2402 insn = new_prog->insnsi + i + delta; 2403 goto next_insn; 2404 } 2405 2406 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML) 2407 /* Implement bpf_get_smp_processor_id() inline. */ 2408 if (insn->imm == BPF_FUNC_get_smp_processor_id && 2409 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2410 /* BPF_FUNC_get_smp_processor_id inlining is an 2411 * optimization, so if cpu_number is ever 2412 * changed in some incompatible and hard to support 2413 * way, it's fine to back out this inlining logic 2414 */ 2415 #ifdef CONFIG_SMP 2416 prog->jit_required = true; 2417 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); 2418 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2419 insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); 2420 cnt = 3; 2421 #else 2422 insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); 2423 cnt = 1; 2424 #endif 2425 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2426 if (!new_prog) 2427 return -ENOMEM; 2428 2429 delta += cnt - 1; 2430 env->prog = prog = new_prog; 2431 insn = new_prog->insnsi + i + delta; 2432 goto next_insn; 2433 } 2434 2435 /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ 2436 if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && 2437 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2438 prog->jit_required = true; 2439 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); 2440 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2441 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); 2442 cnt = 3; 2443 2444 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2445 if (!new_prog) 2446 return -ENOMEM; 2447 2448 delta += cnt - 1; 2449 env->prog = prog = new_prog; 2450 insn = new_prog->insnsi + i + delta; 2451 goto next_insn; 2452 } 2453 #endif 2454 /* Implement bpf_get_func_arg inline. */ 2455 if (prog_type == BPF_PROG_TYPE_TRACING && 2456 insn->imm == BPF_FUNC_get_func_arg) { 2457 if (eatype == BPF_TRACE_RAW_TP) { 2458 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2459 2460 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2461 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2462 cnt = 1; 2463 } else { 2464 /* Load nr_args from ctx - 8 */ 2465 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2466 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2467 cnt = 2; 2468 } 2469 insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 2470 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 2471 insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 2472 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 2473 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2474 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 0); 2475 insn_buf[cnt++] = BPF_JMP_A(1); 2476 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2477 2478 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2479 if (!new_prog) 2480 return -ENOMEM; 2481 2482 delta += cnt - 1; 2483 env->prog = prog = new_prog; 2484 insn = new_prog->insnsi + i + delta; 2485 goto next_insn; 2486 } 2487 2488 /* Implement bpf_get_func_ret inline. */ 2489 if (prog_type == BPF_PROG_TYPE_TRACING && 2490 insn->imm == BPF_FUNC_get_func_ret) { 2491 if (eatype == BPF_TRACE_FEXIT || 2492 eatype == BPF_TRACE_FSESSION || 2493 eatype == BPF_TRACE_FEXIT_MULTI || 2494 eatype == BPF_TRACE_FSESSION_MULTI || 2495 eatype == BPF_MODIFY_RETURN) { 2496 /* Load nr_args from ctx - 8 */ 2497 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2498 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2499 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 2500 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 2501 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2502 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 2503 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 2504 cnt = 7; 2505 } else { 2506 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 2507 cnt = 1; 2508 } 2509 2510 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2511 if (!new_prog) 2512 return -ENOMEM; 2513 2514 delta += cnt - 1; 2515 env->prog = prog = new_prog; 2516 insn = new_prog->insnsi + i + delta; 2517 goto next_insn; 2518 } 2519 2520 /* Implement get_func_arg_cnt inline. */ 2521 if (prog_type == BPF_PROG_TYPE_TRACING && 2522 insn->imm == BPF_FUNC_get_func_arg_cnt) { 2523 if (eatype == BPF_TRACE_RAW_TP) { 2524 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2525 2526 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2527 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2528 cnt = 1; 2529 } else { 2530 /* Load nr_args from ctx - 8 */ 2531 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2532 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2533 cnt = 2; 2534 } 2535 2536 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2537 if (!new_prog) 2538 return -ENOMEM; 2539 2540 delta += cnt - 1; 2541 env->prog = prog = new_prog; 2542 insn = new_prog->insnsi + i + delta; 2543 goto next_insn; 2544 } 2545 2546 /* Implement bpf_get_func_ip inline. */ 2547 if (prog_type == BPF_PROG_TYPE_TRACING && 2548 insn->imm == BPF_FUNC_get_func_ip) { 2549 /* Load IP address from ctx - 16 */ 2550 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 2551 2552 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 2553 if (!new_prog) 2554 return -ENOMEM; 2555 2556 env->prog = prog = new_prog; 2557 insn = new_prog->insnsi + i + delta; 2558 goto next_insn; 2559 } 2560 2561 /* Implement bpf_get_branch_snapshot inline. */ 2562 if (IS_ENABLED(CONFIG_PERF_EVENTS) && 2563 prog->jit_requested && BITS_PER_LONG == 64 && 2564 insn->imm == BPF_FUNC_get_branch_snapshot) { 2565 /* We are dealing with the following func protos: 2566 * u64 bpf_get_branch_snapshot(void *buf, u32 size, u64 flags); 2567 * int perf_snapshot_branch_stack(struct perf_branch_entry *entries, u32 cnt); 2568 */ 2569 const u32 br_entry_size = sizeof(struct perf_branch_entry); 2570 2571 /* struct perf_branch_entry is part of UAPI and is 2572 * used as an array element, so extremely unlikely to 2573 * ever grow or shrink 2574 */ 2575 BUILD_BUG_ON(br_entry_size != 24); 2576 2577 /* if (unlikely(flags)) return -EINVAL */ 2578 insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7); 2579 2580 /* Transform size (bytes) into number of entries (cnt = size / 24). 2581 * But to avoid expensive division instruction, we implement 2582 * divide-by-3 through multiplication, followed by further 2583 * division by 8 through 3-bit right shift. 2584 * Refer to book "Hacker's Delight, 2nd ed." by Henry S. Warren, Jr., 2585 * p. 227, chapter "Unsigned Division by 3" for details and proofs. 2586 * 2587 * N / 3 <=> M * N / 2^33, where M = (2^33 + 1) / 3 = 0xaaaaaaab. 2588 */ 2589 insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab); 2590 insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0); 2591 insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36); 2592 2593 /* call perf_snapshot_branch_stack implementation */ 2594 insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack)); 2595 /* if (entry_cnt == 0) return -ENOENT */ 2596 insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4); 2597 /* return entry_cnt * sizeof(struct perf_branch_entry) */ 2598 insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size); 2599 insn_buf[7] = BPF_JMP_A(3); 2600 /* return -EINVAL; */ 2601 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2602 insn_buf[9] = BPF_JMP_A(1); 2603 /* return -ENOENT; */ 2604 insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT); 2605 cnt = 11; 2606 2607 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2608 if (!new_prog) 2609 return -ENOMEM; 2610 2611 delta += cnt - 1; 2612 env->prog = prog = new_prog; 2613 insn = new_prog->insnsi + i + delta; 2614 goto next_insn; 2615 } 2616 2617 /* Implement bpf_kptr_xchg inline */ 2618 if (prog->jit_requested && BITS_PER_LONG == 64 && 2619 insn->imm == BPF_FUNC_kptr_xchg && 2620 bpf_jit_supports_ptr_xchg()) { 2621 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); 2622 insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); 2623 cnt = 2; 2624 2625 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2626 if (!new_prog) 2627 return -ENOMEM; 2628 2629 delta += cnt - 1; 2630 env->prog = prog = new_prog; 2631 insn = new_prog->insnsi + i + delta; 2632 goto next_insn; 2633 } 2634 patch_call_imm: 2635 fn = env->ops->get_func_proto(insn->imm, env->prog); 2636 /* all functions that have prototype and verifier allowed 2637 * programs to call them, must be real in-kernel functions 2638 */ 2639 if (!fn->func) { 2640 verifier_bug(env, 2641 "not inlined functions %s#%d is missing func", 2642 func_id_name(insn->imm), insn->imm); 2643 return -EFAULT; 2644 } 2645 insn->imm = BPF_CALL_IMM(fn->func); 2646 next_insn: 2647 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2648 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2649 subprogs[cur_subprog].stack_extra = stack_depth_extra; 2650 2651 stack_depth = subprogs[cur_subprog].stack_depth; 2652 if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) { 2653 verbose(env, "stack size %d(extra %d) is too large\n", 2654 stack_depth, stack_depth_extra); 2655 return -EINVAL; 2656 } 2657 cur_subprog++; 2658 stack_depth = subprogs[cur_subprog].stack_depth; 2659 stack_depth_extra = 0; 2660 } 2661 i++; 2662 insn++; 2663 } 2664 2665 env->prog->aux->stack_depth = subprogs[0].stack_depth; 2666 for (i = 0; i < env->subprog_cnt; i++) { 2667 int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1; 2668 int subprog_start = subprogs[i].start; 2669 int stack_slots = subprogs[i].stack_extra / 8; 2670 int slots = delta, cnt = 0; 2671 2672 if (!stack_slots) 2673 continue; 2674 /* We need two slots in case timed may_goto is supported. */ 2675 if (stack_slots > slots) { 2676 verifier_bug(env, "stack_slots supports may_goto only"); 2677 return -EFAULT; 2678 } 2679 2680 stack_depth = subprogs[i].stack_depth; 2681 if (bpf_jit_supports_timed_may_goto()) { 2682 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2683 BPF_MAX_TIMED_LOOPS); 2684 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0); 2685 } else { 2686 /* Add ST insn to subprog prologue to init extra stack */ 2687 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2688 BPF_MAX_LOOPS); 2689 } 2690 /* Copy first actual insn to preserve it */ 2691 insn_buf[cnt] = env->prog->insnsi[subprog_start]; 2692 ret = bpf_adj_moved_insn(&insn_buf[cnt], cnt); 2693 if (ret) { 2694 verbose(env, "insn %d cannot be patched due to 16-bit range\n", 2695 env->insn_aux_data[subprog_start].orig_idx); 2696 return ret; 2697 } 2698 cnt++; 2699 2700 new_prog = __bpf_patch_insn_data(env, subprog_start, insn_buf, cnt, 2701 BPF_PATCH_MOVE_TARGET); 2702 if (!new_prog) 2703 return -ENOMEM; 2704 env->prog = prog = new_prog; 2705 /* 2706 * If may_goto is a first insn of a prog there could be a jmp 2707 * insn that points to it, hence adjust all such jmps to point 2708 * to insn after BPF_ST that inits may_goto count. 2709 * Adjustment will succeed because bpf_patch_insn_data() didn't fail. 2710 */ 2711 WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta)); 2712 } 2713 2714 /* Since poke tab is now finalized, publish aux to tracker. */ 2715 for (i = 0; i < prog->aux->size_poke_tab; i++) { 2716 map_ptr = prog->aux->poke_tab[i].tail_call.map; 2717 if (!map_ptr->ops->map_poke_track || 2718 !map_ptr->ops->map_poke_untrack || 2719 !map_ptr->ops->map_poke_run) { 2720 verifier_bug(env, "poke tab is misconfigured"); 2721 return -EFAULT; 2722 } 2723 2724 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 2725 if (ret < 0) { 2726 verbose(env, "tracking tail call prog failed\n"); 2727 return ret; 2728 } 2729 } 2730 2731 return 0; 2732 } 2733 2734 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 2735 int position, 2736 s32 stack_base, 2737 u32 callback_subprogno, 2738 u32 *total_cnt) 2739 { 2740 s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; 2741 s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; 2742 s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; 2743 int reg_loop_max = BPF_REG_6; 2744 int reg_loop_cnt = BPF_REG_7; 2745 int reg_loop_ctx = BPF_REG_8; 2746 2747 struct bpf_insn *insn_buf = env->insn_buf; 2748 struct bpf_prog *new_prog; 2749 u32 callback_start; 2750 u32 call_insn_offset; 2751 s32 callback_offset; 2752 u32 cnt = 0; 2753 2754 /* This represents an inlined version of bpf_iter.c:bpf_loop, 2755 * be careful to modify this code in sync. 2756 */ 2757 2758 /* Return error and jump to the end of the patch if 2759 * expected number of iterations is too big. 2760 */ 2761 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2); 2762 insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG); 2763 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16); 2764 /* spill R6, R7, R8 to use these as loop vars */ 2765 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset); 2766 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset); 2767 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset); 2768 /* initialize loop vars */ 2769 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1); 2770 insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0); 2771 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3); 2772 /* loop header, 2773 * if reg_loop_cnt >= reg_loop_max skip the loop body 2774 */ 2775 insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5); 2776 /* callback call, 2777 * correct callback offset would be set after patching 2778 */ 2779 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt); 2780 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx); 2781 insn_buf[cnt++] = BPF_CALL_REL(0); 2782 /* increment loop counter */ 2783 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1); 2784 /* jump to loop header if callback returned 0 */ 2785 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6); 2786 /* return value of bpf_loop, 2787 * set R0 to the number of iterations 2788 */ 2789 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt); 2790 /* restore original values of R6, R7, R8 */ 2791 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset); 2792 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset); 2793 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset); 2794 2795 *total_cnt = cnt; 2796 new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); 2797 if (!new_prog) 2798 return new_prog; 2799 2800 /* callback start is known only after patching */ 2801 callback_start = env->subprog_info[callback_subprogno].start; 2802 /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ 2803 call_insn_offset = position + 12; 2804 callback_offset = callback_start - call_insn_offset - 1; 2805 new_prog->insnsi[call_insn_offset].imm = callback_offset; 2806 2807 return new_prog; 2808 } 2809 2810 static bool is_bpf_loop_call(struct bpf_insn *insn) 2811 { 2812 return insn->code == (BPF_JMP | BPF_CALL) && 2813 insn->src_reg == 0 && 2814 insn->imm == BPF_FUNC_loop; 2815 } 2816 2817 /* For all sub-programs in the program (including main) check 2818 * insn_aux_data to see if there are bpf_loop calls that require 2819 * inlining. If such calls are found the calls are replaced with a 2820 * sequence of instructions produced by `inline_bpf_loop` function and 2821 * subprog stack_depth is increased by the size of 3 registers. 2822 * This stack space is used to spill values of the R6, R7, R8. These 2823 * registers are used to store the loop bound, counter and context 2824 * variables. 2825 */ 2826 int bpf_optimize_bpf_loop(struct bpf_verifier_env *env) 2827 { 2828 struct bpf_subprog_info *subprogs = env->subprog_info; 2829 int i, cur_subprog = 0, cnt, delta = 0; 2830 struct bpf_insn *insn = env->prog->insnsi; 2831 int insn_cnt = env->prog->len; 2832 u16 stack_depth = subprogs[cur_subprog].stack_depth; 2833 u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2834 u16 stack_depth_extra = 0; 2835 2836 for (i = 0; i < insn_cnt; i++, insn++) { 2837 struct bpf_loop_inline_state *inline_state = 2838 &env->insn_aux_data[i + delta].loop_inline_state; 2839 2840 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { 2841 struct bpf_prog *new_prog; 2842 2843 stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; 2844 new_prog = inline_bpf_loop(env, 2845 i + delta, 2846 -(stack_depth + stack_depth_extra), 2847 inline_state->callback_subprogno, 2848 &cnt); 2849 if (!new_prog) 2850 return -ENOMEM; 2851 2852 delta += cnt - 1; 2853 env->prog = new_prog; 2854 insn = new_prog->insnsi + i + delta; 2855 } 2856 2857 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2858 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2859 cur_subprog++; 2860 stack_depth = subprogs[cur_subprog].stack_depth; 2861 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2862 stack_depth_extra = 0; 2863 } 2864 } 2865 2866 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 2867 2868 return 0; 2869 } 2870 2871 /* Remove unnecessary spill/fill pairs, members of fastcall pattern, 2872 * adjust subprograms stack depth when possible. 2873 */ 2874 int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) 2875 { 2876 struct bpf_subprog_info *subprog = env->subprog_info; 2877 struct bpf_insn_aux_data *aux = env->insn_aux_data; 2878 struct bpf_insn *insn = env->prog->insnsi; 2879 int insn_cnt = env->prog->len; 2880 u32 spills_num; 2881 bool modified = false; 2882 int i, j; 2883 2884 for (i = 0; i < insn_cnt; i++, insn++) { 2885 if (aux[i].fastcall_spills_num > 0) { 2886 spills_num = aux[i].fastcall_spills_num; 2887 /* NOPs would be removed by opt_remove_nops() */ 2888 for (j = 1; j <= spills_num; ++j) { 2889 *(insn - j) = NOP; 2890 *(insn + j) = NOP; 2891 } 2892 modified = true; 2893 } 2894 if ((subprog + 1)->start == i + 1) { 2895 if (modified && !subprog->keep_fastcall_stack) 2896 subprog->stack_depth = -subprog->fastcall_stack_off; 2897 subprog++; 2898 modified = false; 2899 } 2900 } 2901 2902 return 0; 2903 } 2904