1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/bpf.h> 4 #include <linux/btf.h> 5 #include <linux/bpf_verifier.h> 6 #include <linux/filter.h> 7 #include <linux/vmalloc.h> 8 #include <linux/bsearch.h> 9 #include <linux/sort.h> 10 #include <linux/perf_event.h> 11 #include <net/xdp.h> 12 #include "disasm.h" 13 14 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) 15 16 /* 17 * Matches BPF_PROBE_ATOMIC too: bpf_convert_ctx_accesses() rewrites arena 18 * atomics before bpf_opt_subreg_zext_lo32_rnd_hi32() runs. 19 */ 20 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 21 { 22 return BPF_CLASS(insn->code) == BPF_STX && 23 (BPF_MODE(insn->code) == BPF_ATOMIC || 24 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && 25 insn->imm == BPF_CMPXCHG; 26 } 27 28 /* Returns true if 'insn' is an address space cast instruction translated as BPF_ALU op */ 29 static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *insn) 30 { 31 struct bpf_map *arena = (struct bpf_map *)prog->aux->arena; 32 33 if (insn->code != (BPF_ALU64 | BPF_MOV | BPF_X) || insn->off != BPF_ADDR_SPACE_CAST) 34 return false; 35 36 /* cast from as(1) to as(0) */ 37 if (insn->imm == 1) 38 return true; 39 40 /* cast from as(0) to as(1) */ 41 if (insn->imm == 1 << 16) 42 return arena && arena->map_flags & BPF_F_NO_USER_CONV; 43 44 /* non-BPF_F_NO_USER_CONV cast from as(0) to as(1) should be handled by JIT */ 45 return false; 46 } 47 48 /* Return the regno defined by the insn, or -1. */ 49 static int insn_def_regno(const struct bpf_insn *insn) 50 { 51 switch (BPF_CLASS(insn->code)) { 52 case BPF_JMP: 53 case BPF_JMP32: 54 case BPF_ST: 55 return -1; 56 case BPF_STX: 57 return bpf_atomic_load_reg(insn); 58 default: 59 return insn->dst_reg; 60 } 61 } 62 63 /* 64 * For use only in combination with insn_def_regno() >= 0. 65 * Returns TRUE if the destination register operates on 64-bit, 66 * otherwise return FALSE. 67 */ 68 static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn) 69 { 70 u8 class = BPF_CLASS(insn->code); 71 u8 mode = BPF_MODE(insn->code); 72 u8 size = BPF_SIZE(insn->code); 73 u8 op = BPF_OP(insn->code); 74 bool mode_mem; 75 76 /* subregister endiness swap */ 77 if ((class == BPF_ALU || class == BPF_ALU64) && op == BPF_END && insn->imm != 64) 78 return false; 79 80 /* w0 += 1 */ 81 if (class == BPF_ALU && op != BPF_END) 82 return false; 83 84 /* address space casts converted to BPF_ALU, see bpf_do_misc_fixups() */ 85 if (is_addr_space_cast32(prog, insn)) 86 return false; 87 88 /* non 64-bit, non signed extended loads */ 89 mode_mem = mode == BPF_MEM || mode == BPF_PROBE_MEM || mode == BPF_PROBE_MEM32; 90 if (class == BPF_LDX && mode_mem && size != BPF_DW) 91 return false; 92 93 /* atomics, see insn_def_regno() */ 94 if (class == BPF_STX && size != BPF_DW) 95 return false; 96 97 /* both LD_IND and LD_ABS return 32-bit data. */ 98 if (class == BPF_LD && (mode == BPF_IND || mode == BPF_ABS)) 99 return false; 100 101 /* Conservatively return true at default. */ 102 return true; 103 } 104 105 /* 106 * Return the 32-bit subregister defined by INSN, or -1 if INSN does not 107 * explicitly define a 32-bit value. 108 */ 109 int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) 110 { 111 int dst_reg = insn_def_regno(insn); 112 113 if (dst_reg < 0 || bpf_is_reg64(prog, insn)) 114 return -1; 115 116 return dst_reg; 117 } 118 119 static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) 120 { 121 const struct bpf_kfunc_desc *d0 = a; 122 const struct bpf_kfunc_desc *d1 = b; 123 124 if (d0->imm != d1->imm) 125 return d0->imm < d1->imm ? -1 : 1; 126 if (d0->offset != d1->offset) 127 return d0->offset < d1->offset ? -1 : 1; 128 return 0; 129 } 130 131 const struct btf_func_model * 132 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 133 const struct bpf_insn *insn) 134 { 135 const struct bpf_kfunc_desc desc = { 136 .imm = insn->imm, 137 .offset = insn->off, 138 }; 139 const struct bpf_kfunc_desc *res; 140 struct bpf_kfunc_desc_tab *tab; 141 142 tab = prog->aux->kfunc_tab; 143 res = bsearch(&desc, tab->descs, tab->nr_descs, 144 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off); 145 146 return res ? &res->func_model : NULL; 147 } 148 149 static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc) 150 { 151 unsigned long call_imm; 152 153 if (bpf_jit_supports_far_kfunc_call()) { 154 call_imm = desc->func_id; 155 } else { 156 call_imm = BPF_CALL_IMM(desc->addr); 157 /* Check whether the relative offset overflows desc->imm */ 158 if ((unsigned long)(s32)call_imm != call_imm) { 159 verbose(env, "address of kernel func_id %u is out of range\n", 160 desc->func_id); 161 return -EINVAL; 162 } 163 } 164 desc->imm = call_imm; 165 return 0; 166 } 167 168 static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env) 169 { 170 struct bpf_kfunc_desc_tab *tab; 171 int i, err; 172 173 tab = env->prog->aux->kfunc_tab; 174 if (!tab) 175 return 0; 176 177 for (i = 0; i < tab->nr_descs; i++) { 178 err = set_kfunc_desc_imm(env, &tab->descs[i]); 179 if (err) 180 return err; 181 } 182 183 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 184 kfunc_desc_cmp_by_imm_off, NULL); 185 return 0; 186 } 187 188 static int add_kfunc_in_insns(struct bpf_verifier_env *env, 189 struct bpf_insn *insn, int cnt) 190 { 191 int i, ret; 192 193 for (i = 0; i < cnt; i++, insn++) { 194 if (bpf_pseudo_kfunc_call(insn)) { 195 ret = bpf_add_kfunc_call(env, insn->imm, insn->off); 196 if (ret < 0) 197 return ret; 198 } 199 } 200 return 0; 201 } 202 203 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 204 static int get_callee_stack_depth(struct bpf_verifier_env *env, 205 const struct bpf_insn *insn, int idx) 206 { 207 int start = idx + insn->imm + 1, subprog; 208 209 subprog = bpf_find_subprog(env, start); 210 if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start)) 211 return -EFAULT; 212 return env->subprog_info[subprog].stack_depth; 213 } 214 #endif 215 216 /* single env->prog->insni[off] instruction was replaced with the range 217 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 218 * [0, off) and [off, end) to new locations, so the patched range stays zero 219 */ 220 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 221 struct bpf_prog *new_prog, u32 off, u32 cnt) 222 { 223 struct bpf_insn_aux_data *data = env->insn_aux_data; 224 struct bpf_insn *insn = new_prog->insnsi; 225 u32 old_seen = data[off].seen; 226 u32 prog_len; 227 int i; 228 229 /* aux info at OFF always needs adjustment, no matter fast path 230 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 231 * original insn at old prog. 232 */ 233 data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; 234 235 if (cnt == 1) 236 return; 237 prog_len = new_prog->len; 238 env->insn_aux_data_len = prog_len; 239 240 memmove(data + off + cnt - 1, data + off, 241 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 242 memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1)); 243 for (i = off; i < off + cnt - 1; i++) { 244 /* Expand insni[off]'s seen count to the patched range. */ 245 data[i].seen = old_seen; 246 data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; 247 } 248 249 /* 250 * The indirect_target flag of the original instruction was moved to the last of the 251 * new instructions by the above memmove and memset, but the indirect jump target is 252 * actually the first instruction, so move it back. This also matches with the behavior 253 * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new 254 * instruction. 255 */ 256 if (data[off + cnt - 1].indirect_target) { 257 data[off].indirect_target = 1; 258 data[off + cnt - 1].indirect_target = 0; 259 } 260 } 261 262 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 263 { 264 int i; 265 266 if (len == 1) 267 return; 268 /* NOTE: fake 'exit' subprog should be updated as well. */ 269 for (i = 0; i <= env->subprog_cnt; i++) { 270 if (env->subprog_info[i].start <= off) 271 continue; 272 env->subprog_info[i].start += len - 1; 273 } 274 } 275 276 static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) 277 { 278 int i; 279 280 if (len == 1) 281 return; 282 283 for (i = 0; i < env->insn_array_map_cnt; i++) 284 bpf_insn_array_adjust(env->insn_array_maps[i], off, len); 285 } 286 287 static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) 288 { 289 int i; 290 291 for (i = 0; i < env->insn_array_map_cnt; i++) 292 bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len); 293 } 294 295 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) 296 { 297 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 298 int i, sz = prog->aux->size_poke_tab; 299 struct bpf_jit_poke_descriptor *desc; 300 301 for (i = 0; i < sz; i++) { 302 desc = &tab[i]; 303 if (desc->insn_idx <= off) 304 continue; 305 desc->insn_idx += len - 1; 306 } 307 } 308 309 struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 310 const struct bpf_insn *patch, u32 len) 311 { 312 struct bpf_prog *new_prog; 313 struct bpf_insn_aux_data *new_data = NULL; 314 315 if (len > 1) { 316 new_data = vrealloc(env->insn_aux_data, 317 array_size(env->prog->len + len - 1, 318 sizeof(struct bpf_insn_aux_data)), 319 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 320 if (!new_data) 321 return NULL; 322 323 env->insn_aux_data = new_data; 324 } 325 326 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 327 if (IS_ERR(new_prog)) { 328 if (PTR_ERR(new_prog) == -ERANGE) 329 verbose(env, 330 "insn %d cannot be patched due to 16-bit range\n", 331 env->insn_aux_data[off].orig_idx); 332 return NULL; 333 } 334 adjust_insn_aux_data(env, new_prog, off, len); 335 adjust_subprog_starts(env, off, len); 336 adjust_insn_arrays(env, off, len); 337 adjust_poke_descs(new_prog, off, len); 338 return new_prog; 339 } 340 341 /* 342 * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the 343 * jump offset by 'delta'. 344 */ 345 static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta) 346 { 347 struct bpf_insn *insn = prog->insnsi; 348 u32 insn_cnt = prog->len, i; 349 s32 imm; 350 s16 off; 351 352 for (i = 0; i < insn_cnt; i++, insn++) { 353 u8 code = insn->code; 354 355 if (tgt_idx <= i && i < tgt_idx + delta) 356 continue; 357 358 if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) || 359 BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT) 360 continue; 361 362 if (insn->code == (BPF_JMP32 | BPF_JA)) { 363 if (i + 1 + insn->imm != tgt_idx) 364 continue; 365 if (check_add_overflow(insn->imm, delta, &imm)) 366 return -ERANGE; 367 insn->imm = imm; 368 } else { 369 if (i + 1 + insn->off != tgt_idx) 370 continue; 371 if (check_add_overflow(insn->off, delta, &off)) 372 return -ERANGE; 373 insn->off = off; 374 } 375 } 376 return 0; 377 } 378 379 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 380 u32 off, u32 cnt) 381 { 382 int i, j; 383 384 /* find first prog starting at or after off (first to remove) */ 385 for (i = 0; i < env->subprog_cnt; i++) 386 if (env->subprog_info[i].start >= off) 387 break; 388 /* find first prog starting at or after off + cnt (first to stay) */ 389 for (j = i; j < env->subprog_cnt; j++) 390 if (env->subprog_info[j].start >= off + cnt) 391 break; 392 /* if j doesn't start exactly at off + cnt, we are just removing 393 * the front of previous prog 394 */ 395 if (env->subprog_info[j].start != off + cnt) 396 j--; 397 398 if (j > i) { 399 struct bpf_prog_aux *aux = env->prog->aux; 400 int move; 401 402 /* move fake 'exit' subprog as well */ 403 move = env->subprog_cnt + 1 - j; 404 405 memmove(env->subprog_info + i, 406 env->subprog_info + j, 407 sizeof(*env->subprog_info) * move); 408 env->subprog_cnt -= j - i; 409 410 /* remove func_info and its aux */ 411 if (aux->func_info) { 412 move = aux->func_info_cnt - j; 413 414 memmove(aux->func_info + i, 415 aux->func_info + j, 416 sizeof(*aux->func_info) * move); 417 if (aux->func_info_aux) 418 memmove(aux->func_info_aux + i, 419 aux->func_info_aux + j, 420 sizeof(*aux->func_info_aux) * move); 421 aux->func_info_cnt -= j - i; 422 /* func_info->insn_off is set after all code rewrites, 423 * in adjust_btf_func() - no need to adjust 424 */ 425 } 426 } else { 427 /* convert i from "first prog to remove" to "first to adjust" */ 428 if (env->subprog_info[i].start == off) 429 i++; 430 } 431 432 /* update fake 'exit' subprog as well */ 433 for (; i <= env->subprog_cnt; i++) 434 env->subprog_info[i].start -= cnt; 435 436 return 0; 437 } 438 439 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 440 u32 cnt) 441 { 442 struct bpf_prog *prog = env->prog; 443 u32 i, l_off, l_cnt, nr_linfo; 444 struct bpf_line_info *linfo; 445 446 nr_linfo = prog->aux->nr_linfo; 447 if (!nr_linfo) 448 return 0; 449 450 linfo = prog->aux->linfo; 451 452 /* find first line info to remove, count lines to be removed */ 453 for (i = 0; i < nr_linfo; i++) 454 if (linfo[i].insn_off >= off) 455 break; 456 457 l_off = i; 458 l_cnt = 0; 459 for (; i < nr_linfo; i++) 460 if (linfo[i].insn_off < off + cnt) 461 l_cnt++; 462 else 463 break; 464 465 /* First live insn doesn't match first live linfo, it needs to "inherit" 466 * last removed linfo. prog is already modified, so prog->len == off 467 * means no live instructions after (tail of the program was removed). 468 */ 469 if (prog->len != off && l_cnt && 470 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 471 l_cnt--; 472 linfo[--i].insn_off = off + cnt; 473 } 474 475 /* remove the line info which refer to the removed instructions */ 476 if (l_cnt) { 477 memmove(linfo + l_off, linfo + i, 478 sizeof(*linfo) * (nr_linfo - i)); 479 480 prog->aux->nr_linfo -= l_cnt; 481 nr_linfo = prog->aux->nr_linfo; 482 } 483 484 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 485 for (i = l_off; i < nr_linfo; i++) 486 linfo[i].insn_off -= cnt; 487 488 /* fix up all subprogs (incl. 'exit') which start >= off */ 489 for (i = 0; i <= env->subprog_cnt; i++) 490 if (env->subprog_info[i].linfo_idx > l_off) { 491 /* program may have started in the removed region but 492 * may not be fully removed 493 */ 494 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 495 env->subprog_info[i].linfo_idx -= l_cnt; 496 else 497 env->subprog_info[i].linfo_idx = l_off; 498 } 499 500 return 0; 501 } 502 503 /* 504 * Clean up dynamically allocated fields of aux data for instructions [start, ...] 505 */ 506 void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) 507 { 508 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 509 int end = start + len; 510 int i; 511 512 for (i = start; i < end; i++) { 513 if (aux_data[i].jt) { 514 kvfree(aux_data[i].jt); 515 aux_data[i].jt = NULL; 516 } 517 } 518 } 519 520 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 521 { 522 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 523 unsigned int orig_prog_len = env->prog->len; 524 int err; 525 526 if (bpf_prog_is_offloaded(env->prog->aux)) 527 bpf_prog_offload_remove_insns(env, off, cnt); 528 529 bpf_clear_insn_aux_data(env, off, cnt); 530 531 err = bpf_remove_insns(env->prog, off, cnt); 532 if (err) 533 return err; 534 535 err = adjust_subprog_starts_after_remove(env, off, cnt); 536 if (err) 537 return err; 538 539 err = bpf_adj_linfo_after_remove(env, off, cnt); 540 if (err) 541 return err; 542 543 adjust_insn_arrays_after_remove(env, off, cnt); 544 545 memmove(aux_data + off, aux_data + off + cnt, 546 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 547 env->insn_aux_data_len -= cnt; 548 549 return 0; 550 } 551 552 static const struct bpf_insn NOP = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 553 static const struct bpf_insn MAY_GOTO_0 = BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0, 0); 554 555 bool bpf_insn_is_cond_jump(u8 code) 556 { 557 u8 op; 558 559 op = BPF_OP(code); 560 if (BPF_CLASS(code) == BPF_JMP32) 561 return op != BPF_JA; 562 563 if (BPF_CLASS(code) != BPF_JMP) 564 return false; 565 566 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 567 } 568 569 void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 570 { 571 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 572 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 573 struct bpf_insn *insn = env->prog->insnsi; 574 const int insn_cnt = env->prog->len; 575 int i; 576 577 for (i = 0; i < insn_cnt; i++, insn++) { 578 if (!bpf_insn_is_cond_jump(insn->code)) 579 continue; 580 581 if (!aux_data[i + 1].seen) 582 ja.off = insn->off; 583 else if (!aux_data[i + 1 + insn->off].seen) 584 ja.off = 0; 585 else 586 continue; 587 588 if (bpf_prog_is_offloaded(env->prog->aux)) 589 bpf_prog_offload_replace_insn(env, i, &ja); 590 591 memcpy(insn, &ja, sizeof(ja)); 592 } 593 } 594 595 int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) 596 { 597 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 598 int insn_cnt = env->prog->len; 599 int i, err; 600 601 for (i = 0; i < insn_cnt; i++) { 602 int j; 603 604 j = 0; 605 while (i + j < insn_cnt && !aux_data[i + j].seen) 606 j++; 607 if (!j) 608 continue; 609 610 err = verifier_remove_insns(env, i, j); 611 if (err) 612 return err; 613 insn_cnt = env->prog->len; 614 } 615 616 return 0; 617 } 618 619 int bpf_opt_remove_nops(struct bpf_verifier_env *env) 620 { 621 struct bpf_insn *insn = env->prog->insnsi; 622 int insn_cnt = env->prog->len; 623 bool is_may_goto_0, is_ja; 624 int i, err; 625 626 for (i = 0; i < insn_cnt; i++) { 627 is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0)); 628 is_ja = !memcmp(&insn[i], &NOP, sizeof(NOP)); 629 630 if (!is_may_goto_0 && !is_ja) 631 continue; 632 633 err = verifier_remove_insns(env, i, 1); 634 if (err) 635 return err; 636 insn_cnt--; 637 /* Go back one insn to catch may_goto +1; may_goto +0 sequence */ 638 i -= (is_may_goto_0 && i > 0) ? 2 : 1; 639 } 640 641 return 0; 642 } 643 644 int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 645 const union bpf_attr *attr) 646 { 647 struct bpf_insn *patch; 648 /* use env->insn_buf as two independent buffers */ 649 struct bpf_insn *zext_patch = env->insn_buf; 650 struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2]; 651 struct bpf_insn_aux_data *aux = env->insn_aux_data; 652 int i, patch_len, delta = 0, len = env->prog->len; 653 struct bpf_insn *insns = env->prog->insnsi; 654 struct bpf_prog *new_prog; 655 bool rnd_hi32; 656 657 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 658 zext_patch[1] = BPF_ZEXT_REG(0); 659 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 660 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 661 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 662 for (i = 0; i < len; i++) { 663 int adj_idx = i + delta; 664 struct bpf_insn insn; 665 int load_reg; 666 667 insn = insns[adj_idx]; 668 load_reg = insn_def_regno(&insn); 669 if (!aux[adj_idx].zext_dst) { 670 u8 code, class; 671 u32 imm_rnd; 672 673 if (!rnd_hi32) 674 continue; 675 676 code = insn.code; 677 class = BPF_CLASS(code); 678 if (load_reg == -1) 679 continue; 680 681 if (bpf_is_reg64(env->prog, &insn)) { 682 if (class == BPF_LD && 683 BPF_MODE(code) == BPF_IMM) 684 i++; 685 continue; 686 } 687 688 /* ctx load could be transformed into wider load. */ 689 if (class == BPF_LDX && 690 aux[adj_idx].ptr_type == PTR_TO_CTX) 691 continue; 692 693 imm_rnd = get_random_u32(); 694 rnd_hi32_patch[0] = insn; 695 rnd_hi32_patch[1].imm = imm_rnd; 696 rnd_hi32_patch[3].dst_reg = load_reg; 697 patch = rnd_hi32_patch; 698 patch_len = 4; 699 goto apply_patch_buffer; 700 } 701 702 /* Add in an zero-extend instruction if a) the JIT has requested 703 * it or b) it's a CMPXCHG. 704 * 705 * The latter is because: BPF_CMPXCHG always loads a value into 706 * R0, therefore always zero-extends. However some archs' 707 * equivalent instruction only does this load when the 708 * comparison is successful. This detail of CMPXCHG is 709 * orthogonal to the general zero-extension behaviour of the 710 * CPU, so it's treated independently of bpf_jit_needs_zext. 711 */ 712 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 713 continue; 714 715 /* Zero-extension is done by the caller. */ 716 if (bpf_pseudo_kfunc_call(&insn)) 717 continue; 718 719 if (verifier_bug_if(load_reg == -1, env, 720 "zext_dst is set, but no reg is defined")) 721 return -EFAULT; 722 723 zext_patch[0] = insn; 724 zext_patch[1].dst_reg = load_reg; 725 zext_patch[1].src_reg = load_reg; 726 patch = zext_patch; 727 patch_len = 2; 728 apply_patch_buffer: 729 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 730 if (!new_prog) 731 return -ENOMEM; 732 env->prog = new_prog; 733 insns = new_prog->insnsi; 734 aux = env->insn_aux_data; 735 delta += patch_len - 1; 736 } 737 738 return 0; 739 } 740 741 /* convert load instructions that access fields of a context type into a 742 * sequence of instructions that access fields of the underlying structure: 743 * struct __sk_buff -> struct sk_buff 744 * struct bpf_sock_ops -> struct sock 745 */ 746 int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) 747 { 748 struct bpf_subprog_info *subprogs = env->subprog_info; 749 const struct bpf_verifier_ops *ops = env->ops; 750 int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0; 751 const int insn_cnt = env->prog->len; 752 struct bpf_insn *epilogue_buf = env->epilogue_buf; 753 struct bpf_insn *insn_buf = env->insn_buf; 754 struct bpf_insn *insn; 755 u32 target_size, size_default, off; 756 struct bpf_prog *new_prog; 757 enum bpf_access_type type; 758 bool is_narrower_load; 759 int epilogue_idx = 0; 760 761 if (ops->gen_epilogue) { 762 epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog, 763 -(subprogs[0].stack_depth + 8)); 764 if (epilogue_cnt >= INSN_BUF_SIZE) { 765 verifier_bug(env, "epilogue is too long"); 766 return -EFAULT; 767 } else if (epilogue_cnt) { 768 /* Save the ARG_PTR_TO_CTX for the epilogue to use */ 769 cnt = 0; 770 subprogs[0].stack_depth += 8; 771 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, 772 -subprogs[0].stack_depth); 773 insn_buf[cnt++] = env->prog->insnsi[0]; 774 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 775 if (!new_prog) 776 return -ENOMEM; 777 env->prog = new_prog; 778 delta += cnt - 1; 779 780 ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1); 781 if (ret < 0) 782 return ret; 783 } 784 } 785 786 if (ops->gen_prologue || env->seen_direct_write) { 787 if (!ops->gen_prologue) { 788 verifier_bug(env, "gen_prologue is null"); 789 return -EFAULT; 790 } 791 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 792 env->prog); 793 if (cnt >= INSN_BUF_SIZE) { 794 verifier_bug(env, "prologue is too long"); 795 return -EFAULT; 796 } else if (cnt) { 797 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 798 if (!new_prog) 799 return -ENOMEM; 800 801 env->prog = new_prog; 802 delta += cnt - 1; 803 804 ret = add_kfunc_in_insns(env, insn_buf, cnt - 1); 805 if (ret < 0) 806 return ret; 807 } 808 } 809 810 if (delta) 811 WARN_ON(adjust_jmp_off(env->prog, 0, delta)); 812 813 if (bpf_prog_is_offloaded(env->prog->aux)) 814 return 0; 815 816 insn = env->prog->insnsi + delta; 817 818 for (i = 0; i < insn_cnt; i++, insn++) { 819 bpf_convert_ctx_access_t convert_ctx_access; 820 enum bpf_reg_type ptr_type; 821 u8 mode; 822 823 if (env->insn_aux_data[i + delta].nospec) { 824 WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state); 825 struct bpf_insn *patch = insn_buf; 826 827 *patch++ = BPF_ST_NOSPEC(); 828 *patch++ = *insn; 829 cnt = patch - insn_buf; 830 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 831 if (!new_prog) 832 return -ENOMEM; 833 834 delta += cnt - 1; 835 env->prog = new_prog; 836 insn = new_prog->insnsi + i + delta; 837 /* This can not be easily merged with the 838 * nospec_result-case, because an insn may require a 839 * nospec before and after itself. Therefore also do not 840 * 'continue' here but potentially apply further 841 * patching to insn. *insn should equal patch[1] now. 842 */ 843 } 844 845 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 846 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 847 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 848 insn->code == (BPF_LDX | BPF_MEM | BPF_DW) || 849 insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) || 850 insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) || 851 insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) { 852 type = BPF_READ; 853 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 854 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 855 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 856 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 857 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 858 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 859 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 860 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 861 type = BPF_WRITE; 862 } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) || 863 insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) || 864 insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) || 865 insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) && 866 env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) { 867 insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); 868 env->prog->aux->num_exentries++; 869 continue; 870 } else if (insn->code == (BPF_JMP | BPF_EXIT) && 871 epilogue_cnt && 872 i + delta < subprogs[1].start) { 873 /* Generate epilogue for the main prog */ 874 if (epilogue_idx) { 875 /* jump back to the earlier generated epilogue */ 876 insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1); 877 cnt = 1; 878 } else { 879 memcpy(insn_buf, epilogue_buf, 880 epilogue_cnt * sizeof(*epilogue_buf)); 881 cnt = epilogue_cnt; 882 /* epilogue_idx cannot be 0. It must have at 883 * least one ctx ptr saving insn before the 884 * epilogue. 885 */ 886 epilogue_idx = i + delta; 887 } 888 goto patch_insn_buf; 889 } else { 890 continue; 891 } 892 893 if (type == BPF_WRITE && 894 env->insn_aux_data[i + delta].nospec_result) { 895 /* nospec_result is only used to mitigate Spectre v4 and 896 * to limit verification-time for Spectre v1. 897 */ 898 struct bpf_insn *patch = insn_buf; 899 900 *patch++ = *insn; 901 *patch++ = BPF_ST_NOSPEC(); 902 cnt = patch - insn_buf; 903 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 904 if (!new_prog) 905 return -ENOMEM; 906 907 delta += cnt - 1; 908 env->prog = new_prog; 909 insn = new_prog->insnsi + i + delta; 910 continue; 911 } 912 913 ptr_type = env->insn_aux_data[i + delta].ptr_type; 914 switch ((int)ptr_type) { 915 case PTR_TO_CTX: 916 if (!ops->convert_ctx_access) 917 continue; 918 convert_ctx_access = ops->convert_ctx_access; 919 break; 920 case PTR_TO_SOCKET: 921 case PTR_TO_SOCK_COMMON: 922 convert_ctx_access = bpf_sock_convert_ctx_access; 923 break; 924 case PTR_TO_TCP_SOCK: 925 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 926 break; 927 case PTR_TO_XDP_SOCK: 928 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 929 break; 930 case PTR_TO_ARENA: 931 if (BPF_MODE(insn->code) == BPF_MEMSX) { 932 if (!bpf_jit_supports_insn(insn, true)) { 933 verbose(env, "sign extending loads from arena are not supported yet\n"); 934 return -EOPNOTSUPP; 935 } 936 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code); 937 } else { 938 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code); 939 } 940 env->prog->aux->num_exentries++; 941 continue; 942 default: 943 /* 944 * A pointer which may fault on a dereference must not 945 * be loaded from without fault protection, hence turn 946 * the BPF_LDX into a BPF_PROBE_MEM one so that a bad 947 * address is handled rather than panicking the kernel. 948 * A store through one is rejected earlier, there is no 949 * probed counterpart to rewrite it into. 950 */ 951 if (bpf_is_ptr_to_mem_or_btf_id(ptr_type) && 952 bpf_may_fault_on_deref(ptr_type) && 953 type == BPF_READ) { 954 if (BPF_MODE(insn->code) == BPF_MEM) 955 insn->code = BPF_LDX | BPF_PROBE_MEM | 956 BPF_SIZE(insn->code); 957 else 958 insn->code = BPF_LDX | BPF_PROBE_MEMSX | 959 BPF_SIZE(insn->code); 960 env->prog->aux->num_exentries++; 961 continue; 962 } 963 if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, 964 "access to a fault prone pointer is not rewritten as a probed one")) 965 return -EFAULT; 966 continue; 967 } 968 969 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 970 size = BPF_LDST_BYTES(insn); 971 mode = BPF_MODE(insn->code); 972 973 /* If the read access is a narrower load of the field, 974 * convert to a 4/8-byte load, to minimum program type specific 975 * convert_ctx_access changes. If conversion is successful, 976 * we will apply proper mask to the result. 977 */ 978 is_narrower_load = size < ctx_field_size; 979 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 980 off = insn->off; 981 if (is_narrower_load) { 982 u8 size_code; 983 984 if (type == BPF_WRITE) { 985 verifier_bug(env, "narrow ctx access misconfigured"); 986 return -EFAULT; 987 } 988 989 size_code = BPF_H; 990 if (ctx_field_size == 4) 991 size_code = BPF_W; 992 else if (ctx_field_size == 8) 993 size_code = BPF_DW; 994 995 insn->off = off & ~(size_default - 1); 996 insn->code = BPF_LDX | BPF_MEM | size_code; 997 } 998 999 target_size = 0; 1000 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 1001 &target_size); 1002 if (cnt == 0 || cnt >= INSN_BUF_SIZE || 1003 (ctx_field_size && !target_size)) { 1004 verifier_bug(env, "error during ctx access conversion (%d)", cnt); 1005 return -EFAULT; 1006 } 1007 1008 if (is_narrower_load && size < target_size) { 1009 u8 shift = bpf_ctx_narrow_access_offset( 1010 off, size, size_default) * 8; 1011 if (shift && cnt + 1 >= INSN_BUF_SIZE) { 1012 verifier_bug(env, "narrow ctx load misconfigured"); 1013 return -EFAULT; 1014 } 1015 if (ctx_field_size <= 4) { 1016 if (shift) 1017 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 1018 insn->dst_reg, 1019 shift); 1020 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1021 (1 << size * 8) - 1); 1022 } else { 1023 if (shift) 1024 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 1025 insn->dst_reg, 1026 shift); 1027 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1028 (1ULL << size * 8) - 1); 1029 } 1030 } 1031 if (mode == BPF_MEMSX) 1032 insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, 1033 insn->dst_reg, insn->dst_reg, 1034 size * 8, 0); 1035 1036 patch_insn_buf: 1037 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1038 if (!new_prog) 1039 return -ENOMEM; 1040 1041 delta += cnt - 1; 1042 1043 /* keep walking new program and skip insns we just inserted */ 1044 env->prog = new_prog; 1045 insn = new_prog->insnsi + i + delta; 1046 } 1047 1048 return 0; 1049 } 1050 1051 static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env) 1052 { 1053 u32 *starts = NULL; 1054 1055 starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT); 1056 if (starts) { 1057 for (int i = 0; i < env->subprog_cnt; i++) 1058 starts[i] = env->subprog_info[i].start; 1059 } 1060 return starts; 1061 } 1062 1063 static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts) 1064 { 1065 for (int i = 0; i < env->subprog_cnt; i++) 1066 env->subprog_info[i].start = orig_starts[i]; 1067 /* restore the start of fake 'exit' subprog as well */ 1068 env->subprog_info[env->subprog_cnt].start = env->prog->len; 1069 } 1070 1071 static int jit_subprogs(struct bpf_verifier_env *env) 1072 { 1073 struct bpf_prog *prog = env->prog, **func, *tmp; 1074 int i, j, subprog_start, subprog_end = 0, len, subprog; 1075 struct bpf_map *map_ptr; 1076 struct bpf_insn *insn; 1077 void *old_bpf_func; 1078 int err, num_exentries; 1079 1080 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1081 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 1082 continue; 1083 1084 /* Upon error here we cannot fall back to interpreter but 1085 * need a hard reject of the program. Thus -EFAULT is 1086 * propagated in any case. 1087 */ 1088 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1089 if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d", 1090 i + insn->imm + 1)) 1091 return -EFAULT; 1092 /* temporarily remember subprog id inside insn instead of 1093 * aux_data, since next loop will split up all insns into funcs 1094 */ 1095 insn->off = subprog; 1096 /* remember original imm in case JIT fails and fallback 1097 * to interpreter will be needed 1098 */ 1099 env->insn_aux_data[i].call_imm = insn->imm; 1100 /* point imm to __bpf_call_base+1 from JITs point of view */ 1101 insn->imm = 1; 1102 if (bpf_pseudo_func(insn)) { 1103 #if defined(MODULES_VADDR) 1104 u64 addr = MODULES_VADDR; 1105 #else 1106 u64 addr = VMALLOC_START; 1107 #endif 1108 /* jit (e.g. x86_64) may emit fewer instructions 1109 * if it learns a u32 imm is the same as a u64 imm. 1110 * Set close enough to possible prog address. 1111 */ 1112 insn[0].imm = (u32)addr; 1113 insn[1].imm = addr >> 32; 1114 } 1115 } 1116 1117 err = bpf_prog_alloc_jited_linfo(prog); 1118 if (err) 1119 goto out_undo_insn; 1120 1121 err = -ENOMEM; 1122 func = kzalloc_objs(prog, env->subprog_cnt); 1123 if (!func) 1124 goto out_undo_insn; 1125 1126 for (i = 0; i < env->subprog_cnt; i++) { 1127 subprog_start = subprog_end; 1128 subprog_end = env->subprog_info[i + 1].start; 1129 1130 len = subprog_end - subprog_start; 1131 /* bpf_prog_run() doesn't call subprogs directly, 1132 * hence main prog stats include the runtime of subprogs. 1133 * subprogs don't have IDs and not reachable via prog_get_next_id 1134 * func[i]->stats will never be accessed and stays NULL 1135 */ 1136 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 1137 if (!func[i]) 1138 goto out_free; 1139 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 1140 len * sizeof(struct bpf_insn)); 1141 func[i]->type = prog->type; 1142 func[i]->len = len; 1143 if (bpf_prog_calc_tag(func[i])) 1144 goto out_free; 1145 func[i]->is_func = 1; 1146 func[i]->sleepable = prog->sleepable; 1147 func[i]->blinded = prog->blinded; 1148 func[i]->aux->func_idx = i; 1149 /* Below members will be freed only at prog->aux */ 1150 func[i]->aux->btf = prog->aux->btf; 1151 func[i]->aux->subprog_start = subprog_start; 1152 func[i]->aux->func_info = prog->aux->func_info; 1153 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 1154 func[i]->aux->poke_tab = prog->aux->poke_tab; 1155 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 1156 func[i]->aux->main_prog_aux = prog->aux; 1157 1158 for (j = 0; j < prog->aux->size_poke_tab; j++) { 1159 struct bpf_jit_poke_descriptor *poke; 1160 1161 poke = &prog->aux->poke_tab[j]; 1162 if (poke->insn_idx < subprog_end && 1163 poke->insn_idx >= subprog_start) 1164 poke->aux = func[i]->aux; 1165 } 1166 1167 func[i]->aux->name[0] = 'F'; 1168 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 1169 if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) 1170 func[i]->aux->jits_use_priv_stack = true; 1171 1172 func[i]->jit_requested = 1; 1173 func[i]->blinding_requested = prog->blinding_requested; 1174 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 1175 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 1176 func[i]->aux->linfo = prog->aux->linfo; 1177 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 1178 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 1179 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 1180 func[i]->aux->arena = prog->aux->arena; 1181 func[i]->aux->used_maps = env->used_maps; 1182 func[i]->aux->used_map_cnt = env->used_map_cnt; 1183 num_exentries = 0; 1184 insn = func[i]->insnsi; 1185 for (j = 0; j < func[i]->len; j++, insn++) { 1186 if (BPF_CLASS(insn->code) == BPF_LDX && 1187 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 1188 BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 1189 BPF_MODE(insn->code) == BPF_PROBE_MEM32SX || 1190 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) 1191 num_exentries++; 1192 if ((BPF_CLASS(insn->code) == BPF_STX || 1193 BPF_CLASS(insn->code) == BPF_ST) && 1194 BPF_MODE(insn->code) == BPF_PROBE_MEM32) 1195 num_exentries++; 1196 if (BPF_CLASS(insn->code) == BPF_STX && 1197 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) 1198 num_exentries++; 1199 } 1200 func[i]->aux->num_exentries = num_exentries; 1201 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 1202 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; 1203 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; 1204 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; 1205 func[i]->aux->token = prog->aux->token; 1206 if (!i) 1207 func[i]->aux->exception_boundary = env->seen_exception; 1208 func[i] = bpf_int_jit_compile(env, func[i]); 1209 if (!func[i]->jited) { 1210 err = -ENOTSUPP; 1211 goto out_free; 1212 } 1213 cond_resched(); 1214 } 1215 1216 /* at this point all bpf functions were successfully JITed 1217 * now populate all bpf_calls with correct addresses and 1218 * run last pass of JIT 1219 */ 1220 for (i = 0; i < env->subprog_cnt; i++) { 1221 insn = func[i]->insnsi; 1222 for (j = 0; j < func[i]->len; j++, insn++) { 1223 if (bpf_pseudo_func(insn)) { 1224 subprog = insn->off; 1225 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 1226 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 1227 continue; 1228 } 1229 if (!bpf_pseudo_call(insn)) 1230 continue; 1231 subprog = insn->off; 1232 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 1233 } 1234 1235 /* we use the aux data to keep a list of the start addresses 1236 * of the JITed images for each function in the program 1237 * 1238 * for some architectures, such as powerpc64, the imm field 1239 * might not be large enough to hold the offset of the start 1240 * address of the callee's JITed image from __bpf_call_base 1241 * 1242 * in such cases, we can lookup the start address of a callee 1243 * by using its subprog id, available from the off field of 1244 * the call instruction, as an index for this list 1245 */ 1246 func[i]->aux->func = func; 1247 func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1248 func[i]->aux->real_func_cnt = env->subprog_cnt; 1249 } 1250 for (i = 0; i < env->subprog_cnt; i++) { 1251 old_bpf_func = func[i]->bpf_func; 1252 tmp = bpf_int_jit_compile(env, func[i]); 1253 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 1254 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 1255 err = -ENOTSUPP; 1256 goto out_free; 1257 } 1258 cond_resched(); 1259 } 1260 1261 /* 1262 * Cleanup func[i]->aux fields which aren't required 1263 * or can become invalid in future 1264 */ 1265 for (i = 0; i < env->subprog_cnt; i++) { 1266 func[i]->aux->used_maps = NULL; 1267 func[i]->aux->used_map_cnt = 0; 1268 } 1269 1270 /* finally lock prog and jit images for all functions and 1271 * populate kallsysm. Begin at the first subprogram, since 1272 * bpf_prog_load will add the kallsyms for the main program. 1273 */ 1274 for (i = 1; i < env->subprog_cnt; i++) { 1275 err = bpf_prog_lock_ro(func[i]); 1276 if (err) 1277 goto out_free; 1278 } 1279 1280 for (i = 1; i < env->subprog_cnt; i++) 1281 bpf_prog_kallsyms_add(func[i]); 1282 1283 /* Last step: make now unused interpreter insns from main 1284 * prog consistent for later dump requests, so they can 1285 * later look the same as if they were interpreted only. 1286 */ 1287 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1288 if (bpf_pseudo_func(insn)) { 1289 insn[0].imm = env->insn_aux_data[i].call_imm; 1290 insn[1].imm = insn->off; 1291 insn->off = 0; 1292 continue; 1293 } 1294 if (!bpf_pseudo_call(insn)) 1295 continue; 1296 insn->imm = env->insn_aux_data[i].call_imm; 1297 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1298 insn->off = subprog; 1299 } 1300 1301 prog->jited = 1; 1302 prog->bpf_func = func[0]->bpf_func; 1303 prog->jited_len = func[0]->jited_len; 1304 prog->aux->extable = func[0]->aux->extable; 1305 prog->aux->num_exentries = func[0]->aux->num_exentries; 1306 prog->aux->func = func; 1307 prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1308 prog->aux->real_func_cnt = env->subprog_cnt; 1309 prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; 1310 prog->aux->exception_boundary = func[0]->aux->exception_boundary; 1311 prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust; 1312 bpf_prog_jit_attempt_done(prog); 1313 return 0; 1314 out_free: 1315 /* We failed JIT'ing, so at this point we need to unregister poke 1316 * descriptors from subprogs, so that kernel is not attempting to 1317 * patch it anymore as we're freeing the subprog JIT memory. 1318 */ 1319 for (i = 0; i < prog->aux->size_poke_tab; i++) { 1320 map_ptr = prog->aux->poke_tab[i].tail_call.map; 1321 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 1322 } 1323 /* At this point we're guaranteed that poke descriptors are not 1324 * live anymore. We can just unlink its descriptor table as it's 1325 * released with the main prog. 1326 */ 1327 for (i = 0; i < env->subprog_cnt; i++) { 1328 if (!func[i]) 1329 continue; 1330 func[i]->aux->poke_tab = NULL; 1331 bpf_jit_free(func[i]); 1332 } 1333 kfree(func); 1334 out_undo_insn: 1335 bpf_prog_jit_attempt_done(prog); 1336 return err; 1337 } 1338 1339 int bpf_jit_subprogs(struct bpf_verifier_env *env) 1340 { 1341 int err, i; 1342 bool blinded = false; 1343 struct bpf_insn *insn; 1344 struct bpf_prog *prog, *orig_prog; 1345 u32 *orig_subprog_starts; 1346 1347 if (env->subprog_cnt <= 1) 1348 return 0; 1349 1350 prog = orig_prog = env->prog; 1351 if (bpf_prog_need_blind(prog)) { 1352 orig_subprog_starts = bpf_dup_subprog_starts(env); 1353 if (!orig_subprog_starts) { 1354 err = -ENOMEM; 1355 goto out_cleanup; 1356 } 1357 prog = bpf_jit_blind_constants(env, prog); 1358 if (IS_ERR(prog)) { 1359 err = -ENOMEM; 1360 prog = orig_prog; 1361 goto out_restore; 1362 } 1363 blinded = true; 1364 } 1365 1366 err = jit_subprogs(env); 1367 if (err) 1368 goto out_jit_err; 1369 1370 if (blinded) { 1371 bpf_jit_prog_release_other(prog, orig_prog); 1372 kvfree(orig_subprog_starts); 1373 } 1374 1375 return 0; 1376 1377 out_jit_err: 1378 if (blinded) { 1379 bpf_jit_prog_release_other(orig_prog, prog); 1380 /* roll back to the clean original prog */ 1381 prog = env->prog = orig_prog; 1382 goto out_restore; 1383 } else { 1384 if (err != -EFAULT) { 1385 /* 1386 * We will fall back to interpreter mode when err is not -EFAULT, before 1387 * that, insn->off and insn->imm should be restored to their original 1388 * values since they were modified by jit_subprogs. 1389 */ 1390 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1391 if (!bpf_pseudo_call(insn)) 1392 continue; 1393 insn->off = 0; 1394 insn->imm = env->insn_aux_data[i].call_imm; 1395 } 1396 } 1397 goto out_cleanup; 1398 } 1399 1400 out_restore: 1401 bpf_restore_subprog_starts(env, orig_subprog_starts); 1402 kvfree(orig_subprog_starts); 1403 out_cleanup: 1404 /* cleanup main prog to be interpreted */ 1405 prog->jit_requested = 0; 1406 prog->blinding_requested = 0; 1407 return err; 1408 } 1409 1410 int bpf_fixup_call_args(struct bpf_verifier_env *env) 1411 { 1412 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1413 struct bpf_prog *prog = env->prog; 1414 struct bpf_insn *insn = prog->insnsi; 1415 int depth; 1416 #endif 1417 int i, err = 0; 1418 1419 for (i = 0; i < env->subprog_cnt; i++) { 1420 struct bpf_subprog_info *subprog = &env->subprog_info[i]; 1421 u16 outgoing = subprog->stack_arg_cnt - bpf_in_stack_arg_cnt(subprog); 1422 1423 if (subprog->max_out_stack_arg_cnt > outgoing) { 1424 verbose(env, 1425 "func#%d writes %u stack arg slots, but calls only require %u\n", 1426 i, subprog->max_out_stack_arg_cnt, outgoing); 1427 return -EINVAL; 1428 } 1429 } 1430 1431 if (env->prog->jit_requested && 1432 !bpf_prog_is_offloaded(env->prog->aux)) { 1433 err = bpf_jit_subprogs(env); 1434 if (err == 0) 1435 return 0; 1436 if (err == -EFAULT) 1437 return err; 1438 } 1439 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1440 if (prog->jit_required) { 1441 verbose(env, "program requires BPF JIT compiler but it is not available\n"); 1442 return -EINVAL; 1443 } 1444 for (i = 0; i < env->subprog_cnt; i++) { 1445 if (bpf_in_stack_arg_cnt(&env->subprog_info[i])) { 1446 verbose(env, "stack args are not supported in non-JITed programs\n"); 1447 return -EINVAL; 1448 } 1449 } 1450 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 1451 /* When JIT fails the progs with bpf2bpf calls and tail_calls 1452 * have to be rejected, since interpreter doesn't support them yet. 1453 */ 1454 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 1455 return -EINVAL; 1456 } 1457 for (i = 0; i < prog->len; i++, insn++) { 1458 if (bpf_pseudo_func(insn)) { 1459 /* When JIT fails the progs with callback calls 1460 * have to be rejected, since interpreter doesn't support them yet. 1461 */ 1462 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 1463 return -EINVAL; 1464 } 1465 1466 if (!bpf_pseudo_call(insn)) 1467 continue; 1468 depth = get_callee_stack_depth(env, insn, i); 1469 if (depth < 0) 1470 return depth; 1471 err = bpf_patch_call_args(insn, depth); 1472 if (err) { 1473 verbose(env, "stack depth %d exceeds interpreter stack depth limit\n", 1474 depth); 1475 return err; 1476 } 1477 } 1478 err = 0; 1479 #endif 1480 return err; 1481 } 1482 1483 /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */ 1484 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) 1485 { 1486 struct bpf_subprog_info *info = env->subprog_info; 1487 int cnt = env->subprog_cnt; 1488 struct bpf_prog *prog; 1489 1490 /* We only reserve one slot for hidden subprogs in subprog_info. */ 1491 if (env->hidden_subprog_cnt) { 1492 verifier_bug(env, "only one hidden subprog supported"); 1493 return -EFAULT; 1494 } 1495 /* We're not patching any existing instruction, just appending the new 1496 * ones for the hidden subprog. Hence all of the adjustment operations 1497 * in bpf_patch_insn_data are no-ops. 1498 */ 1499 prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); 1500 if (!prog) 1501 return -ENOMEM; 1502 env->prog = prog; 1503 info[cnt + 1].start = info[cnt].start; 1504 info[cnt].start = prog->len - len + 1; 1505 env->subprog_cnt++; 1506 env->hidden_subprog_cnt++; 1507 return 0; 1508 } 1509 1510 /* Do various post-verification rewrites in a single program pass. 1511 * These rewrites simplify JIT and interpreter implementations. 1512 */ 1513 int bpf_do_misc_fixups(struct bpf_verifier_env *env) 1514 { 1515 struct bpf_prog *prog = env->prog; 1516 enum bpf_attach_type eatype = prog->expected_attach_type; 1517 enum bpf_prog_type prog_type = resolve_prog_type(prog); 1518 struct bpf_insn *insn = prog->insnsi; 1519 const struct bpf_func_proto *fn; 1520 const int insn_cnt = prog->len; 1521 const struct bpf_map_ops *ops; 1522 struct bpf_insn_aux_data *aux; 1523 struct bpf_insn *insn_buf = env->insn_buf; 1524 struct bpf_prog *new_prog; 1525 struct bpf_map *map_ptr; 1526 int i, ret, cnt, delta = 0, cur_subprog = 0; 1527 struct bpf_subprog_info *subprogs = env->subprog_info; 1528 u16 stack_depth = subprogs[cur_subprog].stack_depth; 1529 u16 stack_depth_extra = 0; 1530 1531 if (env->seen_exception && !env->exception_callback_subprog) { 1532 struct bpf_insn *patch = insn_buf; 1533 1534 *patch++ = env->prog->insnsi[insn_cnt - 1]; 1535 *patch++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 1536 *patch++ = BPF_EXIT_INSN(); 1537 ret = add_hidden_subprog(env, insn_buf, patch - insn_buf); 1538 if (ret < 0) 1539 return ret; 1540 prog = env->prog; 1541 insn = prog->insnsi; 1542 1543 env->exception_callback_subprog = env->subprog_cnt - 1; 1544 /* Don't update insn_cnt, as add_hidden_subprog always appends insns */ 1545 bpf_mark_subprog_exc_cb(env, env->exception_callback_subprog); 1546 } 1547 1548 for (i = 0; i < insn_cnt;) { 1549 if (is_addr_space_cast32(env->prog, insn)) { 1550 /* convert to 32-bit mov that clears upper 32-bit */ 1551 insn->code = BPF_ALU | BPF_MOV | BPF_X; 1552 /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ 1553 insn->off = 0; 1554 insn->imm = 0; 1555 goto next_insn; 1556 } 1557 1558 if (env->insn_aux_data[i + delta].needs_zext) 1559 /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ 1560 insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); 1561 1562 /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ 1563 if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || 1564 insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || 1565 insn->code == (BPF_ALU | BPF_MOD | BPF_K) || 1566 insn->code == (BPF_ALU | BPF_DIV | BPF_K)) && 1567 insn->off == 1 && insn->imm == -1) { 1568 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1569 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1570 struct bpf_insn *patch = insn_buf; 1571 1572 if (isdiv) 1573 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1574 BPF_NEG | BPF_K, insn->dst_reg, 1575 0, 0, 0); 1576 else 1577 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1578 1579 cnt = patch - insn_buf; 1580 1581 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1582 if (!new_prog) 1583 return -ENOMEM; 1584 1585 delta += cnt - 1; 1586 env->prog = prog = new_prog; 1587 insn = new_prog->insnsi + i + delta; 1588 goto next_insn; 1589 } 1590 1591 /* Make divide-by-zero and divide-by-minus-one exceptions impossible. */ 1592 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 1593 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 1594 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 1595 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 1596 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1597 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1598 bool is_sdiv = isdiv && insn->off == 1; 1599 bool is_smod = !isdiv && insn->off == 1; 1600 struct bpf_insn *patch = insn_buf; 1601 1602 if (is_sdiv) { 1603 /* [R,W]x sdiv 0 -> 0 1604 * LLONG_MIN sdiv -1 -> LLONG_MIN 1605 * INT_MIN sdiv -1 -> INT_MIN 1606 */ 1607 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1608 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1609 BPF_ADD | BPF_K, BPF_REG_AX, 1610 0, 0, 1); 1611 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1612 BPF_JGT | BPF_K, BPF_REG_AX, 1613 0, 4, 1); 1614 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1615 BPF_JEQ | BPF_K, BPF_REG_AX, 1616 0, 1, 0); 1617 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1618 BPF_MOV | BPF_K, insn->dst_reg, 1619 0, 0, 0); 1620 /* BPF_NEG(LLONG_MIN) == -LLONG_MIN == LLONG_MIN */ 1621 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1622 BPF_NEG | BPF_K, insn->dst_reg, 1623 0, 0, 0); 1624 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1625 *patch++ = *insn; 1626 cnt = patch - insn_buf; 1627 } else if (is_smod) { 1628 /* [R,W]x mod 0 -> [R,W]x */ 1629 /* [R,W]x mod -1 -> 0 */ 1630 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1631 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1632 BPF_ADD | BPF_K, BPF_REG_AX, 1633 0, 0, 1); 1634 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1635 BPF_JGT | BPF_K, BPF_REG_AX, 1636 0, 3, 1); 1637 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1638 BPF_JEQ | BPF_K, BPF_REG_AX, 1639 0, 3 + (is64 ? 0 : 1), 1); 1640 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1641 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1642 *patch++ = *insn; 1643 1644 if (!is64) { 1645 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1646 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 1647 } 1648 cnt = patch - insn_buf; 1649 } else if (isdiv) { 1650 /* [R,W]x div 0 -> 0 */ 1651 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1652 BPF_JNE | BPF_K, insn->src_reg, 1653 0, 2, 0); 1654 *patch++ = BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg); 1655 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1656 *patch++ = *insn; 1657 cnt = patch - insn_buf; 1658 } else { 1659 /* [R,W]x mod 0 -> [R,W]x */ 1660 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1661 BPF_JEQ | BPF_K, insn->src_reg, 1662 0, 1 + (is64 ? 0 : 1), 0); 1663 *patch++ = *insn; 1664 1665 if (!is64) { 1666 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1667 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 1668 } 1669 cnt = patch - insn_buf; 1670 } 1671 1672 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1673 if (!new_prog) 1674 return -ENOMEM; 1675 1676 delta += cnt - 1; 1677 env->prog = prog = new_prog; 1678 insn = new_prog->insnsi + i + delta; 1679 goto next_insn; 1680 } 1681 1682 /* Make it impossible to de-reference a userspace address */ 1683 if (BPF_CLASS(insn->code) == BPF_LDX && 1684 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 1685 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) { 1686 struct bpf_insn *patch = insn_buf; 1687 u64 uaddress_limit = bpf_arch_uaddress_limit(); 1688 1689 if (!uaddress_limit) 1690 goto next_insn; 1691 1692 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1693 if (insn->off) 1694 *patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off); 1695 *patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32); 1696 *patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2); 1697 *patch++ = *insn; 1698 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1699 *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0); 1700 1701 cnt = patch - insn_buf; 1702 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1703 if (!new_prog) 1704 return -ENOMEM; 1705 1706 delta += cnt - 1; 1707 env->prog = prog = new_prog; 1708 insn = new_prog->insnsi + i + delta; 1709 goto next_insn; 1710 } 1711 1712 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 1713 if (BPF_CLASS(insn->code) == BPF_LD && 1714 (BPF_MODE(insn->code) == BPF_ABS || 1715 BPF_MODE(insn->code) == BPF_IND)) { 1716 cnt = env->ops->gen_ld_abs(insn, insn_buf); 1717 if (cnt == 0 || cnt >= INSN_BUF_SIZE) { 1718 verifier_bug(env, "%d insns generated for ld_abs", cnt); 1719 return -EFAULT; 1720 } 1721 1722 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1723 if (!new_prog) 1724 return -ENOMEM; 1725 1726 delta += cnt - 1; 1727 env->prog = prog = new_prog; 1728 insn = new_prog->insnsi + i + delta; 1729 goto next_insn; 1730 } 1731 1732 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 1733 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 1734 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 1735 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 1736 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 1737 struct bpf_insn *patch = insn_buf; 1738 bool issrc, isneg, isimm; 1739 u32 off_reg; 1740 1741 aux = &env->insn_aux_data[i + delta]; 1742 if (!aux->alu_state || 1743 aux->alu_state == BPF_ALU_NON_POINTER) 1744 goto next_insn; 1745 1746 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 1747 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 1748 BPF_ALU_SANITIZE_SRC; 1749 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 1750 1751 off_reg = issrc ? insn->src_reg : insn->dst_reg; 1752 if (isimm) { 1753 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 1754 } else { 1755 if (isneg) 1756 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 1757 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 1758 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 1759 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 1760 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 1761 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 1762 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 1763 } 1764 if (!issrc) 1765 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 1766 insn->src_reg = BPF_REG_AX; 1767 if (isneg) 1768 insn->code = insn->code == code_add ? 1769 code_sub : code_add; 1770 *patch++ = *insn; 1771 if (issrc && isneg && !isimm) 1772 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 1773 cnt = patch - insn_buf; 1774 1775 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1776 if (!new_prog) 1777 return -ENOMEM; 1778 1779 delta += cnt - 1; 1780 env->prog = prog = new_prog; 1781 insn = new_prog->insnsi + i + delta; 1782 goto next_insn; 1783 } 1784 1785 if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { 1786 int stack_off_cnt = -stack_depth - 16; 1787 1788 /* 1789 * Two 8 byte slots, depth-16 stores the count, and 1790 * depth-8 stores the start timestamp of the loop. 1791 * 1792 * The starting value of count is BPF_MAX_TIMED_LOOPS 1793 * (0xffff). Every iteration loads it and subs it by 1, 1794 * until the value becomes 0 in AX (thus, 1 in stack), 1795 * after which we call arch_bpf_timed_may_goto, which 1796 * either sets AX to 0xffff to keep looping, or to 0 1797 * upon timeout. AX is then stored into the stack. In 1798 * the next iteration, we either see 0 and break out, or 1799 * continue iterating until the next time value is 0 1800 * after subtraction, rinse and repeat. 1801 */ 1802 stack_depth_extra = 16; 1803 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt); 1804 if (insn->off >= 0) 1805 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5); 1806 else 1807 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); 1808 insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); 1809 insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); 1810 /* 1811 * AX is used as an argument to pass in stack_off_cnt 1812 * (to add to r10/fp), and also as the return value of 1813 * the call to arch_bpf_timed_may_goto. 1814 */ 1815 insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt); 1816 insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); 1817 insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); 1818 cnt = 7; 1819 1820 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1821 if (!new_prog) 1822 return -ENOMEM; 1823 1824 delta += cnt - 1; 1825 env->prog = prog = new_prog; 1826 insn = new_prog->insnsi + i + delta; 1827 goto next_insn; 1828 } else if (bpf_is_may_goto_insn(insn)) { 1829 int stack_off = -stack_depth - 8; 1830 1831 stack_depth_extra = 8; 1832 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); 1833 if (insn->off >= 0) 1834 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); 1835 else 1836 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); 1837 insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); 1838 insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); 1839 cnt = 4; 1840 1841 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1842 if (!new_prog) 1843 return -ENOMEM; 1844 1845 delta += cnt - 1; 1846 env->prog = prog = new_prog; 1847 insn = new_prog->insnsi + i + delta; 1848 goto next_insn; 1849 } 1850 1851 if (bpf_jit_supports_percpu_insn() && 1852 insn->code == (BPF_LD | BPF_IMM | BPF_DW) && 1853 (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 1854 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) { 1855 struct bpf_map *map; 1856 1857 aux = &env->insn_aux_data[i + delta]; 1858 map = env->used_maps[aux->map_index]; 1859 if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) 1860 goto next_insn; 1861 1862 prog->jit_required = true; 1863 1864 /* 1865 * We are *skipping* first half of ld_imm64 insn 1866 * with 'i++;', patching over second half of it 1867 * with that same half + mov64_percpu_reg insn. 1868 * All because bpf_patch_insn_data() can only 1869 * replace one 8-byte insn, which does not work 1870 * well for ld_imm64 insn. 1871 */ 1872 1873 insn_buf[0] = insn[1]; 1874 insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg); 1875 cnt = 2; 1876 1877 i++; 1878 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1879 if (!new_prog) 1880 return -ENOMEM; 1881 1882 delta += cnt - 1; 1883 env->prog = prog = new_prog; 1884 insn = new_prog->insnsi + i + delta; 1885 goto next_insn; 1886 } 1887 1888 if (insn->code != (BPF_JMP | BPF_CALL)) 1889 goto next_insn; 1890 if (insn->src_reg == BPF_PSEUDO_CALL) 1891 goto next_insn; 1892 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1893 ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); 1894 if (ret) 1895 return ret; 1896 if (cnt == 0) 1897 goto next_insn; 1898 1899 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1900 if (!new_prog) 1901 return -ENOMEM; 1902 1903 delta += cnt - 1; 1904 env->prog = prog = new_prog; 1905 insn = new_prog->insnsi + i + delta; 1906 goto next_insn; 1907 } 1908 1909 /* Skip inlining the helper call if the JIT does it. */ 1910 if (bpf_jit_inlines_helper_call(insn->imm)) { 1911 prog->jit_required = 1; 1912 goto next_insn; 1913 } 1914 1915 if (insn->imm == BPF_FUNC_get_route_realm) 1916 prog->dst_needed = 1; 1917 if (insn->imm == BPF_FUNC_get_prandom_u32) 1918 bpf_user_rnd_init_once(); 1919 if (insn->imm == BPF_FUNC_override_return) 1920 prog->kprobe_override = 1; 1921 if (insn->imm == BPF_FUNC_tail_call) { 1922 /* If we tail call into other programs, we 1923 * cannot make any assumptions since they can 1924 * be replaced dynamically during runtime in 1925 * the program array. 1926 */ 1927 prog->cb_access = 1; 1928 if (!bpf_allow_tail_call_in_subprogs(env)) 1929 prog->aux->stack_depth = MAX_BPF_STACK; 1930 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 1931 1932 /* mark bpf_tail_call as different opcode to avoid 1933 * conditional branch in the interpreter for every normal 1934 * call and to prevent accidental JITing by JIT compiler 1935 * that doesn't support bpf_tail_call yet 1936 */ 1937 insn->imm = 0; 1938 insn->code = BPF_JMP | BPF_TAIL_CALL; 1939 1940 aux = &env->insn_aux_data[i + delta]; 1941 if (env->bpf_capable && !prog->blinding_requested && 1942 prog->jit_requested && 1943 !bpf_map_key_poisoned(aux) && 1944 !bpf_map_ptr_poisoned(aux) && 1945 !bpf_map_ptr_unpriv(aux)) { 1946 struct bpf_jit_poke_descriptor desc = { 1947 .reason = BPF_POKE_REASON_TAIL_CALL, 1948 .tail_call.map = aux->map_ptr_state.map_ptr, 1949 .tail_call.key = bpf_map_key_immediate(aux), 1950 .insn_idx = i + delta, 1951 }; 1952 1953 ret = bpf_jit_add_poke_descriptor(prog, &desc); 1954 if (ret < 0) { 1955 verbose(env, "adding tail call poke descriptor failed\n"); 1956 return ret; 1957 } 1958 1959 insn->imm = ret + 1; 1960 goto next_insn; 1961 } 1962 1963 if (!bpf_map_ptr_unpriv(aux)) 1964 goto next_insn; 1965 1966 /* instead of changing every JIT dealing with tail_call 1967 * emit two extra insns: 1968 * if (index >= max_entries) goto out; 1969 * index &= array->index_mask; 1970 * to avoid out-of-bounds cpu speculation 1971 */ 1972 if (bpf_map_ptr_poisoned(aux)) { 1973 verbose(env, "tail_call abusing map_ptr\n"); 1974 return -EINVAL; 1975 } 1976 1977 map_ptr = aux->map_ptr_state.map_ptr; 1978 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 1979 map_ptr->max_entries, 2); 1980 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 1981 container_of(map_ptr, 1982 struct bpf_array, 1983 map)->index_mask); 1984 insn_buf[2] = *insn; 1985 cnt = 3; 1986 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1987 if (!new_prog) 1988 return -ENOMEM; 1989 1990 delta += cnt - 1; 1991 env->prog = prog = new_prog; 1992 insn = new_prog->insnsi + i + delta; 1993 goto next_insn; 1994 } 1995 1996 if (insn->imm == BPF_FUNC_timer_set_callback) { 1997 /* The verifier will process callback_fn as many times as necessary 1998 * with different maps and the register states prepared by 1999 * set_timer_callback_state will be accurate. 2000 * 2001 * The following use case is valid: 2002 * map1 is shared by prog1, prog2, prog3. 2003 * prog1 calls bpf_timer_init for some map1 elements 2004 * prog2 calls bpf_timer_set_callback for some map1 elements. 2005 * Those that were not bpf_timer_init-ed will return -EINVAL. 2006 * prog3 calls bpf_timer_start for some map1 elements. 2007 * Those that were not both bpf_timer_init-ed and 2008 * bpf_timer_set_callback-ed will return -EINVAL. 2009 */ 2010 struct bpf_insn ld_addrs[2] = { 2011 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), 2012 }; 2013 2014 insn_buf[0] = ld_addrs[0]; 2015 insn_buf[1] = ld_addrs[1]; 2016 insn_buf[2] = *insn; 2017 cnt = 3; 2018 2019 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2020 if (!new_prog) 2021 return -ENOMEM; 2022 2023 delta += cnt - 1; 2024 env->prog = prog = new_prog; 2025 insn = new_prog->insnsi + i + delta; 2026 goto patch_call_imm; 2027 } 2028 2029 /* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */ 2030 if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) { 2031 /* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data, 2032 * bpf_mem_alloc() returns a ptr to the percpu data ptr. 2033 */ 2034 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 2035 insn_buf[1] = *insn; 2036 cnt = 2; 2037 2038 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2039 if (!new_prog) 2040 return -ENOMEM; 2041 2042 delta += cnt - 1; 2043 env->prog = prog = new_prog; 2044 insn = new_prog->insnsi + i + delta; 2045 goto patch_call_imm; 2046 } 2047 2048 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 2049 * and other inlining handlers are currently limited to 64 bit 2050 * only. 2051 */ 2052 if (prog->jit_requested && BITS_PER_LONG == 64 && 2053 (insn->imm == BPF_FUNC_map_lookup_elem || 2054 insn->imm == BPF_FUNC_map_update_elem || 2055 insn->imm == BPF_FUNC_map_delete_elem || 2056 insn->imm == BPF_FUNC_map_push_elem || 2057 insn->imm == BPF_FUNC_map_pop_elem || 2058 insn->imm == BPF_FUNC_map_peek_elem || 2059 insn->imm == BPF_FUNC_redirect_map || 2060 insn->imm == BPF_FUNC_for_each_map_elem || 2061 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { 2062 aux = &env->insn_aux_data[i + delta]; 2063 if (bpf_map_ptr_poisoned(aux)) 2064 goto patch_call_imm; 2065 2066 map_ptr = aux->map_ptr_state.map_ptr; 2067 ops = map_ptr->ops; 2068 if (insn->imm == BPF_FUNC_map_lookup_elem && 2069 ops->map_gen_lookup) { 2070 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 2071 if (cnt == -EOPNOTSUPP) 2072 goto patch_map_ops_generic; 2073 if (cnt <= 0 || cnt >= INSN_BUF_SIZE) { 2074 verifier_bug(env, "%d insns generated for map lookup", cnt); 2075 return -EFAULT; 2076 } 2077 2078 if (bpf_map_is_percpu_map(map_ptr->map_type)) 2079 prog->jit_required = true; 2080 2081 new_prog = bpf_patch_insn_data(env, i + delta, 2082 insn_buf, cnt); 2083 if (!new_prog) 2084 return -ENOMEM; 2085 2086 delta += cnt - 1; 2087 env->prog = prog = new_prog; 2088 insn = new_prog->insnsi + i + delta; 2089 goto next_insn; 2090 } 2091 2092 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 2093 (void *(*)(struct bpf_map *map, void *key))NULL)); 2094 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 2095 (long (*)(struct bpf_map *map, void *key))NULL)); 2096 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 2097 (long (*)(struct bpf_map *map, void *key, void *value, 2098 u64 flags))NULL)); 2099 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 2100 (long (*)(struct bpf_map *map, void *value, 2101 u64 flags))NULL)); 2102 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 2103 (long (*)(struct bpf_map *map, void *value))NULL)); 2104 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 2105 (long (*)(struct bpf_map *map, void *value))NULL)); 2106 BUILD_BUG_ON(!__same_type(ops->map_redirect, 2107 (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); 2108 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 2109 (long (*)(struct bpf_map *map, 2110 bpf_callback_t callback_fn, 2111 void *callback_ctx, 2112 u64 flags))NULL)); 2113 BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, 2114 (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL)); 2115 2116 patch_map_ops_generic: 2117 switch (insn->imm) { 2118 case BPF_FUNC_map_lookup_elem: 2119 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 2120 goto next_insn; 2121 case BPF_FUNC_map_update_elem: 2122 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 2123 goto next_insn; 2124 case BPF_FUNC_map_delete_elem: 2125 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 2126 goto next_insn; 2127 case BPF_FUNC_map_push_elem: 2128 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 2129 goto next_insn; 2130 case BPF_FUNC_map_pop_elem: 2131 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 2132 goto next_insn; 2133 case BPF_FUNC_map_peek_elem: 2134 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 2135 goto next_insn; 2136 case BPF_FUNC_redirect_map: 2137 insn->imm = BPF_CALL_IMM(ops->map_redirect); 2138 goto next_insn; 2139 case BPF_FUNC_for_each_map_elem: 2140 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 2141 goto next_insn; 2142 case BPF_FUNC_map_lookup_percpu_elem: 2143 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); 2144 goto next_insn; 2145 } 2146 2147 goto patch_call_imm; 2148 } 2149 2150 /* Implement bpf_jiffies64 inline. */ 2151 if (prog->jit_requested && BITS_PER_LONG == 64 && 2152 insn->imm == BPF_FUNC_jiffies64) { 2153 struct bpf_insn ld_jiffies_addr[2] = { 2154 BPF_LD_IMM64(BPF_REG_0, 2155 (unsigned long)&jiffies), 2156 }; 2157 2158 insn_buf[0] = ld_jiffies_addr[0]; 2159 insn_buf[1] = ld_jiffies_addr[1]; 2160 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 2161 BPF_REG_0, 0); 2162 cnt = 3; 2163 2164 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 2165 cnt); 2166 if (!new_prog) 2167 return -ENOMEM; 2168 2169 delta += cnt - 1; 2170 env->prog = prog = new_prog; 2171 insn = new_prog->insnsi + i + delta; 2172 goto next_insn; 2173 } 2174 2175 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML) 2176 /* Implement bpf_get_smp_processor_id() inline. */ 2177 if (insn->imm == BPF_FUNC_get_smp_processor_id && 2178 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2179 /* BPF_FUNC_get_smp_processor_id inlining is an 2180 * optimization, so if cpu_number is ever 2181 * changed in some incompatible and hard to support 2182 * way, it's fine to back out this inlining logic 2183 */ 2184 #ifdef CONFIG_SMP 2185 prog->jit_required = true; 2186 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); 2187 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2188 insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); 2189 cnt = 3; 2190 #else 2191 insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); 2192 cnt = 1; 2193 #endif 2194 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2195 if (!new_prog) 2196 return -ENOMEM; 2197 2198 delta += cnt - 1; 2199 env->prog = prog = new_prog; 2200 insn = new_prog->insnsi + i + delta; 2201 goto next_insn; 2202 } 2203 2204 /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ 2205 if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && 2206 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2207 prog->jit_required = true; 2208 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); 2209 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2210 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); 2211 cnt = 3; 2212 2213 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2214 if (!new_prog) 2215 return -ENOMEM; 2216 2217 delta += cnt - 1; 2218 env->prog = prog = new_prog; 2219 insn = new_prog->insnsi + i + delta; 2220 goto next_insn; 2221 } 2222 #endif 2223 /* Implement bpf_get_func_arg inline. */ 2224 if (prog_type == BPF_PROG_TYPE_TRACING && 2225 insn->imm == BPF_FUNC_get_func_arg) { 2226 if (eatype == BPF_TRACE_RAW_TP) { 2227 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2228 2229 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2230 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2231 cnt = 1; 2232 } else { 2233 /* Load nr_args from ctx - 8 */ 2234 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2235 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2236 cnt = 2; 2237 } 2238 insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 2239 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 2240 insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 2241 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 2242 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2243 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 0); 2244 insn_buf[cnt++] = BPF_JMP_A(1); 2245 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2246 2247 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2248 if (!new_prog) 2249 return -ENOMEM; 2250 2251 delta += cnt - 1; 2252 env->prog = prog = new_prog; 2253 insn = new_prog->insnsi + i + delta; 2254 goto next_insn; 2255 } 2256 2257 /* Implement bpf_get_func_ret inline. */ 2258 if (prog_type == BPF_PROG_TYPE_TRACING && 2259 insn->imm == BPF_FUNC_get_func_ret) { 2260 if (eatype == BPF_TRACE_FEXIT || 2261 eatype == BPF_TRACE_FSESSION || 2262 eatype == BPF_TRACE_FEXIT_MULTI || 2263 eatype == BPF_TRACE_FSESSION_MULTI || 2264 eatype == BPF_MODIFY_RETURN) { 2265 /* Load nr_args from ctx - 8 */ 2266 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2267 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2268 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 2269 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 2270 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2271 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 2272 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 2273 cnt = 7; 2274 } else { 2275 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 2276 cnt = 1; 2277 } 2278 2279 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2280 if (!new_prog) 2281 return -ENOMEM; 2282 2283 delta += cnt - 1; 2284 env->prog = prog = new_prog; 2285 insn = new_prog->insnsi + i + delta; 2286 goto next_insn; 2287 } 2288 2289 /* Implement get_func_arg_cnt inline. */ 2290 if (prog_type == BPF_PROG_TYPE_TRACING && 2291 insn->imm == BPF_FUNC_get_func_arg_cnt) { 2292 if (eatype == BPF_TRACE_RAW_TP) { 2293 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2294 2295 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2296 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2297 cnt = 1; 2298 } else { 2299 /* Load nr_args from ctx - 8 */ 2300 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2301 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2302 cnt = 2; 2303 } 2304 2305 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2306 if (!new_prog) 2307 return -ENOMEM; 2308 2309 delta += cnt - 1; 2310 env->prog = prog = new_prog; 2311 insn = new_prog->insnsi + i + delta; 2312 goto next_insn; 2313 } 2314 2315 /* Implement bpf_get_func_ip inline. */ 2316 if (prog_type == BPF_PROG_TYPE_TRACING && 2317 insn->imm == BPF_FUNC_get_func_ip) { 2318 /* Load IP address from ctx - 16 */ 2319 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 2320 2321 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 2322 if (!new_prog) 2323 return -ENOMEM; 2324 2325 env->prog = prog = new_prog; 2326 insn = new_prog->insnsi + i + delta; 2327 goto next_insn; 2328 } 2329 2330 /* Implement bpf_get_branch_snapshot inline. */ 2331 if (IS_ENABLED(CONFIG_PERF_EVENTS) && 2332 prog->jit_requested && BITS_PER_LONG == 64 && 2333 insn->imm == BPF_FUNC_get_branch_snapshot) { 2334 /* We are dealing with the following func protos: 2335 * u64 bpf_get_branch_snapshot(void *buf, u32 size, u64 flags); 2336 * int perf_snapshot_branch_stack(struct perf_branch_entry *entries, u32 cnt); 2337 */ 2338 const u32 br_entry_size = sizeof(struct perf_branch_entry); 2339 2340 /* struct perf_branch_entry is part of UAPI and is 2341 * used as an array element, so extremely unlikely to 2342 * ever grow or shrink 2343 */ 2344 BUILD_BUG_ON(br_entry_size != 24); 2345 2346 /* if (unlikely(flags)) return -EINVAL */ 2347 insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7); 2348 2349 /* Transform size (bytes) into number of entries (cnt = size / 24). 2350 * But to avoid expensive division instruction, we implement 2351 * divide-by-3 through multiplication, followed by further 2352 * division by 8 through 3-bit right shift. 2353 * Refer to book "Hacker's Delight, 2nd ed." by Henry S. Warren, Jr., 2354 * p. 227, chapter "Unsigned Division by 3" for details and proofs. 2355 * 2356 * N / 3 <=> M * N / 2^33, where M = (2^33 + 1) / 3 = 0xaaaaaaab. 2357 */ 2358 insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab); 2359 insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0); 2360 insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36); 2361 2362 /* call perf_snapshot_branch_stack implementation */ 2363 insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack)); 2364 /* if (entry_cnt == 0) return -ENOENT */ 2365 insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4); 2366 /* return entry_cnt * sizeof(struct perf_branch_entry) */ 2367 insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size); 2368 insn_buf[7] = BPF_JMP_A(3); 2369 /* return -EINVAL; */ 2370 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2371 insn_buf[9] = BPF_JMP_A(1); 2372 /* return -ENOENT; */ 2373 insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT); 2374 cnt = 11; 2375 2376 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2377 if (!new_prog) 2378 return -ENOMEM; 2379 2380 delta += cnt - 1; 2381 env->prog = prog = new_prog; 2382 insn = new_prog->insnsi + i + delta; 2383 goto next_insn; 2384 } 2385 2386 /* Implement bpf_kptr_xchg inline */ 2387 if (prog->jit_requested && BITS_PER_LONG == 64 && 2388 insn->imm == BPF_FUNC_kptr_xchg && 2389 bpf_jit_supports_ptr_xchg()) { 2390 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); 2391 insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); 2392 cnt = 2; 2393 2394 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2395 if (!new_prog) 2396 return -ENOMEM; 2397 2398 delta += cnt - 1; 2399 env->prog = prog = new_prog; 2400 insn = new_prog->insnsi + i + delta; 2401 goto next_insn; 2402 } 2403 patch_call_imm: 2404 fn = env->ops->get_func_proto(insn->imm, env->prog); 2405 /* all functions that have prototype and verifier allowed 2406 * programs to call them, must be real in-kernel functions 2407 */ 2408 if (!fn->func) { 2409 verifier_bug(env, 2410 "not inlined functions %s#%d is missing func", 2411 func_id_name(insn->imm), insn->imm); 2412 return -EFAULT; 2413 } 2414 insn->imm = BPF_CALL_IMM(fn->func); 2415 next_insn: 2416 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2417 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2418 subprogs[cur_subprog].stack_extra = stack_depth_extra; 2419 2420 stack_depth = subprogs[cur_subprog].stack_depth; 2421 if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) { 2422 verbose(env, "stack size %d(extra %d) is too large\n", 2423 stack_depth, stack_depth_extra); 2424 return -EINVAL; 2425 } 2426 cur_subprog++; 2427 stack_depth = subprogs[cur_subprog].stack_depth; 2428 stack_depth_extra = 0; 2429 } 2430 i++; 2431 insn++; 2432 } 2433 2434 env->prog->aux->stack_depth = subprogs[0].stack_depth; 2435 for (i = 0; i < env->subprog_cnt; i++) { 2436 int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1; 2437 int subprog_start = subprogs[i].start; 2438 int stack_slots = subprogs[i].stack_extra / 8; 2439 int slots = delta, cnt = 0; 2440 2441 if (!stack_slots) 2442 continue; 2443 /* We need two slots in case timed may_goto is supported. */ 2444 if (stack_slots > slots) { 2445 verifier_bug(env, "stack_slots supports may_goto only"); 2446 return -EFAULT; 2447 } 2448 2449 stack_depth = subprogs[i].stack_depth; 2450 if (bpf_jit_supports_timed_may_goto()) { 2451 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2452 BPF_MAX_TIMED_LOOPS); 2453 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0); 2454 } else { 2455 /* Add ST insn to subprog prologue to init extra stack */ 2456 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2457 BPF_MAX_LOOPS); 2458 } 2459 /* Copy first actual insn to preserve it */ 2460 insn_buf[cnt++] = env->prog->insnsi[subprog_start]; 2461 2462 new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); 2463 if (!new_prog) 2464 return -ENOMEM; 2465 env->prog = prog = new_prog; 2466 /* 2467 * If may_goto is a first insn of a prog there could be a jmp 2468 * insn that points to it, hence adjust all such jmps to point 2469 * to insn after BPF_ST that inits may_goto count. 2470 * Adjustment will succeed because bpf_patch_insn_data() didn't fail. 2471 */ 2472 WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta)); 2473 } 2474 2475 /* Since poke tab is now finalized, publish aux to tracker. */ 2476 for (i = 0; i < prog->aux->size_poke_tab; i++) { 2477 map_ptr = prog->aux->poke_tab[i].tail_call.map; 2478 if (!map_ptr->ops->map_poke_track || 2479 !map_ptr->ops->map_poke_untrack || 2480 !map_ptr->ops->map_poke_run) { 2481 verifier_bug(env, "poke tab is misconfigured"); 2482 return -EFAULT; 2483 } 2484 2485 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 2486 if (ret < 0) { 2487 verbose(env, "tracking tail call prog failed\n"); 2488 return ret; 2489 } 2490 } 2491 2492 ret = sort_kfunc_descs_by_imm_off(env); 2493 if (ret) 2494 return ret; 2495 2496 return 0; 2497 } 2498 2499 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 2500 int position, 2501 s32 stack_base, 2502 u32 callback_subprogno, 2503 u32 *total_cnt) 2504 { 2505 s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; 2506 s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; 2507 s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; 2508 int reg_loop_max = BPF_REG_6; 2509 int reg_loop_cnt = BPF_REG_7; 2510 int reg_loop_ctx = BPF_REG_8; 2511 2512 struct bpf_insn *insn_buf = env->insn_buf; 2513 struct bpf_prog *new_prog; 2514 u32 callback_start; 2515 u32 call_insn_offset; 2516 s32 callback_offset; 2517 u32 cnt = 0; 2518 2519 /* This represents an inlined version of bpf_iter.c:bpf_loop, 2520 * be careful to modify this code in sync. 2521 */ 2522 2523 /* Return error and jump to the end of the patch if 2524 * expected number of iterations is too big. 2525 */ 2526 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2); 2527 insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG); 2528 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16); 2529 /* spill R6, R7, R8 to use these as loop vars */ 2530 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset); 2531 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset); 2532 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset); 2533 /* initialize loop vars */ 2534 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1); 2535 insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0); 2536 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3); 2537 /* loop header, 2538 * if reg_loop_cnt >= reg_loop_max skip the loop body 2539 */ 2540 insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5); 2541 /* callback call, 2542 * correct callback offset would be set after patching 2543 */ 2544 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt); 2545 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx); 2546 insn_buf[cnt++] = BPF_CALL_REL(0); 2547 /* increment loop counter */ 2548 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1); 2549 /* jump to loop header if callback returned 0 */ 2550 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6); 2551 /* return value of bpf_loop, 2552 * set R0 to the number of iterations 2553 */ 2554 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt); 2555 /* restore original values of R6, R7, R8 */ 2556 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset); 2557 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset); 2558 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset); 2559 2560 *total_cnt = cnt; 2561 new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); 2562 if (!new_prog) 2563 return new_prog; 2564 2565 /* callback start is known only after patching */ 2566 callback_start = env->subprog_info[callback_subprogno].start; 2567 /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ 2568 call_insn_offset = position + 12; 2569 callback_offset = callback_start - call_insn_offset - 1; 2570 new_prog->insnsi[call_insn_offset].imm = callback_offset; 2571 2572 return new_prog; 2573 } 2574 2575 static bool is_bpf_loop_call(struct bpf_insn *insn) 2576 { 2577 return insn->code == (BPF_JMP | BPF_CALL) && 2578 insn->src_reg == 0 && 2579 insn->imm == BPF_FUNC_loop; 2580 } 2581 2582 /* For all sub-programs in the program (including main) check 2583 * insn_aux_data to see if there are bpf_loop calls that require 2584 * inlining. If such calls are found the calls are replaced with a 2585 * sequence of instructions produced by `inline_bpf_loop` function and 2586 * subprog stack_depth is increased by the size of 3 registers. 2587 * This stack space is used to spill values of the R6, R7, R8. These 2588 * registers are used to store the loop bound, counter and context 2589 * variables. 2590 */ 2591 int bpf_optimize_bpf_loop(struct bpf_verifier_env *env) 2592 { 2593 struct bpf_subprog_info *subprogs = env->subprog_info; 2594 int i, cur_subprog = 0, cnt, delta = 0; 2595 struct bpf_insn *insn = env->prog->insnsi; 2596 int insn_cnt = env->prog->len; 2597 u16 stack_depth = subprogs[cur_subprog].stack_depth; 2598 u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2599 u16 stack_depth_extra = 0; 2600 2601 for (i = 0; i < insn_cnt; i++, insn++) { 2602 struct bpf_loop_inline_state *inline_state = 2603 &env->insn_aux_data[i + delta].loop_inline_state; 2604 2605 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { 2606 struct bpf_prog *new_prog; 2607 2608 stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; 2609 new_prog = inline_bpf_loop(env, 2610 i + delta, 2611 -(stack_depth + stack_depth_extra), 2612 inline_state->callback_subprogno, 2613 &cnt); 2614 if (!new_prog) 2615 return -ENOMEM; 2616 2617 delta += cnt - 1; 2618 env->prog = new_prog; 2619 insn = new_prog->insnsi + i + delta; 2620 } 2621 2622 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2623 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2624 cur_subprog++; 2625 stack_depth = subprogs[cur_subprog].stack_depth; 2626 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2627 stack_depth_extra = 0; 2628 } 2629 } 2630 2631 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 2632 2633 return 0; 2634 } 2635 2636 /* Remove unnecessary spill/fill pairs, members of fastcall pattern, 2637 * adjust subprograms stack depth when possible. 2638 */ 2639 int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) 2640 { 2641 struct bpf_subprog_info *subprog = env->subprog_info; 2642 struct bpf_insn_aux_data *aux = env->insn_aux_data; 2643 struct bpf_insn *insn = env->prog->insnsi; 2644 int insn_cnt = env->prog->len; 2645 u32 spills_num; 2646 bool modified = false; 2647 int i, j; 2648 2649 for (i = 0; i < insn_cnt; i++, insn++) { 2650 if (aux[i].fastcall_spills_num > 0) { 2651 spills_num = aux[i].fastcall_spills_num; 2652 /* NOPs would be removed by opt_remove_nops() */ 2653 for (j = 1; j <= spills_num; ++j) { 2654 *(insn - j) = NOP; 2655 *(insn + j) = NOP; 2656 } 2657 modified = true; 2658 } 2659 if ((subprog + 1)->start == i + 1) { 2660 if (modified && !subprog->keep_fastcall_stack) 2661 subprog->stack_depth = -subprog->fastcall_stack_off; 2662 subprog++; 2663 modified = false; 2664 } 2665 } 2666 2667 return 0; 2668 } 2669 2670