1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/bpf.h> 4 #include <linux/btf.h> 5 #include <linux/bpf_verifier.h> 6 #include <linux/filter.h> 7 #include <linux/vmalloc.h> 8 #include <linux/bsearch.h> 9 #include <linux/sort.h> 10 #include <linux/perf_event.h> 11 #include <net/xdp.h> 12 #include "disasm.h" 13 14 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) 15 16 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 17 { 18 return BPF_CLASS(insn->code) == BPF_STX && 19 BPF_MODE(insn->code) == BPF_ATOMIC && 20 insn->imm == BPF_CMPXCHG; 21 } 22 23 /* Returns true if 'insn' is an address space cast instruction translated as BPF_ALU op */ 24 static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *insn) 25 { 26 struct bpf_map *arena = (struct bpf_map *)prog->aux->arena; 27 28 if (insn->code != (BPF_ALU64 | BPF_MOV | BPF_X) || insn->off != BPF_ADDR_SPACE_CAST) 29 return false; 30 31 /* cast from as(1) to as(0) */ 32 if (insn->imm == 1) 33 return true; 34 35 /* cast from as(0) to as(1) */ 36 if (insn->imm == 1 << 16) 37 return arena && arena->map_flags & BPF_F_NO_USER_CONV; 38 39 /* non-BPF_F_NO_USER_CONV cast from as(0) to as(1) should be handled by JIT */ 40 return false; 41 } 42 43 /* Return the regno defined by the insn, or -1. */ 44 static int insn_def_regno(const struct bpf_insn *insn) 45 { 46 switch (BPF_CLASS(insn->code)) { 47 case BPF_JMP: 48 case BPF_JMP32: 49 case BPF_ST: 50 return -1; 51 case BPF_STX: 52 return bpf_atomic_load_reg(insn); 53 default: 54 return insn->dst_reg; 55 } 56 } 57 58 /* 59 * For use only in combination with insn_def_regno() >= 0. 60 * Returns TRUE if the destination register operates on 64-bit, 61 * otherwise return FALSE. 62 */ 63 static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn) 64 { 65 u8 class = BPF_CLASS(insn->code); 66 u8 mode = BPF_MODE(insn->code); 67 u8 size = BPF_SIZE(insn->code); 68 u8 op = BPF_OP(insn->code); 69 bool mode_mem; 70 71 /* subregister endiness swap */ 72 if ((class == BPF_ALU || class == BPF_ALU64) && op == BPF_END && insn->imm != 64) 73 return false; 74 75 /* w0 += 1 */ 76 if (class == BPF_ALU && op != BPF_END) 77 return false; 78 79 /* address space casts converted to BPF_ALU, see bpf_do_misc_fixups() */ 80 if (is_addr_space_cast32(prog, insn)) 81 return false; 82 83 /* non 64-bit, non signed extended loads */ 84 mode_mem = mode == BPF_MEM || mode == BPF_PROBE_MEM || mode == BPF_PROBE_MEM32; 85 if (class == BPF_LDX && mode_mem && size != BPF_DW) 86 return false; 87 88 /* atomics, see insn_def_regno() */ 89 if (class == BPF_STX && size != BPF_DW) 90 return false; 91 92 /* both LD_IND and LD_ABS return 32-bit data. */ 93 if (class == BPF_LD && (mode == BPF_IND || mode == BPF_ABS)) 94 return false; 95 96 /* Conservatively return true at default. */ 97 return true; 98 } 99 100 /* 101 * Return the 32-bit subregister defined by INSN, or -1 if INSN does not 102 * explicitly define a 32-bit value. 103 */ 104 int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) 105 { 106 int dst_reg = insn_def_regno(insn); 107 108 if (dst_reg < 0 || bpf_is_reg64(prog, insn)) 109 return -1; 110 111 return dst_reg; 112 } 113 114 static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) 115 { 116 const struct bpf_kfunc_desc *d0 = a; 117 const struct bpf_kfunc_desc *d1 = b; 118 119 if (d0->imm != d1->imm) 120 return d0->imm < d1->imm ? -1 : 1; 121 if (d0->offset != d1->offset) 122 return d0->offset < d1->offset ? -1 : 1; 123 return 0; 124 } 125 126 const struct btf_func_model * 127 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 128 const struct bpf_insn *insn) 129 { 130 const struct bpf_kfunc_desc desc = { 131 .imm = insn->imm, 132 .offset = insn->off, 133 }; 134 const struct bpf_kfunc_desc *res; 135 struct bpf_kfunc_desc_tab *tab; 136 137 tab = prog->aux->kfunc_tab; 138 res = bsearch(&desc, tab->descs, tab->nr_descs, 139 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off); 140 141 return res ? &res->func_model : NULL; 142 } 143 144 static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc) 145 { 146 unsigned long call_imm; 147 148 if (bpf_jit_supports_far_kfunc_call()) { 149 call_imm = desc->func_id; 150 } else { 151 call_imm = BPF_CALL_IMM(desc->addr); 152 /* Check whether the relative offset overflows desc->imm */ 153 if ((unsigned long)(s32)call_imm != call_imm) { 154 verbose(env, "address of kernel func_id %u is out of range\n", 155 desc->func_id); 156 return -EINVAL; 157 } 158 } 159 desc->imm = call_imm; 160 return 0; 161 } 162 163 static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env) 164 { 165 struct bpf_kfunc_desc_tab *tab; 166 int i, err; 167 168 tab = env->prog->aux->kfunc_tab; 169 if (!tab) 170 return 0; 171 172 for (i = 0; i < tab->nr_descs; i++) { 173 err = set_kfunc_desc_imm(env, &tab->descs[i]); 174 if (err) 175 return err; 176 } 177 178 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 179 kfunc_desc_cmp_by_imm_off, NULL); 180 return 0; 181 } 182 183 static int add_kfunc_in_insns(struct bpf_verifier_env *env, 184 struct bpf_insn *insn, int cnt) 185 { 186 int i, ret; 187 188 for (i = 0; i < cnt; i++, insn++) { 189 if (bpf_pseudo_kfunc_call(insn)) { 190 ret = bpf_add_kfunc_call(env, insn->imm, insn->off); 191 if (ret < 0) 192 return ret; 193 } 194 } 195 return 0; 196 } 197 198 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 199 static int get_callee_stack_depth(struct bpf_verifier_env *env, 200 const struct bpf_insn *insn, int idx) 201 { 202 int start = idx + insn->imm + 1, subprog; 203 204 subprog = bpf_find_subprog(env, start); 205 if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start)) 206 return -EFAULT; 207 return env->subprog_info[subprog].stack_depth; 208 } 209 #endif 210 211 /* single env->prog->insni[off] instruction was replaced with the range 212 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 213 * [0, off) and [off, end) to new locations, so the patched range stays zero 214 */ 215 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 216 struct bpf_prog *new_prog, u32 off, u32 cnt) 217 { 218 struct bpf_insn_aux_data *data = env->insn_aux_data; 219 struct bpf_insn *insn = new_prog->insnsi; 220 u32 old_seen = data[off].seen; 221 u32 prog_len; 222 int i; 223 224 /* aux info at OFF always needs adjustment, no matter fast path 225 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 226 * original insn at old prog. 227 */ 228 data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; 229 230 if (cnt == 1) 231 return; 232 prog_len = new_prog->len; 233 env->insn_aux_data_len = prog_len; 234 235 memmove(data + off + cnt - 1, data + off, 236 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 237 memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1)); 238 for (i = off; i < off + cnt - 1; i++) { 239 /* Expand insni[off]'s seen count to the patched range. */ 240 data[i].seen = old_seen; 241 data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; 242 } 243 244 /* 245 * The indirect_target flag of the original instruction was moved to the last of the 246 * new instructions by the above memmove and memset, but the indirect jump target is 247 * actually the first instruction, so move it back. This also matches with the behavior 248 * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new 249 * instruction. 250 */ 251 if (data[off + cnt - 1].indirect_target) { 252 data[off].indirect_target = 1; 253 data[off + cnt - 1].indirect_target = 0; 254 } 255 } 256 257 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 258 { 259 int i; 260 261 if (len == 1) 262 return; 263 /* NOTE: fake 'exit' subprog should be updated as well. */ 264 for (i = 0; i <= env->subprog_cnt; i++) { 265 if (env->subprog_info[i].start <= off) 266 continue; 267 env->subprog_info[i].start += len - 1; 268 } 269 } 270 271 static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) 272 { 273 int i; 274 275 if (len == 1) 276 return; 277 278 for (i = 0; i < env->insn_array_map_cnt; i++) 279 bpf_insn_array_adjust(env->insn_array_maps[i], off, len); 280 } 281 282 static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) 283 { 284 int i; 285 286 for (i = 0; i < env->insn_array_map_cnt; i++) 287 bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len); 288 } 289 290 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) 291 { 292 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 293 int i, sz = prog->aux->size_poke_tab; 294 struct bpf_jit_poke_descriptor *desc; 295 296 for (i = 0; i < sz; i++) { 297 desc = &tab[i]; 298 if (desc->insn_idx <= off) 299 continue; 300 desc->insn_idx += len - 1; 301 } 302 } 303 304 struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 305 const struct bpf_insn *patch, u32 len) 306 { 307 struct bpf_prog *new_prog; 308 struct bpf_insn_aux_data *new_data = NULL; 309 310 if (len > 1) { 311 new_data = vrealloc(env->insn_aux_data, 312 array_size(env->prog->len + len - 1, 313 sizeof(struct bpf_insn_aux_data)), 314 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 315 if (!new_data) 316 return NULL; 317 318 env->insn_aux_data = new_data; 319 } 320 321 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 322 if (IS_ERR(new_prog)) { 323 if (PTR_ERR(new_prog) == -ERANGE) 324 verbose(env, 325 "insn %d cannot be patched due to 16-bit range\n", 326 env->insn_aux_data[off].orig_idx); 327 return NULL; 328 } 329 adjust_insn_aux_data(env, new_prog, off, len); 330 adjust_subprog_starts(env, off, len); 331 adjust_insn_arrays(env, off, len); 332 adjust_poke_descs(new_prog, off, len); 333 return new_prog; 334 } 335 336 /* 337 * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the 338 * jump offset by 'delta'. 339 */ 340 static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta) 341 { 342 struct bpf_insn *insn = prog->insnsi; 343 u32 insn_cnt = prog->len, i; 344 s32 imm; 345 s16 off; 346 347 for (i = 0; i < insn_cnt; i++, insn++) { 348 u8 code = insn->code; 349 350 if (tgt_idx <= i && i < tgt_idx + delta) 351 continue; 352 353 if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) || 354 BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT) 355 continue; 356 357 if (insn->code == (BPF_JMP32 | BPF_JA)) { 358 if (i + 1 + insn->imm != tgt_idx) 359 continue; 360 if (check_add_overflow(insn->imm, delta, &imm)) 361 return -ERANGE; 362 insn->imm = imm; 363 } else { 364 if (i + 1 + insn->off != tgt_idx) 365 continue; 366 if (check_add_overflow(insn->off, delta, &off)) 367 return -ERANGE; 368 insn->off = off; 369 } 370 } 371 return 0; 372 } 373 374 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 375 u32 off, u32 cnt) 376 { 377 int i, j; 378 379 /* find first prog starting at or after off (first to remove) */ 380 for (i = 0; i < env->subprog_cnt; i++) 381 if (env->subprog_info[i].start >= off) 382 break; 383 /* find first prog starting at or after off + cnt (first to stay) */ 384 for (j = i; j < env->subprog_cnt; j++) 385 if (env->subprog_info[j].start >= off + cnt) 386 break; 387 /* if j doesn't start exactly at off + cnt, we are just removing 388 * the front of previous prog 389 */ 390 if (env->subprog_info[j].start != off + cnt) 391 j--; 392 393 if (j > i) { 394 struct bpf_prog_aux *aux = env->prog->aux; 395 int move; 396 397 /* move fake 'exit' subprog as well */ 398 move = env->subprog_cnt + 1 - j; 399 400 memmove(env->subprog_info + i, 401 env->subprog_info + j, 402 sizeof(*env->subprog_info) * move); 403 env->subprog_cnt -= j - i; 404 405 /* remove func_info and its aux */ 406 if (aux->func_info) { 407 move = aux->func_info_cnt - j; 408 409 memmove(aux->func_info + i, 410 aux->func_info + j, 411 sizeof(*aux->func_info) * move); 412 if (aux->func_info_aux) 413 memmove(aux->func_info_aux + i, 414 aux->func_info_aux + j, 415 sizeof(*aux->func_info_aux) * move); 416 aux->func_info_cnt -= j - i; 417 /* func_info->insn_off is set after all code rewrites, 418 * in adjust_btf_func() - no need to adjust 419 */ 420 } 421 } else { 422 /* convert i from "first prog to remove" to "first to adjust" */ 423 if (env->subprog_info[i].start == off) 424 i++; 425 } 426 427 /* update fake 'exit' subprog as well */ 428 for (; i <= env->subprog_cnt; i++) 429 env->subprog_info[i].start -= cnt; 430 431 return 0; 432 } 433 434 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 435 u32 cnt) 436 { 437 struct bpf_prog *prog = env->prog; 438 u32 i, l_off, l_cnt, nr_linfo; 439 struct bpf_line_info *linfo; 440 441 nr_linfo = prog->aux->nr_linfo; 442 if (!nr_linfo) 443 return 0; 444 445 linfo = prog->aux->linfo; 446 447 /* find first line info to remove, count lines to be removed */ 448 for (i = 0; i < nr_linfo; i++) 449 if (linfo[i].insn_off >= off) 450 break; 451 452 l_off = i; 453 l_cnt = 0; 454 for (; i < nr_linfo; i++) 455 if (linfo[i].insn_off < off + cnt) 456 l_cnt++; 457 else 458 break; 459 460 /* First live insn doesn't match first live linfo, it needs to "inherit" 461 * last removed linfo. prog is already modified, so prog->len == off 462 * means no live instructions after (tail of the program was removed). 463 */ 464 if (prog->len != off && l_cnt && 465 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 466 l_cnt--; 467 linfo[--i].insn_off = off + cnt; 468 } 469 470 /* remove the line info which refer to the removed instructions */ 471 if (l_cnt) { 472 memmove(linfo + l_off, linfo + i, 473 sizeof(*linfo) * (nr_linfo - i)); 474 475 prog->aux->nr_linfo -= l_cnt; 476 nr_linfo = prog->aux->nr_linfo; 477 } 478 479 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 480 for (i = l_off; i < nr_linfo; i++) 481 linfo[i].insn_off -= cnt; 482 483 /* fix up all subprogs (incl. 'exit') which start >= off */ 484 for (i = 0; i <= env->subprog_cnt; i++) 485 if (env->subprog_info[i].linfo_idx > l_off) { 486 /* program may have started in the removed region but 487 * may not be fully removed 488 */ 489 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 490 env->subprog_info[i].linfo_idx -= l_cnt; 491 else 492 env->subprog_info[i].linfo_idx = l_off; 493 } 494 495 return 0; 496 } 497 498 /* 499 * Clean up dynamically allocated fields of aux data for instructions [start, ...] 500 */ 501 void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) 502 { 503 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 504 int end = start + len; 505 int i; 506 507 for (i = start; i < end; i++) { 508 if (aux_data[i].jt) { 509 kvfree(aux_data[i].jt); 510 aux_data[i].jt = NULL; 511 } 512 } 513 } 514 515 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 516 { 517 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 518 unsigned int orig_prog_len = env->prog->len; 519 int err; 520 521 if (bpf_prog_is_offloaded(env->prog->aux)) 522 bpf_prog_offload_remove_insns(env, off, cnt); 523 524 bpf_clear_insn_aux_data(env, off, cnt); 525 526 err = bpf_remove_insns(env->prog, off, cnt); 527 if (err) 528 return err; 529 530 err = adjust_subprog_starts_after_remove(env, off, cnt); 531 if (err) 532 return err; 533 534 err = bpf_adj_linfo_after_remove(env, off, cnt); 535 if (err) 536 return err; 537 538 adjust_insn_arrays_after_remove(env, off, cnt); 539 540 memmove(aux_data + off, aux_data + off + cnt, 541 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 542 env->insn_aux_data_len -= cnt; 543 544 return 0; 545 } 546 547 static const struct bpf_insn NOP = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 548 static const struct bpf_insn MAY_GOTO_0 = BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0, 0); 549 550 bool bpf_insn_is_cond_jump(u8 code) 551 { 552 u8 op; 553 554 op = BPF_OP(code); 555 if (BPF_CLASS(code) == BPF_JMP32) 556 return op != BPF_JA; 557 558 if (BPF_CLASS(code) != BPF_JMP) 559 return false; 560 561 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 562 } 563 564 void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 565 { 566 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 567 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 568 struct bpf_insn *insn = env->prog->insnsi; 569 const int insn_cnt = env->prog->len; 570 int i; 571 572 for (i = 0; i < insn_cnt; i++, insn++) { 573 if (!bpf_insn_is_cond_jump(insn->code)) 574 continue; 575 576 if (!aux_data[i + 1].seen) 577 ja.off = insn->off; 578 else if (!aux_data[i + 1 + insn->off].seen) 579 ja.off = 0; 580 else 581 continue; 582 583 if (bpf_prog_is_offloaded(env->prog->aux)) 584 bpf_prog_offload_replace_insn(env, i, &ja); 585 586 memcpy(insn, &ja, sizeof(ja)); 587 } 588 } 589 590 int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) 591 { 592 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 593 int insn_cnt = env->prog->len; 594 int i, err; 595 596 for (i = 0; i < insn_cnt; i++) { 597 int j; 598 599 j = 0; 600 while (i + j < insn_cnt && !aux_data[i + j].seen) 601 j++; 602 if (!j) 603 continue; 604 605 err = verifier_remove_insns(env, i, j); 606 if (err) 607 return err; 608 insn_cnt = env->prog->len; 609 } 610 611 return 0; 612 } 613 614 int bpf_opt_remove_nops(struct bpf_verifier_env *env) 615 { 616 struct bpf_insn *insn = env->prog->insnsi; 617 int insn_cnt = env->prog->len; 618 bool is_may_goto_0, is_ja; 619 int i, err; 620 621 for (i = 0; i < insn_cnt; i++) { 622 is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0)); 623 is_ja = !memcmp(&insn[i], &NOP, sizeof(NOP)); 624 625 if (!is_may_goto_0 && !is_ja) 626 continue; 627 628 err = verifier_remove_insns(env, i, 1); 629 if (err) 630 return err; 631 insn_cnt--; 632 /* Go back one insn to catch may_goto +1; may_goto +0 sequence */ 633 i -= (is_may_goto_0 && i > 0) ? 2 : 1; 634 } 635 636 return 0; 637 } 638 639 int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 640 const union bpf_attr *attr) 641 { 642 struct bpf_insn *patch; 643 /* use env->insn_buf as two independent buffers */ 644 struct bpf_insn *zext_patch = env->insn_buf; 645 struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2]; 646 struct bpf_insn_aux_data *aux = env->insn_aux_data; 647 int i, patch_len, delta = 0, len = env->prog->len; 648 struct bpf_insn *insns = env->prog->insnsi; 649 struct bpf_prog *new_prog; 650 bool rnd_hi32; 651 652 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 653 zext_patch[1] = BPF_ZEXT_REG(0); 654 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 655 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 656 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 657 for (i = 0; i < len; i++) { 658 int adj_idx = i + delta; 659 struct bpf_insn insn; 660 int load_reg; 661 662 insn = insns[adj_idx]; 663 load_reg = insn_def_regno(&insn); 664 if (!aux[adj_idx].zext_dst) { 665 u8 code, class; 666 u32 imm_rnd; 667 668 if (!rnd_hi32) 669 continue; 670 671 code = insn.code; 672 class = BPF_CLASS(code); 673 if (load_reg == -1) 674 continue; 675 676 if (bpf_is_reg64(env->prog, &insn)) { 677 if (class == BPF_LD && 678 BPF_MODE(code) == BPF_IMM) 679 i++; 680 continue; 681 } 682 683 /* ctx load could be transformed into wider load. */ 684 if (class == BPF_LDX && 685 aux[adj_idx].ptr_type == PTR_TO_CTX) 686 continue; 687 688 imm_rnd = get_random_u32(); 689 rnd_hi32_patch[0] = insn; 690 rnd_hi32_patch[1].imm = imm_rnd; 691 rnd_hi32_patch[3].dst_reg = load_reg; 692 patch = rnd_hi32_patch; 693 patch_len = 4; 694 goto apply_patch_buffer; 695 } 696 697 /* Add in an zero-extend instruction if a) the JIT has requested 698 * it or b) it's a CMPXCHG. 699 * 700 * The latter is because: BPF_CMPXCHG always loads a value into 701 * R0, therefore always zero-extends. However some archs' 702 * equivalent instruction only does this load when the 703 * comparison is successful. This detail of CMPXCHG is 704 * orthogonal to the general zero-extension behaviour of the 705 * CPU, so it's treated independently of bpf_jit_needs_zext. 706 */ 707 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 708 continue; 709 710 /* Zero-extension is done by the caller. */ 711 if (bpf_pseudo_kfunc_call(&insn)) 712 continue; 713 714 if (verifier_bug_if(load_reg == -1, env, 715 "zext_dst is set, but no reg is defined")) 716 return -EFAULT; 717 718 zext_patch[0] = insn; 719 zext_patch[1].dst_reg = load_reg; 720 zext_patch[1].src_reg = load_reg; 721 patch = zext_patch; 722 patch_len = 2; 723 apply_patch_buffer: 724 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 725 if (!new_prog) 726 return -ENOMEM; 727 env->prog = new_prog; 728 insns = new_prog->insnsi; 729 aux = env->insn_aux_data; 730 delta += patch_len - 1; 731 } 732 733 return 0; 734 } 735 736 /* convert load instructions that access fields of a context type into a 737 * sequence of instructions that access fields of the underlying structure: 738 * struct __sk_buff -> struct sk_buff 739 * struct bpf_sock_ops -> struct sock 740 */ 741 int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) 742 { 743 struct bpf_subprog_info *subprogs = env->subprog_info; 744 const struct bpf_verifier_ops *ops = env->ops; 745 int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0; 746 const int insn_cnt = env->prog->len; 747 struct bpf_insn *epilogue_buf = env->epilogue_buf; 748 struct bpf_insn *insn_buf = env->insn_buf; 749 struct bpf_insn *insn; 750 u32 target_size, size_default, off; 751 struct bpf_prog *new_prog; 752 enum bpf_access_type type; 753 bool is_narrower_load; 754 int epilogue_idx = 0; 755 756 if (ops->gen_epilogue) { 757 epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog, 758 -(subprogs[0].stack_depth + 8)); 759 if (epilogue_cnt >= INSN_BUF_SIZE) { 760 verifier_bug(env, "epilogue is too long"); 761 return -EFAULT; 762 } else if (epilogue_cnt) { 763 /* Save the ARG_PTR_TO_CTX for the epilogue to use */ 764 cnt = 0; 765 subprogs[0].stack_depth += 8; 766 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, 767 -subprogs[0].stack_depth); 768 insn_buf[cnt++] = env->prog->insnsi[0]; 769 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 770 if (!new_prog) 771 return -ENOMEM; 772 env->prog = new_prog; 773 delta += cnt - 1; 774 775 ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1); 776 if (ret < 0) 777 return ret; 778 } 779 } 780 781 if (ops->gen_prologue || env->seen_direct_write) { 782 if (!ops->gen_prologue) { 783 verifier_bug(env, "gen_prologue is null"); 784 return -EFAULT; 785 } 786 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 787 env->prog); 788 if (cnt >= INSN_BUF_SIZE) { 789 verifier_bug(env, "prologue is too long"); 790 return -EFAULT; 791 } else if (cnt) { 792 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 793 if (!new_prog) 794 return -ENOMEM; 795 796 env->prog = new_prog; 797 delta += cnt - 1; 798 799 ret = add_kfunc_in_insns(env, insn_buf, cnt - 1); 800 if (ret < 0) 801 return ret; 802 } 803 } 804 805 if (delta) 806 WARN_ON(adjust_jmp_off(env->prog, 0, delta)); 807 808 if (bpf_prog_is_offloaded(env->prog->aux)) 809 return 0; 810 811 insn = env->prog->insnsi + delta; 812 813 for (i = 0; i < insn_cnt; i++, insn++) { 814 bpf_convert_ctx_access_t convert_ctx_access; 815 enum bpf_reg_type ptr_type; 816 u8 mode; 817 818 if (env->insn_aux_data[i + delta].nospec) { 819 WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state); 820 struct bpf_insn *patch = insn_buf; 821 822 *patch++ = BPF_ST_NOSPEC(); 823 *patch++ = *insn; 824 cnt = patch - insn_buf; 825 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 826 if (!new_prog) 827 return -ENOMEM; 828 829 delta += cnt - 1; 830 env->prog = new_prog; 831 insn = new_prog->insnsi + i + delta; 832 /* This can not be easily merged with the 833 * nospec_result-case, because an insn may require a 834 * nospec before and after itself. Therefore also do not 835 * 'continue' here but potentially apply further 836 * patching to insn. *insn should equal patch[1] now. 837 */ 838 } 839 840 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 841 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 842 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 843 insn->code == (BPF_LDX | BPF_MEM | BPF_DW) || 844 insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) || 845 insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) || 846 insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) { 847 type = BPF_READ; 848 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 849 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 850 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 851 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 852 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 853 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 854 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 855 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 856 type = BPF_WRITE; 857 } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) || 858 insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) || 859 insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) || 860 insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) && 861 env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) { 862 insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); 863 env->prog->aux->num_exentries++; 864 continue; 865 } else if (insn->code == (BPF_JMP | BPF_EXIT) && 866 epilogue_cnt && 867 i + delta < subprogs[1].start) { 868 /* Generate epilogue for the main prog */ 869 if (epilogue_idx) { 870 /* jump back to the earlier generated epilogue */ 871 insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1); 872 cnt = 1; 873 } else { 874 memcpy(insn_buf, epilogue_buf, 875 epilogue_cnt * sizeof(*epilogue_buf)); 876 cnt = epilogue_cnt; 877 /* epilogue_idx cannot be 0. It must have at 878 * least one ctx ptr saving insn before the 879 * epilogue. 880 */ 881 epilogue_idx = i + delta; 882 } 883 goto patch_insn_buf; 884 } else { 885 continue; 886 } 887 888 if (type == BPF_WRITE && 889 env->insn_aux_data[i + delta].nospec_result) { 890 /* nospec_result is only used to mitigate Spectre v4 and 891 * to limit verification-time for Spectre v1. 892 */ 893 struct bpf_insn *patch = insn_buf; 894 895 *patch++ = *insn; 896 *patch++ = BPF_ST_NOSPEC(); 897 cnt = patch - insn_buf; 898 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 899 if (!new_prog) 900 return -ENOMEM; 901 902 delta += cnt - 1; 903 env->prog = new_prog; 904 insn = new_prog->insnsi + i + delta; 905 continue; 906 } 907 908 ptr_type = env->insn_aux_data[i + delta].ptr_type; 909 switch ((int)ptr_type) { 910 case PTR_TO_CTX: 911 if (!ops->convert_ctx_access) 912 continue; 913 convert_ctx_access = ops->convert_ctx_access; 914 break; 915 case PTR_TO_SOCKET: 916 case PTR_TO_SOCK_COMMON: 917 convert_ctx_access = bpf_sock_convert_ctx_access; 918 break; 919 case PTR_TO_TCP_SOCK: 920 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 921 break; 922 case PTR_TO_XDP_SOCK: 923 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 924 break; 925 case PTR_TO_ARENA: 926 if (BPF_MODE(insn->code) == BPF_MEMSX) { 927 if (!bpf_jit_supports_insn(insn, true)) { 928 verbose(env, "sign extending loads from arena are not supported yet\n"); 929 return -EOPNOTSUPP; 930 } 931 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code); 932 } else { 933 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code); 934 } 935 env->prog->aux->num_exentries++; 936 continue; 937 default: 938 /* 939 * A pointer which may fault on a dereference must not 940 * be loaded from without fault protection, hence turn 941 * the BPF_LDX into a BPF_PROBE_MEM one so that a bad 942 * address is handled rather than panicking the kernel. 943 * A store through one is rejected earlier, there is no 944 * probed counterpart to rewrite it into. 945 */ 946 if (bpf_is_ptr_to_mem_or_btf_id(ptr_type) && 947 bpf_may_fault_on_deref(ptr_type) && 948 type == BPF_READ) { 949 if (BPF_MODE(insn->code) == BPF_MEM) 950 insn->code = BPF_LDX | BPF_PROBE_MEM | 951 BPF_SIZE(insn->code); 952 else 953 insn->code = BPF_LDX | BPF_PROBE_MEMSX | 954 BPF_SIZE(insn->code); 955 env->prog->aux->num_exentries++; 956 continue; 957 } 958 if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, 959 "access to a fault prone pointer is not rewritten as a probed one")) 960 return -EFAULT; 961 continue; 962 } 963 964 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 965 size = BPF_LDST_BYTES(insn); 966 mode = BPF_MODE(insn->code); 967 968 /* If the read access is a narrower load of the field, 969 * convert to a 4/8-byte load, to minimum program type specific 970 * convert_ctx_access changes. If conversion is successful, 971 * we will apply proper mask to the result. 972 */ 973 is_narrower_load = size < ctx_field_size; 974 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 975 off = insn->off; 976 if (is_narrower_load) { 977 u8 size_code; 978 979 if (type == BPF_WRITE) { 980 verifier_bug(env, "narrow ctx access misconfigured"); 981 return -EFAULT; 982 } 983 984 size_code = BPF_H; 985 if (ctx_field_size == 4) 986 size_code = BPF_W; 987 else if (ctx_field_size == 8) 988 size_code = BPF_DW; 989 990 insn->off = off & ~(size_default - 1); 991 insn->code = BPF_LDX | BPF_MEM | size_code; 992 } 993 994 target_size = 0; 995 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 996 &target_size); 997 if (cnt == 0 || cnt >= INSN_BUF_SIZE || 998 (ctx_field_size && !target_size)) { 999 verifier_bug(env, "error during ctx access conversion (%d)", cnt); 1000 return -EFAULT; 1001 } 1002 1003 if (is_narrower_load && size < target_size) { 1004 u8 shift = bpf_ctx_narrow_access_offset( 1005 off, size, size_default) * 8; 1006 if (shift && cnt + 1 >= INSN_BUF_SIZE) { 1007 verifier_bug(env, "narrow ctx load misconfigured"); 1008 return -EFAULT; 1009 } 1010 if (ctx_field_size <= 4) { 1011 if (shift) 1012 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 1013 insn->dst_reg, 1014 shift); 1015 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1016 (1 << size * 8) - 1); 1017 } else { 1018 if (shift) 1019 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 1020 insn->dst_reg, 1021 shift); 1022 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1023 (1ULL << size * 8) - 1); 1024 } 1025 } 1026 if (mode == BPF_MEMSX) 1027 insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, 1028 insn->dst_reg, insn->dst_reg, 1029 size * 8, 0); 1030 1031 patch_insn_buf: 1032 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1033 if (!new_prog) 1034 return -ENOMEM; 1035 1036 delta += cnt - 1; 1037 1038 /* keep walking new program and skip insns we just inserted */ 1039 env->prog = new_prog; 1040 insn = new_prog->insnsi + i + delta; 1041 } 1042 1043 return 0; 1044 } 1045 1046 static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env) 1047 { 1048 u32 *starts = NULL; 1049 1050 starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT); 1051 if (starts) { 1052 for (int i = 0; i < env->subprog_cnt; i++) 1053 starts[i] = env->subprog_info[i].start; 1054 } 1055 return starts; 1056 } 1057 1058 static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts) 1059 { 1060 for (int i = 0; i < env->subprog_cnt; i++) 1061 env->subprog_info[i].start = orig_starts[i]; 1062 /* restore the start of fake 'exit' subprog as well */ 1063 env->subprog_info[env->subprog_cnt].start = env->prog->len; 1064 } 1065 1066 static int jit_subprogs(struct bpf_verifier_env *env) 1067 { 1068 struct bpf_prog *prog = env->prog, **func, *tmp; 1069 int i, j, subprog_start, subprog_end = 0, len, subprog; 1070 struct bpf_map *map_ptr; 1071 struct bpf_insn *insn; 1072 void *old_bpf_func; 1073 int err, num_exentries; 1074 1075 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1076 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 1077 continue; 1078 1079 /* Upon error here we cannot fall back to interpreter but 1080 * need a hard reject of the program. Thus -EFAULT is 1081 * propagated in any case. 1082 */ 1083 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1084 if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d", 1085 i + insn->imm + 1)) 1086 return -EFAULT; 1087 /* temporarily remember subprog id inside insn instead of 1088 * aux_data, since next loop will split up all insns into funcs 1089 */ 1090 insn->off = subprog; 1091 /* remember original imm in case JIT fails and fallback 1092 * to interpreter will be needed 1093 */ 1094 env->insn_aux_data[i].call_imm = insn->imm; 1095 /* point imm to __bpf_call_base+1 from JITs point of view */ 1096 insn->imm = 1; 1097 if (bpf_pseudo_func(insn)) { 1098 #if defined(MODULES_VADDR) 1099 u64 addr = MODULES_VADDR; 1100 #else 1101 u64 addr = VMALLOC_START; 1102 #endif 1103 /* jit (e.g. x86_64) may emit fewer instructions 1104 * if it learns a u32 imm is the same as a u64 imm. 1105 * Set close enough to possible prog address. 1106 */ 1107 insn[0].imm = (u32)addr; 1108 insn[1].imm = addr >> 32; 1109 } 1110 } 1111 1112 err = bpf_prog_alloc_jited_linfo(prog); 1113 if (err) 1114 goto out_undo_insn; 1115 1116 err = -ENOMEM; 1117 func = kzalloc_objs(prog, env->subprog_cnt); 1118 if (!func) 1119 goto out_undo_insn; 1120 1121 for (i = 0; i < env->subprog_cnt; i++) { 1122 subprog_start = subprog_end; 1123 subprog_end = env->subprog_info[i + 1].start; 1124 1125 len = subprog_end - subprog_start; 1126 /* bpf_prog_run() doesn't call subprogs directly, 1127 * hence main prog stats include the runtime of subprogs. 1128 * subprogs don't have IDs and not reachable via prog_get_next_id 1129 * func[i]->stats will never be accessed and stays NULL 1130 */ 1131 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 1132 if (!func[i]) 1133 goto out_free; 1134 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 1135 len * sizeof(struct bpf_insn)); 1136 func[i]->type = prog->type; 1137 func[i]->len = len; 1138 if (bpf_prog_calc_tag(func[i])) 1139 goto out_free; 1140 func[i]->is_func = 1; 1141 func[i]->sleepable = prog->sleepable; 1142 func[i]->blinded = prog->blinded; 1143 func[i]->aux->func_idx = i; 1144 /* Below members will be freed only at prog->aux */ 1145 func[i]->aux->btf = prog->aux->btf; 1146 func[i]->aux->subprog_start = subprog_start; 1147 func[i]->aux->func_info = prog->aux->func_info; 1148 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 1149 func[i]->aux->poke_tab = prog->aux->poke_tab; 1150 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 1151 func[i]->aux->main_prog_aux = prog->aux; 1152 1153 for (j = 0; j < prog->aux->size_poke_tab; j++) { 1154 struct bpf_jit_poke_descriptor *poke; 1155 1156 poke = &prog->aux->poke_tab[j]; 1157 if (poke->insn_idx < subprog_end && 1158 poke->insn_idx >= subprog_start) 1159 poke->aux = func[i]->aux; 1160 } 1161 1162 func[i]->aux->name[0] = 'F'; 1163 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 1164 if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) 1165 func[i]->aux->jits_use_priv_stack = true; 1166 1167 func[i]->jit_requested = 1; 1168 func[i]->blinding_requested = prog->blinding_requested; 1169 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 1170 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 1171 func[i]->aux->linfo = prog->aux->linfo; 1172 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 1173 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 1174 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 1175 func[i]->aux->arena = prog->aux->arena; 1176 func[i]->aux->used_maps = env->used_maps; 1177 func[i]->aux->used_map_cnt = env->used_map_cnt; 1178 num_exentries = 0; 1179 insn = func[i]->insnsi; 1180 for (j = 0; j < func[i]->len; j++, insn++) { 1181 if (BPF_CLASS(insn->code) == BPF_LDX && 1182 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 1183 BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 1184 BPF_MODE(insn->code) == BPF_PROBE_MEM32SX || 1185 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) 1186 num_exentries++; 1187 if ((BPF_CLASS(insn->code) == BPF_STX || 1188 BPF_CLASS(insn->code) == BPF_ST) && 1189 BPF_MODE(insn->code) == BPF_PROBE_MEM32) 1190 num_exentries++; 1191 if (BPF_CLASS(insn->code) == BPF_STX && 1192 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) 1193 num_exentries++; 1194 } 1195 func[i]->aux->num_exentries = num_exentries; 1196 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 1197 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; 1198 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; 1199 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; 1200 func[i]->aux->token = prog->aux->token; 1201 if (!i) 1202 func[i]->aux->exception_boundary = env->seen_exception; 1203 func[i] = bpf_int_jit_compile(env, func[i]); 1204 if (!func[i]->jited) { 1205 err = -ENOTSUPP; 1206 goto out_free; 1207 } 1208 cond_resched(); 1209 } 1210 1211 /* at this point all bpf functions were successfully JITed 1212 * now populate all bpf_calls with correct addresses and 1213 * run last pass of JIT 1214 */ 1215 for (i = 0; i < env->subprog_cnt; i++) { 1216 insn = func[i]->insnsi; 1217 for (j = 0; j < func[i]->len; j++, insn++) { 1218 if (bpf_pseudo_func(insn)) { 1219 subprog = insn->off; 1220 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 1221 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 1222 continue; 1223 } 1224 if (!bpf_pseudo_call(insn)) 1225 continue; 1226 subprog = insn->off; 1227 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 1228 } 1229 1230 /* we use the aux data to keep a list of the start addresses 1231 * of the JITed images for each function in the program 1232 * 1233 * for some architectures, such as powerpc64, the imm field 1234 * might not be large enough to hold the offset of the start 1235 * address of the callee's JITed image from __bpf_call_base 1236 * 1237 * in such cases, we can lookup the start address of a callee 1238 * by using its subprog id, available from the off field of 1239 * the call instruction, as an index for this list 1240 */ 1241 func[i]->aux->func = func; 1242 func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1243 func[i]->aux->real_func_cnt = env->subprog_cnt; 1244 } 1245 for (i = 0; i < env->subprog_cnt; i++) { 1246 old_bpf_func = func[i]->bpf_func; 1247 tmp = bpf_int_jit_compile(env, func[i]); 1248 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 1249 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 1250 err = -ENOTSUPP; 1251 goto out_free; 1252 } 1253 cond_resched(); 1254 } 1255 1256 /* 1257 * Cleanup func[i]->aux fields which aren't required 1258 * or can become invalid in future 1259 */ 1260 for (i = 0; i < env->subprog_cnt; i++) { 1261 func[i]->aux->used_maps = NULL; 1262 func[i]->aux->used_map_cnt = 0; 1263 } 1264 1265 /* finally lock prog and jit images for all functions and 1266 * populate kallsysm. Begin at the first subprogram, since 1267 * bpf_prog_load will add the kallsyms for the main program. 1268 */ 1269 for (i = 1; i < env->subprog_cnt; i++) { 1270 err = bpf_prog_lock_ro(func[i]); 1271 if (err) 1272 goto out_free; 1273 } 1274 1275 for (i = 1; i < env->subprog_cnt; i++) 1276 bpf_prog_kallsyms_add(func[i]); 1277 1278 /* Last step: make now unused interpreter insns from main 1279 * prog consistent for later dump requests, so they can 1280 * later look the same as if they were interpreted only. 1281 */ 1282 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1283 if (bpf_pseudo_func(insn)) { 1284 insn[0].imm = env->insn_aux_data[i].call_imm; 1285 insn[1].imm = insn->off; 1286 insn->off = 0; 1287 continue; 1288 } 1289 if (!bpf_pseudo_call(insn)) 1290 continue; 1291 insn->imm = env->insn_aux_data[i].call_imm; 1292 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1293 insn->off = subprog; 1294 } 1295 1296 prog->jited = 1; 1297 prog->bpf_func = func[0]->bpf_func; 1298 prog->jited_len = func[0]->jited_len; 1299 prog->aux->extable = func[0]->aux->extable; 1300 prog->aux->num_exentries = func[0]->aux->num_exentries; 1301 prog->aux->func = func; 1302 prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1303 prog->aux->real_func_cnt = env->subprog_cnt; 1304 prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; 1305 prog->aux->exception_boundary = func[0]->aux->exception_boundary; 1306 prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust; 1307 bpf_prog_jit_attempt_done(prog); 1308 return 0; 1309 out_free: 1310 /* We failed JIT'ing, so at this point we need to unregister poke 1311 * descriptors from subprogs, so that kernel is not attempting to 1312 * patch it anymore as we're freeing the subprog JIT memory. 1313 */ 1314 for (i = 0; i < prog->aux->size_poke_tab; i++) { 1315 map_ptr = prog->aux->poke_tab[i].tail_call.map; 1316 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 1317 } 1318 /* At this point we're guaranteed that poke descriptors are not 1319 * live anymore. We can just unlink its descriptor table as it's 1320 * released with the main prog. 1321 */ 1322 for (i = 0; i < env->subprog_cnt; i++) { 1323 if (!func[i]) 1324 continue; 1325 func[i]->aux->poke_tab = NULL; 1326 bpf_jit_free(func[i]); 1327 } 1328 kfree(func); 1329 out_undo_insn: 1330 bpf_prog_jit_attempt_done(prog); 1331 return err; 1332 } 1333 1334 int bpf_jit_subprogs(struct bpf_verifier_env *env) 1335 { 1336 int err, i; 1337 bool blinded = false; 1338 struct bpf_insn *insn; 1339 struct bpf_prog *prog, *orig_prog; 1340 u32 *orig_subprog_starts; 1341 1342 if (env->subprog_cnt <= 1) 1343 return 0; 1344 1345 prog = orig_prog = env->prog; 1346 if (bpf_prog_need_blind(prog)) { 1347 orig_subprog_starts = bpf_dup_subprog_starts(env); 1348 if (!orig_subprog_starts) { 1349 err = -ENOMEM; 1350 goto out_cleanup; 1351 } 1352 prog = bpf_jit_blind_constants(env, prog); 1353 if (IS_ERR(prog)) { 1354 err = -ENOMEM; 1355 prog = orig_prog; 1356 goto out_restore; 1357 } 1358 blinded = true; 1359 } 1360 1361 err = jit_subprogs(env); 1362 if (err) 1363 goto out_jit_err; 1364 1365 if (blinded) { 1366 bpf_jit_prog_release_other(prog, orig_prog); 1367 kvfree(orig_subprog_starts); 1368 } 1369 1370 return 0; 1371 1372 out_jit_err: 1373 if (blinded) { 1374 bpf_jit_prog_release_other(orig_prog, prog); 1375 /* roll back to the clean original prog */ 1376 prog = env->prog = orig_prog; 1377 goto out_restore; 1378 } else { 1379 if (err != -EFAULT) { 1380 /* 1381 * We will fall back to interpreter mode when err is not -EFAULT, before 1382 * that, insn->off and insn->imm should be restored to their original 1383 * values since they were modified by jit_subprogs. 1384 */ 1385 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1386 if (!bpf_pseudo_call(insn)) 1387 continue; 1388 insn->off = 0; 1389 insn->imm = env->insn_aux_data[i].call_imm; 1390 } 1391 } 1392 goto out_cleanup; 1393 } 1394 1395 out_restore: 1396 bpf_restore_subprog_starts(env, orig_subprog_starts); 1397 kvfree(orig_subprog_starts); 1398 out_cleanup: 1399 /* cleanup main prog to be interpreted */ 1400 prog->jit_requested = 0; 1401 prog->blinding_requested = 0; 1402 return err; 1403 } 1404 1405 int bpf_fixup_call_args(struct bpf_verifier_env *env) 1406 { 1407 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1408 struct bpf_prog *prog = env->prog; 1409 struct bpf_insn *insn = prog->insnsi; 1410 int depth; 1411 #endif 1412 int i, err = 0; 1413 1414 for (i = 0; i < env->subprog_cnt; i++) { 1415 struct bpf_subprog_info *subprog = &env->subprog_info[i]; 1416 u16 outgoing = subprog->stack_arg_cnt - bpf_in_stack_arg_cnt(subprog); 1417 1418 if (subprog->max_out_stack_arg_cnt > outgoing) { 1419 verbose(env, 1420 "func#%d writes %u stack arg slots, but calls only require %u\n", 1421 i, subprog->max_out_stack_arg_cnt, outgoing); 1422 return -EINVAL; 1423 } 1424 } 1425 1426 if (env->prog->jit_requested && 1427 !bpf_prog_is_offloaded(env->prog->aux)) { 1428 err = bpf_jit_subprogs(env); 1429 if (err == 0) 1430 return 0; 1431 if (err == -EFAULT) 1432 return err; 1433 } 1434 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1435 if (prog->jit_required) { 1436 verbose(env, "program requires BPF JIT compiler but it is not available\n"); 1437 return -EINVAL; 1438 } 1439 for (i = 0; i < env->subprog_cnt; i++) { 1440 if (bpf_in_stack_arg_cnt(&env->subprog_info[i])) { 1441 verbose(env, "stack args are not supported in non-JITed programs\n"); 1442 return -EINVAL; 1443 } 1444 } 1445 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 1446 /* When JIT fails the progs with bpf2bpf calls and tail_calls 1447 * have to be rejected, since interpreter doesn't support them yet. 1448 */ 1449 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 1450 return -EINVAL; 1451 } 1452 for (i = 0; i < prog->len; i++, insn++) { 1453 if (bpf_pseudo_func(insn)) { 1454 /* When JIT fails the progs with callback calls 1455 * have to be rejected, since interpreter doesn't support them yet. 1456 */ 1457 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 1458 return -EINVAL; 1459 } 1460 1461 if (!bpf_pseudo_call(insn)) 1462 continue; 1463 depth = get_callee_stack_depth(env, insn, i); 1464 if (depth < 0) 1465 return depth; 1466 err = bpf_patch_call_args(insn, depth); 1467 if (err) { 1468 verbose(env, "stack depth %d exceeds interpreter stack depth limit\n", 1469 depth); 1470 return err; 1471 } 1472 } 1473 err = 0; 1474 #endif 1475 return err; 1476 } 1477 1478 /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */ 1479 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) 1480 { 1481 struct bpf_subprog_info *info = env->subprog_info; 1482 int cnt = env->subprog_cnt; 1483 struct bpf_prog *prog; 1484 1485 /* We only reserve one slot for hidden subprogs in subprog_info. */ 1486 if (env->hidden_subprog_cnt) { 1487 verifier_bug(env, "only one hidden subprog supported"); 1488 return -EFAULT; 1489 } 1490 /* We're not patching any existing instruction, just appending the new 1491 * ones for the hidden subprog. Hence all of the adjustment operations 1492 * in bpf_patch_insn_data are no-ops. 1493 */ 1494 prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); 1495 if (!prog) 1496 return -ENOMEM; 1497 env->prog = prog; 1498 info[cnt + 1].start = info[cnt].start; 1499 info[cnt].start = prog->len - len + 1; 1500 env->subprog_cnt++; 1501 env->hidden_subprog_cnt++; 1502 return 0; 1503 } 1504 1505 /* Do various post-verification rewrites in a single program pass. 1506 * These rewrites simplify JIT and interpreter implementations. 1507 */ 1508 int bpf_do_misc_fixups(struct bpf_verifier_env *env) 1509 { 1510 struct bpf_prog *prog = env->prog; 1511 enum bpf_attach_type eatype = prog->expected_attach_type; 1512 enum bpf_prog_type prog_type = resolve_prog_type(prog); 1513 struct bpf_insn *insn = prog->insnsi; 1514 const struct bpf_func_proto *fn; 1515 const int insn_cnt = prog->len; 1516 const struct bpf_map_ops *ops; 1517 struct bpf_insn_aux_data *aux; 1518 struct bpf_insn *insn_buf = env->insn_buf; 1519 struct bpf_prog *new_prog; 1520 struct bpf_map *map_ptr; 1521 int i, ret, cnt, delta = 0, cur_subprog = 0; 1522 struct bpf_subprog_info *subprogs = env->subprog_info; 1523 u16 stack_depth = subprogs[cur_subprog].stack_depth; 1524 u16 stack_depth_extra = 0; 1525 1526 if (env->seen_exception && !env->exception_callback_subprog) { 1527 struct bpf_insn *patch = insn_buf; 1528 1529 *patch++ = env->prog->insnsi[insn_cnt - 1]; 1530 *patch++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 1531 *patch++ = BPF_EXIT_INSN(); 1532 ret = add_hidden_subprog(env, insn_buf, patch - insn_buf); 1533 if (ret < 0) 1534 return ret; 1535 prog = env->prog; 1536 insn = prog->insnsi; 1537 1538 env->exception_callback_subprog = env->subprog_cnt - 1; 1539 /* Don't update insn_cnt, as add_hidden_subprog always appends insns */ 1540 bpf_mark_subprog_exc_cb(env, env->exception_callback_subprog); 1541 } 1542 1543 for (i = 0; i < insn_cnt;) { 1544 if (is_addr_space_cast32(env->prog, insn)) { 1545 /* convert to 32-bit mov that clears upper 32-bit */ 1546 insn->code = BPF_ALU | BPF_MOV | BPF_X; 1547 /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ 1548 insn->off = 0; 1549 insn->imm = 0; 1550 goto next_insn; 1551 } 1552 1553 if (env->insn_aux_data[i + delta].needs_zext) 1554 /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ 1555 insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); 1556 1557 /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ 1558 if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || 1559 insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || 1560 insn->code == (BPF_ALU | BPF_MOD | BPF_K) || 1561 insn->code == (BPF_ALU | BPF_DIV | BPF_K)) && 1562 insn->off == 1 && insn->imm == -1) { 1563 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1564 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1565 struct bpf_insn *patch = insn_buf; 1566 1567 if (isdiv) 1568 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1569 BPF_NEG | BPF_K, insn->dst_reg, 1570 0, 0, 0); 1571 else 1572 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1573 1574 cnt = patch - insn_buf; 1575 1576 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1577 if (!new_prog) 1578 return -ENOMEM; 1579 1580 delta += cnt - 1; 1581 env->prog = prog = new_prog; 1582 insn = new_prog->insnsi + i + delta; 1583 goto next_insn; 1584 } 1585 1586 /* Make divide-by-zero and divide-by-minus-one exceptions impossible. */ 1587 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 1588 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 1589 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 1590 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 1591 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1592 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1593 bool is_sdiv = isdiv && insn->off == 1; 1594 bool is_smod = !isdiv && insn->off == 1; 1595 struct bpf_insn *patch = insn_buf; 1596 1597 if (is_sdiv) { 1598 /* [R,W]x sdiv 0 -> 0 1599 * LLONG_MIN sdiv -1 -> LLONG_MIN 1600 * INT_MIN sdiv -1 -> INT_MIN 1601 */ 1602 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1603 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1604 BPF_ADD | BPF_K, BPF_REG_AX, 1605 0, 0, 1); 1606 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1607 BPF_JGT | BPF_K, BPF_REG_AX, 1608 0, 4, 1); 1609 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1610 BPF_JEQ | BPF_K, BPF_REG_AX, 1611 0, 1, 0); 1612 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1613 BPF_MOV | BPF_K, insn->dst_reg, 1614 0, 0, 0); 1615 /* BPF_NEG(LLONG_MIN) == -LLONG_MIN == LLONG_MIN */ 1616 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1617 BPF_NEG | BPF_K, insn->dst_reg, 1618 0, 0, 0); 1619 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1620 *patch++ = *insn; 1621 cnt = patch - insn_buf; 1622 } else if (is_smod) { 1623 /* [R,W]x mod 0 -> [R,W]x */ 1624 /* [R,W]x mod -1 -> 0 */ 1625 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1626 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1627 BPF_ADD | BPF_K, BPF_REG_AX, 1628 0, 0, 1); 1629 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1630 BPF_JGT | BPF_K, BPF_REG_AX, 1631 0, 3, 1); 1632 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1633 BPF_JEQ | BPF_K, BPF_REG_AX, 1634 0, 3 + (is64 ? 0 : 1), 1); 1635 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1636 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1637 *patch++ = *insn; 1638 1639 if (!is64) { 1640 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1641 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 1642 } 1643 cnt = patch - insn_buf; 1644 } else if (isdiv) { 1645 /* [R,W]x div 0 -> 0 */ 1646 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1647 BPF_JNE | BPF_K, insn->src_reg, 1648 0, 2, 0); 1649 *patch++ = BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg); 1650 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1651 *patch++ = *insn; 1652 cnt = patch - insn_buf; 1653 } else { 1654 /* [R,W]x mod 0 -> [R,W]x */ 1655 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1656 BPF_JEQ | BPF_K, insn->src_reg, 1657 0, 1 + (is64 ? 0 : 1), 0); 1658 *patch++ = *insn; 1659 1660 if (!is64) { 1661 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1662 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 1663 } 1664 cnt = patch - insn_buf; 1665 } 1666 1667 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1668 if (!new_prog) 1669 return -ENOMEM; 1670 1671 delta += cnt - 1; 1672 env->prog = prog = new_prog; 1673 insn = new_prog->insnsi + i + delta; 1674 goto next_insn; 1675 } 1676 1677 /* Make it impossible to de-reference a userspace address */ 1678 if (BPF_CLASS(insn->code) == BPF_LDX && 1679 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 1680 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) { 1681 struct bpf_insn *patch = insn_buf; 1682 u64 uaddress_limit = bpf_arch_uaddress_limit(); 1683 1684 if (!uaddress_limit) 1685 goto next_insn; 1686 1687 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1688 if (insn->off) 1689 *patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off); 1690 *patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32); 1691 *patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2); 1692 *patch++ = *insn; 1693 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1694 *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0); 1695 1696 cnt = patch - insn_buf; 1697 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1698 if (!new_prog) 1699 return -ENOMEM; 1700 1701 delta += cnt - 1; 1702 env->prog = prog = new_prog; 1703 insn = new_prog->insnsi + i + delta; 1704 goto next_insn; 1705 } 1706 1707 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 1708 if (BPF_CLASS(insn->code) == BPF_LD && 1709 (BPF_MODE(insn->code) == BPF_ABS || 1710 BPF_MODE(insn->code) == BPF_IND)) { 1711 cnt = env->ops->gen_ld_abs(insn, insn_buf); 1712 if (cnt == 0 || cnt >= INSN_BUF_SIZE) { 1713 verifier_bug(env, "%d insns generated for ld_abs", cnt); 1714 return -EFAULT; 1715 } 1716 1717 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1718 if (!new_prog) 1719 return -ENOMEM; 1720 1721 delta += cnt - 1; 1722 env->prog = prog = new_prog; 1723 insn = new_prog->insnsi + i + delta; 1724 goto next_insn; 1725 } 1726 1727 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 1728 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 1729 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 1730 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 1731 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 1732 struct bpf_insn *patch = insn_buf; 1733 bool issrc, isneg, isimm; 1734 u32 off_reg; 1735 1736 aux = &env->insn_aux_data[i + delta]; 1737 if (!aux->alu_state || 1738 aux->alu_state == BPF_ALU_NON_POINTER) 1739 goto next_insn; 1740 1741 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 1742 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 1743 BPF_ALU_SANITIZE_SRC; 1744 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 1745 1746 off_reg = issrc ? insn->src_reg : insn->dst_reg; 1747 if (isimm) { 1748 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 1749 } else { 1750 if (isneg) 1751 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 1752 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 1753 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 1754 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 1755 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 1756 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 1757 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 1758 } 1759 if (!issrc) 1760 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 1761 insn->src_reg = BPF_REG_AX; 1762 if (isneg) 1763 insn->code = insn->code == code_add ? 1764 code_sub : code_add; 1765 *patch++ = *insn; 1766 if (issrc && isneg && !isimm) 1767 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 1768 cnt = patch - insn_buf; 1769 1770 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1771 if (!new_prog) 1772 return -ENOMEM; 1773 1774 delta += cnt - 1; 1775 env->prog = prog = new_prog; 1776 insn = new_prog->insnsi + i + delta; 1777 goto next_insn; 1778 } 1779 1780 if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { 1781 int stack_off_cnt = -stack_depth - 16; 1782 1783 /* 1784 * Two 8 byte slots, depth-16 stores the count, and 1785 * depth-8 stores the start timestamp of the loop. 1786 * 1787 * The starting value of count is BPF_MAX_TIMED_LOOPS 1788 * (0xffff). Every iteration loads it and subs it by 1, 1789 * until the value becomes 0 in AX (thus, 1 in stack), 1790 * after which we call arch_bpf_timed_may_goto, which 1791 * either sets AX to 0xffff to keep looping, or to 0 1792 * upon timeout. AX is then stored into the stack. In 1793 * the next iteration, we either see 0 and break out, or 1794 * continue iterating until the next time value is 0 1795 * after subtraction, rinse and repeat. 1796 */ 1797 stack_depth_extra = 16; 1798 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt); 1799 if (insn->off >= 0) 1800 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5); 1801 else 1802 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); 1803 insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); 1804 insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); 1805 /* 1806 * AX is used as an argument to pass in stack_off_cnt 1807 * (to add to r10/fp), and also as the return value of 1808 * the call to arch_bpf_timed_may_goto. 1809 */ 1810 insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt); 1811 insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); 1812 insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); 1813 cnt = 7; 1814 1815 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1816 if (!new_prog) 1817 return -ENOMEM; 1818 1819 delta += cnt - 1; 1820 env->prog = prog = new_prog; 1821 insn = new_prog->insnsi + i + delta; 1822 goto next_insn; 1823 } else if (bpf_is_may_goto_insn(insn)) { 1824 int stack_off = -stack_depth - 8; 1825 1826 stack_depth_extra = 8; 1827 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); 1828 if (insn->off >= 0) 1829 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); 1830 else 1831 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); 1832 insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); 1833 insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); 1834 cnt = 4; 1835 1836 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1837 if (!new_prog) 1838 return -ENOMEM; 1839 1840 delta += cnt - 1; 1841 env->prog = prog = new_prog; 1842 insn = new_prog->insnsi + i + delta; 1843 goto next_insn; 1844 } 1845 1846 if (bpf_jit_supports_percpu_insn() && 1847 insn->code == (BPF_LD | BPF_IMM | BPF_DW) && 1848 (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 1849 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) { 1850 struct bpf_map *map; 1851 1852 aux = &env->insn_aux_data[i + delta]; 1853 map = env->used_maps[aux->map_index]; 1854 if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) 1855 goto next_insn; 1856 1857 prog->jit_required = true; 1858 1859 /* 1860 * We are *skipping* first half of ld_imm64 insn 1861 * with 'i++;', patching over second half of it 1862 * with that same half + mov64_percpu_reg insn. 1863 * All because bpf_patch_insn_data() can only 1864 * replace one 8-byte insn, which does not work 1865 * well for ld_imm64 insn. 1866 */ 1867 1868 insn_buf[0] = insn[1]; 1869 insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg); 1870 cnt = 2; 1871 1872 i++; 1873 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1874 if (!new_prog) 1875 return -ENOMEM; 1876 1877 delta += cnt - 1; 1878 env->prog = prog = new_prog; 1879 insn = new_prog->insnsi + i + delta; 1880 goto next_insn; 1881 } 1882 1883 if (insn->code != (BPF_JMP | BPF_CALL)) 1884 goto next_insn; 1885 if (insn->src_reg == BPF_PSEUDO_CALL) 1886 goto next_insn; 1887 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1888 ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); 1889 if (ret) 1890 return ret; 1891 if (cnt == 0) 1892 goto next_insn; 1893 1894 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1895 if (!new_prog) 1896 return -ENOMEM; 1897 1898 delta += cnt - 1; 1899 env->prog = prog = new_prog; 1900 insn = new_prog->insnsi + i + delta; 1901 goto next_insn; 1902 } 1903 1904 /* Skip inlining the helper call if the JIT does it. */ 1905 if (bpf_jit_inlines_helper_call(insn->imm)) { 1906 prog->jit_required = 1; 1907 goto next_insn; 1908 } 1909 1910 if (insn->imm == BPF_FUNC_get_route_realm) 1911 prog->dst_needed = 1; 1912 if (insn->imm == BPF_FUNC_get_prandom_u32) 1913 bpf_user_rnd_init_once(); 1914 if (insn->imm == BPF_FUNC_override_return) 1915 prog->kprobe_override = 1; 1916 if (insn->imm == BPF_FUNC_tail_call) { 1917 /* If we tail call into other programs, we 1918 * cannot make any assumptions since they can 1919 * be replaced dynamically during runtime in 1920 * the program array. 1921 */ 1922 prog->cb_access = 1; 1923 if (!bpf_allow_tail_call_in_subprogs(env)) 1924 prog->aux->stack_depth = MAX_BPF_STACK; 1925 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 1926 1927 /* mark bpf_tail_call as different opcode to avoid 1928 * conditional branch in the interpreter for every normal 1929 * call and to prevent accidental JITing by JIT compiler 1930 * that doesn't support bpf_tail_call yet 1931 */ 1932 insn->imm = 0; 1933 insn->code = BPF_JMP | BPF_TAIL_CALL; 1934 1935 aux = &env->insn_aux_data[i + delta]; 1936 if (env->bpf_capable && !prog->blinding_requested && 1937 prog->jit_requested && 1938 !bpf_map_key_poisoned(aux) && 1939 !bpf_map_ptr_poisoned(aux) && 1940 !bpf_map_ptr_unpriv(aux)) { 1941 struct bpf_jit_poke_descriptor desc = { 1942 .reason = BPF_POKE_REASON_TAIL_CALL, 1943 .tail_call.map = aux->map_ptr_state.map_ptr, 1944 .tail_call.key = bpf_map_key_immediate(aux), 1945 .insn_idx = i + delta, 1946 }; 1947 1948 ret = bpf_jit_add_poke_descriptor(prog, &desc); 1949 if (ret < 0) { 1950 verbose(env, "adding tail call poke descriptor failed\n"); 1951 return ret; 1952 } 1953 1954 insn->imm = ret + 1; 1955 goto next_insn; 1956 } 1957 1958 if (!bpf_map_ptr_unpriv(aux)) 1959 goto next_insn; 1960 1961 /* instead of changing every JIT dealing with tail_call 1962 * emit two extra insns: 1963 * if (index >= max_entries) goto out; 1964 * index &= array->index_mask; 1965 * to avoid out-of-bounds cpu speculation 1966 */ 1967 if (bpf_map_ptr_poisoned(aux)) { 1968 verbose(env, "tail_call abusing map_ptr\n"); 1969 return -EINVAL; 1970 } 1971 1972 map_ptr = aux->map_ptr_state.map_ptr; 1973 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 1974 map_ptr->max_entries, 2); 1975 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 1976 container_of(map_ptr, 1977 struct bpf_array, 1978 map)->index_mask); 1979 insn_buf[2] = *insn; 1980 cnt = 3; 1981 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1982 if (!new_prog) 1983 return -ENOMEM; 1984 1985 delta += cnt - 1; 1986 env->prog = prog = new_prog; 1987 insn = new_prog->insnsi + i + delta; 1988 goto next_insn; 1989 } 1990 1991 if (insn->imm == BPF_FUNC_timer_set_callback) { 1992 /* The verifier will process callback_fn as many times as necessary 1993 * with different maps and the register states prepared by 1994 * set_timer_callback_state will be accurate. 1995 * 1996 * The following use case is valid: 1997 * map1 is shared by prog1, prog2, prog3. 1998 * prog1 calls bpf_timer_init for some map1 elements 1999 * prog2 calls bpf_timer_set_callback for some map1 elements. 2000 * Those that were not bpf_timer_init-ed will return -EINVAL. 2001 * prog3 calls bpf_timer_start for some map1 elements. 2002 * Those that were not both bpf_timer_init-ed and 2003 * bpf_timer_set_callback-ed will return -EINVAL. 2004 */ 2005 struct bpf_insn ld_addrs[2] = { 2006 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), 2007 }; 2008 2009 insn_buf[0] = ld_addrs[0]; 2010 insn_buf[1] = ld_addrs[1]; 2011 insn_buf[2] = *insn; 2012 cnt = 3; 2013 2014 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2015 if (!new_prog) 2016 return -ENOMEM; 2017 2018 delta += cnt - 1; 2019 env->prog = prog = new_prog; 2020 insn = new_prog->insnsi + i + delta; 2021 goto patch_call_imm; 2022 } 2023 2024 /* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */ 2025 if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) { 2026 /* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data, 2027 * bpf_mem_alloc() returns a ptr to the percpu data ptr. 2028 */ 2029 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 2030 insn_buf[1] = *insn; 2031 cnt = 2; 2032 2033 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2034 if (!new_prog) 2035 return -ENOMEM; 2036 2037 delta += cnt - 1; 2038 env->prog = prog = new_prog; 2039 insn = new_prog->insnsi + i + delta; 2040 goto patch_call_imm; 2041 } 2042 2043 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 2044 * and other inlining handlers are currently limited to 64 bit 2045 * only. 2046 */ 2047 if (prog->jit_requested && BITS_PER_LONG == 64 && 2048 (insn->imm == BPF_FUNC_map_lookup_elem || 2049 insn->imm == BPF_FUNC_map_update_elem || 2050 insn->imm == BPF_FUNC_map_delete_elem || 2051 insn->imm == BPF_FUNC_map_push_elem || 2052 insn->imm == BPF_FUNC_map_pop_elem || 2053 insn->imm == BPF_FUNC_map_peek_elem || 2054 insn->imm == BPF_FUNC_redirect_map || 2055 insn->imm == BPF_FUNC_for_each_map_elem || 2056 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { 2057 aux = &env->insn_aux_data[i + delta]; 2058 if (bpf_map_ptr_poisoned(aux)) 2059 goto patch_call_imm; 2060 2061 map_ptr = aux->map_ptr_state.map_ptr; 2062 ops = map_ptr->ops; 2063 if (insn->imm == BPF_FUNC_map_lookup_elem && 2064 ops->map_gen_lookup) { 2065 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 2066 if (cnt == -EOPNOTSUPP) 2067 goto patch_map_ops_generic; 2068 if (cnt <= 0 || cnt >= INSN_BUF_SIZE) { 2069 verifier_bug(env, "%d insns generated for map lookup", cnt); 2070 return -EFAULT; 2071 } 2072 2073 if (bpf_map_is_percpu_map(map_ptr->map_type)) 2074 prog->jit_required = true; 2075 2076 new_prog = bpf_patch_insn_data(env, i + delta, 2077 insn_buf, cnt); 2078 if (!new_prog) 2079 return -ENOMEM; 2080 2081 delta += cnt - 1; 2082 env->prog = prog = new_prog; 2083 insn = new_prog->insnsi + i + delta; 2084 goto next_insn; 2085 } 2086 2087 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 2088 (void *(*)(struct bpf_map *map, void *key))NULL)); 2089 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 2090 (long (*)(struct bpf_map *map, void *key))NULL)); 2091 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 2092 (long (*)(struct bpf_map *map, void *key, void *value, 2093 u64 flags))NULL)); 2094 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 2095 (long (*)(struct bpf_map *map, void *value, 2096 u64 flags))NULL)); 2097 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 2098 (long (*)(struct bpf_map *map, void *value))NULL)); 2099 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 2100 (long (*)(struct bpf_map *map, void *value))NULL)); 2101 BUILD_BUG_ON(!__same_type(ops->map_redirect, 2102 (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); 2103 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 2104 (long (*)(struct bpf_map *map, 2105 bpf_callback_t callback_fn, 2106 void *callback_ctx, 2107 u64 flags))NULL)); 2108 BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, 2109 (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL)); 2110 2111 patch_map_ops_generic: 2112 switch (insn->imm) { 2113 case BPF_FUNC_map_lookup_elem: 2114 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 2115 goto next_insn; 2116 case BPF_FUNC_map_update_elem: 2117 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 2118 goto next_insn; 2119 case BPF_FUNC_map_delete_elem: 2120 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 2121 goto next_insn; 2122 case BPF_FUNC_map_push_elem: 2123 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 2124 goto next_insn; 2125 case BPF_FUNC_map_pop_elem: 2126 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 2127 goto next_insn; 2128 case BPF_FUNC_map_peek_elem: 2129 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 2130 goto next_insn; 2131 case BPF_FUNC_redirect_map: 2132 insn->imm = BPF_CALL_IMM(ops->map_redirect); 2133 goto next_insn; 2134 case BPF_FUNC_for_each_map_elem: 2135 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 2136 goto next_insn; 2137 case BPF_FUNC_map_lookup_percpu_elem: 2138 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); 2139 goto next_insn; 2140 } 2141 2142 goto patch_call_imm; 2143 } 2144 2145 /* Implement bpf_jiffies64 inline. */ 2146 if (prog->jit_requested && BITS_PER_LONG == 64 && 2147 insn->imm == BPF_FUNC_jiffies64) { 2148 struct bpf_insn ld_jiffies_addr[2] = { 2149 BPF_LD_IMM64(BPF_REG_0, 2150 (unsigned long)&jiffies), 2151 }; 2152 2153 insn_buf[0] = ld_jiffies_addr[0]; 2154 insn_buf[1] = ld_jiffies_addr[1]; 2155 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 2156 BPF_REG_0, 0); 2157 cnt = 3; 2158 2159 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 2160 cnt); 2161 if (!new_prog) 2162 return -ENOMEM; 2163 2164 delta += cnt - 1; 2165 env->prog = prog = new_prog; 2166 insn = new_prog->insnsi + i + delta; 2167 goto next_insn; 2168 } 2169 2170 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML) 2171 /* Implement bpf_get_smp_processor_id() inline. */ 2172 if (insn->imm == BPF_FUNC_get_smp_processor_id && 2173 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2174 /* BPF_FUNC_get_smp_processor_id inlining is an 2175 * optimization, so if cpu_number is ever 2176 * changed in some incompatible and hard to support 2177 * way, it's fine to back out this inlining logic 2178 */ 2179 #ifdef CONFIG_SMP 2180 prog->jit_required = true; 2181 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); 2182 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2183 insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); 2184 cnt = 3; 2185 #else 2186 insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); 2187 cnt = 1; 2188 #endif 2189 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2190 if (!new_prog) 2191 return -ENOMEM; 2192 2193 delta += cnt - 1; 2194 env->prog = prog = new_prog; 2195 insn = new_prog->insnsi + i + delta; 2196 goto next_insn; 2197 } 2198 2199 /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ 2200 if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && 2201 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2202 prog->jit_required = true; 2203 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); 2204 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2205 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); 2206 cnt = 3; 2207 2208 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2209 if (!new_prog) 2210 return -ENOMEM; 2211 2212 delta += cnt - 1; 2213 env->prog = prog = new_prog; 2214 insn = new_prog->insnsi + i + delta; 2215 goto next_insn; 2216 } 2217 #endif 2218 /* Implement bpf_get_func_arg inline. */ 2219 if (prog_type == BPF_PROG_TYPE_TRACING && 2220 insn->imm == BPF_FUNC_get_func_arg) { 2221 if (eatype == BPF_TRACE_RAW_TP) { 2222 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2223 2224 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2225 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2226 cnt = 1; 2227 } else { 2228 /* Load nr_args from ctx - 8 */ 2229 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2230 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2231 cnt = 2; 2232 } 2233 insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 2234 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 2235 insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 2236 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 2237 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2238 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 0); 2239 insn_buf[cnt++] = BPF_JMP_A(1); 2240 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2241 2242 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2243 if (!new_prog) 2244 return -ENOMEM; 2245 2246 delta += cnt - 1; 2247 env->prog = prog = new_prog; 2248 insn = new_prog->insnsi + i + delta; 2249 goto next_insn; 2250 } 2251 2252 /* Implement bpf_get_func_ret inline. */ 2253 if (prog_type == BPF_PROG_TYPE_TRACING && 2254 insn->imm == BPF_FUNC_get_func_ret) { 2255 if (eatype == BPF_TRACE_FEXIT || 2256 eatype == BPF_TRACE_FSESSION || 2257 eatype == BPF_TRACE_FEXIT_MULTI || 2258 eatype == BPF_TRACE_FSESSION_MULTI || 2259 eatype == BPF_MODIFY_RETURN) { 2260 /* Load nr_args from ctx - 8 */ 2261 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2262 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2263 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 2264 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 2265 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2266 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 2267 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 2268 cnt = 7; 2269 } else { 2270 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 2271 cnt = 1; 2272 } 2273 2274 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2275 if (!new_prog) 2276 return -ENOMEM; 2277 2278 delta += cnt - 1; 2279 env->prog = prog = new_prog; 2280 insn = new_prog->insnsi + i + delta; 2281 goto next_insn; 2282 } 2283 2284 /* Implement get_func_arg_cnt inline. */ 2285 if (prog_type == BPF_PROG_TYPE_TRACING && 2286 insn->imm == BPF_FUNC_get_func_arg_cnt) { 2287 if (eatype == BPF_TRACE_RAW_TP) { 2288 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2289 2290 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2291 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2292 cnt = 1; 2293 } else { 2294 /* Load nr_args from ctx - 8 */ 2295 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2296 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2297 cnt = 2; 2298 } 2299 2300 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2301 if (!new_prog) 2302 return -ENOMEM; 2303 2304 delta += cnt - 1; 2305 env->prog = prog = new_prog; 2306 insn = new_prog->insnsi + i + delta; 2307 goto next_insn; 2308 } 2309 2310 /* Implement bpf_get_func_ip inline. */ 2311 if (prog_type == BPF_PROG_TYPE_TRACING && 2312 insn->imm == BPF_FUNC_get_func_ip) { 2313 /* Load IP address from ctx - 16 */ 2314 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 2315 2316 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 2317 if (!new_prog) 2318 return -ENOMEM; 2319 2320 env->prog = prog = new_prog; 2321 insn = new_prog->insnsi + i + delta; 2322 goto next_insn; 2323 } 2324 2325 /* Implement bpf_get_branch_snapshot inline. */ 2326 if (IS_ENABLED(CONFIG_PERF_EVENTS) && 2327 prog->jit_requested && BITS_PER_LONG == 64 && 2328 insn->imm == BPF_FUNC_get_branch_snapshot) { 2329 /* We are dealing with the following func protos: 2330 * u64 bpf_get_branch_snapshot(void *buf, u32 size, u64 flags); 2331 * int perf_snapshot_branch_stack(struct perf_branch_entry *entries, u32 cnt); 2332 */ 2333 const u32 br_entry_size = sizeof(struct perf_branch_entry); 2334 2335 /* struct perf_branch_entry is part of UAPI and is 2336 * used as an array element, so extremely unlikely to 2337 * ever grow or shrink 2338 */ 2339 BUILD_BUG_ON(br_entry_size != 24); 2340 2341 /* if (unlikely(flags)) return -EINVAL */ 2342 insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7); 2343 2344 /* Transform size (bytes) into number of entries (cnt = size / 24). 2345 * But to avoid expensive division instruction, we implement 2346 * divide-by-3 through multiplication, followed by further 2347 * division by 8 through 3-bit right shift. 2348 * Refer to book "Hacker's Delight, 2nd ed." by Henry S. Warren, Jr., 2349 * p. 227, chapter "Unsigned Division by 3" for details and proofs. 2350 * 2351 * N / 3 <=> M * N / 2^33, where M = (2^33 + 1) / 3 = 0xaaaaaaab. 2352 */ 2353 insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab); 2354 insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0); 2355 insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36); 2356 2357 /* call perf_snapshot_branch_stack implementation */ 2358 insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack)); 2359 /* if (entry_cnt == 0) return -ENOENT */ 2360 insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4); 2361 /* return entry_cnt * sizeof(struct perf_branch_entry) */ 2362 insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size); 2363 insn_buf[7] = BPF_JMP_A(3); 2364 /* return -EINVAL; */ 2365 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2366 insn_buf[9] = BPF_JMP_A(1); 2367 /* return -ENOENT; */ 2368 insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT); 2369 cnt = 11; 2370 2371 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2372 if (!new_prog) 2373 return -ENOMEM; 2374 2375 delta += cnt - 1; 2376 env->prog = prog = new_prog; 2377 insn = new_prog->insnsi + i + delta; 2378 goto next_insn; 2379 } 2380 2381 /* Implement bpf_kptr_xchg inline */ 2382 if (prog->jit_requested && BITS_PER_LONG == 64 && 2383 insn->imm == BPF_FUNC_kptr_xchg && 2384 bpf_jit_supports_ptr_xchg()) { 2385 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); 2386 insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); 2387 cnt = 2; 2388 2389 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2390 if (!new_prog) 2391 return -ENOMEM; 2392 2393 delta += cnt - 1; 2394 env->prog = prog = new_prog; 2395 insn = new_prog->insnsi + i + delta; 2396 goto next_insn; 2397 } 2398 patch_call_imm: 2399 fn = env->ops->get_func_proto(insn->imm, env->prog); 2400 /* all functions that have prototype and verifier allowed 2401 * programs to call them, must be real in-kernel functions 2402 */ 2403 if (!fn->func) { 2404 verifier_bug(env, 2405 "not inlined functions %s#%d is missing func", 2406 func_id_name(insn->imm), insn->imm); 2407 return -EFAULT; 2408 } 2409 insn->imm = BPF_CALL_IMM(fn->func); 2410 next_insn: 2411 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2412 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2413 subprogs[cur_subprog].stack_extra = stack_depth_extra; 2414 2415 stack_depth = subprogs[cur_subprog].stack_depth; 2416 if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) { 2417 verbose(env, "stack size %d(extra %d) is too large\n", 2418 stack_depth, stack_depth_extra); 2419 return -EINVAL; 2420 } 2421 cur_subprog++; 2422 stack_depth = subprogs[cur_subprog].stack_depth; 2423 stack_depth_extra = 0; 2424 } 2425 i++; 2426 insn++; 2427 } 2428 2429 env->prog->aux->stack_depth = subprogs[0].stack_depth; 2430 for (i = 0; i < env->subprog_cnt; i++) { 2431 int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1; 2432 int subprog_start = subprogs[i].start; 2433 int stack_slots = subprogs[i].stack_extra / 8; 2434 int slots = delta, cnt = 0; 2435 2436 if (!stack_slots) 2437 continue; 2438 /* We need two slots in case timed may_goto is supported. */ 2439 if (stack_slots > slots) { 2440 verifier_bug(env, "stack_slots supports may_goto only"); 2441 return -EFAULT; 2442 } 2443 2444 stack_depth = subprogs[i].stack_depth; 2445 if (bpf_jit_supports_timed_may_goto()) { 2446 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2447 BPF_MAX_TIMED_LOOPS); 2448 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0); 2449 } else { 2450 /* Add ST insn to subprog prologue to init extra stack */ 2451 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2452 BPF_MAX_LOOPS); 2453 } 2454 /* Copy first actual insn to preserve it */ 2455 insn_buf[cnt++] = env->prog->insnsi[subprog_start]; 2456 2457 new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); 2458 if (!new_prog) 2459 return -ENOMEM; 2460 env->prog = prog = new_prog; 2461 /* 2462 * If may_goto is a first insn of a prog there could be a jmp 2463 * insn that points to it, hence adjust all such jmps to point 2464 * to insn after BPF_ST that inits may_goto count. 2465 * Adjustment will succeed because bpf_patch_insn_data() didn't fail. 2466 */ 2467 WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta)); 2468 } 2469 2470 /* Since poke tab is now finalized, publish aux to tracker. */ 2471 for (i = 0; i < prog->aux->size_poke_tab; i++) { 2472 map_ptr = prog->aux->poke_tab[i].tail_call.map; 2473 if (!map_ptr->ops->map_poke_track || 2474 !map_ptr->ops->map_poke_untrack || 2475 !map_ptr->ops->map_poke_run) { 2476 verifier_bug(env, "poke tab is misconfigured"); 2477 return -EFAULT; 2478 } 2479 2480 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 2481 if (ret < 0) { 2482 verbose(env, "tracking tail call prog failed\n"); 2483 return ret; 2484 } 2485 } 2486 2487 ret = sort_kfunc_descs_by_imm_off(env); 2488 if (ret) 2489 return ret; 2490 2491 return 0; 2492 } 2493 2494 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 2495 int position, 2496 s32 stack_base, 2497 u32 callback_subprogno, 2498 u32 *total_cnt) 2499 { 2500 s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; 2501 s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; 2502 s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; 2503 int reg_loop_max = BPF_REG_6; 2504 int reg_loop_cnt = BPF_REG_7; 2505 int reg_loop_ctx = BPF_REG_8; 2506 2507 struct bpf_insn *insn_buf = env->insn_buf; 2508 struct bpf_prog *new_prog; 2509 u32 callback_start; 2510 u32 call_insn_offset; 2511 s32 callback_offset; 2512 u32 cnt = 0; 2513 2514 /* This represents an inlined version of bpf_iter.c:bpf_loop, 2515 * be careful to modify this code in sync. 2516 */ 2517 2518 /* Return error and jump to the end of the patch if 2519 * expected number of iterations is too big. 2520 */ 2521 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2); 2522 insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG); 2523 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16); 2524 /* spill R6, R7, R8 to use these as loop vars */ 2525 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset); 2526 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset); 2527 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset); 2528 /* initialize loop vars */ 2529 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1); 2530 insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0); 2531 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3); 2532 /* loop header, 2533 * if reg_loop_cnt >= reg_loop_max skip the loop body 2534 */ 2535 insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5); 2536 /* callback call, 2537 * correct callback offset would be set after patching 2538 */ 2539 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt); 2540 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx); 2541 insn_buf[cnt++] = BPF_CALL_REL(0); 2542 /* increment loop counter */ 2543 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1); 2544 /* jump to loop header if callback returned 0 */ 2545 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6); 2546 /* return value of bpf_loop, 2547 * set R0 to the number of iterations 2548 */ 2549 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt); 2550 /* restore original values of R6, R7, R8 */ 2551 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset); 2552 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset); 2553 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset); 2554 2555 *total_cnt = cnt; 2556 new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); 2557 if (!new_prog) 2558 return new_prog; 2559 2560 /* callback start is known only after patching */ 2561 callback_start = env->subprog_info[callback_subprogno].start; 2562 /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ 2563 call_insn_offset = position + 12; 2564 callback_offset = callback_start - call_insn_offset - 1; 2565 new_prog->insnsi[call_insn_offset].imm = callback_offset; 2566 2567 return new_prog; 2568 } 2569 2570 static bool is_bpf_loop_call(struct bpf_insn *insn) 2571 { 2572 return insn->code == (BPF_JMP | BPF_CALL) && 2573 insn->src_reg == 0 && 2574 insn->imm == BPF_FUNC_loop; 2575 } 2576 2577 /* For all sub-programs in the program (including main) check 2578 * insn_aux_data to see if there are bpf_loop calls that require 2579 * inlining. If such calls are found the calls are replaced with a 2580 * sequence of instructions produced by `inline_bpf_loop` function and 2581 * subprog stack_depth is increased by the size of 3 registers. 2582 * This stack space is used to spill values of the R6, R7, R8. These 2583 * registers are used to store the loop bound, counter and context 2584 * variables. 2585 */ 2586 int bpf_optimize_bpf_loop(struct bpf_verifier_env *env) 2587 { 2588 struct bpf_subprog_info *subprogs = env->subprog_info; 2589 int i, cur_subprog = 0, cnt, delta = 0; 2590 struct bpf_insn *insn = env->prog->insnsi; 2591 int insn_cnt = env->prog->len; 2592 u16 stack_depth = subprogs[cur_subprog].stack_depth; 2593 u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2594 u16 stack_depth_extra = 0; 2595 2596 for (i = 0; i < insn_cnt; i++, insn++) { 2597 struct bpf_loop_inline_state *inline_state = 2598 &env->insn_aux_data[i + delta].loop_inline_state; 2599 2600 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { 2601 struct bpf_prog *new_prog; 2602 2603 stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; 2604 new_prog = inline_bpf_loop(env, 2605 i + delta, 2606 -(stack_depth + stack_depth_extra), 2607 inline_state->callback_subprogno, 2608 &cnt); 2609 if (!new_prog) 2610 return -ENOMEM; 2611 2612 delta += cnt - 1; 2613 env->prog = new_prog; 2614 insn = new_prog->insnsi + i + delta; 2615 } 2616 2617 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2618 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2619 cur_subprog++; 2620 stack_depth = subprogs[cur_subprog].stack_depth; 2621 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2622 stack_depth_extra = 0; 2623 } 2624 } 2625 2626 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 2627 2628 return 0; 2629 } 2630 2631 /* Remove unnecessary spill/fill pairs, members of fastcall pattern, 2632 * adjust subprograms stack depth when possible. 2633 */ 2634 int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) 2635 { 2636 struct bpf_subprog_info *subprog = env->subprog_info; 2637 struct bpf_insn_aux_data *aux = env->insn_aux_data; 2638 struct bpf_insn *insn = env->prog->insnsi; 2639 int insn_cnt = env->prog->len; 2640 u32 spills_num; 2641 bool modified = false; 2642 int i, j; 2643 2644 for (i = 0; i < insn_cnt; i++, insn++) { 2645 if (aux[i].fastcall_spills_num > 0) { 2646 spills_num = aux[i].fastcall_spills_num; 2647 /* NOPs would be removed by opt_remove_nops() */ 2648 for (j = 1; j <= spills_num; ++j) { 2649 *(insn - j) = NOP; 2650 *(insn + j) = NOP; 2651 } 2652 modified = true; 2653 } 2654 if ((subprog + 1)->start == i + 1) { 2655 if (modified && !subprog->keep_fastcall_stack) 2656 subprog->stack_depth = -subprog->fastcall_stack_off; 2657 subprog++; 2658 modified = false; 2659 } 2660 } 2661 2662 return 0; 2663 } 2664 2665