1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/bpf.h> 4 #include <linux/btf.h> 5 #include <linux/bpf_verifier.h> 6 #include <linux/filter.h> 7 #include <linux/vmalloc.h> 8 #include <linux/bsearch.h> 9 #include <linux/sort.h> 10 #include <linux/perf_event.h> 11 #include <net/xdp.h> 12 #include "disasm.h" 13 14 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) 15 16 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 17 { 18 return BPF_CLASS(insn->code) == BPF_STX && 19 BPF_MODE(insn->code) == BPF_ATOMIC && 20 insn->imm == BPF_CMPXCHG; 21 } 22 23 /* Returns true if 'insn' is an address space cast instruction translated as BPF_ALU op */ 24 static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *insn) 25 { 26 struct bpf_map *arena = (struct bpf_map *)prog->aux->arena; 27 28 if (insn->code != (BPF_ALU64 | BPF_MOV | BPF_X) || insn->off != BPF_ADDR_SPACE_CAST) 29 return false; 30 31 /* cast from as(1) to as(0) */ 32 if (insn->imm == 1) 33 return true; 34 35 /* cast from as(0) to as(1) */ 36 if (insn->imm == 1 << 16) 37 return arena && arena->map_flags & BPF_F_NO_USER_CONV; 38 39 /* non-BPF_F_NO_USER_CONV cast from as(0) to as(1) should be handled by JIT */ 40 return false; 41 } 42 43 /* Return the regno defined by the insn, or -1. */ 44 static int insn_def_regno(const struct bpf_insn *insn) 45 { 46 switch (BPF_CLASS(insn->code)) { 47 case BPF_JMP: 48 case BPF_JMP32: 49 case BPF_ST: 50 return -1; 51 case BPF_STX: 52 if (BPF_MODE(insn->code) == BPF_ATOMIC || 53 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { 54 if (insn->imm == BPF_CMPXCHG) 55 return BPF_REG_0; 56 else if (insn->imm == BPF_LOAD_ACQ) 57 return insn->dst_reg; 58 else if (insn->imm & BPF_FETCH) 59 return insn->src_reg; 60 } 61 return -1; 62 default: 63 return insn->dst_reg; 64 } 65 } 66 67 /* 68 * For use only in combination with insn_def_regno() >= 0. 69 * Returns TRUE if the destination register operates on 64-bit, 70 * otherwise return FALSE. 71 */ 72 static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn) 73 { 74 u8 class = BPF_CLASS(insn->code); 75 u8 mode = BPF_MODE(insn->code); 76 u8 size = BPF_SIZE(insn->code); 77 u8 op = BPF_OP(insn->code); 78 bool mode_mem; 79 80 /* subregister endiness swap */ 81 if ((class == BPF_ALU || class == BPF_ALU64) && op == BPF_END && insn->imm != 64) 82 return false; 83 84 /* w0 += 1 */ 85 if (class == BPF_ALU && op != BPF_END) 86 return false; 87 88 /* address space casts converted to BPF_ALU, see bpf_do_misc_fixups() */ 89 if (is_addr_space_cast32(prog, insn)) 90 return false; 91 92 /* non 64-bit, non signed extended loads */ 93 mode_mem = mode == BPF_MEM || mode == BPF_PROBE_MEM || mode == BPF_PROBE_MEM32; 94 if (class == BPF_LDX && mode_mem && size != BPF_DW) 95 return false; 96 97 /* atomics, see insn_def_regno() */ 98 if (class == BPF_STX && size != BPF_DW) 99 return false; 100 101 /* both LD_IND and LD_ABS return 32-bit data. */ 102 if (class == BPF_LD && (mode == BPF_IND || mode == BPF_ABS)) 103 return false; 104 105 /* Conservatively return true at default. */ 106 return true; 107 } 108 109 /* 110 * Return the 32-bit subregister defined by INSN, or -1 if INSN does not 111 * explicitly define a 32-bit value. 112 */ 113 int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) 114 { 115 int dst_reg = insn_def_regno(insn); 116 117 if (dst_reg < 0 || bpf_is_reg64(prog, insn)) 118 return -1; 119 120 return dst_reg; 121 } 122 123 static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) 124 { 125 const struct bpf_kfunc_desc *d0 = a; 126 const struct bpf_kfunc_desc *d1 = b; 127 128 if (d0->imm != d1->imm) 129 return d0->imm < d1->imm ? -1 : 1; 130 if (d0->offset != d1->offset) 131 return d0->offset < d1->offset ? -1 : 1; 132 return 0; 133 } 134 135 const struct btf_func_model * 136 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 137 const struct bpf_insn *insn) 138 { 139 const struct bpf_kfunc_desc desc = { 140 .imm = insn->imm, 141 .offset = insn->off, 142 }; 143 const struct bpf_kfunc_desc *res; 144 struct bpf_kfunc_desc_tab *tab; 145 146 tab = prog->aux->kfunc_tab; 147 res = bsearch(&desc, tab->descs, tab->nr_descs, 148 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off); 149 150 return res ? &res->func_model : NULL; 151 } 152 153 static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc) 154 { 155 unsigned long call_imm; 156 157 if (bpf_jit_supports_far_kfunc_call()) { 158 call_imm = desc->func_id; 159 } else { 160 call_imm = BPF_CALL_IMM(desc->addr); 161 /* Check whether the relative offset overflows desc->imm */ 162 if ((unsigned long)(s32)call_imm != call_imm) { 163 verbose(env, "address of kernel func_id %u is out of range\n", 164 desc->func_id); 165 return -EINVAL; 166 } 167 } 168 desc->imm = call_imm; 169 return 0; 170 } 171 172 static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env) 173 { 174 struct bpf_kfunc_desc_tab *tab; 175 int i, err; 176 177 tab = env->prog->aux->kfunc_tab; 178 if (!tab) 179 return 0; 180 181 for (i = 0; i < tab->nr_descs; i++) { 182 err = set_kfunc_desc_imm(env, &tab->descs[i]); 183 if (err) 184 return err; 185 } 186 187 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 188 kfunc_desc_cmp_by_imm_off, NULL); 189 return 0; 190 } 191 192 static int add_kfunc_in_insns(struct bpf_verifier_env *env, 193 struct bpf_insn *insn, int cnt) 194 { 195 int i, ret; 196 197 for (i = 0; i < cnt; i++, insn++) { 198 if (bpf_pseudo_kfunc_call(insn)) { 199 ret = bpf_add_kfunc_call(env, insn->imm, insn->off); 200 if (ret < 0) 201 return ret; 202 } 203 } 204 return 0; 205 } 206 207 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 208 static int get_callee_stack_depth(struct bpf_verifier_env *env, 209 const struct bpf_insn *insn, int idx) 210 { 211 int start = idx + insn->imm + 1, subprog; 212 213 subprog = bpf_find_subprog(env, start); 214 if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start)) 215 return -EFAULT; 216 return env->subprog_info[subprog].stack_depth; 217 } 218 #endif 219 220 /* single env->prog->insni[off] instruction was replaced with the range 221 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 222 * [0, off) and [off, end) to new locations, so the patched range stays zero 223 */ 224 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 225 struct bpf_prog *new_prog, u32 off, u32 cnt) 226 { 227 struct bpf_insn_aux_data *data = env->insn_aux_data; 228 struct bpf_insn *insn = new_prog->insnsi; 229 u32 old_seen = data[off].seen; 230 u32 prog_len; 231 int i; 232 233 /* aux info at OFF always needs adjustment, no matter fast path 234 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 235 * original insn at old prog. 236 */ 237 data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; 238 239 if (cnt == 1) 240 return; 241 prog_len = new_prog->len; 242 243 memmove(data + off + cnt - 1, data + off, 244 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 245 memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1)); 246 for (i = off; i < off + cnt - 1; i++) { 247 /* Expand insni[off]'s seen count to the patched range. */ 248 data[i].seen = old_seen; 249 data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; 250 } 251 252 /* 253 * The indirect_target flag of the original instruction was moved to the last of the 254 * new instructions by the above memmove and memset, but the indirect jump target is 255 * actually the first instruction, so move it back. This also matches with the behavior 256 * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new 257 * instruction. 258 */ 259 if (data[off + cnt - 1].indirect_target) { 260 data[off].indirect_target = 1; 261 data[off + cnt - 1].indirect_target = 0; 262 } 263 } 264 265 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 266 { 267 int i; 268 269 if (len == 1) 270 return; 271 /* NOTE: fake 'exit' subprog should be updated as well. */ 272 for (i = 0; i <= env->subprog_cnt; i++) { 273 if (env->subprog_info[i].start <= off) 274 continue; 275 env->subprog_info[i].start += len - 1; 276 } 277 } 278 279 static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) 280 { 281 int i; 282 283 if (len == 1) 284 return; 285 286 for (i = 0; i < env->insn_array_map_cnt; i++) 287 bpf_insn_array_adjust(env->insn_array_maps[i], off, len); 288 } 289 290 static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) 291 { 292 int i; 293 294 for (i = 0; i < env->insn_array_map_cnt; i++) 295 bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len); 296 } 297 298 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) 299 { 300 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 301 int i, sz = prog->aux->size_poke_tab; 302 struct bpf_jit_poke_descriptor *desc; 303 304 for (i = 0; i < sz; i++) { 305 desc = &tab[i]; 306 if (desc->insn_idx <= off) 307 continue; 308 desc->insn_idx += len - 1; 309 } 310 } 311 312 struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 313 const struct bpf_insn *patch, u32 len) 314 { 315 struct bpf_prog *new_prog; 316 struct bpf_insn_aux_data *new_data = NULL; 317 318 if (len > 1) { 319 new_data = vrealloc(env->insn_aux_data, 320 array_size(env->prog->len + len - 1, 321 sizeof(struct bpf_insn_aux_data)), 322 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 323 if (!new_data) 324 return NULL; 325 326 env->insn_aux_data = new_data; 327 } 328 329 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 330 if (IS_ERR(new_prog)) { 331 if (PTR_ERR(new_prog) == -ERANGE) 332 verbose(env, 333 "insn %d cannot be patched due to 16-bit range\n", 334 env->insn_aux_data[off].orig_idx); 335 return NULL; 336 } 337 adjust_insn_aux_data(env, new_prog, off, len); 338 adjust_subprog_starts(env, off, len); 339 adjust_insn_arrays(env, off, len); 340 adjust_poke_descs(new_prog, off, len); 341 return new_prog; 342 } 343 344 /* 345 * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the 346 * jump offset by 'delta'. 347 */ 348 static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta) 349 { 350 struct bpf_insn *insn = prog->insnsi; 351 u32 insn_cnt = prog->len, i; 352 s32 imm; 353 s16 off; 354 355 for (i = 0; i < insn_cnt; i++, insn++) { 356 u8 code = insn->code; 357 358 if (tgt_idx <= i && i < tgt_idx + delta) 359 continue; 360 361 if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) || 362 BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT) 363 continue; 364 365 if (insn->code == (BPF_JMP32 | BPF_JA)) { 366 if (i + 1 + insn->imm != tgt_idx) 367 continue; 368 if (check_add_overflow(insn->imm, delta, &imm)) 369 return -ERANGE; 370 insn->imm = imm; 371 } else { 372 if (i + 1 + insn->off != tgt_idx) 373 continue; 374 if (check_add_overflow(insn->off, delta, &off)) 375 return -ERANGE; 376 insn->off = off; 377 } 378 } 379 return 0; 380 } 381 382 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 383 u32 off, u32 cnt) 384 { 385 int i, j; 386 387 /* find first prog starting at or after off (first to remove) */ 388 for (i = 0; i < env->subprog_cnt; i++) 389 if (env->subprog_info[i].start >= off) 390 break; 391 /* find first prog starting at or after off + cnt (first to stay) */ 392 for (j = i; j < env->subprog_cnt; j++) 393 if (env->subprog_info[j].start >= off + cnt) 394 break; 395 /* if j doesn't start exactly at off + cnt, we are just removing 396 * the front of previous prog 397 */ 398 if (env->subprog_info[j].start != off + cnt) 399 j--; 400 401 if (j > i) { 402 struct bpf_prog_aux *aux = env->prog->aux; 403 int move; 404 405 /* move fake 'exit' subprog as well */ 406 move = env->subprog_cnt + 1 - j; 407 408 memmove(env->subprog_info + i, 409 env->subprog_info + j, 410 sizeof(*env->subprog_info) * move); 411 env->subprog_cnt -= j - i; 412 413 /* remove func_info */ 414 if (aux->func_info) { 415 move = aux->func_info_cnt - j; 416 417 memmove(aux->func_info + i, 418 aux->func_info + j, 419 sizeof(*aux->func_info) * move); 420 aux->func_info_cnt -= j - i; 421 /* func_info->insn_off is set after all code rewrites, 422 * in adjust_btf_func() - no need to adjust 423 */ 424 } 425 } else { 426 /* convert i from "first prog to remove" to "first to adjust" */ 427 if (env->subprog_info[i].start == off) 428 i++; 429 } 430 431 /* update fake 'exit' subprog as well */ 432 for (; i <= env->subprog_cnt; i++) 433 env->subprog_info[i].start -= cnt; 434 435 return 0; 436 } 437 438 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 439 u32 cnt) 440 { 441 struct bpf_prog *prog = env->prog; 442 u32 i, l_off, l_cnt, nr_linfo; 443 struct bpf_line_info *linfo; 444 445 nr_linfo = prog->aux->nr_linfo; 446 if (!nr_linfo) 447 return 0; 448 449 linfo = prog->aux->linfo; 450 451 /* find first line info to remove, count lines to be removed */ 452 for (i = 0; i < nr_linfo; i++) 453 if (linfo[i].insn_off >= off) 454 break; 455 456 l_off = i; 457 l_cnt = 0; 458 for (; i < nr_linfo; i++) 459 if (linfo[i].insn_off < off + cnt) 460 l_cnt++; 461 else 462 break; 463 464 /* First live insn doesn't match first live linfo, it needs to "inherit" 465 * last removed linfo. prog is already modified, so prog->len == off 466 * means no live instructions after (tail of the program was removed). 467 */ 468 if (prog->len != off && l_cnt && 469 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 470 l_cnt--; 471 linfo[--i].insn_off = off + cnt; 472 } 473 474 /* remove the line info which refer to the removed instructions */ 475 if (l_cnt) { 476 memmove(linfo + l_off, linfo + i, 477 sizeof(*linfo) * (nr_linfo - i)); 478 479 prog->aux->nr_linfo -= l_cnt; 480 nr_linfo = prog->aux->nr_linfo; 481 } 482 483 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 484 for (i = l_off; i < nr_linfo; i++) 485 linfo[i].insn_off -= cnt; 486 487 /* fix up all subprogs (incl. 'exit') which start >= off */ 488 for (i = 0; i <= env->subprog_cnt; i++) 489 if (env->subprog_info[i].linfo_idx > l_off) { 490 /* program may have started in the removed region but 491 * may not be fully removed 492 */ 493 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 494 env->subprog_info[i].linfo_idx -= l_cnt; 495 else 496 env->subprog_info[i].linfo_idx = l_off; 497 } 498 499 return 0; 500 } 501 502 /* 503 * Clean up dynamically allocated fields of aux data for instructions [start, ...] 504 */ 505 void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) 506 { 507 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 508 struct bpf_insn *insns = env->prog->insnsi; 509 int end = start + len; 510 int i; 511 512 for (i = start; i < end; i++) { 513 if (aux_data[i].jt) { 514 kvfree(aux_data[i].jt); 515 aux_data[i].jt = NULL; 516 } 517 518 if (bpf_is_ldimm64(&insns[i])) 519 i++; 520 } 521 } 522 523 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 524 { 525 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 526 unsigned int orig_prog_len = env->prog->len; 527 int err; 528 529 if (bpf_prog_is_offloaded(env->prog->aux)) 530 bpf_prog_offload_remove_insns(env, off, cnt); 531 532 /* Should be called before bpf_remove_insns, as it uses prog->insnsi */ 533 bpf_clear_insn_aux_data(env, off, cnt); 534 535 err = bpf_remove_insns(env->prog, off, cnt); 536 if (err) 537 return err; 538 539 err = adjust_subprog_starts_after_remove(env, off, cnt); 540 if (err) 541 return err; 542 543 err = bpf_adj_linfo_after_remove(env, off, cnt); 544 if (err) 545 return err; 546 547 adjust_insn_arrays_after_remove(env, off, cnt); 548 549 memmove(aux_data + off, aux_data + off + cnt, 550 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 551 552 return 0; 553 } 554 555 static const struct bpf_insn NOP = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 556 static const struct bpf_insn MAY_GOTO_0 = BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0, 0); 557 558 bool bpf_insn_is_cond_jump(u8 code) 559 { 560 u8 op; 561 562 op = BPF_OP(code); 563 if (BPF_CLASS(code) == BPF_JMP32) 564 return op != BPF_JA; 565 566 if (BPF_CLASS(code) != BPF_JMP) 567 return false; 568 569 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 570 } 571 572 void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 573 { 574 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 575 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 576 struct bpf_insn *insn = env->prog->insnsi; 577 const int insn_cnt = env->prog->len; 578 int i; 579 580 for (i = 0; i < insn_cnt; i++, insn++) { 581 if (!bpf_insn_is_cond_jump(insn->code)) 582 continue; 583 584 if (!aux_data[i + 1].seen) 585 ja.off = insn->off; 586 else if (!aux_data[i + 1 + insn->off].seen) 587 ja.off = 0; 588 else 589 continue; 590 591 if (bpf_prog_is_offloaded(env->prog->aux)) 592 bpf_prog_offload_replace_insn(env, i, &ja); 593 594 memcpy(insn, &ja, sizeof(ja)); 595 } 596 } 597 598 int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) 599 { 600 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 601 int insn_cnt = env->prog->len; 602 int i, err; 603 604 for (i = 0; i < insn_cnt; i++) { 605 int j; 606 607 j = 0; 608 while (i + j < insn_cnt && !aux_data[i + j].seen) 609 j++; 610 if (!j) 611 continue; 612 613 err = verifier_remove_insns(env, i, j); 614 if (err) 615 return err; 616 insn_cnt = env->prog->len; 617 } 618 619 return 0; 620 } 621 622 int bpf_opt_remove_nops(struct bpf_verifier_env *env) 623 { 624 struct bpf_insn *insn = env->prog->insnsi; 625 int insn_cnt = env->prog->len; 626 bool is_may_goto_0, is_ja; 627 int i, err; 628 629 for (i = 0; i < insn_cnt; i++) { 630 is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0)); 631 is_ja = !memcmp(&insn[i], &NOP, sizeof(NOP)); 632 633 if (!is_may_goto_0 && !is_ja) 634 continue; 635 636 err = verifier_remove_insns(env, i, 1); 637 if (err) 638 return err; 639 insn_cnt--; 640 /* Go back one insn to catch may_goto +1; may_goto +0 sequence */ 641 i -= (is_may_goto_0 && i > 0) ? 2 : 1; 642 } 643 644 return 0; 645 } 646 647 int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 648 const union bpf_attr *attr) 649 { 650 struct bpf_insn *patch; 651 /* use env->insn_buf as two independent buffers */ 652 struct bpf_insn *zext_patch = env->insn_buf; 653 struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2]; 654 struct bpf_insn_aux_data *aux = env->insn_aux_data; 655 int i, patch_len, delta = 0, len = env->prog->len; 656 struct bpf_insn *insns = env->prog->insnsi; 657 struct bpf_prog *new_prog; 658 bool rnd_hi32; 659 660 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 661 zext_patch[1] = BPF_ZEXT_REG(0); 662 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 663 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 664 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 665 for (i = 0; i < len; i++) { 666 int adj_idx = i + delta; 667 struct bpf_insn insn; 668 int load_reg; 669 670 insn = insns[adj_idx]; 671 load_reg = insn_def_regno(&insn); 672 if (!aux[adj_idx].zext_dst) { 673 u8 code, class; 674 u32 imm_rnd; 675 676 if (!rnd_hi32) 677 continue; 678 679 code = insn.code; 680 class = BPF_CLASS(code); 681 if (load_reg == -1) 682 continue; 683 684 if (bpf_is_reg64(env->prog, &insn)) { 685 if (class == BPF_LD && 686 BPF_MODE(code) == BPF_IMM) 687 i++; 688 continue; 689 } 690 691 /* ctx load could be transformed into wider load. */ 692 if (class == BPF_LDX && 693 aux[adj_idx].ptr_type == PTR_TO_CTX) 694 continue; 695 696 imm_rnd = get_random_u32(); 697 rnd_hi32_patch[0] = insn; 698 rnd_hi32_patch[1].imm = imm_rnd; 699 rnd_hi32_patch[3].dst_reg = load_reg; 700 patch = rnd_hi32_patch; 701 patch_len = 4; 702 goto apply_patch_buffer; 703 } 704 705 /* Add in an zero-extend instruction if a) the JIT has requested 706 * it or b) it's a CMPXCHG. 707 * 708 * The latter is because: BPF_CMPXCHG always loads a value into 709 * R0, therefore always zero-extends. However some archs' 710 * equivalent instruction only does this load when the 711 * comparison is successful. This detail of CMPXCHG is 712 * orthogonal to the general zero-extension behaviour of the 713 * CPU, so it's treated independently of bpf_jit_needs_zext. 714 */ 715 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 716 continue; 717 718 /* Zero-extension is done by the caller. */ 719 if (bpf_pseudo_kfunc_call(&insn)) 720 continue; 721 722 if (verifier_bug_if(load_reg == -1, env, 723 "zext_dst is set, but no reg is defined")) 724 return -EFAULT; 725 726 zext_patch[0] = insn; 727 zext_patch[1].dst_reg = load_reg; 728 zext_patch[1].src_reg = load_reg; 729 patch = zext_patch; 730 patch_len = 2; 731 apply_patch_buffer: 732 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 733 if (!new_prog) 734 return -ENOMEM; 735 env->prog = new_prog; 736 insns = new_prog->insnsi; 737 aux = env->insn_aux_data; 738 delta += patch_len - 1; 739 } 740 741 return 0; 742 } 743 744 /* convert load instructions that access fields of a context type into a 745 * sequence of instructions that access fields of the underlying structure: 746 * struct __sk_buff -> struct sk_buff 747 * struct bpf_sock_ops -> struct sock 748 */ 749 int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) 750 { 751 struct bpf_subprog_info *subprogs = env->subprog_info; 752 const struct bpf_verifier_ops *ops = env->ops; 753 int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0; 754 const int insn_cnt = env->prog->len; 755 struct bpf_insn *epilogue_buf = env->epilogue_buf; 756 struct bpf_insn *insn_buf = env->insn_buf; 757 struct bpf_insn *insn; 758 u32 target_size, size_default, off; 759 struct bpf_prog *new_prog; 760 enum bpf_access_type type; 761 bool is_narrower_load; 762 int epilogue_idx = 0; 763 764 if (ops->gen_epilogue) { 765 epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog, 766 -(subprogs[0].stack_depth + 8)); 767 if (epilogue_cnt >= INSN_BUF_SIZE) { 768 verifier_bug(env, "epilogue is too long"); 769 return -EFAULT; 770 } else if (epilogue_cnt) { 771 /* Save the ARG_PTR_TO_CTX for the epilogue to use */ 772 cnt = 0; 773 subprogs[0].stack_depth += 8; 774 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, 775 -subprogs[0].stack_depth); 776 insn_buf[cnt++] = env->prog->insnsi[0]; 777 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 778 if (!new_prog) 779 return -ENOMEM; 780 env->prog = new_prog; 781 delta += cnt - 1; 782 783 ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1); 784 if (ret < 0) 785 return ret; 786 } 787 } 788 789 if (ops->gen_prologue || env->seen_direct_write) { 790 if (!ops->gen_prologue) { 791 verifier_bug(env, "gen_prologue is null"); 792 return -EFAULT; 793 } 794 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 795 env->prog); 796 if (cnt >= INSN_BUF_SIZE) { 797 verifier_bug(env, "prologue is too long"); 798 return -EFAULT; 799 } else if (cnt) { 800 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 801 if (!new_prog) 802 return -ENOMEM; 803 804 env->prog = new_prog; 805 delta += cnt - 1; 806 807 ret = add_kfunc_in_insns(env, insn_buf, cnt - 1); 808 if (ret < 0) 809 return ret; 810 } 811 } 812 813 if (delta) 814 WARN_ON(adjust_jmp_off(env->prog, 0, delta)); 815 816 if (bpf_prog_is_offloaded(env->prog->aux)) 817 return 0; 818 819 insn = env->prog->insnsi + delta; 820 821 for (i = 0; i < insn_cnt; i++, insn++) { 822 bpf_convert_ctx_access_t convert_ctx_access; 823 u8 mode; 824 825 if (env->insn_aux_data[i + delta].nospec) { 826 WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state); 827 struct bpf_insn *patch = insn_buf; 828 829 *patch++ = BPF_ST_NOSPEC(); 830 *patch++ = *insn; 831 cnt = patch - insn_buf; 832 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 833 if (!new_prog) 834 return -ENOMEM; 835 836 delta += cnt - 1; 837 env->prog = new_prog; 838 insn = new_prog->insnsi + i + delta; 839 /* This can not be easily merged with the 840 * nospec_result-case, because an insn may require a 841 * nospec before and after itself. Therefore also do not 842 * 'continue' here but potentially apply further 843 * patching to insn. *insn should equal patch[1] now. 844 */ 845 } 846 847 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 848 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 849 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 850 insn->code == (BPF_LDX | BPF_MEM | BPF_DW) || 851 insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) || 852 insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) || 853 insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) { 854 type = BPF_READ; 855 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 856 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 857 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 858 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 859 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 860 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 861 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 862 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 863 type = BPF_WRITE; 864 } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) || 865 insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) || 866 insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) || 867 insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) && 868 env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) { 869 insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); 870 env->prog->aux->num_exentries++; 871 continue; 872 } else if (insn->code == (BPF_JMP | BPF_EXIT) && 873 epilogue_cnt && 874 i + delta < subprogs[1].start) { 875 /* Generate epilogue for the main prog */ 876 if (epilogue_idx) { 877 /* jump back to the earlier generated epilogue */ 878 insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1); 879 cnt = 1; 880 } else { 881 memcpy(insn_buf, epilogue_buf, 882 epilogue_cnt * sizeof(*epilogue_buf)); 883 cnt = epilogue_cnt; 884 /* epilogue_idx cannot be 0. It must have at 885 * least one ctx ptr saving insn before the 886 * epilogue. 887 */ 888 epilogue_idx = i + delta; 889 } 890 goto patch_insn_buf; 891 } else { 892 continue; 893 } 894 895 if (type == BPF_WRITE && 896 env->insn_aux_data[i + delta].nospec_result) { 897 /* nospec_result is only used to mitigate Spectre v4 and 898 * to limit verification-time for Spectre v1. 899 */ 900 struct bpf_insn *patch = insn_buf; 901 902 *patch++ = *insn; 903 *patch++ = BPF_ST_NOSPEC(); 904 cnt = patch - insn_buf; 905 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 906 if (!new_prog) 907 return -ENOMEM; 908 909 delta += cnt - 1; 910 env->prog = new_prog; 911 insn = new_prog->insnsi + i + delta; 912 continue; 913 } 914 915 switch ((int)env->insn_aux_data[i + delta].ptr_type) { 916 case PTR_TO_CTX: 917 if (!ops->convert_ctx_access) 918 continue; 919 convert_ctx_access = ops->convert_ctx_access; 920 break; 921 case PTR_TO_SOCKET: 922 case PTR_TO_SOCK_COMMON: 923 convert_ctx_access = bpf_sock_convert_ctx_access; 924 break; 925 case PTR_TO_TCP_SOCK: 926 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 927 break; 928 case PTR_TO_XDP_SOCK: 929 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 930 break; 931 case PTR_TO_BTF_ID: 932 case PTR_TO_BTF_ID | PTR_UNTRUSTED: 933 /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike 934 * PTR_TO_BTF_ID, and an active referenced id, but the same cannot 935 * be said once it is marked PTR_UNTRUSTED, hence we must handle 936 * any faults for loads into such types. BPF_WRITE is disallowed 937 * for this case. 938 */ 939 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: 940 case PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED: 941 if (type == BPF_READ) { 942 if (BPF_MODE(insn->code) == BPF_MEM) 943 insn->code = BPF_LDX | BPF_PROBE_MEM | 944 BPF_SIZE((insn)->code); 945 else 946 insn->code = BPF_LDX | BPF_PROBE_MEMSX | 947 BPF_SIZE((insn)->code); 948 env->prog->aux->num_exentries++; 949 } 950 continue; 951 case PTR_TO_ARENA: 952 if (BPF_MODE(insn->code) == BPF_MEMSX) { 953 if (!bpf_jit_supports_insn(insn, true)) { 954 verbose(env, "sign extending loads from arena are not supported yet\n"); 955 return -EOPNOTSUPP; 956 } 957 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code); 958 } else { 959 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code); 960 } 961 env->prog->aux->num_exentries++; 962 continue; 963 default: 964 continue; 965 } 966 967 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 968 size = BPF_LDST_BYTES(insn); 969 mode = BPF_MODE(insn->code); 970 971 /* If the read access is a narrower load of the field, 972 * convert to a 4/8-byte load, to minimum program type specific 973 * convert_ctx_access changes. If conversion is successful, 974 * we will apply proper mask to the result. 975 */ 976 is_narrower_load = size < ctx_field_size; 977 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 978 off = insn->off; 979 if (is_narrower_load) { 980 u8 size_code; 981 982 if (type == BPF_WRITE) { 983 verifier_bug(env, "narrow ctx access misconfigured"); 984 return -EFAULT; 985 } 986 987 size_code = BPF_H; 988 if (ctx_field_size == 4) 989 size_code = BPF_W; 990 else if (ctx_field_size == 8) 991 size_code = BPF_DW; 992 993 insn->off = off & ~(size_default - 1); 994 insn->code = BPF_LDX | BPF_MEM | size_code; 995 } 996 997 target_size = 0; 998 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 999 &target_size); 1000 if (cnt == 0 || cnt >= INSN_BUF_SIZE || 1001 (ctx_field_size && !target_size)) { 1002 verifier_bug(env, "error during ctx access conversion (%d)", cnt); 1003 return -EFAULT; 1004 } 1005 1006 if (is_narrower_load && size < target_size) { 1007 u8 shift = bpf_ctx_narrow_access_offset( 1008 off, size, size_default) * 8; 1009 if (shift && cnt + 1 >= INSN_BUF_SIZE) { 1010 verifier_bug(env, "narrow ctx load misconfigured"); 1011 return -EFAULT; 1012 } 1013 if (ctx_field_size <= 4) { 1014 if (shift) 1015 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 1016 insn->dst_reg, 1017 shift); 1018 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1019 (1 << size * 8) - 1); 1020 } else { 1021 if (shift) 1022 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 1023 insn->dst_reg, 1024 shift); 1025 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 1026 (1ULL << size * 8) - 1); 1027 } 1028 } 1029 if (mode == BPF_MEMSX) 1030 insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, 1031 insn->dst_reg, insn->dst_reg, 1032 size * 8, 0); 1033 1034 patch_insn_buf: 1035 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1036 if (!new_prog) 1037 return -ENOMEM; 1038 1039 delta += cnt - 1; 1040 1041 /* keep walking new program and skip insns we just inserted */ 1042 env->prog = new_prog; 1043 insn = new_prog->insnsi + i + delta; 1044 } 1045 1046 return 0; 1047 } 1048 1049 static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env) 1050 { 1051 u32 *starts = NULL; 1052 1053 starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT); 1054 if (starts) { 1055 for (int i = 0; i < env->subprog_cnt; i++) 1056 starts[i] = env->subprog_info[i].start; 1057 } 1058 return starts; 1059 } 1060 1061 static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts) 1062 { 1063 for (int i = 0; i < env->subprog_cnt; i++) 1064 env->subprog_info[i].start = orig_starts[i]; 1065 /* restore the start of fake 'exit' subprog as well */ 1066 env->subprog_info[env->subprog_cnt].start = env->prog->len; 1067 } 1068 1069 struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) 1070 { 1071 size_t size; 1072 void *new_aux; 1073 1074 size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len); 1075 new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT); 1076 if (new_aux) 1077 memcpy(new_aux, env->insn_aux_data, size); 1078 return new_aux; 1079 } 1080 1081 void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, 1082 struct bpf_insn_aux_data *orig_insn_aux) 1083 { 1084 /* the expanded elements are zero-filled, so no special handling is required */ 1085 vfree(env->insn_aux_data); 1086 env->insn_aux_data = orig_insn_aux; 1087 } 1088 1089 static int jit_subprogs(struct bpf_verifier_env *env) 1090 { 1091 struct bpf_prog *prog = env->prog, **func, *tmp; 1092 int i, j, subprog_start, subprog_end = 0, len, subprog; 1093 struct bpf_map *map_ptr; 1094 struct bpf_insn *insn; 1095 void *old_bpf_func; 1096 int err, num_exentries; 1097 1098 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1099 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 1100 continue; 1101 1102 /* Upon error here we cannot fall back to interpreter but 1103 * need a hard reject of the program. Thus -EFAULT is 1104 * propagated in any case. 1105 */ 1106 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1107 if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d", 1108 i + insn->imm + 1)) 1109 return -EFAULT; 1110 /* temporarily remember subprog id inside insn instead of 1111 * aux_data, since next loop will split up all insns into funcs 1112 */ 1113 insn->off = subprog; 1114 /* remember original imm in case JIT fails and fallback 1115 * to interpreter will be needed 1116 */ 1117 env->insn_aux_data[i].call_imm = insn->imm; 1118 /* point imm to __bpf_call_base+1 from JITs point of view */ 1119 insn->imm = 1; 1120 if (bpf_pseudo_func(insn)) { 1121 #if defined(MODULES_VADDR) 1122 u64 addr = MODULES_VADDR; 1123 #else 1124 u64 addr = VMALLOC_START; 1125 #endif 1126 /* jit (e.g. x86_64) may emit fewer instructions 1127 * if it learns a u32 imm is the same as a u64 imm. 1128 * Set close enough to possible prog address. 1129 */ 1130 insn[0].imm = (u32)addr; 1131 insn[1].imm = addr >> 32; 1132 } 1133 } 1134 1135 err = bpf_prog_alloc_jited_linfo(prog); 1136 if (err) 1137 goto out_undo_insn; 1138 1139 err = -ENOMEM; 1140 func = kzalloc_objs(prog, env->subprog_cnt); 1141 if (!func) 1142 goto out_undo_insn; 1143 1144 for (i = 0; i < env->subprog_cnt; i++) { 1145 subprog_start = subprog_end; 1146 subprog_end = env->subprog_info[i + 1].start; 1147 1148 len = subprog_end - subprog_start; 1149 /* bpf_prog_run() doesn't call subprogs directly, 1150 * hence main prog stats include the runtime of subprogs. 1151 * subprogs don't have IDs and not reachable via prog_get_next_id 1152 * func[i]->stats will never be accessed and stays NULL 1153 */ 1154 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 1155 if (!func[i]) 1156 goto out_free; 1157 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 1158 len * sizeof(struct bpf_insn)); 1159 func[i]->type = prog->type; 1160 func[i]->len = len; 1161 if (bpf_prog_calc_tag(func[i])) 1162 goto out_free; 1163 func[i]->is_func = 1; 1164 func[i]->sleepable = prog->sleepable; 1165 func[i]->blinded = prog->blinded; 1166 func[i]->aux->func_idx = i; 1167 /* Below members will be freed only at prog->aux */ 1168 func[i]->aux->btf = prog->aux->btf; 1169 func[i]->aux->subprog_start = subprog_start; 1170 func[i]->aux->func_info = prog->aux->func_info; 1171 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 1172 func[i]->aux->poke_tab = prog->aux->poke_tab; 1173 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 1174 func[i]->aux->main_prog_aux = prog->aux; 1175 1176 for (j = 0; j < prog->aux->size_poke_tab; j++) { 1177 struct bpf_jit_poke_descriptor *poke; 1178 1179 poke = &prog->aux->poke_tab[j]; 1180 if (poke->insn_idx < subprog_end && 1181 poke->insn_idx >= subprog_start) 1182 poke->aux = func[i]->aux; 1183 } 1184 1185 func[i]->aux->name[0] = 'F'; 1186 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 1187 if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) 1188 func[i]->aux->jits_use_priv_stack = true; 1189 1190 func[i]->jit_requested = 1; 1191 func[i]->blinding_requested = prog->blinding_requested; 1192 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 1193 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 1194 func[i]->aux->linfo = prog->aux->linfo; 1195 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 1196 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 1197 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 1198 func[i]->aux->arena = prog->aux->arena; 1199 func[i]->aux->used_maps = env->used_maps; 1200 func[i]->aux->used_map_cnt = env->used_map_cnt; 1201 num_exentries = 0; 1202 insn = func[i]->insnsi; 1203 for (j = 0; j < func[i]->len; j++, insn++) { 1204 if (BPF_CLASS(insn->code) == BPF_LDX && 1205 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 1206 BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 1207 BPF_MODE(insn->code) == BPF_PROBE_MEM32SX || 1208 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) 1209 num_exentries++; 1210 if ((BPF_CLASS(insn->code) == BPF_STX || 1211 BPF_CLASS(insn->code) == BPF_ST) && 1212 BPF_MODE(insn->code) == BPF_PROBE_MEM32) 1213 num_exentries++; 1214 if (BPF_CLASS(insn->code) == BPF_STX && 1215 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) 1216 num_exentries++; 1217 } 1218 func[i]->aux->num_exentries = num_exentries; 1219 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 1220 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; 1221 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; 1222 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; 1223 func[i]->aux->token = prog->aux->token; 1224 if (!i) 1225 func[i]->aux->exception_boundary = env->seen_exception; 1226 func[i] = bpf_int_jit_compile(env, func[i]); 1227 if (!func[i]->jited) { 1228 err = -ENOTSUPP; 1229 goto out_free; 1230 } 1231 cond_resched(); 1232 } 1233 1234 /* at this point all bpf functions were successfully JITed 1235 * now populate all bpf_calls with correct addresses and 1236 * run last pass of JIT 1237 */ 1238 for (i = 0; i < env->subprog_cnt; i++) { 1239 insn = func[i]->insnsi; 1240 for (j = 0; j < func[i]->len; j++, insn++) { 1241 if (bpf_pseudo_func(insn)) { 1242 subprog = insn->off; 1243 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 1244 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 1245 continue; 1246 } 1247 if (!bpf_pseudo_call(insn)) 1248 continue; 1249 subprog = insn->off; 1250 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 1251 } 1252 1253 /* we use the aux data to keep a list of the start addresses 1254 * of the JITed images for each function in the program 1255 * 1256 * for some architectures, such as powerpc64, the imm field 1257 * might not be large enough to hold the offset of the start 1258 * address of the callee's JITed image from __bpf_call_base 1259 * 1260 * in such cases, we can lookup the start address of a callee 1261 * by using its subprog id, available from the off field of 1262 * the call instruction, as an index for this list 1263 */ 1264 func[i]->aux->func = func; 1265 func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1266 func[i]->aux->real_func_cnt = env->subprog_cnt; 1267 } 1268 for (i = 0; i < env->subprog_cnt; i++) { 1269 old_bpf_func = func[i]->bpf_func; 1270 tmp = bpf_int_jit_compile(env, func[i]); 1271 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 1272 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 1273 err = -ENOTSUPP; 1274 goto out_free; 1275 } 1276 cond_resched(); 1277 } 1278 1279 /* 1280 * Cleanup func[i]->aux fields which aren't required 1281 * or can become invalid in future 1282 */ 1283 for (i = 0; i < env->subprog_cnt; i++) { 1284 func[i]->aux->used_maps = NULL; 1285 func[i]->aux->used_map_cnt = 0; 1286 } 1287 1288 /* finally lock prog and jit images for all functions and 1289 * populate kallsysm. Begin at the first subprogram, since 1290 * bpf_prog_load will add the kallsyms for the main program. 1291 */ 1292 for (i = 1; i < env->subprog_cnt; i++) { 1293 err = bpf_prog_lock_ro(func[i]); 1294 if (err) 1295 goto out_free; 1296 } 1297 1298 for (i = 1; i < env->subprog_cnt; i++) 1299 bpf_prog_kallsyms_add(func[i]); 1300 1301 /* Last step: make now unused interpreter insns from main 1302 * prog consistent for later dump requests, so they can 1303 * later look the same as if they were interpreted only. 1304 */ 1305 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1306 if (bpf_pseudo_func(insn)) { 1307 insn[0].imm = env->insn_aux_data[i].call_imm; 1308 insn[1].imm = insn->off; 1309 insn->off = 0; 1310 continue; 1311 } 1312 if (!bpf_pseudo_call(insn)) 1313 continue; 1314 insn->imm = env->insn_aux_data[i].call_imm; 1315 subprog = bpf_find_subprog(env, i + insn->imm + 1); 1316 insn->off = subprog; 1317 } 1318 1319 prog->jited = 1; 1320 prog->bpf_func = func[0]->bpf_func; 1321 prog->jited_len = func[0]->jited_len; 1322 prog->aux->extable = func[0]->aux->extable; 1323 prog->aux->num_exentries = func[0]->aux->num_exentries; 1324 prog->aux->func = func; 1325 prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 1326 prog->aux->real_func_cnt = env->subprog_cnt; 1327 prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; 1328 prog->aux->exception_boundary = func[0]->aux->exception_boundary; 1329 prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust; 1330 bpf_prog_jit_attempt_done(prog); 1331 return 0; 1332 out_free: 1333 /* We failed JIT'ing, so at this point we need to unregister poke 1334 * descriptors from subprogs, so that kernel is not attempting to 1335 * patch it anymore as we're freeing the subprog JIT memory. 1336 */ 1337 for (i = 0; i < prog->aux->size_poke_tab; i++) { 1338 map_ptr = prog->aux->poke_tab[i].tail_call.map; 1339 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 1340 } 1341 /* At this point we're guaranteed that poke descriptors are not 1342 * live anymore. We can just unlink its descriptor table as it's 1343 * released with the main prog. 1344 */ 1345 for (i = 0; i < env->subprog_cnt; i++) { 1346 if (!func[i]) 1347 continue; 1348 func[i]->aux->poke_tab = NULL; 1349 bpf_jit_free(func[i]); 1350 } 1351 kfree(func); 1352 out_undo_insn: 1353 bpf_prog_jit_attempt_done(prog); 1354 return err; 1355 } 1356 1357 int bpf_jit_subprogs(struct bpf_verifier_env *env) 1358 { 1359 int err, i; 1360 bool blinded = false; 1361 struct bpf_insn *insn; 1362 struct bpf_prog *prog, *orig_prog; 1363 struct bpf_insn_aux_data *orig_insn_aux; 1364 u32 *orig_subprog_starts; 1365 1366 if (env->subprog_cnt <= 1) 1367 return 0; 1368 1369 prog = orig_prog = env->prog; 1370 if (bpf_prog_need_blind(prog)) { 1371 orig_insn_aux = bpf_dup_insn_aux_data(env); 1372 if (!orig_insn_aux) { 1373 err = -ENOMEM; 1374 goto out_cleanup; 1375 } 1376 orig_subprog_starts = bpf_dup_subprog_starts(env); 1377 if (!orig_subprog_starts) { 1378 vfree(orig_insn_aux); 1379 err = -ENOMEM; 1380 goto out_cleanup; 1381 } 1382 prog = bpf_jit_blind_constants(env, prog); 1383 if (IS_ERR(prog)) { 1384 err = -ENOMEM; 1385 prog = orig_prog; 1386 goto out_restore; 1387 } 1388 blinded = true; 1389 } 1390 1391 err = jit_subprogs(env); 1392 if (err) 1393 goto out_jit_err; 1394 1395 if (blinded) { 1396 bpf_jit_prog_release_other(prog, orig_prog); 1397 kvfree(orig_subprog_starts); 1398 vfree(orig_insn_aux); 1399 } 1400 1401 return 0; 1402 1403 out_jit_err: 1404 if (blinded) { 1405 bpf_jit_prog_release_other(orig_prog, prog); 1406 /* roll back to the clean original prog */ 1407 prog = env->prog = orig_prog; 1408 goto out_restore; 1409 } else { 1410 if (err != -EFAULT) { 1411 /* 1412 * We will fall back to interpreter mode when err is not -EFAULT, before 1413 * that, insn->off and insn->imm should be restored to their original 1414 * values since they were modified by jit_subprogs. 1415 */ 1416 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1417 if (!bpf_pseudo_call(insn)) 1418 continue; 1419 insn->off = 0; 1420 insn->imm = env->insn_aux_data[i].call_imm; 1421 } 1422 } 1423 goto out_cleanup; 1424 } 1425 1426 out_restore: 1427 bpf_restore_subprog_starts(env, orig_subprog_starts); 1428 bpf_restore_insn_aux_data(env, orig_insn_aux); 1429 kvfree(orig_subprog_starts); 1430 out_cleanup: 1431 /* cleanup main prog to be interpreted */ 1432 prog->jit_requested = 0; 1433 prog->blinding_requested = 0; 1434 return err; 1435 } 1436 1437 int bpf_fixup_call_args(struct bpf_verifier_env *env) 1438 { 1439 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1440 struct bpf_prog *prog = env->prog; 1441 struct bpf_insn *insn = prog->insnsi; 1442 int depth; 1443 #endif 1444 int i, err = 0; 1445 1446 for (i = 0; i < env->subprog_cnt; i++) { 1447 struct bpf_subprog_info *subprog = &env->subprog_info[i]; 1448 u16 outgoing = subprog->stack_arg_cnt - bpf_in_stack_arg_cnt(subprog); 1449 1450 if (subprog->max_out_stack_arg_cnt > outgoing) { 1451 verbose(env, 1452 "func#%d writes %u stack arg slots, but calls only require %u\n", 1453 i, subprog->max_out_stack_arg_cnt, outgoing); 1454 return -EINVAL; 1455 } 1456 } 1457 1458 if (env->prog->jit_requested && 1459 !bpf_prog_is_offloaded(env->prog->aux)) { 1460 err = bpf_jit_subprogs(env); 1461 if (err == 0) 1462 return 0; 1463 if (err == -EFAULT) 1464 return err; 1465 } 1466 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1467 if (prog->jit_required) { 1468 verbose(env, "program requires BPF JIT compiler but it is not available\n"); 1469 return -EINVAL; 1470 } 1471 for (i = 0; i < env->subprog_cnt; i++) { 1472 if (bpf_in_stack_arg_cnt(&env->subprog_info[i])) { 1473 verbose(env, "stack args are not supported in non-JITed programs\n"); 1474 return -EINVAL; 1475 } 1476 } 1477 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 1478 /* When JIT fails the progs with bpf2bpf calls and tail_calls 1479 * have to be rejected, since interpreter doesn't support them yet. 1480 */ 1481 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 1482 return -EINVAL; 1483 } 1484 for (i = 0; i < prog->len; i++, insn++) { 1485 if (bpf_pseudo_func(insn)) { 1486 /* When JIT fails the progs with callback calls 1487 * have to be rejected, since interpreter doesn't support them yet. 1488 */ 1489 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 1490 return -EINVAL; 1491 } 1492 1493 if (!bpf_pseudo_call(insn)) 1494 continue; 1495 depth = get_callee_stack_depth(env, insn, i); 1496 if (depth < 0) 1497 return depth; 1498 err = bpf_patch_call_args(insn, depth); 1499 if (err) { 1500 verbose(env, "stack depth %d exceeds interpreter stack depth limit\n", 1501 depth); 1502 return err; 1503 } 1504 } 1505 err = 0; 1506 #endif 1507 return err; 1508 } 1509 1510 1511 /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */ 1512 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) 1513 { 1514 struct bpf_subprog_info *info = env->subprog_info; 1515 int cnt = env->subprog_cnt; 1516 struct bpf_prog *prog; 1517 1518 /* We only reserve one slot for hidden subprogs in subprog_info. */ 1519 if (env->hidden_subprog_cnt) { 1520 verifier_bug(env, "only one hidden subprog supported"); 1521 return -EFAULT; 1522 } 1523 /* We're not patching any existing instruction, just appending the new 1524 * ones for the hidden subprog. Hence all of the adjustment operations 1525 * in bpf_patch_insn_data are no-ops. 1526 */ 1527 prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); 1528 if (!prog) 1529 return -ENOMEM; 1530 env->prog = prog; 1531 info[cnt + 1].start = info[cnt].start; 1532 info[cnt].start = prog->len - len + 1; 1533 env->subprog_cnt++; 1534 env->hidden_subprog_cnt++; 1535 return 0; 1536 } 1537 1538 /* Do various post-verification rewrites in a single program pass. 1539 * These rewrites simplify JIT and interpreter implementations. 1540 */ 1541 int bpf_do_misc_fixups(struct bpf_verifier_env *env) 1542 { 1543 struct bpf_prog *prog = env->prog; 1544 enum bpf_attach_type eatype = prog->expected_attach_type; 1545 enum bpf_prog_type prog_type = resolve_prog_type(prog); 1546 struct bpf_insn *insn = prog->insnsi; 1547 const struct bpf_func_proto *fn; 1548 const int insn_cnt = prog->len; 1549 const struct bpf_map_ops *ops; 1550 struct bpf_insn_aux_data *aux; 1551 struct bpf_insn *insn_buf = env->insn_buf; 1552 struct bpf_prog *new_prog; 1553 struct bpf_map *map_ptr; 1554 int i, ret, cnt, delta = 0, cur_subprog = 0; 1555 struct bpf_subprog_info *subprogs = env->subprog_info; 1556 u16 stack_depth = subprogs[cur_subprog].stack_depth; 1557 u16 stack_depth_extra = 0; 1558 1559 if (env->seen_exception && !env->exception_callback_subprog) { 1560 struct bpf_insn *patch = insn_buf; 1561 1562 *patch++ = env->prog->insnsi[insn_cnt - 1]; 1563 *patch++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 1564 *patch++ = BPF_EXIT_INSN(); 1565 ret = add_hidden_subprog(env, insn_buf, patch - insn_buf); 1566 if (ret < 0) 1567 return ret; 1568 prog = env->prog; 1569 insn = prog->insnsi; 1570 1571 env->exception_callback_subprog = env->subprog_cnt - 1; 1572 /* Don't update insn_cnt, as add_hidden_subprog always appends insns */ 1573 bpf_mark_subprog_exc_cb(env, env->exception_callback_subprog); 1574 } 1575 1576 for (i = 0; i < insn_cnt;) { 1577 if (is_addr_space_cast32(env->prog, insn)) { 1578 /* convert to 32-bit mov that clears upper 32-bit */ 1579 insn->code = BPF_ALU | BPF_MOV | BPF_X; 1580 /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ 1581 insn->off = 0; 1582 insn->imm = 0; 1583 goto next_insn; 1584 } 1585 1586 if (env->insn_aux_data[i + delta].needs_zext) 1587 /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ 1588 insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); 1589 1590 /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ 1591 if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || 1592 insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || 1593 insn->code == (BPF_ALU | BPF_MOD | BPF_K) || 1594 insn->code == (BPF_ALU | BPF_DIV | BPF_K)) && 1595 insn->off == 1 && insn->imm == -1) { 1596 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1597 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1598 struct bpf_insn *patch = insn_buf; 1599 1600 if (isdiv) 1601 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1602 BPF_NEG | BPF_K, insn->dst_reg, 1603 0, 0, 0); 1604 else 1605 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1606 1607 cnt = patch - insn_buf; 1608 1609 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1610 if (!new_prog) 1611 return -ENOMEM; 1612 1613 delta += cnt - 1; 1614 env->prog = prog = new_prog; 1615 insn = new_prog->insnsi + i + delta; 1616 goto next_insn; 1617 } 1618 1619 /* Make divide-by-zero and divide-by-minus-one exceptions impossible. */ 1620 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 1621 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 1622 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 1623 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 1624 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 1625 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 1626 bool is_sdiv = isdiv && insn->off == 1; 1627 bool is_smod = !isdiv && insn->off == 1; 1628 struct bpf_insn *patch = insn_buf; 1629 1630 if (is_sdiv) { 1631 /* [R,W]x sdiv 0 -> 0 1632 * LLONG_MIN sdiv -1 -> LLONG_MIN 1633 * INT_MIN sdiv -1 -> INT_MIN 1634 */ 1635 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1636 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1637 BPF_ADD | BPF_K, BPF_REG_AX, 1638 0, 0, 1); 1639 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1640 BPF_JGT | BPF_K, BPF_REG_AX, 1641 0, 4, 1); 1642 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1643 BPF_JEQ | BPF_K, BPF_REG_AX, 1644 0, 1, 0); 1645 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1646 BPF_MOV | BPF_K, insn->dst_reg, 1647 0, 0, 0); 1648 /* BPF_NEG(LLONG_MIN) == -LLONG_MIN == LLONG_MIN */ 1649 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1650 BPF_NEG | BPF_K, insn->dst_reg, 1651 0, 0, 0); 1652 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1653 *patch++ = *insn; 1654 cnt = patch - insn_buf; 1655 } else if (is_smod) { 1656 /* [R,W]x mod 0 -> [R,W]x */ 1657 /* [R,W]x mod -1 -> 0 */ 1658 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1659 *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | 1660 BPF_ADD | BPF_K, BPF_REG_AX, 1661 0, 0, 1); 1662 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1663 BPF_JGT | BPF_K, BPF_REG_AX, 1664 0, 3, 1); 1665 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1666 BPF_JEQ | BPF_K, BPF_REG_AX, 1667 0, 3 + (is64 ? 0 : 1), 1); 1668 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); 1669 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1670 *patch++ = *insn; 1671 1672 if (!is64) { 1673 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1674 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 1675 } 1676 cnt = patch - insn_buf; 1677 } else if (isdiv) { 1678 /* [R,W]x div 0 -> 0 */ 1679 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1680 BPF_JNE | BPF_K, insn->src_reg, 1681 0, 2, 0); 1682 *patch++ = BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg); 1683 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1684 *patch++ = *insn; 1685 cnt = patch - insn_buf; 1686 } else { 1687 /* [R,W]x mod 0 -> [R,W]x */ 1688 *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 1689 BPF_JEQ | BPF_K, insn->src_reg, 1690 0, 1 + (is64 ? 0 : 1), 0); 1691 *patch++ = *insn; 1692 1693 if (!is64) { 1694 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1695 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); 1696 } 1697 cnt = patch - insn_buf; 1698 } 1699 1700 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1701 if (!new_prog) 1702 return -ENOMEM; 1703 1704 delta += cnt - 1; 1705 env->prog = prog = new_prog; 1706 insn = new_prog->insnsi + i + delta; 1707 goto next_insn; 1708 } 1709 1710 /* Make it impossible to de-reference a userspace address */ 1711 if (BPF_CLASS(insn->code) == BPF_LDX && 1712 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 1713 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) { 1714 struct bpf_insn *patch = insn_buf; 1715 u64 uaddress_limit = bpf_arch_uaddress_limit(); 1716 1717 if (!uaddress_limit) 1718 goto next_insn; 1719 1720 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 1721 if (insn->off) 1722 *patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off); 1723 *patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32); 1724 *patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2); 1725 *patch++ = *insn; 1726 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 1727 *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0); 1728 1729 cnt = patch - insn_buf; 1730 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1731 if (!new_prog) 1732 return -ENOMEM; 1733 1734 delta += cnt - 1; 1735 env->prog = prog = new_prog; 1736 insn = new_prog->insnsi + i + delta; 1737 goto next_insn; 1738 } 1739 1740 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 1741 if (BPF_CLASS(insn->code) == BPF_LD && 1742 (BPF_MODE(insn->code) == BPF_ABS || 1743 BPF_MODE(insn->code) == BPF_IND)) { 1744 cnt = env->ops->gen_ld_abs(insn, insn_buf); 1745 if (cnt == 0 || cnt >= INSN_BUF_SIZE) { 1746 verifier_bug(env, "%d insns generated for ld_abs", cnt); 1747 return -EFAULT; 1748 } 1749 1750 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1751 if (!new_prog) 1752 return -ENOMEM; 1753 1754 delta += cnt - 1; 1755 env->prog = prog = new_prog; 1756 insn = new_prog->insnsi + i + delta; 1757 goto next_insn; 1758 } 1759 1760 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 1761 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 1762 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 1763 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 1764 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 1765 struct bpf_insn *patch = insn_buf; 1766 bool issrc, isneg, isimm; 1767 u32 off_reg; 1768 1769 aux = &env->insn_aux_data[i + delta]; 1770 if (!aux->alu_state || 1771 aux->alu_state == BPF_ALU_NON_POINTER) 1772 goto next_insn; 1773 1774 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 1775 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 1776 BPF_ALU_SANITIZE_SRC; 1777 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 1778 1779 off_reg = issrc ? insn->src_reg : insn->dst_reg; 1780 if (isimm) { 1781 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 1782 } else { 1783 if (isneg) 1784 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 1785 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 1786 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 1787 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 1788 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 1789 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 1790 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 1791 } 1792 if (!issrc) 1793 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 1794 insn->src_reg = BPF_REG_AX; 1795 if (isneg) 1796 insn->code = insn->code == code_add ? 1797 code_sub : code_add; 1798 *patch++ = *insn; 1799 if (issrc && isneg && !isimm) 1800 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 1801 cnt = patch - insn_buf; 1802 1803 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1804 if (!new_prog) 1805 return -ENOMEM; 1806 1807 delta += cnt - 1; 1808 env->prog = prog = new_prog; 1809 insn = new_prog->insnsi + i + delta; 1810 goto next_insn; 1811 } 1812 1813 if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { 1814 int stack_off_cnt = -stack_depth - 16; 1815 1816 /* 1817 * Two 8 byte slots, depth-16 stores the count, and 1818 * depth-8 stores the start timestamp of the loop. 1819 * 1820 * The starting value of count is BPF_MAX_TIMED_LOOPS 1821 * (0xffff). Every iteration loads it and subs it by 1, 1822 * until the value becomes 0 in AX (thus, 1 in stack), 1823 * after which we call arch_bpf_timed_may_goto, which 1824 * either sets AX to 0xffff to keep looping, or to 0 1825 * upon timeout. AX is then stored into the stack. In 1826 * the next iteration, we either see 0 and break out, or 1827 * continue iterating until the next time value is 0 1828 * after subtraction, rinse and repeat. 1829 */ 1830 stack_depth_extra = 16; 1831 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt); 1832 if (insn->off >= 0) 1833 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5); 1834 else 1835 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); 1836 insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); 1837 insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); 1838 /* 1839 * AX is used as an argument to pass in stack_off_cnt 1840 * (to add to r10/fp), and also as the return value of 1841 * the call to arch_bpf_timed_may_goto. 1842 */ 1843 insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt); 1844 insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); 1845 insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); 1846 cnt = 7; 1847 1848 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1849 if (!new_prog) 1850 return -ENOMEM; 1851 1852 delta += cnt - 1; 1853 env->prog = prog = new_prog; 1854 insn = new_prog->insnsi + i + delta; 1855 goto next_insn; 1856 } else if (bpf_is_may_goto_insn(insn)) { 1857 int stack_off = -stack_depth - 8; 1858 1859 stack_depth_extra = 8; 1860 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); 1861 if (insn->off >= 0) 1862 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); 1863 else 1864 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); 1865 insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); 1866 insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); 1867 cnt = 4; 1868 1869 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1870 if (!new_prog) 1871 return -ENOMEM; 1872 1873 delta += cnt - 1; 1874 env->prog = prog = new_prog; 1875 insn = new_prog->insnsi + i + delta; 1876 goto next_insn; 1877 } 1878 1879 if (insn->code != (BPF_JMP | BPF_CALL)) 1880 goto next_insn; 1881 if (insn->src_reg == BPF_PSEUDO_CALL) 1882 goto next_insn; 1883 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1884 ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); 1885 if (ret) 1886 return ret; 1887 if (cnt == 0) 1888 goto next_insn; 1889 1890 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1891 if (!new_prog) 1892 return -ENOMEM; 1893 1894 delta += cnt - 1; 1895 env->prog = prog = new_prog; 1896 insn = new_prog->insnsi + i + delta; 1897 goto next_insn; 1898 } 1899 1900 /* Skip inlining the helper call if the JIT does it. */ 1901 if (bpf_jit_inlines_helper_call(insn->imm)) { 1902 prog->jit_required = 1; 1903 goto next_insn; 1904 } 1905 1906 if (insn->imm == BPF_FUNC_get_route_realm) 1907 prog->dst_needed = 1; 1908 if (insn->imm == BPF_FUNC_get_prandom_u32) 1909 bpf_user_rnd_init_once(); 1910 if (insn->imm == BPF_FUNC_override_return) 1911 prog->kprobe_override = 1; 1912 if (insn->imm == BPF_FUNC_tail_call) { 1913 /* If we tail call into other programs, we 1914 * cannot make any assumptions since they can 1915 * be replaced dynamically during runtime in 1916 * the program array. 1917 */ 1918 prog->cb_access = 1; 1919 if (!bpf_allow_tail_call_in_subprogs(env)) 1920 prog->aux->stack_depth = MAX_BPF_STACK; 1921 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 1922 1923 /* mark bpf_tail_call as different opcode to avoid 1924 * conditional branch in the interpreter for every normal 1925 * call and to prevent accidental JITing by JIT compiler 1926 * that doesn't support bpf_tail_call yet 1927 */ 1928 insn->imm = 0; 1929 insn->code = BPF_JMP | BPF_TAIL_CALL; 1930 1931 aux = &env->insn_aux_data[i + delta]; 1932 if (env->bpf_capable && !prog->blinding_requested && 1933 prog->jit_requested && 1934 !bpf_map_key_poisoned(aux) && 1935 !bpf_map_ptr_poisoned(aux) && 1936 !bpf_map_ptr_unpriv(aux)) { 1937 struct bpf_jit_poke_descriptor desc = { 1938 .reason = BPF_POKE_REASON_TAIL_CALL, 1939 .tail_call.map = aux->map_ptr_state.map_ptr, 1940 .tail_call.key = bpf_map_key_immediate(aux), 1941 .insn_idx = i + delta, 1942 }; 1943 1944 ret = bpf_jit_add_poke_descriptor(prog, &desc); 1945 if (ret < 0) { 1946 verbose(env, "adding tail call poke descriptor failed\n"); 1947 return ret; 1948 } 1949 1950 insn->imm = ret + 1; 1951 goto next_insn; 1952 } 1953 1954 if (!bpf_map_ptr_unpriv(aux)) 1955 goto next_insn; 1956 1957 /* instead of changing every JIT dealing with tail_call 1958 * emit two extra insns: 1959 * if (index >= max_entries) goto out; 1960 * index &= array->index_mask; 1961 * to avoid out-of-bounds cpu speculation 1962 */ 1963 if (bpf_map_ptr_poisoned(aux)) { 1964 verbose(env, "tail_call abusing map_ptr\n"); 1965 return -EINVAL; 1966 } 1967 1968 map_ptr = aux->map_ptr_state.map_ptr; 1969 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 1970 map_ptr->max_entries, 2); 1971 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 1972 container_of(map_ptr, 1973 struct bpf_array, 1974 map)->index_mask); 1975 insn_buf[2] = *insn; 1976 cnt = 3; 1977 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 1978 if (!new_prog) 1979 return -ENOMEM; 1980 1981 delta += cnt - 1; 1982 env->prog = prog = new_prog; 1983 insn = new_prog->insnsi + i + delta; 1984 goto next_insn; 1985 } 1986 1987 if (insn->imm == BPF_FUNC_timer_set_callback) { 1988 /* The verifier will process callback_fn as many times as necessary 1989 * with different maps and the register states prepared by 1990 * set_timer_callback_state will be accurate. 1991 * 1992 * The following use case is valid: 1993 * map1 is shared by prog1, prog2, prog3. 1994 * prog1 calls bpf_timer_init for some map1 elements 1995 * prog2 calls bpf_timer_set_callback for some map1 elements. 1996 * Those that were not bpf_timer_init-ed will return -EINVAL. 1997 * prog3 calls bpf_timer_start for some map1 elements. 1998 * Those that were not both bpf_timer_init-ed and 1999 * bpf_timer_set_callback-ed will return -EINVAL. 2000 */ 2001 struct bpf_insn ld_addrs[2] = { 2002 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), 2003 }; 2004 2005 insn_buf[0] = ld_addrs[0]; 2006 insn_buf[1] = ld_addrs[1]; 2007 insn_buf[2] = *insn; 2008 cnt = 3; 2009 2010 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2011 if (!new_prog) 2012 return -ENOMEM; 2013 2014 delta += cnt - 1; 2015 env->prog = prog = new_prog; 2016 insn = new_prog->insnsi + i + delta; 2017 goto patch_call_imm; 2018 } 2019 2020 /* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */ 2021 if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) { 2022 /* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data, 2023 * bpf_mem_alloc() returns a ptr to the percpu data ptr. 2024 */ 2025 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 2026 insn_buf[1] = *insn; 2027 cnt = 2; 2028 2029 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2030 if (!new_prog) 2031 return -ENOMEM; 2032 2033 delta += cnt - 1; 2034 env->prog = prog = new_prog; 2035 insn = new_prog->insnsi + i + delta; 2036 goto patch_call_imm; 2037 } 2038 2039 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 2040 * and other inlining handlers are currently limited to 64 bit 2041 * only. 2042 */ 2043 if (prog->jit_requested && BITS_PER_LONG == 64 && 2044 (insn->imm == BPF_FUNC_map_lookup_elem || 2045 insn->imm == BPF_FUNC_map_update_elem || 2046 insn->imm == BPF_FUNC_map_delete_elem || 2047 insn->imm == BPF_FUNC_map_push_elem || 2048 insn->imm == BPF_FUNC_map_pop_elem || 2049 insn->imm == BPF_FUNC_map_peek_elem || 2050 insn->imm == BPF_FUNC_redirect_map || 2051 insn->imm == BPF_FUNC_for_each_map_elem || 2052 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { 2053 aux = &env->insn_aux_data[i + delta]; 2054 if (bpf_map_ptr_poisoned(aux)) 2055 goto patch_call_imm; 2056 2057 map_ptr = aux->map_ptr_state.map_ptr; 2058 ops = map_ptr->ops; 2059 if (insn->imm == BPF_FUNC_map_lookup_elem && 2060 ops->map_gen_lookup) { 2061 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 2062 if (cnt == -EOPNOTSUPP) 2063 goto patch_map_ops_generic; 2064 if (cnt <= 0 || cnt >= INSN_BUF_SIZE) { 2065 verifier_bug(env, "%d insns generated for map lookup", cnt); 2066 return -EFAULT; 2067 } 2068 2069 if (bpf_map_is_percpu_map(map_ptr->map_type)) 2070 prog->jit_required = true; 2071 2072 new_prog = bpf_patch_insn_data(env, i + delta, 2073 insn_buf, cnt); 2074 if (!new_prog) 2075 return -ENOMEM; 2076 2077 delta += cnt - 1; 2078 env->prog = prog = new_prog; 2079 insn = new_prog->insnsi + i + delta; 2080 goto next_insn; 2081 } 2082 2083 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 2084 (void *(*)(struct bpf_map *map, void *key))NULL)); 2085 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 2086 (long (*)(struct bpf_map *map, void *key))NULL)); 2087 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 2088 (long (*)(struct bpf_map *map, void *key, void *value, 2089 u64 flags))NULL)); 2090 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 2091 (long (*)(struct bpf_map *map, void *value, 2092 u64 flags))NULL)); 2093 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 2094 (long (*)(struct bpf_map *map, void *value))NULL)); 2095 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 2096 (long (*)(struct bpf_map *map, void *value))NULL)); 2097 BUILD_BUG_ON(!__same_type(ops->map_redirect, 2098 (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); 2099 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 2100 (long (*)(struct bpf_map *map, 2101 bpf_callback_t callback_fn, 2102 void *callback_ctx, 2103 u64 flags))NULL)); 2104 BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, 2105 (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL)); 2106 2107 patch_map_ops_generic: 2108 switch (insn->imm) { 2109 case BPF_FUNC_map_lookup_elem: 2110 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 2111 goto next_insn; 2112 case BPF_FUNC_map_update_elem: 2113 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 2114 goto next_insn; 2115 case BPF_FUNC_map_delete_elem: 2116 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 2117 goto next_insn; 2118 case BPF_FUNC_map_push_elem: 2119 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 2120 goto next_insn; 2121 case BPF_FUNC_map_pop_elem: 2122 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 2123 goto next_insn; 2124 case BPF_FUNC_map_peek_elem: 2125 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 2126 goto next_insn; 2127 case BPF_FUNC_redirect_map: 2128 insn->imm = BPF_CALL_IMM(ops->map_redirect); 2129 goto next_insn; 2130 case BPF_FUNC_for_each_map_elem: 2131 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 2132 goto next_insn; 2133 case BPF_FUNC_map_lookup_percpu_elem: 2134 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); 2135 goto next_insn; 2136 } 2137 2138 goto patch_call_imm; 2139 } 2140 2141 /* Implement bpf_jiffies64 inline. */ 2142 if (prog->jit_requested && BITS_PER_LONG == 64 && 2143 insn->imm == BPF_FUNC_jiffies64) { 2144 struct bpf_insn ld_jiffies_addr[2] = { 2145 BPF_LD_IMM64(BPF_REG_0, 2146 (unsigned long)&jiffies), 2147 }; 2148 2149 insn_buf[0] = ld_jiffies_addr[0]; 2150 insn_buf[1] = ld_jiffies_addr[1]; 2151 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 2152 BPF_REG_0, 0); 2153 cnt = 3; 2154 2155 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 2156 cnt); 2157 if (!new_prog) 2158 return -ENOMEM; 2159 2160 delta += cnt - 1; 2161 env->prog = prog = new_prog; 2162 insn = new_prog->insnsi + i + delta; 2163 goto next_insn; 2164 } 2165 2166 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML) 2167 /* Implement bpf_get_smp_processor_id() inline. */ 2168 if (insn->imm == BPF_FUNC_get_smp_processor_id && 2169 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2170 /* BPF_FUNC_get_smp_processor_id inlining is an 2171 * optimization, so if cpu_number is ever 2172 * changed in some incompatible and hard to support 2173 * way, it's fine to back out this inlining logic 2174 */ 2175 #ifdef CONFIG_SMP 2176 prog->jit_required = true; 2177 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); 2178 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2179 insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); 2180 cnt = 3; 2181 #else 2182 insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); 2183 cnt = 1; 2184 #endif 2185 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2186 if (!new_prog) 2187 return -ENOMEM; 2188 2189 delta += cnt - 1; 2190 env->prog = prog = new_prog; 2191 insn = new_prog->insnsi + i + delta; 2192 goto next_insn; 2193 } 2194 2195 /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ 2196 if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && 2197 bpf_verifier_inlines_helper_call(env, insn->imm)) { 2198 prog->jit_required = true; 2199 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); 2200 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 2201 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); 2202 cnt = 3; 2203 2204 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2205 if (!new_prog) 2206 return -ENOMEM; 2207 2208 delta += cnt - 1; 2209 env->prog = prog = new_prog; 2210 insn = new_prog->insnsi + i + delta; 2211 goto next_insn; 2212 } 2213 #endif 2214 /* Implement bpf_get_func_arg inline. */ 2215 if (prog_type == BPF_PROG_TYPE_TRACING && 2216 insn->imm == BPF_FUNC_get_func_arg) { 2217 if (eatype == BPF_TRACE_RAW_TP) { 2218 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2219 2220 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2221 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2222 cnt = 1; 2223 } else { 2224 /* Load nr_args from ctx - 8 */ 2225 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2226 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2227 cnt = 2; 2228 } 2229 insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 2230 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 2231 insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 2232 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 2233 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2234 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 0); 2235 insn_buf[cnt++] = BPF_JMP_A(1); 2236 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2237 2238 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2239 if (!new_prog) 2240 return -ENOMEM; 2241 2242 delta += cnt - 1; 2243 env->prog = prog = new_prog; 2244 insn = new_prog->insnsi + i + delta; 2245 goto next_insn; 2246 } 2247 2248 /* Implement bpf_get_func_ret inline. */ 2249 if (prog_type == BPF_PROG_TYPE_TRACING && 2250 insn->imm == BPF_FUNC_get_func_ret) { 2251 if (eatype == BPF_TRACE_FEXIT || 2252 eatype == BPF_TRACE_FSESSION || 2253 eatype == BPF_TRACE_FEXIT_MULTI || 2254 eatype == BPF_TRACE_FSESSION_MULTI || 2255 eatype == BPF_MODIFY_RETURN) { 2256 /* Load nr_args from ctx - 8 */ 2257 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2258 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2259 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 2260 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 2261 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 2262 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 2263 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 2264 cnt = 7; 2265 } else { 2266 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 2267 cnt = 1; 2268 } 2269 2270 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2271 if (!new_prog) 2272 return -ENOMEM; 2273 2274 delta += cnt - 1; 2275 env->prog = prog = new_prog; 2276 insn = new_prog->insnsi + i + delta; 2277 goto next_insn; 2278 } 2279 2280 /* Implement get_func_arg_cnt inline. */ 2281 if (prog_type == BPF_PROG_TYPE_TRACING && 2282 insn->imm == BPF_FUNC_get_func_arg_cnt) { 2283 if (eatype == BPF_TRACE_RAW_TP) { 2284 int nr_args = btf_type_vlen(prog->aux->attach_func_proto); 2285 2286 /* skip 'void *__data' in btf_trace_##name() and save to reg0 */ 2287 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); 2288 cnt = 1; 2289 } else { 2290 /* Load nr_args from ctx - 8 */ 2291 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 2292 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 2293 cnt = 2; 2294 } 2295 2296 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2297 if (!new_prog) 2298 return -ENOMEM; 2299 2300 delta += cnt - 1; 2301 env->prog = prog = new_prog; 2302 insn = new_prog->insnsi + i + delta; 2303 goto next_insn; 2304 } 2305 2306 /* Implement bpf_get_func_ip inline. */ 2307 if (prog_type == BPF_PROG_TYPE_TRACING && 2308 insn->imm == BPF_FUNC_get_func_ip) { 2309 /* Load IP address from ctx - 16 */ 2310 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 2311 2312 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 2313 if (!new_prog) 2314 return -ENOMEM; 2315 2316 env->prog = prog = new_prog; 2317 insn = new_prog->insnsi + i + delta; 2318 goto next_insn; 2319 } 2320 2321 /* Implement bpf_get_branch_snapshot inline. */ 2322 if (IS_ENABLED(CONFIG_PERF_EVENTS) && 2323 prog->jit_requested && BITS_PER_LONG == 64 && 2324 insn->imm == BPF_FUNC_get_branch_snapshot) { 2325 /* We are dealing with the following func protos: 2326 * u64 bpf_get_branch_snapshot(void *buf, u32 size, u64 flags); 2327 * int perf_snapshot_branch_stack(struct perf_branch_entry *entries, u32 cnt); 2328 */ 2329 const u32 br_entry_size = sizeof(struct perf_branch_entry); 2330 2331 /* struct perf_branch_entry is part of UAPI and is 2332 * used as an array element, so extremely unlikely to 2333 * ever grow or shrink 2334 */ 2335 BUILD_BUG_ON(br_entry_size != 24); 2336 2337 /* if (unlikely(flags)) return -EINVAL */ 2338 insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7); 2339 2340 /* Transform size (bytes) into number of entries (cnt = size / 24). 2341 * But to avoid expensive division instruction, we implement 2342 * divide-by-3 through multiplication, followed by further 2343 * division by 8 through 3-bit right shift. 2344 * Refer to book "Hacker's Delight, 2nd ed." by Henry S. Warren, Jr., 2345 * p. 227, chapter "Unsigned Division by 3" for details and proofs. 2346 * 2347 * N / 3 <=> M * N / 2^33, where M = (2^33 + 1) / 3 = 0xaaaaaaab. 2348 */ 2349 insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab); 2350 insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0); 2351 insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36); 2352 2353 /* call perf_snapshot_branch_stack implementation */ 2354 insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack)); 2355 /* if (entry_cnt == 0) return -ENOENT */ 2356 insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4); 2357 /* return entry_cnt * sizeof(struct perf_branch_entry) */ 2358 insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size); 2359 insn_buf[7] = BPF_JMP_A(3); 2360 /* return -EINVAL; */ 2361 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 2362 insn_buf[9] = BPF_JMP_A(1); 2363 /* return -ENOENT; */ 2364 insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT); 2365 cnt = 11; 2366 2367 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2368 if (!new_prog) 2369 return -ENOMEM; 2370 2371 delta += cnt - 1; 2372 env->prog = prog = new_prog; 2373 insn = new_prog->insnsi + i + delta; 2374 goto next_insn; 2375 } 2376 2377 /* Implement bpf_kptr_xchg inline */ 2378 if (prog->jit_requested && BITS_PER_LONG == 64 && 2379 insn->imm == BPF_FUNC_kptr_xchg && 2380 bpf_jit_supports_ptr_xchg()) { 2381 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); 2382 insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); 2383 cnt = 2; 2384 2385 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 2386 if (!new_prog) 2387 return -ENOMEM; 2388 2389 delta += cnt - 1; 2390 env->prog = prog = new_prog; 2391 insn = new_prog->insnsi + i + delta; 2392 goto next_insn; 2393 } 2394 patch_call_imm: 2395 fn = env->ops->get_func_proto(insn->imm, env->prog); 2396 /* all functions that have prototype and verifier allowed 2397 * programs to call them, must be real in-kernel functions 2398 */ 2399 if (!fn->func) { 2400 verifier_bug(env, 2401 "not inlined functions %s#%d is missing func", 2402 func_id_name(insn->imm), insn->imm); 2403 return -EFAULT; 2404 } 2405 insn->imm = BPF_CALL_IMM(fn->func); 2406 next_insn: 2407 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2408 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2409 subprogs[cur_subprog].stack_extra = stack_depth_extra; 2410 2411 stack_depth = subprogs[cur_subprog].stack_depth; 2412 if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) { 2413 verbose(env, "stack size %d(extra %d) is too large\n", 2414 stack_depth, stack_depth_extra); 2415 return -EINVAL; 2416 } 2417 cur_subprog++; 2418 stack_depth = subprogs[cur_subprog].stack_depth; 2419 stack_depth_extra = 0; 2420 } 2421 i++; 2422 insn++; 2423 } 2424 2425 env->prog->aux->stack_depth = subprogs[0].stack_depth; 2426 for (i = 0; i < env->subprog_cnt; i++) { 2427 int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1; 2428 int subprog_start = subprogs[i].start; 2429 int stack_slots = subprogs[i].stack_extra / 8; 2430 int slots = delta, cnt = 0; 2431 2432 if (!stack_slots) 2433 continue; 2434 /* We need two slots in case timed may_goto is supported. */ 2435 if (stack_slots > slots) { 2436 verifier_bug(env, "stack_slots supports may_goto only"); 2437 return -EFAULT; 2438 } 2439 2440 stack_depth = subprogs[i].stack_depth; 2441 if (bpf_jit_supports_timed_may_goto()) { 2442 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2443 BPF_MAX_TIMED_LOOPS); 2444 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0); 2445 } else { 2446 /* Add ST insn to subprog prologue to init extra stack */ 2447 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, 2448 BPF_MAX_LOOPS); 2449 } 2450 /* Copy first actual insn to preserve it */ 2451 insn_buf[cnt++] = env->prog->insnsi[subprog_start]; 2452 2453 new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); 2454 if (!new_prog) 2455 return -ENOMEM; 2456 env->prog = prog = new_prog; 2457 /* 2458 * If may_goto is a first insn of a prog there could be a jmp 2459 * insn that points to it, hence adjust all such jmps to point 2460 * to insn after BPF_ST that inits may_goto count. 2461 * Adjustment will succeed because bpf_patch_insn_data() didn't fail. 2462 */ 2463 WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta)); 2464 } 2465 2466 /* Since poke tab is now finalized, publish aux to tracker. */ 2467 for (i = 0; i < prog->aux->size_poke_tab; i++) { 2468 map_ptr = prog->aux->poke_tab[i].tail_call.map; 2469 if (!map_ptr->ops->map_poke_track || 2470 !map_ptr->ops->map_poke_untrack || 2471 !map_ptr->ops->map_poke_run) { 2472 verifier_bug(env, "poke tab is misconfigured"); 2473 return -EFAULT; 2474 } 2475 2476 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 2477 if (ret < 0) { 2478 verbose(env, "tracking tail call prog failed\n"); 2479 return ret; 2480 } 2481 } 2482 2483 ret = sort_kfunc_descs_by_imm_off(env); 2484 if (ret) 2485 return ret; 2486 2487 return 0; 2488 } 2489 2490 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 2491 int position, 2492 s32 stack_base, 2493 u32 callback_subprogno, 2494 u32 *total_cnt) 2495 { 2496 s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; 2497 s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; 2498 s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; 2499 int reg_loop_max = BPF_REG_6; 2500 int reg_loop_cnt = BPF_REG_7; 2501 int reg_loop_ctx = BPF_REG_8; 2502 2503 struct bpf_insn *insn_buf = env->insn_buf; 2504 struct bpf_prog *new_prog; 2505 u32 callback_start; 2506 u32 call_insn_offset; 2507 s32 callback_offset; 2508 u32 cnt = 0; 2509 2510 /* This represents an inlined version of bpf_iter.c:bpf_loop, 2511 * be careful to modify this code in sync. 2512 */ 2513 2514 /* Return error and jump to the end of the patch if 2515 * expected number of iterations is too big. 2516 */ 2517 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2); 2518 insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG); 2519 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16); 2520 /* spill R6, R7, R8 to use these as loop vars */ 2521 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset); 2522 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset); 2523 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset); 2524 /* initialize loop vars */ 2525 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1); 2526 insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0); 2527 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3); 2528 /* loop header, 2529 * if reg_loop_cnt >= reg_loop_max skip the loop body 2530 */ 2531 insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5); 2532 /* callback call, 2533 * correct callback offset would be set after patching 2534 */ 2535 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt); 2536 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx); 2537 insn_buf[cnt++] = BPF_CALL_REL(0); 2538 /* increment loop counter */ 2539 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1); 2540 /* jump to loop header if callback returned 0 */ 2541 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6); 2542 /* return value of bpf_loop, 2543 * set R0 to the number of iterations 2544 */ 2545 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt); 2546 /* restore original values of R6, R7, R8 */ 2547 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset); 2548 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset); 2549 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset); 2550 2551 *total_cnt = cnt; 2552 new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); 2553 if (!new_prog) 2554 return new_prog; 2555 2556 /* callback start is known only after patching */ 2557 callback_start = env->subprog_info[callback_subprogno].start; 2558 /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ 2559 call_insn_offset = position + 12; 2560 callback_offset = callback_start - call_insn_offset - 1; 2561 new_prog->insnsi[call_insn_offset].imm = callback_offset; 2562 2563 return new_prog; 2564 } 2565 2566 static bool is_bpf_loop_call(struct bpf_insn *insn) 2567 { 2568 return insn->code == (BPF_JMP | BPF_CALL) && 2569 insn->src_reg == 0 && 2570 insn->imm == BPF_FUNC_loop; 2571 } 2572 2573 /* For all sub-programs in the program (including main) check 2574 * insn_aux_data to see if there are bpf_loop calls that require 2575 * inlining. If such calls are found the calls are replaced with a 2576 * sequence of instructions produced by `inline_bpf_loop` function and 2577 * subprog stack_depth is increased by the size of 3 registers. 2578 * This stack space is used to spill values of the R6, R7, R8. These 2579 * registers are used to store the loop bound, counter and context 2580 * variables. 2581 */ 2582 int bpf_optimize_bpf_loop(struct bpf_verifier_env *env) 2583 { 2584 struct bpf_subprog_info *subprogs = env->subprog_info; 2585 int i, cur_subprog = 0, cnt, delta = 0; 2586 struct bpf_insn *insn = env->prog->insnsi; 2587 int insn_cnt = env->prog->len; 2588 u16 stack_depth = subprogs[cur_subprog].stack_depth; 2589 u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2590 u16 stack_depth_extra = 0; 2591 2592 for (i = 0; i < insn_cnt; i++, insn++) { 2593 struct bpf_loop_inline_state *inline_state = 2594 &env->insn_aux_data[i + delta].loop_inline_state; 2595 2596 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { 2597 struct bpf_prog *new_prog; 2598 2599 stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; 2600 new_prog = inline_bpf_loop(env, 2601 i + delta, 2602 -(stack_depth + stack_depth_extra), 2603 inline_state->callback_subprogno, 2604 &cnt); 2605 if (!new_prog) 2606 return -ENOMEM; 2607 2608 delta += cnt - 1; 2609 env->prog = new_prog; 2610 insn = new_prog->insnsi + i + delta; 2611 } 2612 2613 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 2614 subprogs[cur_subprog].stack_depth += stack_depth_extra; 2615 cur_subprog++; 2616 stack_depth = subprogs[cur_subprog].stack_depth; 2617 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 2618 stack_depth_extra = 0; 2619 } 2620 } 2621 2622 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 2623 2624 return 0; 2625 } 2626 2627 /* Remove unnecessary spill/fill pairs, members of fastcall pattern, 2628 * adjust subprograms stack depth when possible. 2629 */ 2630 int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) 2631 { 2632 struct bpf_subprog_info *subprog = env->subprog_info; 2633 struct bpf_insn_aux_data *aux = env->insn_aux_data; 2634 struct bpf_insn *insn = env->prog->insnsi; 2635 int insn_cnt = env->prog->len; 2636 u32 spills_num; 2637 bool modified = false; 2638 int i, j; 2639 2640 for (i = 0; i < insn_cnt; i++, insn++) { 2641 if (aux[i].fastcall_spills_num > 0) { 2642 spills_num = aux[i].fastcall_spills_num; 2643 /* NOPs would be removed by opt_remove_nops() */ 2644 for (j = 1; j <= spills_num; ++j) { 2645 *(insn - j) = NOP; 2646 *(insn + j) = NOP; 2647 } 2648 modified = true; 2649 } 2650 if ((subprog + 1)->start == i + 1) { 2651 if (modified && !subprog->keep_fastcall_stack) 2652 subprog->stack_depth = -subprog->fastcall_stack_off; 2653 subprog++; 2654 modified = false; 2655 } 2656 } 2657 2658 return 0; 2659 } 2660 2661