Lines Matching refs:insn
21 static bool is_cmpxchg_insn(const struct bpf_insn *insn) in is_cmpxchg_insn() argument
23 return BPF_CLASS(insn->code) == BPF_STX && in is_cmpxchg_insn()
24 (BPF_MODE(insn->code) == BPF_ATOMIC || in is_cmpxchg_insn()
25 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && in is_cmpxchg_insn()
26 insn->imm == BPF_CMPXCHG; in is_cmpxchg_insn()
30 static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *insn) in is_addr_space_cast32() argument
34 if (insn->code != (BPF_ALU64 | BPF_MOV | BPF_X) || insn->off != BPF_ADDR_SPACE_CAST) in is_addr_space_cast32()
38 if (insn->imm == 1) in is_addr_space_cast32()
42 if (insn->imm == 1 << 16) in is_addr_space_cast32()
50 static int insn_def_regno(const struct bpf_insn *insn) in insn_def_regno() argument
52 switch (BPF_CLASS(insn->code)) { in insn_def_regno()
58 return bpf_atomic_load_reg(insn); in insn_def_regno()
60 return insn->dst_reg; in insn_def_regno()
69 static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn) in bpf_is_reg64() argument
71 u8 class = BPF_CLASS(insn->code); in bpf_is_reg64()
72 u8 mode = BPF_MODE(insn->code); in bpf_is_reg64()
73 u8 size = BPF_SIZE(insn->code); in bpf_is_reg64()
74 u8 op = BPF_OP(insn->code); in bpf_is_reg64()
78 if ((class == BPF_ALU || class == BPF_ALU64) && op == BPF_END && insn->imm != 64) in bpf_is_reg64()
86 if (is_addr_space_cast32(prog, insn)) in bpf_is_reg64()
110 int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) in bpf_insn_def32() argument
112 int dst_reg = insn_def_regno(insn); in bpf_insn_def32()
114 if (dst_reg < 0 || bpf_is_reg64(prog, insn)) in bpf_insn_def32()
134 const struct bpf_insn *insn) in bpf_jit_find_kfunc_model() argument
137 .imm = insn->imm, in bpf_jit_find_kfunc_model()
138 .offset = insn->off, in bpf_jit_find_kfunc_model()
190 struct bpf_insn *insn, int cnt) in add_kfunc_in_insns() argument
194 for (i = 0; i < cnt; i++, insn++) { in add_kfunc_in_insns()
195 if (bpf_pseudo_kfunc_call(insn)) { in add_kfunc_in_insns()
196 ret = bpf_add_kfunc_call(env, insn->imm, insn->off); in add_kfunc_in_insns()
206 const struct bpf_insn *insn, int idx) in get_callee_stack_depth() argument
208 int start = idx + insn->imm + 1, subprog; in get_callee_stack_depth()
225 struct bpf_insn *insn = new_prog->insnsi; in adjust_insn_aux_data() local
234 data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; in adjust_insn_aux_data()
247 data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; in adjust_insn_aux_data()
364 struct bpf_insn *insn = prog->insnsi; in adjust_jmp_off() local
369 for (i = 0; i < insn_cnt; i++, insn++) { in adjust_jmp_off()
370 u8 code = insn->code; in adjust_jmp_off()
379 if (insn->code == (BPF_JMP32 | BPF_JA)) { in adjust_jmp_off()
380 if (i + 1 + insn->imm != tgt_idx) in adjust_jmp_off()
382 if (check_add_overflow(insn->imm, delta, &imm)) in adjust_jmp_off()
384 insn->imm = imm; in adjust_jmp_off()
386 if (i + 1 + insn->off != tgt_idx) in adjust_jmp_off()
388 if (check_add_overflow(insn->off, delta, &off)) in adjust_jmp_off()
390 insn->off = off; in adjust_jmp_off()
593 struct bpf_insn *insn = env->prog->insnsi; in bpf_opt_hard_wire_dead_code_branches() local
597 for (i = 0; i < insn_cnt; i++, insn++) { in bpf_opt_hard_wire_dead_code_branches()
598 if (!bpf_insn_is_cond_jump(insn->code)) in bpf_opt_hard_wire_dead_code_branches()
602 ja.off = insn->off; in bpf_opt_hard_wire_dead_code_branches()
603 else if (!aux_data[i + 1 + insn->off].seen) in bpf_opt_hard_wire_dead_code_branches()
611 memcpy(insn, &ja, sizeof(ja)); in bpf_opt_hard_wire_dead_code_branches()
641 struct bpf_insn *insn = env->prog->insnsi; in bpf_opt_remove_nops() local
647 is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0)); in bpf_opt_remove_nops()
648 is_ja = !memcmp(&insn[i], &NOP, sizeof(NOP)); in bpf_opt_remove_nops()
684 struct bpf_insn insn; in bpf_opt_subreg_zext_lo32_rnd_hi32() local
687 insn = insns[adj_idx]; in bpf_opt_subreg_zext_lo32_rnd_hi32()
688 load_reg = insn_def_regno(&insn); in bpf_opt_subreg_zext_lo32_rnd_hi32()
696 code = insn.code; in bpf_opt_subreg_zext_lo32_rnd_hi32()
701 if (bpf_is_reg64(env->prog, &insn)) { in bpf_opt_subreg_zext_lo32_rnd_hi32()
714 rnd_hi32_patch[0] = insn; in bpf_opt_subreg_zext_lo32_rnd_hi32()
732 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) in bpf_opt_subreg_zext_lo32_rnd_hi32()
736 if (bpf_pseudo_kfunc_call(&insn)) in bpf_opt_subreg_zext_lo32_rnd_hi32()
743 zext_patch[0] = insn; in bpf_opt_subreg_zext_lo32_rnd_hi32()
774 struct bpf_insn *insn; in bpf_convert_ctx_accesses() local
836 insn = env->prog->insnsi + delta; in bpf_convert_ctx_accesses()
838 for (i = 0; i < insn_cnt; i++, insn++) { in bpf_convert_ctx_accesses()
848 *patch++ = *insn; in bpf_convert_ctx_accesses()
856 insn = new_prog->insnsi + i + delta; in bpf_convert_ctx_accesses()
865 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || in bpf_convert_ctx_accesses()
866 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || in bpf_convert_ctx_accesses()
867 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || in bpf_convert_ctx_accesses()
868 insn->code == (BPF_LDX | BPF_MEM | BPF_DW) || in bpf_convert_ctx_accesses()
869 insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) || in bpf_convert_ctx_accesses()
870 insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) || in bpf_convert_ctx_accesses()
871 insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) { in bpf_convert_ctx_accesses()
873 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || in bpf_convert_ctx_accesses()
874 insn->code == (BPF_STX | BPF_MEM | BPF_H) || in bpf_convert_ctx_accesses()
875 insn->code == (BPF_STX | BPF_MEM | BPF_W) || in bpf_convert_ctx_accesses()
876 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || in bpf_convert_ctx_accesses()
877 insn->code == (BPF_ST | BPF_MEM | BPF_B) || in bpf_convert_ctx_accesses()
878 insn->code == (BPF_ST | BPF_MEM | BPF_H) || in bpf_convert_ctx_accesses()
879 insn->code == (BPF_ST | BPF_MEM | BPF_W) || in bpf_convert_ctx_accesses()
880 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { in bpf_convert_ctx_accesses()
882 } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) || in bpf_convert_ctx_accesses()
883 insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) || in bpf_convert_ctx_accesses()
884 insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) || in bpf_convert_ctx_accesses()
885 insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) && in bpf_convert_ctx_accesses()
887 insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); in bpf_convert_ctx_accesses()
890 } else if (insn->code == (BPF_JMP | BPF_EXIT) && in bpf_convert_ctx_accesses()
920 *patch++ = *insn; in bpf_convert_ctx_accesses()
929 insn = new_prog->insnsi + i + delta; in bpf_convert_ctx_accesses()
951 if (BPF_MODE(insn->code) == BPF_MEMSX) { in bpf_convert_ctx_accesses()
952 if (!bpf_jit_supports_insn(insn, true)) { in bpf_convert_ctx_accesses()
956 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code); in bpf_convert_ctx_accesses()
958 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code); in bpf_convert_ctx_accesses()
974 if (BPF_MODE(insn->code) == BPF_MEM) in bpf_convert_ctx_accesses()
975 insn->code = BPF_LDX | BPF_PROBE_MEM | in bpf_convert_ctx_accesses()
976 BPF_SIZE(insn->code); in bpf_convert_ctx_accesses()
978 insn->code = BPF_LDX | BPF_PROBE_MEMSX | in bpf_convert_ctx_accesses()
979 BPF_SIZE(insn->code); in bpf_convert_ctx_accesses()
990 size = BPF_LDST_BYTES(insn); in bpf_convert_ctx_accesses()
991 mode = BPF_MODE(insn->code); in bpf_convert_ctx_accesses()
1000 off = insn->off; in bpf_convert_ctx_accesses()
1015 insn->off = off & ~(size_default - 1); in bpf_convert_ctx_accesses()
1016 insn->code = BPF_LDX | BPF_MEM | size_code; in bpf_convert_ctx_accesses()
1020 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, in bpf_convert_ctx_accesses()
1038 insn->dst_reg, in bpf_convert_ctx_accesses()
1040 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in bpf_convert_ctx_accesses()
1045 insn->dst_reg, in bpf_convert_ctx_accesses()
1047 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in bpf_convert_ctx_accesses()
1053 insn->dst_reg, insn->dst_reg, in bpf_convert_ctx_accesses()
1065 insn = new_prog->insnsi + i + delta; in bpf_convert_ctx_accesses()
1096 struct bpf_insn *insn; in jit_subprogs() local
1100 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { in jit_subprogs()
1101 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) in jit_subprogs()
1108 subprog = bpf_find_subprog(env, i + insn->imm + 1); in jit_subprogs()
1110 i + insn->imm + 1)) in jit_subprogs()
1115 insn->off = subprog; in jit_subprogs()
1119 env->insn_aux_data[i].call_imm = insn->imm; in jit_subprogs()
1121 insn->imm = 1; in jit_subprogs()
1122 if (bpf_pseudo_func(insn)) { in jit_subprogs()
1132 insn[0].imm = (u32)addr; in jit_subprogs()
1133 insn[1].imm = addr >> 32; in jit_subprogs()
1204 insn = func[i]->insnsi; in jit_subprogs()
1205 for (j = 0; j < func[i]->len; j++, insn++) { in jit_subprogs()
1206 if (BPF_CLASS(insn->code) == BPF_LDX && in jit_subprogs()
1207 (BPF_MODE(insn->code) == BPF_PROBE_MEM || in jit_subprogs()
1208 BPF_MODE(insn->code) == BPF_PROBE_MEM32 || in jit_subprogs()
1209 BPF_MODE(insn->code) == BPF_PROBE_MEM32SX || in jit_subprogs()
1210 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) in jit_subprogs()
1212 if ((BPF_CLASS(insn->code) == BPF_STX || in jit_subprogs()
1213 BPF_CLASS(insn->code) == BPF_ST) && in jit_subprogs()
1214 BPF_MODE(insn->code) == BPF_PROBE_MEM32) in jit_subprogs()
1216 if (BPF_CLASS(insn->code) == BPF_STX && in jit_subprogs()
1217 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) in jit_subprogs()
1241 insn = func[i]->insnsi; in jit_subprogs()
1242 for (j = 0; j < func[i]->len; j++, insn++) { in jit_subprogs()
1243 if (bpf_pseudo_func(insn)) { in jit_subprogs()
1244 subprog = insn->off; in jit_subprogs()
1245 insn[0].imm = (u32)(long)func[subprog]->bpf_func; in jit_subprogs()
1246 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; in jit_subprogs()
1249 if (!bpf_pseudo_call(insn)) in jit_subprogs()
1251 subprog = insn->off; in jit_subprogs()
1252 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); in jit_subprogs()
1307 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { in jit_subprogs()
1308 if (bpf_pseudo_func(insn)) { in jit_subprogs()
1309 insn[0].imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
1310 insn[1].imm = insn->off; in jit_subprogs()
1311 insn->off = 0; in jit_subprogs()
1314 if (!bpf_pseudo_call(insn)) in jit_subprogs()
1316 insn->imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
1317 subprog = bpf_find_subprog(env, i + insn->imm + 1); in jit_subprogs()
1318 insn->off = subprog; in jit_subprogs()
1363 struct bpf_insn *insn; in bpf_jit_subprogs() local
1410 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { in bpf_jit_subprogs()
1411 if (!bpf_pseudo_call(insn)) in bpf_jit_subprogs()
1413 insn->off = 0; in bpf_jit_subprogs()
1414 insn->imm = env->insn_aux_data[i].call_imm; in bpf_jit_subprogs()
1434 struct bpf_insn *insn = prog->insnsi; in bpf_fixup_call_args() local
1477 for (i = 0; i < prog->len; i++, insn++) { in bpf_fixup_call_args()
1478 if (bpf_pseudo_func(insn)) { in bpf_fixup_call_args()
1486 if (!bpf_pseudo_call(insn)) in bpf_fixup_call_args()
1488 depth = get_callee_stack_depth(env, insn, i); in bpf_fixup_call_args()
1491 err = bpf_patch_call_args(insn, depth); in bpf_fixup_call_args()
1538 struct bpf_insn *insn = prog->insnsi; in bpf_do_misc_fixups() local
1561 insn = prog->insnsi; in bpf_do_misc_fixups()
1569 if (is_addr_space_cast32(env->prog, insn)) { in bpf_do_misc_fixups()
1571 insn->code = BPF_ALU | BPF_MOV | BPF_X; in bpf_do_misc_fixups()
1573 insn->off = 0; in bpf_do_misc_fixups()
1574 insn->imm = 0; in bpf_do_misc_fixups()
1580 insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); in bpf_do_misc_fixups()
1583 if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || in bpf_do_misc_fixups()
1584 insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || in bpf_do_misc_fixups()
1585 insn->code == (BPF_ALU | BPF_MOD | BPF_K) || in bpf_do_misc_fixups()
1586 insn->code == (BPF_ALU | BPF_DIV | BPF_K)) && in bpf_do_misc_fixups()
1587 insn->off == 1 && insn->imm == -1) { in bpf_do_misc_fixups()
1588 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; in bpf_do_misc_fixups()
1589 bool isdiv = BPF_OP(insn->code) == BPF_DIV; in bpf_do_misc_fixups()
1594 BPF_NEG | BPF_K, insn->dst_reg, in bpf_do_misc_fixups()
1597 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); in bpf_do_misc_fixups()
1607 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
1612 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || in bpf_do_misc_fixups()
1613 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || in bpf_do_misc_fixups()
1614 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || in bpf_do_misc_fixups()
1615 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { in bpf_do_misc_fixups()
1616 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; in bpf_do_misc_fixups()
1617 bool isdiv = BPF_OP(insn->code) == BPF_DIV; in bpf_do_misc_fixups()
1618 bool is_sdiv = isdiv && insn->off == 1; in bpf_do_misc_fixups()
1619 bool is_smod = !isdiv && insn->off == 1; in bpf_do_misc_fixups()
1627 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); in bpf_do_misc_fixups()
1638 BPF_MOV | BPF_K, insn->dst_reg, in bpf_do_misc_fixups()
1642 BPF_NEG | BPF_K, insn->dst_reg, in bpf_do_misc_fixups()
1645 *patch++ = *insn; in bpf_do_misc_fixups()
1650 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); in bpf_do_misc_fixups()
1660 *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); in bpf_do_misc_fixups()
1662 *patch++ = *insn; in bpf_do_misc_fixups()
1666 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); in bpf_do_misc_fixups()
1672 BPF_JNE | BPF_K, insn->src_reg, in bpf_do_misc_fixups()
1674 *patch++ = BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg); in bpf_do_misc_fixups()
1676 *patch++ = *insn; in bpf_do_misc_fixups()
1681 BPF_JEQ | BPF_K, insn->src_reg, in bpf_do_misc_fixups()
1683 *patch++ = *insn; in bpf_do_misc_fixups()
1687 *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); in bpf_do_misc_fixups()
1698 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
1703 if (BPF_CLASS(insn->code) == BPF_LDX && in bpf_do_misc_fixups()
1704 (BPF_MODE(insn->code) == BPF_PROBE_MEM || in bpf_do_misc_fixups()
1705 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) { in bpf_do_misc_fixups()
1712 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); in bpf_do_misc_fixups()
1713 if (insn->off) in bpf_do_misc_fixups()
1714 *patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off); in bpf_do_misc_fixups()
1717 *patch++ = *insn; in bpf_do_misc_fixups()
1719 *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0); in bpf_do_misc_fixups()
1728 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
1733 if (BPF_CLASS(insn->code) == BPF_LD && in bpf_do_misc_fixups()
1734 (BPF_MODE(insn->code) == BPF_ABS || in bpf_do_misc_fixups()
1735 BPF_MODE(insn->code) == BPF_IND)) { in bpf_do_misc_fixups()
1736 cnt = env->ops->gen_ld_abs(insn, insn_buf); in bpf_do_misc_fixups()
1748 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
1753 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || in bpf_do_misc_fixups()
1754 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { in bpf_do_misc_fixups()
1771 off_reg = issrc ? insn->src_reg : insn->dst_reg; in bpf_do_misc_fixups()
1785 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); in bpf_do_misc_fixups()
1786 insn->src_reg = BPF_REG_AX; in bpf_do_misc_fixups()
1788 insn->code = insn->code == code_add ? in bpf_do_misc_fixups()
1790 *patch++ = *insn; in bpf_do_misc_fixups()
1801 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
1805 if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { in bpf_do_misc_fixups()
1824 if (insn->off >= 0) in bpf_do_misc_fixups()
1825 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5); in bpf_do_misc_fixups()
1827 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); in bpf_do_misc_fixups()
1846 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
1848 } else if (bpf_is_may_goto_insn(insn)) { in bpf_do_misc_fixups()
1853 if (insn->off >= 0) in bpf_do_misc_fixups()
1854 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); in bpf_do_misc_fixups()
1856 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); in bpf_do_misc_fixups()
1867 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
1872 insn->code == (BPF_LD | BPF_IMM | BPF_DW) && in bpf_do_misc_fixups()
1873 (insn->src_reg == BPF_PSEUDO_MAP_VALUE || in bpf_do_misc_fixups()
1874 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) { in bpf_do_misc_fixups()
1893 insn_buf[0] = insn[1]; in bpf_do_misc_fixups()
1894 insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg); in bpf_do_misc_fixups()
1904 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
1908 if (insn->code != (BPF_JMP | BPF_CALL)) in bpf_do_misc_fixups()
1910 if (insn->src_reg == BPF_PSEUDO_CALL) in bpf_do_misc_fixups()
1912 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { in bpf_do_misc_fixups()
1913 ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); in bpf_do_misc_fixups()
1925 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
1930 if (bpf_jit_inlines_helper_call(insn->imm)) { in bpf_do_misc_fixups()
1935 if (insn->imm == BPF_FUNC_get_route_realm) in bpf_do_misc_fixups()
1937 if (insn->imm == BPF_FUNC_get_prandom_u32) in bpf_do_misc_fixups()
1939 if (insn->imm == BPF_FUNC_override_return) in bpf_do_misc_fixups()
1941 if (insn->imm == BPF_FUNC_tail_call) { in bpf_do_misc_fixups()
1957 insn->imm = 0; in bpf_do_misc_fixups()
1958 insn->code = BPF_JMP | BPF_TAIL_CALL; in bpf_do_misc_fixups()
1979 insn->imm = ret + 1; in bpf_do_misc_fixups()
2004 insn_buf[2] = *insn; in bpf_do_misc_fixups()
2012 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2016 if (insn->imm == BPF_FUNC_timer_set_callback) { in bpf_do_misc_fixups()
2036 insn_buf[2] = *insn; in bpf_do_misc_fixups()
2045 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2055 insn_buf[1] = *insn; in bpf_do_misc_fixups()
2064 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2073 (insn->imm == BPF_FUNC_map_lookup_elem || in bpf_do_misc_fixups()
2074 insn->imm == BPF_FUNC_map_update_elem || in bpf_do_misc_fixups()
2075 insn->imm == BPF_FUNC_map_delete_elem || in bpf_do_misc_fixups()
2076 insn->imm == BPF_FUNC_map_push_elem || in bpf_do_misc_fixups()
2077 insn->imm == BPF_FUNC_map_pop_elem || in bpf_do_misc_fixups()
2078 insn->imm == BPF_FUNC_map_peek_elem || in bpf_do_misc_fixups()
2079 insn->imm == BPF_FUNC_redirect_map || in bpf_do_misc_fixups()
2080 insn->imm == BPF_FUNC_for_each_map_elem || in bpf_do_misc_fixups()
2081 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { in bpf_do_misc_fixups()
2088 if (insn->imm == BPF_FUNC_map_lookup_elem && in bpf_do_misc_fixups()
2108 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2137 switch (insn->imm) { in bpf_do_misc_fixups()
2139 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); in bpf_do_misc_fixups()
2142 insn->imm = BPF_CALL_IMM(ops->map_update_elem); in bpf_do_misc_fixups()
2145 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); in bpf_do_misc_fixups()
2148 insn->imm = BPF_CALL_IMM(ops->map_push_elem); in bpf_do_misc_fixups()
2151 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); in bpf_do_misc_fixups()
2154 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); in bpf_do_misc_fixups()
2157 insn->imm = BPF_CALL_IMM(ops->map_redirect); in bpf_do_misc_fixups()
2160 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); in bpf_do_misc_fixups()
2163 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); in bpf_do_misc_fixups()
2172 insn->imm == BPF_FUNC_jiffies64) { in bpf_do_misc_fixups()
2191 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2197 if (insn->imm == BPF_FUNC_get_smp_processor_id && in bpf_do_misc_fixups()
2198 bpf_verifier_inlines_helper_call(env, insn->imm)) { in bpf_do_misc_fixups()
2220 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2225 if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && in bpf_do_misc_fixups()
2226 bpf_verifier_inlines_helper_call(env, insn->imm)) { in bpf_do_misc_fixups()
2239 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2245 insn->imm == BPF_FUNC_get_func_arg) { in bpf_do_misc_fixups()
2273 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2279 insn->imm == BPF_FUNC_get_func_ret) { in bpf_do_misc_fixups()
2305 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2311 insn->imm == BPF_FUNC_get_func_arg_cnt) { in bpf_do_misc_fixups()
2331 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2337 insn->imm == BPF_FUNC_get_func_ip) { in bpf_do_misc_fixups()
2346 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2353 insn->imm == BPF_FUNC_get_branch_snapshot) { in bpf_do_misc_fixups()
2402 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2408 insn->imm == BPF_FUNC_kptr_xchg && in bpf_do_misc_fixups()
2420 insn = new_prog->insnsi + i + delta; in bpf_do_misc_fixups()
2424 fn = env->ops->get_func_proto(insn->imm, env->prog); in bpf_do_misc_fixups()
2431 func_id_name(insn->imm), insn->imm); in bpf_do_misc_fixups()
2434 insn->imm = BPF_CALL_IMM(fn->func); in bpf_do_misc_fixups()
2451 insn++; in bpf_do_misc_fixups()
2595 static bool is_bpf_loop_call(struct bpf_insn *insn) in is_bpf_loop_call() argument
2597 return insn->code == (BPF_JMP | BPF_CALL) && in is_bpf_loop_call()
2598 insn->src_reg == 0 && in is_bpf_loop_call()
2599 insn->imm == BPF_FUNC_loop; in is_bpf_loop_call()
2615 struct bpf_insn *insn = env->prog->insnsi; in bpf_optimize_bpf_loop() local
2621 for (i = 0; i < insn_cnt; i++, insn++) { in bpf_optimize_bpf_loop()
2625 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { in bpf_optimize_bpf_loop()
2639 insn = new_prog->insnsi + i + delta; in bpf_optimize_bpf_loop()
2663 struct bpf_insn *insn = env->prog->insnsi; in bpf_remove_fastcall_spills_fills() local
2669 for (i = 0; i < insn_cnt; i++, insn++) { in bpf_remove_fastcall_spills_fills()
2674 *(insn - j) = NOP; in bpf_remove_fastcall_spills_fills()
2675 *(insn + j) = NOP; in bpf_remove_fastcall_spills_fills()