Lines Matching refs:insn_buf
644 struct bpf_insn *zext_patch = env->insn_buf; in bpf_opt_subreg_zext_lo32_rnd_hi32()
645 struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2]; in bpf_opt_subreg_zext_lo32_rnd_hi32()
748 struct bpf_insn *insn_buf = env->insn_buf; in bpf_convert_ctx_accesses() local
766 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, in bpf_convert_ctx_accesses()
768 insn_buf[cnt++] = env->prog->insnsi[0]; in bpf_convert_ctx_accesses()
769 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in bpf_convert_ctx_accesses()
786 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, in bpf_convert_ctx_accesses()
792 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in bpf_convert_ctx_accesses()
799 ret = add_kfunc_in_insns(env, insn_buf, cnt - 1); in bpf_convert_ctx_accesses()
820 struct bpf_insn *patch = insn_buf; in bpf_convert_ctx_accesses()
824 cnt = patch - insn_buf; in bpf_convert_ctx_accesses()
825 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_convert_ctx_accesses()
871 insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1); in bpf_convert_ctx_accesses()
874 memcpy(insn_buf, epilogue_buf, in bpf_convert_ctx_accesses()
893 struct bpf_insn *patch = insn_buf; in bpf_convert_ctx_accesses()
897 cnt = patch - insn_buf; in bpf_convert_ctx_accesses()
898 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_convert_ctx_accesses()
995 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, in bpf_convert_ctx_accesses()
1012 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, in bpf_convert_ctx_accesses()
1015 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in bpf_convert_ctx_accesses()
1019 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, in bpf_convert_ctx_accesses()
1022 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in bpf_convert_ctx_accesses()
1027 insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, in bpf_convert_ctx_accesses()
1032 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_convert_ctx_accesses()
1518 struct bpf_insn *insn_buf = env->insn_buf; in bpf_do_misc_fixups() local
1527 struct bpf_insn *patch = insn_buf; in bpf_do_misc_fixups()
1532 ret = add_hidden_subprog(env, insn_buf, patch - insn_buf); in bpf_do_misc_fixups()
1565 struct bpf_insn *patch = insn_buf; in bpf_do_misc_fixups()
1574 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1576 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1595 struct bpf_insn *patch = insn_buf; in bpf_do_misc_fixups()
1621 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1643 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1652 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1664 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1667 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1681 struct bpf_insn *patch = insn_buf; in bpf_do_misc_fixups()
1696 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1697 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1711 cnt = env->ops->gen_ld_abs(insn, insn_buf); in bpf_do_misc_fixups()
1717 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1732 struct bpf_insn *patch = insn_buf; in bpf_do_misc_fixups()
1768 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1770 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1798 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt); in bpf_do_misc_fixups()
1800 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5); in bpf_do_misc_fixups()
1802 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); in bpf_do_misc_fixups()
1803 insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); in bpf_do_misc_fixups()
1804 insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); in bpf_do_misc_fixups()
1810 insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt); in bpf_do_misc_fixups()
1811 insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); in bpf_do_misc_fixups()
1812 insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); in bpf_do_misc_fixups()
1815 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1827 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); in bpf_do_misc_fixups()
1829 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); in bpf_do_misc_fixups()
1831 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); in bpf_do_misc_fixups()
1832 insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); in bpf_do_misc_fixups()
1833 insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); in bpf_do_misc_fixups()
1836 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1868 insn_buf[0] = insn[1]; in bpf_do_misc_fixups()
1869 insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg); in bpf_do_misc_fixups()
1873 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1888 ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); in bpf_do_misc_fixups()
1894 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1973 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, in bpf_do_misc_fixups()
1975 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, in bpf_do_misc_fixups()
1979 insn_buf[2] = *insn; in bpf_do_misc_fixups()
1981 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2009 insn_buf[0] = ld_addrs[0]; in bpf_do_misc_fixups()
2010 insn_buf[1] = ld_addrs[1]; in bpf_do_misc_fixups()
2011 insn_buf[2] = *insn; in bpf_do_misc_fixups()
2014 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2029 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); in bpf_do_misc_fixups()
2030 insn_buf[1] = *insn; in bpf_do_misc_fixups()
2033 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2065 cnt = ops->map_gen_lookup(map_ptr, insn_buf); in bpf_do_misc_fixups()
2077 insn_buf, cnt); in bpf_do_misc_fixups()
2153 insn_buf[0] = ld_jiffies_addr[0]; in bpf_do_misc_fixups()
2154 insn_buf[1] = ld_jiffies_addr[1]; in bpf_do_misc_fixups()
2155 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, in bpf_do_misc_fixups()
2159 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, in bpf_do_misc_fixups()
2181 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); in bpf_do_misc_fixups()
2182 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); in bpf_do_misc_fixups()
2183 insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); in bpf_do_misc_fixups()
2186 insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); in bpf_do_misc_fixups()
2189 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2203 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); in bpf_do_misc_fixups()
2204 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); in bpf_do_misc_fixups()
2205 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); in bpf_do_misc_fixups()
2208 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2225 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); in bpf_do_misc_fixups()
2229 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); in bpf_do_misc_fixups()
2230 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); in bpf_do_misc_fixups()
2233 insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); in bpf_do_misc_fixups()
2234 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); in bpf_do_misc_fixups()
2235 insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); in bpf_do_misc_fixups()
2236 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); in bpf_do_misc_fixups()
2237 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); in bpf_do_misc_fixups()
2238 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 0); in bpf_do_misc_fixups()
2239 insn_buf[cnt++] = BPF_JMP_A(1); in bpf_do_misc_fixups()
2240 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); in bpf_do_misc_fixups()
2242 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2261 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); in bpf_do_misc_fixups()
2262 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); in bpf_do_misc_fixups()
2263 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); in bpf_do_misc_fixups()
2264 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); in bpf_do_misc_fixups()
2265 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); in bpf_do_misc_fixups()
2266 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); in bpf_do_misc_fixups()
2267 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); in bpf_do_misc_fixups()
2270 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); in bpf_do_misc_fixups()
2274 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2291 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); in bpf_do_misc_fixups()
2295 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); in bpf_do_misc_fixups()
2296 insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); in bpf_do_misc_fixups()
2300 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2314 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); in bpf_do_misc_fixups()
2316 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); in bpf_do_misc_fixups()
2342 insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7); in bpf_do_misc_fixups()
2353 insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab); in bpf_do_misc_fixups()
2354 insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0); in bpf_do_misc_fixups()
2355 insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36); in bpf_do_misc_fixups()
2358 insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack)); in bpf_do_misc_fixups()
2360 insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4); in bpf_do_misc_fixups()
2362 insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size); in bpf_do_misc_fixups()
2363 insn_buf[7] = BPF_JMP_A(3); in bpf_do_misc_fixups()
2365 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); in bpf_do_misc_fixups()
2366 insn_buf[9] = BPF_JMP_A(1); in bpf_do_misc_fixups()
2368 insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT); in bpf_do_misc_fixups()
2371 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2385 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); in bpf_do_misc_fixups()
2386 insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); in bpf_do_misc_fixups()
2389 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2446 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, in bpf_do_misc_fixups()
2448 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0); in bpf_do_misc_fixups()
2451 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, in bpf_do_misc_fixups()
2455 insn_buf[cnt++] = env->prog->insnsi[subprog_start]; in bpf_do_misc_fixups()
2457 new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); in bpf_do_misc_fixups()
2507 struct bpf_insn *insn_buf = env->insn_buf; in inline_bpf_loop() local
2521 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2); in inline_bpf_loop()
2522 insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG); in inline_bpf_loop()
2523 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16); in inline_bpf_loop()
2525 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset); in inline_bpf_loop()
2526 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset); in inline_bpf_loop()
2527 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset); in inline_bpf_loop()
2529 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1); in inline_bpf_loop()
2530 insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0); in inline_bpf_loop()
2531 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3); in inline_bpf_loop()
2535 insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5); in inline_bpf_loop()
2539 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt); in inline_bpf_loop()
2540 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx); in inline_bpf_loop()
2541 insn_buf[cnt++] = BPF_CALL_REL(0); in inline_bpf_loop()
2543 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1); in inline_bpf_loop()
2545 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6); in inline_bpf_loop()
2549 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt); in inline_bpf_loop()
2551 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset); in inline_bpf_loop()
2552 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset); in inline_bpf_loop()
2553 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset); in inline_bpf_loop()
2556 new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); in inline_bpf_loop()