Lines Matching refs:cnt
190 struct bpf_insn *insn, int cnt) in add_kfunc_in_insns() argument
194 for (i = 0; i < cnt; i++, insn++) { in add_kfunc_in_insns()
222 struct bpf_prog *new_prog, u32 off, u32 cnt) in adjust_insn_aux_data() argument
234 data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; in adjust_insn_aux_data()
236 if (cnt == 1) in adjust_insn_aux_data()
241 memmove(data + off + cnt - 1, data + off, in adjust_insn_aux_data()
242 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); in adjust_insn_aux_data()
243 memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1)); in adjust_insn_aux_data()
244 for (i = off; i < off + cnt - 1; i++) { in adjust_insn_aux_data()
257 if (data[off + cnt - 1].indirect_target) { in adjust_insn_aux_data()
259 data[off + cnt - 1].indirect_target = 0; in adjust_insn_aux_data()
397 u32 off, u32 cnt) in adjust_subprog_starts_after_remove() argument
407 if (env->subprog_info[j].start >= off + cnt) in adjust_subprog_starts_after_remove()
412 if (env->subprog_info[j].start != off + cnt) in adjust_subprog_starts_after_remove()
451 env->subprog_info[i].start -= cnt; in adjust_subprog_starts_after_remove()
457 u32 cnt) in bpf_adj_linfo_after_remove() argument
477 if (linfo[i].insn_off < off + cnt) in bpf_adj_linfo_after_remove()
487 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { in bpf_adj_linfo_after_remove()
489 linfo[--i].insn_off = off + cnt; in bpf_adj_linfo_after_remove()
503 linfo[i].insn_off -= cnt; in bpf_adj_linfo_after_remove()
537 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) in verifier_remove_insns() argument
547 bpf_prog_offload_remove_insns(env, off, cnt); in verifier_remove_insns()
549 bpf_clear_insn_aux_data(env, off, cnt); in verifier_remove_insns()
551 err = bpf_remove_insns(env->prog, off, cnt); in verifier_remove_insns()
555 err = adjust_subprog_starts_after_remove(env, off, cnt); in verifier_remove_insns()
559 err = bpf_adj_linfo_after_remove(env, off, cnt); in verifier_remove_insns()
563 adjust_insn_arrays_after_remove(env, off, cnt); in verifier_remove_insns()
565 memmove(aux_data + off, aux_data + off + cnt, in verifier_remove_insns()
566 sizeof(*aux_data) * (orig_prog_len - off - cnt)); in verifier_remove_insns()
567 env->insn_aux_data_len -= cnt; in verifier_remove_insns()
770 int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0; in bpf_convert_ctx_accesses() local
789 cnt = 0; in bpf_convert_ctx_accesses()
791 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, in bpf_convert_ctx_accesses()
793 insn_buf[cnt++] = env->prog->insnsi[0]; in bpf_convert_ctx_accesses()
794 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in bpf_convert_ctx_accesses()
798 delta += cnt - 1; in bpf_convert_ctx_accesses()
811 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, in bpf_convert_ctx_accesses()
813 if (cnt >= INSN_BUF_SIZE) { in bpf_convert_ctx_accesses()
816 } else if (cnt) { in bpf_convert_ctx_accesses()
817 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in bpf_convert_ctx_accesses()
822 delta += cnt - 1; in bpf_convert_ctx_accesses()
824 ret = add_kfunc_in_insns(env, insn_buf, cnt - 1); in bpf_convert_ctx_accesses()
849 cnt = patch - insn_buf; in bpf_convert_ctx_accesses()
850 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_convert_ctx_accesses()
854 delta += cnt - 1; in bpf_convert_ctx_accesses()
897 cnt = 1; in bpf_convert_ctx_accesses()
901 cnt = epilogue_cnt; in bpf_convert_ctx_accesses()
922 cnt = patch - insn_buf; in bpf_convert_ctx_accesses()
923 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_convert_ctx_accesses()
927 delta += cnt - 1; in bpf_convert_ctx_accesses()
1020 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, in bpf_convert_ctx_accesses()
1022 if (cnt == 0 || cnt >= INSN_BUF_SIZE || in bpf_convert_ctx_accesses()
1024 verifier_bug(env, "error during ctx access conversion (%d)", cnt); in bpf_convert_ctx_accesses()
1031 if (shift && cnt + 1 >= INSN_BUF_SIZE) { in bpf_convert_ctx_accesses()
1037 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, in bpf_convert_ctx_accesses()
1040 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in bpf_convert_ctx_accesses()
1044 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, in bpf_convert_ctx_accesses()
1047 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in bpf_convert_ctx_accesses()
1052 insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, in bpf_convert_ctx_accesses()
1057 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_convert_ctx_accesses()
1061 delta += cnt - 1; in bpf_convert_ctx_accesses()
1507 int cnt = env->subprog_cnt; in add_hidden_subprog() local
1523 info[cnt + 1].start = info[cnt].start; in add_hidden_subprog()
1524 info[cnt].start = prog->len - len + 1; in add_hidden_subprog()
1546 int i, ret, cnt, delta = 0, cur_subprog = 0; in bpf_do_misc_fixups() local
1599 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1601 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1605 delta += cnt - 1; in bpf_do_misc_fixups()
1646 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1668 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1677 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1689 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1692 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1696 delta += cnt - 1; in bpf_do_misc_fixups()
1721 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1722 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1726 delta += cnt - 1; in bpf_do_misc_fixups()
1736 cnt = env->ops->gen_ld_abs(insn, insn_buf); in bpf_do_misc_fixups()
1737 if (cnt == 0 || cnt >= INSN_BUF_SIZE) { in bpf_do_misc_fixups()
1738 verifier_bug(env, "%d insns generated for ld_abs", cnt); in bpf_do_misc_fixups()
1742 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1746 delta += cnt - 1; in bpf_do_misc_fixups()
1793 cnt = patch - insn_buf; in bpf_do_misc_fixups()
1795 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1799 delta += cnt - 1; in bpf_do_misc_fixups()
1838 cnt = 7; in bpf_do_misc_fixups()
1840 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1844 delta += cnt - 1; in bpf_do_misc_fixups()
1859 cnt = 4; in bpf_do_misc_fixups()
1861 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1865 delta += cnt - 1; in bpf_do_misc_fixups()
1895 cnt = 2; in bpf_do_misc_fixups()
1898 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1902 delta += cnt - 1; in bpf_do_misc_fixups()
1913 ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); in bpf_do_misc_fixups()
1916 if (cnt == 0) in bpf_do_misc_fixups()
1919 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
1923 delta += cnt - 1; in bpf_do_misc_fixups()
2005 cnt = 3; in bpf_do_misc_fixups()
2006 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2010 delta += cnt - 1; in bpf_do_misc_fixups()
2037 cnt = 3; in bpf_do_misc_fixups()
2039 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2043 delta += cnt - 1; in bpf_do_misc_fixups()
2056 cnt = 2; in bpf_do_misc_fixups()
2058 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2062 delta += cnt - 1; in bpf_do_misc_fixups()
2090 cnt = ops->map_gen_lookup(map_ptr, insn_buf); in bpf_do_misc_fixups()
2091 if (cnt == -EOPNOTSUPP) in bpf_do_misc_fixups()
2093 if (cnt <= 0 || cnt >= INSN_BUF_SIZE) { in bpf_do_misc_fixups()
2094 verifier_bug(env, "%d insns generated for map lookup", cnt); in bpf_do_misc_fixups()
2102 insn_buf, cnt); in bpf_do_misc_fixups()
2106 delta += cnt - 1; in bpf_do_misc_fixups()
2182 cnt = 3; in bpf_do_misc_fixups()
2185 cnt); in bpf_do_misc_fixups()
2189 delta += cnt - 1; in bpf_do_misc_fixups()
2209 cnt = 3; in bpf_do_misc_fixups()
2212 cnt = 1; in bpf_do_misc_fixups()
2214 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2218 delta += cnt - 1; in bpf_do_misc_fixups()
2231 cnt = 3; in bpf_do_misc_fixups()
2233 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2237 delta += cnt - 1; in bpf_do_misc_fixups()
2251 cnt = 1; in bpf_do_misc_fixups()
2256 cnt = 2; in bpf_do_misc_fixups()
2258 insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); in bpf_do_misc_fixups()
2259 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); in bpf_do_misc_fixups()
2260 insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); in bpf_do_misc_fixups()
2261 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); in bpf_do_misc_fixups()
2262 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); in bpf_do_misc_fixups()
2263 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 0); in bpf_do_misc_fixups()
2264 insn_buf[cnt++] = BPF_JMP_A(1); in bpf_do_misc_fixups()
2265 insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); in bpf_do_misc_fixups()
2267 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2271 delta += cnt - 1; in bpf_do_misc_fixups()
2293 cnt = 7; in bpf_do_misc_fixups()
2296 cnt = 1; in bpf_do_misc_fixups()
2299 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2303 delta += cnt - 1; in bpf_do_misc_fixups()
2317 cnt = 1; in bpf_do_misc_fixups()
2322 cnt = 2; in bpf_do_misc_fixups()
2325 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2329 delta += cnt - 1; in bpf_do_misc_fixups()
2394 cnt = 11; in bpf_do_misc_fixups()
2396 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2400 delta += cnt - 1; in bpf_do_misc_fixups()
2412 cnt = 2; in bpf_do_misc_fixups()
2414 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in bpf_do_misc_fixups()
2418 delta += cnt - 1; in bpf_do_misc_fixups()
2459 int slots = delta, cnt = 0; in bpf_do_misc_fixups() local
2471 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, in bpf_do_misc_fixups()
2473 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0); in bpf_do_misc_fixups()
2476 insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, in bpf_do_misc_fixups()
2480 insn_buf[cnt++] = env->prog->insnsi[subprog_start]; in bpf_do_misc_fixups()
2482 new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); in bpf_do_misc_fixups()
2537 u32 cnt = 0; in inline_bpf_loop() local
2546 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2); in inline_bpf_loop()
2547 insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG); in inline_bpf_loop()
2548 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16); in inline_bpf_loop()
2550 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset); in inline_bpf_loop()
2551 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset); in inline_bpf_loop()
2552 insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset); in inline_bpf_loop()
2554 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1); in inline_bpf_loop()
2555 insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0); in inline_bpf_loop()
2556 insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3); in inline_bpf_loop()
2560 insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5); in inline_bpf_loop()
2564 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt); in inline_bpf_loop()
2565 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx); in inline_bpf_loop()
2566 insn_buf[cnt++] = BPF_CALL_REL(0); in inline_bpf_loop()
2568 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1); in inline_bpf_loop()
2570 insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6); in inline_bpf_loop()
2574 insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt); in inline_bpf_loop()
2576 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset); in inline_bpf_loop()
2577 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset); in inline_bpf_loop()
2578 insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset); in inline_bpf_loop()
2580 *total_cnt = cnt; in inline_bpf_loop()
2581 new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); in inline_bpf_loop()
2614 int i, cur_subprog = 0, cnt, delta = 0; in bpf_optimize_bpf_loop() local
2633 &cnt); in bpf_optimize_bpf_loop()
2637 delta += cnt - 1; in bpf_optimize_bpf_loop()