Lines Matching refs:subprog
431 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog) in subprog_is_global() argument
435 return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; in subprog_is_global()
438 static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) in subprog_name() argument
445 info = &env->prog->aux->func_info[subprog]; in subprog_name()
449 static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog) in mark_subprog_exc_cb() argument
451 struct bpf_subprog_info *info = subprog_info(env, subprog); in mark_subprog_exc_cb()
458 static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog) in subprog_is_exc_cb() argument
460 return subprog_info(env, subprog)->is_exception_cb; in subprog_is_exc_cb()
2926 int subprog, bool is_sleepable) in push_async_cb() argument
2960 subprog /* subprog number within this prog */); in push_async_cb()
3473 struct bpf_subprog_info *subprog = env->subprog_info; in add_subprog_and_kfunc() local
3525 subprog[env->subprog_cnt].start = insn_cnt; in add_subprog_and_kfunc()
3529 verbose(env, "func#%d @%d\n", i, subprog[i].start); in add_subprog_and_kfunc()
3537 struct bpf_subprog_info *subprog = env->subprog_info; in check_subprogs() local
3542 subprog_start = subprog[cur_subprog].start; in check_subprogs()
3543 subprog_end = subprog[cur_subprog + 1].start; in check_subprogs()
3550 subprog[cur_subprog].has_tail_call = true; in check_subprogs()
3551 subprog[cur_subprog].tail_call_reachable = true; in check_subprogs()
3555 subprog[cur_subprog].has_ld_abs = true; in check_subprogs()
3561 subprog[cur_subprog].exit_idx = i; in check_subprogs()
3584 subprog_end = subprog[cur_subprog + 1].start; in check_subprogs()
4344 int subprog_insn_idx, subprog; in backtrack_insn() local
4347 subprog = find_subprog(env, subprog_insn_idx); in backtrack_insn()
4348 if (subprog < 0) in backtrack_insn()
4351 if (subprog_is_global(env, subprog)) { in backtrack_insn()
5057 struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; in check_fastcall_stack_contract() local
5061 if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern) in check_fastcall_stack_contract()
5068 subprog->fastcall_stack_off = S16_MIN; in check_fastcall_stack_contract()
5072 for (i = subprog->start; i < (subprog + 1)->start; ++i) { in check_fastcall_stack_contract()
6568 struct bpf_subprog_info *subprog = env->subprog_info; in check_max_stack_depth_subprog() local
6576 i = subprog[idx].start; in check_max_stack_depth_subprog()
6578 subprog[idx].priv_stack_mode = NO_PRIV_STACK; in check_max_stack_depth_subprog()
6599 if (idx && subprog[idx].has_tail_call && depth >= 256) { in check_max_stack_depth_subprog()
6606 subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth); in check_max_stack_depth_subprog()
6612 if (subprog[idx].priv_stack_mode == PRIV_STACK_UNKNOWN && in check_max_stack_depth_subprog()
6614 subprog[idx].priv_stack_mode = PRIV_STACK_ADAPTIVE; in check_max_stack_depth_subprog()
6617 if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) { in check_max_stack_depth_subprog()
6632 subprog_end = subprog[idx + 1].start; in check_max_stack_depth_subprog()
6641 if (subprog[idx].is_cb) in check_max_stack_depth_subprog()
6644 if (subprog[ret_prog[c]].is_cb) { in check_max_stack_depth_subprog()
6668 if (subprog[sidx].is_async_cb) { in check_max_stack_depth_subprog()
6669 if (subprog[sidx].has_tail_call) { in check_max_stack_depth_subprog()
6676 if (subprog[sidx].is_exception_cb) { in check_max_stack_depth_subprog()
6684 subprog[idx].priv_stack_mode = NO_PRIV_STACK; in check_max_stack_depth_subprog()
6686 if (subprog[idx].has_tail_call) in check_max_stack_depth_subprog()
6704 if (subprog[ret_prog[j]].is_exception_cb) { in check_max_stack_depth_subprog()
6708 subprog[ret_prog[j]].tail_call_reachable = true; in check_max_stack_depth_subprog()
6710 if (subprog[0].tail_call_reachable) in check_max_stack_depth_subprog()
6718 if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE) in check_max_stack_depth_subprog()
6719 depth -= round_up_stack_depth(env, subprog[idx].stack_depth); in check_max_stack_depth_subprog()
6774 int start = idx + insn->imm + 1, subprog; in get_callee_stack_depth() local
6776 subprog = find_subprog(env, start); in get_callee_stack_depth()
6777 if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start)) in get_callee_stack_depth()
6779 return env->subprog_info[subprog].stack_depth; in get_callee_stack_depth()
10478 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite, in setup_func_entry() argument
10510 subprog /* subprog number within this prog */); in setup_func_entry()
10526 static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, in btf_check_func_arg_match() argument
10530 struct bpf_subprog_info *sub = subprog_info(env, subprog); in btf_check_func_arg_match()
10535 ret = btf_prepare_func_args(env, subprog); in btf_check_func_arg_match()
10627 static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, in btf_check_subprog_call() argument
10638 btf_id = prog->aux->func_info[subprog].type_id; in btf_check_subprog_call()
10642 if (prog->aux->func_info_aux[subprog].unreliable) in btf_check_subprog_call()
10645 err = btf_check_func_arg_match(env, subprog, btf, regs); in btf_check_subprog_call()
10651 prog->aux->func_info_aux[subprog].unreliable = true; in btf_check_subprog_call()
10656 int insn_idx, int subprog, in push_callback_call() argument
10664 err = btf_check_subprog_call(env, subprog, caller->regs); in push_callback_call()
10672 env->subprog_info[subprog].is_cb = true; in push_callback_call()
10689 env->subprog_info[subprog].is_async_cb = true; in push_callback_call()
10690 async_cb = push_async_cb(env, env->subprog_info[subprog].start, in push_callback_call()
10691 insn_idx, subprog, in push_callback_call()
10709 callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false); in push_callback_call()
10713 err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb, in push_callback_call()
10729 int err, subprog, target_insn; in check_func_call() local
10732 subprog = find_subprog(env, target_insn); in check_func_call()
10733 if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program", in check_func_call()
10738 err = btf_check_subprog_call(env, subprog, caller->regs); in check_func_call()
10741 if (subprog_is_global(env, subprog)) { in check_func_call()
10742 const char *sub_name = subprog_name(env, subprog); in check_func_call()
10750 if (env->subprog_info[subprog].might_sleep && in check_func_call()
10761 subprog, sub_name); in check_func_call()
10767 subprog, sub_name); in check_func_call()
10768 if (env->subprog_info[subprog].changes_pkt_data) in check_func_call()
10771 subprog_aux(env, subprog)->called = true; in check_func_call()
10785 err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); in check_func_call()
10792 *insn_idx = env->subprog_info[subprog].start - 1; in check_func_call()
17501 struct bpf_subprog_info *subprog; in mark_subprog_changes_pkt_data() local
17503 subprog = bpf_find_containing_subprog(env, off); in mark_subprog_changes_pkt_data()
17504 subprog->changes_pkt_data = true; in mark_subprog_changes_pkt_data()
17509 struct bpf_subprog_info *subprog; in mark_subprog_might_sleep() local
17511 subprog = bpf_find_containing_subprog(env, off); in mark_subprog_might_sleep()
17512 subprog->might_sleep = true; in mark_subprog_might_sleep()
17828 struct bpf_subprog_info *subprog, in mark_fastcall_pattern_for_call() argument
17910 subprog->keep_fastcall_stack = 1; in mark_fastcall_pattern_for_call()
17911 subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off); in mark_fastcall_pattern_for_call()
17916 struct bpf_subprog_info *subprog = env->subprog_info; in mark_fastcall_patterns() local
17921 for (s = 0; s < env->subprog_cnt; ++s, ++subprog) { in mark_fastcall_patterns()
17924 for (i = subprog->start; i < (subprog + 1)->start; ++i) { in mark_fastcall_patterns()
17932 for (i = subprog->start; i < (subprog + 1)->start; ++i) { in mark_fastcall_patterns()
17936 mark_fastcall_pattern_for_call(env, subprog, i, lowest_off); in mark_fastcall_patterns()
18096 static struct bpf_subprog_info *subprog; in create_jt() local
18101 subprog = bpf_find_containing_subprog(env, t); in create_jt()
18102 subprog_start = subprog->start; in create_jt()
18103 subprog_end = (subprog + 1)->start; in create_jt()
18166 static struct bpf_subprog_info *subprog; in visit_tailcall_insn() local
18176 subprog = bpf_find_containing_subprog(env, t); in visit_tailcall_insn()
18178 jt->items[1] = subprog->exit_idx; in visit_tailcall_insn()
22076 int i, j, subprog_start, subprog_end = 0, len, subprog; in jit_subprogs() local
22094 subprog = find_subprog(env, i + insn->imm + 1); in jit_subprogs()
22095 if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d", in jit_subprogs()
22101 insn->off = subprog; in jit_subprogs()
22236 subprog = insn->off; in jit_subprogs()
22237 insn[0].imm = (u32)(long)func[subprog]->bpf_func; in jit_subprogs()
22238 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; in jit_subprogs()
22243 subprog = insn->off; in jit_subprogs()
22244 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); in jit_subprogs()
22309 subprog = find_subprog(env, i + insn->off + 1); in jit_subprogs()
22310 insn->imm = subprog; in jit_subprogs()
23676 struct bpf_subprog_info *subprog = env->subprog_info; in remove_fastcall_spills_fills() local
23694 if ((subprog + 1)->start == i + 1) { in remove_fastcall_spills_fills()
23695 if (modified && !subprog->keep_fastcall_stack) in remove_fastcall_spills_fills()
23696 subprog->stack_depth = -subprog->fastcall_stack_off; in remove_fastcall_spills_fills()
23697 subprog++; in remove_fastcall_spills_fills()
23748 static int do_check_common(struct bpf_verifier_env *env, int subprog) in do_check_common() argument
23751 struct bpf_subprog_info *sub = subprog_info(env, subprog); in do_check_common()
23776 subprog); in do_check_common()
23777 state->first_insn_idx = env->subprog_info[subprog].start; in do_check_common()
23781 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { in do_check_common()
23782 const char *sub_name = subprog_name(env, subprog); in do_check_common()
23787 verbose(env, "Validating %s() func#%d...\n", sub_name, subprog); in do_check_common()
23788 ret = btf_prepare_func_args(env, subprog); in do_check_common()
23792 if (subprog_is_exc_cb(env, subprog)) { in do_check_common()
23864 if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) { in do_check_common()
24145 int ret = 0, subprog = -1, i; in bpf_check_attach_target() local
24186 subprog = i; in bpf_check_attach_target()
24189 if (subprog == -1) { in bpf_check_attach_target()
24193 if (aux->func && aux->func[subprog]->aux->exception_cb) { in bpf_check_attach_target()
24199 conservative = aux->func_info_aux[subprog].unreliable; in bpf_check_attach_target()
24212 ? aux->func[subprog]->aux->changes_pkt_data in bpf_check_attach_target()
24221 ? aux->func[subprog]->aux->might_sleep in bpf_check_attach_target()
24379 if (subprog == 0) in bpf_check_attach_target()
24382 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; in bpf_check_attach_target()