Lines Matching refs:env
199 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx);
201 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
202 static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
203 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);
204 static int ref_set_non_owning(struct bpf_verifier_env *env,
206 static void specialize_kfunc(struct bpf_verifier_env *env,
360 struct bpf_verifier_env *env = private_data; in verbose() local
363 if (!bpf_verifier_log_needed(&env->log)) in verbose()
367 bpf_verifier_vlog(&env->log, fmt, args); in verbose()
371 static void verbose_invalid_scalar(struct bpf_verifier_env *env, in verbose_invalid_scalar() argument
378 verbose(env, "%s the register %s has", ctx, reg_name); in verbose_invalid_scalar()
380 verbose(env, " smin=%lld", reg->smin_value); in verbose_invalid_scalar()
384 verbose(env, " smax=%lld", reg->smax_value); in verbose_invalid_scalar()
388 verbose(env, " unknown scalar value"); in verbose_invalid_scalar()
389 verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval); in verbose_invalid_scalar()
425 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog) in subprog_is_global() argument
427 struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux; in subprog_is_global()
432 static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) in subprog_name() argument
436 if (!env->prog->aux->func_info) in subprog_name()
439 info = &env->prog->aux->func_info[subprog]; in subprog_name()
440 return btf_type_name(env->prog->aux->btf, info->type_id); in subprog_name()
443 static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog) in mark_subprog_exc_cb() argument
445 struct bpf_subprog_info *info = subprog_info(env, subprog); in mark_subprog_exc_cb()
452 static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog) in subprog_is_exc_cb() argument
454 return subprog_info(env, subprog)->is_exception_cb; in subprog_is_exc_cb()
547 static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx) in is_may_goto_insn_at() argument
549 return is_may_goto_insn(&env->prog->insnsi[insn_idx]); in is_may_goto_insn_at()
587 static struct bpf_func_state *func(struct bpf_verifier_env *env, in func() argument
590 struct bpf_verifier_state *cur = env->cur_state; in func()
609 static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in stack_slot_obj_get_spi() argument
615 verbose(env, "%s has to be at a constant offset\n", obj_kind); in stack_slot_obj_get_spi()
621 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); in stack_slot_obj_get_spi()
627 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); in stack_slot_obj_get_spi()
631 if (!is_spi_bounds_valid(func(env, reg), spi, nr_slots)) in stack_slot_obj_get_spi()
636 static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in dynptr_get_spi() argument
638 return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS); in dynptr_get_spi()
641 static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) in iter_get_spi() argument
643 return stack_slot_obj_get_spi(env, reg, "iter", nr_slots); in iter_get_spi()
646 static int irq_flag_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in irq_flag_get_spi() argument
648 return stack_slot_obj_get_spi(env, reg, "irq_flag", 1); in irq_flag_get_spi()
692 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
695 static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, in mark_dynptr_stack_regs() argument
700 int id = ++env->id_gen; in mark_dynptr_stack_regs()
706 static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, in mark_dynptr_cb_reg() argument
710 __mark_dynptr_reg(reg, type, true, ++env->id_gen); in mark_dynptr_cb_reg()
713 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
716 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in mark_stack_slots_dynptr() argument
719 struct bpf_func_state *state = func(env, reg); in mark_stack_slots_dynptr()
723 spi = dynptr_get_spi(env, reg); in mark_stack_slots_dynptr()
736 err = destroy_if_dynptr_stack_slot(env, state, spi); in mark_stack_slots_dynptr()
739 err = destroy_if_dynptr_stack_slot(env, state, spi - 1); in mark_stack_slots_dynptr()
752 mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, in mark_stack_slots_dynptr()
762 id = acquire_reference(env, insn_idx); in mark_stack_slots_dynptr()
777 static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi) in invalidate_dynptr() argument
786 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); in invalidate_dynptr()
787 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); in invalidate_dynptr()
814 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in unmark_stack_slots_dynptr() argument
816 struct bpf_func_state *state = func(env, reg); in unmark_stack_slots_dynptr()
819 spi = dynptr_get_spi(env, reg); in unmark_stack_slots_dynptr()
824 invalidate_dynptr(env, state, spi); in unmark_stack_slots_dynptr()
838 WARN_ON_ONCE(release_reference(env, ref_obj_id)); in unmark_stack_slots_dynptr()
850 verbose(env, "verifier internal error: misconfigured ref_obj_id\n"); in unmark_stack_slots_dynptr()
854 invalidate_dynptr(env, state, i); in unmark_stack_slots_dynptr()
860 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
863 static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) in mark_reg_invalid() argument
865 if (!env->allow_ptr_leaks) in mark_reg_invalid()
866 __mark_reg_not_init(env, reg); in mark_reg_invalid()
868 __mark_reg_unknown(env, reg); in mark_reg_invalid()
871 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, in destroy_if_dynptr_stack_slot() argument
891 verbose(env, "cannot overwrite referenced dynptr\n"); in destroy_if_dynptr_stack_slot()
895 mark_stack_slot_scratched(env, spi); in destroy_if_dynptr_stack_slot()
896 mark_stack_slot_scratched(env, spi - 1); in destroy_if_dynptr_stack_slot()
906 bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({ in destroy_if_dynptr_stack_slot()
911 mark_reg_invalid(env, dreg); in destroy_if_dynptr_stack_slot()
917 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); in destroy_if_dynptr_stack_slot()
918 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); in destroy_if_dynptr_stack_slot()
927 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_dynptr_reg_valid_uninit() argument
934 spi = dynptr_get_spi(env, reg); in is_dynptr_reg_valid_uninit()
955 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_dynptr_reg_valid_init() argument
957 struct bpf_func_state *state = func(env, reg); in is_dynptr_reg_valid_init()
969 spi = dynptr_get_spi(env, reg); in is_dynptr_reg_valid_init()
984 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in is_dynptr_type_expected() argument
987 struct bpf_func_state *state = func(env, reg); in is_dynptr_type_expected()
999 spi = dynptr_get_spi(env, reg); in is_dynptr_type_expected()
1008 static bool in_rcu_cs(struct bpf_verifier_env *env);
1012 static int mark_stack_slots_iter(struct bpf_verifier_env *env, in mark_stack_slots_iter() argument
1017 struct bpf_func_state *state = func(env, reg); in mark_stack_slots_iter()
1020 spi = iter_get_spi(env, reg, nr_slots); in mark_stack_slots_iter()
1024 id = acquire_reference(env, insn_idx); in mark_stack_slots_iter()
1035 if (in_rcu_cs(env)) in mark_stack_slots_iter()
1050 mark_stack_slot_scratched(env, spi - i); in mark_stack_slots_iter()
1056 static int unmark_stack_slots_iter(struct bpf_verifier_env *env, in unmark_stack_slots_iter() argument
1059 struct bpf_func_state *state = func(env, reg); in unmark_stack_slots_iter()
1062 spi = iter_get_spi(env, reg, nr_slots); in unmark_stack_slots_iter()
1071 WARN_ON_ONCE(release_reference(env, st->ref_obj_id)); in unmark_stack_slots_iter()
1073 __mark_reg_not_init(env, st); in unmark_stack_slots_iter()
1081 mark_stack_slot_scratched(env, spi - i); in unmark_stack_slots_iter()
1087 static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env, in is_iter_reg_valid_uninit() argument
1090 struct bpf_func_state *state = func(env, reg); in is_iter_reg_valid_uninit()
1097 spi = iter_get_spi(env, reg, nr_slots); in is_iter_reg_valid_uninit()
1114 static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in is_iter_reg_valid_init() argument
1117 struct bpf_func_state *state = func(env, reg); in is_iter_reg_valid_init()
1120 spi = iter_get_spi(env, reg, nr_slots); in is_iter_reg_valid_init()
1146 static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx);
1149 static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, in mark_stack_slot_irq_flag() argument
1153 struct bpf_func_state *state = func(env, reg); in mark_stack_slot_irq_flag()
1158 spi = irq_flag_get_spi(env, reg); in mark_stack_slot_irq_flag()
1162 id = acquire_irq_state(env, insn_idx); in mark_stack_slot_irq_flag()
1177 mark_stack_slot_scratched(env, spi); in mark_stack_slot_irq_flag()
1181 static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in unmark_stack_slot_irq_flag() argument
1183 struct bpf_func_state *state = func(env, reg); in unmark_stack_slot_irq_flag()
1188 spi = irq_flag_get_spi(env, reg); in unmark_stack_slot_irq_flag()
1195 err = release_irq_state(env->cur_state, st->ref_obj_id); in unmark_stack_slot_irq_flag()
1200 for (int i = 0; i < env->cur_state->acquired_refs; i++) { in unmark_stack_slot_irq_flag()
1201 if (env->cur_state->refs[i].id == env->cur_state->active_irq_id) { in unmark_stack_slot_irq_flag()
1202 insn_idx = env->cur_state->refs[i].insn_idx; in unmark_stack_slot_irq_flag()
1207 verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n", in unmark_stack_slot_irq_flag()
1208 env->cur_state->active_irq_id, insn_idx); in unmark_stack_slot_irq_flag()
1212 __mark_reg_not_init(env, st); in unmark_stack_slot_irq_flag()
1220 mark_stack_slot_scratched(env, spi); in unmark_stack_slot_irq_flag()
1224 static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_irq_flag_reg_valid_uninit() argument
1226 struct bpf_func_state *state = func(env, reg); in is_irq_flag_reg_valid_uninit()
1234 spi = irq_flag_get_spi(env, reg); in is_irq_flag_reg_valid_uninit()
1248 static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_irq_flag_reg_valid_init() argument
1250 struct bpf_func_state *state = func(env, reg); in is_irq_flag_reg_valid_init()
1255 spi = irq_flag_get_spi(env, reg); in is_irq_flag_reg_valid_init()
1326 static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype) in mark_stack_slot_misc() argument
1442 static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) in grow_stack_state() argument
1460 if (env->subprog_info[state->subprogno].stack_depth < size) in grow_stack_state()
1461 env->subprog_info[state->subprogno].stack_depth = size; in grow_stack_state()
1471 static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_env *env, int insn_i… in acquire_reference_state() argument
1473 struct bpf_verifier_state *state = env->cur_state; in acquire_reference_state()
1485 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx) in acquire_reference() argument
1489 s = acquire_reference_state(env, insn_idx); in acquire_reference()
1493 s->id = ++env->id_gen; in acquire_reference()
1497 static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type, in acquire_lock_state() argument
1500 struct bpf_verifier_state *state = env->cur_state; in acquire_lock_state()
1503 s = acquire_reference_state(env, insn_idx); in acquire_lock_state()
1514 static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx) in acquire_irq_state() argument
1516 struct bpf_verifier_state *state = env->cur_state; in acquire_irq_state()
1519 s = acquire_reference_state(env, insn_idx); in acquire_irq_state()
1523 s->id = ++env->id_gen; in acquire_irq_state()
1679 static u32 state_htab_size(struct bpf_verifier_env *env) in state_htab_size() argument
1681 return env->prog->len; in state_htab_size()
1684 static struct bpf_verifier_state_list **explored_state(struct bpf_verifier_env *env, int idx) in explored_state() argument
1686 struct bpf_verifier_state *cur = env->cur_state; in explored_state()
1689 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; in explored_state()
1863 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in update_branch_counts() argument
1888 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, in pop_stack() argument
1891 struct bpf_verifier_state *cur = env->cur_state; in pop_stack()
1892 struct bpf_verifier_stack_elem *elem, *head = env->head; in pop_stack()
1895 if (env->head == NULL) in pop_stack()
1904 bpf_vlog_reset(&env->log, head->log_pos); in pop_stack()
1912 env->head = elem; in pop_stack()
1913 env->stack_size--; in pop_stack()
1917 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, in push_stack() argument
1921 struct bpf_verifier_state *cur = env->cur_state; in push_stack()
1931 elem->next = env->head; in push_stack()
1932 elem->log_pos = env->log.end_pos; in push_stack()
1933 env->head = elem; in push_stack()
1934 env->stack_size++; in push_stack()
1939 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { in push_stack()
1940 verbose(env, "The sequence of %d jumps is too complex.\n", in push_stack()
1941 env->stack_size); in push_stack()
1958 free_verifier_state(env->cur_state, true); in push_stack()
1959 env->cur_state = NULL; in push_stack()
1961 while (!pop_stack(env, NULL, NULL, false)); in push_stack()
2015 static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) in __mark_reg_const_zero() argument
2022 reg->precise = !env->bpf_capable; in __mark_reg_const_zero()
2025 static void mark_reg_known_zero(struct bpf_verifier_env *env, in mark_reg_known_zero() argument
2029 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); in mark_reg_known_zero()
2032 __mark_reg_not_init(env, regs + regno); in mark_reg_known_zero()
2475 static int reg_bounds_sanity_check(struct bpf_verifier_env *env, in reg_bounds_sanity_check() argument
2512 verbose(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] " in reg_bounds_sanity_check()
2519 if (env->test_reg_invariants) in reg_bounds_sanity_check()
2569 static void __mark_reg_unknown(const struct bpf_verifier_env *env, in __mark_reg_unknown() argument
2573 reg->precise = !env->bpf_capable; in __mark_reg_unknown()
2576 static void mark_reg_unknown(struct bpf_verifier_env *env, in mark_reg_unknown() argument
2580 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); in mark_reg_unknown()
2583 __mark_reg_not_init(env, regs + regno); in mark_reg_unknown()
2586 __mark_reg_unknown(env, regs + regno); in mark_reg_unknown()
2589 static int __mark_reg_s32_range(struct bpf_verifier_env *env, in __mark_reg_s32_range() argument
2605 return reg_bounds_sanity_check(env, reg, "s32_range"); in __mark_reg_s32_range()
2608 static void __mark_reg_not_init(const struct bpf_verifier_env *env, in __mark_reg_not_init() argument
2611 __mark_reg_unknown(env, reg); in __mark_reg_not_init()
2615 static void mark_reg_not_init(struct bpf_verifier_env *env, in mark_reg_not_init() argument
2619 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); in mark_reg_not_init()
2622 __mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
2625 __mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
2628 static void mark_btf_ld_reg(struct bpf_verifier_env *env, in mark_btf_ld_reg() argument
2635 mark_reg_unknown(env, regs, regno); in mark_btf_ld_reg()
2638 mark_reg_known_zero(env, regs, regno); in mark_btf_ld_reg()
2643 regs[regno].id = ++env->id_gen; in mark_btf_ld_reg()
2647 static void init_reg_state(struct bpf_verifier_env *env, in init_reg_state() argument
2654 mark_reg_not_init(env, regs, i); in init_reg_state()
2662 mark_reg_known_zero(env, regs, BPF_REG_FP); in init_reg_state()
2672 static void init_func_state(struct bpf_verifier_env *env, in init_func_state() argument
2680 init_reg_state(env, state); in init_func_state()
2681 mark_verifier_state_scratched(env); in init_func_state()
2685 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, in push_async_cb() argument
2698 elem->next = env->head; in push_async_cb()
2699 elem->log_pos = env->log.end_pos; in push_async_cb()
2700 env->head = elem; in push_async_cb()
2701 env->stack_size++; in push_async_cb()
2702 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { in push_async_cb()
2703 verbose(env, in push_async_cb()
2705 env->stack_size); in push_async_cb()
2718 elem->st.insn_hist_start = env->cur_state->insn_hist_end; in push_async_cb()
2723 init_func_state(env, frame, in push_async_cb()
2730 free_verifier_state(env->cur_state, true); in push_async_cb()
2731 env->cur_state = NULL; in push_async_cb()
2733 while (!pop_stack(env, NULL, NULL, false)); in push_async_cb()
2751 static struct bpf_subprog_info *find_containing_subprog(struct bpf_verifier_env *env, int off) in find_containing_subprog() argument
2753 struct bpf_subprog_info *vals = env->subprog_info; in find_containing_subprog()
2756 if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0) in find_containing_subprog()
2760 r = env->subprog_cnt - 1; in find_containing_subprog()
2772 static int find_subprog(struct bpf_verifier_env *env, int off) in find_subprog() argument
2776 p = find_containing_subprog(env, off); in find_subprog()
2779 return p - env->subprog_info; in find_subprog()
2782 static int add_subprog(struct bpf_verifier_env *env, int off) in add_subprog() argument
2784 int insn_cnt = env->prog->len; in add_subprog()
2788 verbose(env, "call to invalid destination\n"); in add_subprog()
2791 ret = find_subprog(env, off); in add_subprog()
2794 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { in add_subprog()
2795 verbose(env, "too many subprograms\n"); in add_subprog()
2799 env->subprog_info[env->subprog_cnt++].start = off; in add_subprog()
2800 sort(env->subprog_info, env->subprog_cnt, in add_subprog()
2801 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); in add_subprog()
2802 return env->subprog_cnt - 1; in add_subprog()
2805 static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env) in bpf_find_exception_callback_insn_off() argument
2807 struct bpf_prog_aux *aux = env->prog->aux; in bpf_find_exception_callback_insn_off()
2821 verbose(env, "invalid btf id for main subprog in func_info\n"); in bpf_find_exception_callback_insn_off()
2832 verbose(env, "multiple exception callback tags for main subprog\n"); in bpf_find_exception_callback_insn_off()
2838 verbose(env, "exception callback '%s' could not be found in BTF\n", name); in bpf_find_exception_callback_insn_off()
2844 verbose(env, "exception callback '%s' must have global linkage\n", name); in bpf_find_exception_callback_insn_off()
2856 verbose(env, "invalid exception callback insn_off in func_info: 0\n"); in bpf_find_exception_callback_insn_off()
2861 verbose(env, "exception callback type id not found in func_info\n"); in bpf_find_exception_callback_insn_off()
2943 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, in __find_kfunc_desc_btf() argument
2953 tab = env->prog->aux->kfunc_btf_tab; in __find_kfunc_desc_btf()
2958 verbose(env, "too many different module BTFs\n"); in __find_kfunc_desc_btf()
2962 if (bpfptr_is_null(env->fd_array)) { in __find_kfunc_desc_btf()
2963 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); in __find_kfunc_desc_btf()
2967 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, in __find_kfunc_desc_btf()
2974 verbose(env, "invalid module BTF fd specified\n"); in __find_kfunc_desc_btf()
2979 verbose(env, "BTF fd for kfunc is not a module BTF\n"); in __find_kfunc_desc_btf()
3019 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) in find_kfunc_desc_btf() argument
3026 verbose(env, "negative offset disallowed for kernel module function call\n"); in find_kfunc_desc_btf()
3030 return __find_kfunc_desc_btf(env, offset); in find_kfunc_desc_btf()
3035 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) in add_kfunc_call() argument
3048 prog_aux = env->prog->aux; in add_kfunc_call()
3053 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); in add_kfunc_call()
3057 if (!env->prog->jit_requested) { in add_kfunc_call()
3058 verbose(env, "JIT is required for calling kernel function\n"); in add_kfunc_call()
3063 verbose(env, "JIT does not support calling kernel function\n"); in add_kfunc_call()
3067 if (!env->prog->gpl_compatible) { in add_kfunc_call()
3068 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); in add_kfunc_call()
3094 desc_btf = find_kfunc_desc_btf(env, offset); in add_kfunc_call()
3096 verbose(env, "failed to find BTF for kernel function\n"); in add_kfunc_call()
3100 if (find_kfunc_desc(env->prog, func_id, offset)) in add_kfunc_call()
3104 verbose(env, "too many different kernel function calls\n"); in add_kfunc_call()
3110 verbose(env, "kernel btf_id %u is not a function\n", in add_kfunc_call()
3116 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", in add_kfunc_call()
3124 verbose(env, "cannot find address for kernel function %s\n", in add_kfunc_call()
3128 specialize_kfunc(env, func_id, offset, &addr); in add_kfunc_call()
3136 verbose(env, "address of kernel function %s is out of range\n", in add_kfunc_call()
3143 err = bpf_dev_bound_kfunc_check(&env->log, prog_aux); in add_kfunc_call()
3153 err = btf_distill_func_proto(&env->log, desc_btf, in add_kfunc_call()
3209 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) in add_subprog_and_kfunc() argument
3211 struct bpf_subprog_info *subprog = env->subprog_info; in add_subprog_and_kfunc()
3212 int i, ret, insn_cnt = env->prog->len, ex_cb_insn; in add_subprog_and_kfunc()
3213 struct bpf_insn *insn = env->prog->insnsi; in add_subprog_and_kfunc()
3216 ret = add_subprog(env, 0); in add_subprog_and_kfunc()
3225 if (!env->bpf_capable) { in add_subprog_and_kfunc()
3226 …verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_AD… in add_subprog_and_kfunc()
3231 ret = add_subprog(env, i + insn->imm + 1); in add_subprog_and_kfunc()
3233 ret = add_kfunc_call(env, insn->imm, insn->off); in add_subprog_and_kfunc()
3239 ret = bpf_find_exception_callback_insn_off(env); in add_subprog_and_kfunc()
3248 ret = add_subprog(env, ex_cb_insn); in add_subprog_and_kfunc()
3251 for (i = 1; i < env->subprog_cnt; i++) { in add_subprog_and_kfunc()
3252 if (env->subprog_info[i].start != ex_cb_insn) in add_subprog_and_kfunc()
3254 env->exception_callback_subprog = i; in add_subprog_and_kfunc()
3255 mark_subprog_exc_cb(env, i); in add_subprog_and_kfunc()
3263 subprog[env->subprog_cnt].start = insn_cnt; in add_subprog_and_kfunc()
3265 if (env->log.level & BPF_LOG_LEVEL2) in add_subprog_and_kfunc()
3266 for (i = 0; i < env->subprog_cnt; i++) in add_subprog_and_kfunc()
3267 verbose(env, "func#%d @%d\n", i, subprog[i].start); in add_subprog_and_kfunc()
3272 static int check_subprogs(struct bpf_verifier_env *env) in check_subprogs() argument
3275 struct bpf_subprog_info *subprog = env->subprog_info; in check_subprogs()
3276 struct bpf_insn *insn = env->prog->insnsi; in check_subprogs()
3277 int insn_cnt = env->prog->len; in check_subprogs()
3303 verbose(env, "jump out of range from insn %d to %d\n", i, off); in check_subprogs()
3315 verbose(env, "last insn is not an exit or jmp\n"); in check_subprogs()
3320 if (cur_subprog < env->subprog_cnt) in check_subprogs()
3330 static int mark_reg_read(struct bpf_verifier_env *env, in mark_reg_read() argument
3342 verbose(env, "verifier BUG type %s var_off %lld off %d\n", in mark_reg_read()
3343 reg_type_str(env, parent->type), in mark_reg_read()
3373 if (env->longest_mark_read_walk < cnt) in mark_reg_read()
3374 env->longest_mark_read_walk = cnt; in mark_reg_read()
3378 static int mark_stack_slot_obj_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in mark_stack_slot_obj_read() argument
3381 struct bpf_func_state *state = func(env, reg); in mark_stack_slot_obj_read()
3387 err = mark_reg_read(env, st, st->parent, REG_LIVE_READ64); in mark_stack_slot_obj_read()
3391 mark_stack_slot_scratched(env, spi - i); in mark_stack_slot_obj_read()
3396 static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in mark_dynptr_read() argument
3406 spi = dynptr_get_spi(env, reg); in mark_dynptr_read()
3413 return mark_stack_slot_obj_read(env, reg, spi, BPF_DYNPTR_NR_SLOTS); in mark_dynptr_read()
3416 static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in mark_iter_read() argument
3419 return mark_stack_slot_obj_read(env, reg, spi, nr_slots); in mark_iter_read()
3422 static int mark_irq_flag_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in mark_irq_flag_read() argument
3426 spi = irq_flag_get_spi(env, reg); in mark_irq_flag_read()
3429 return mark_stack_slot_obj_read(env, reg, spi, 1); in mark_irq_flag_read()
3436 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, in is_reg64() argument
3547 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) in insn_has_def32() argument
3554 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); in insn_has_def32()
3557 static void mark_insn_zext(struct bpf_verifier_env *env, in mark_insn_zext() argument
3565 env->insn_aux_data[def_idx - 1].zext_dst = true; in mark_insn_zext()
3570 static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, in __check_reg_arg() argument
3573 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; in __check_reg_arg()
3578 verbose(env, "R%d is invalid\n", regno); in __check_reg_arg()
3582 mark_reg_scratched(env, regno); in __check_reg_arg()
3585 rw64 = is_reg64(env, insn, regno, reg, t); in __check_reg_arg()
3589 verbose(env, "R%d !read_ok\n", regno); in __check_reg_arg()
3597 mark_insn_zext(env, reg); in __check_reg_arg()
3599 return mark_reg_read(env, reg, reg->parent, in __check_reg_arg()
3604 verbose(env, "frame pointer is read only\n"); in __check_reg_arg()
3608 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; in __check_reg_arg()
3610 mark_reg_unknown(env, regs, regno); in __check_reg_arg()
3615 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, in check_reg_arg() argument
3618 struct bpf_verifier_state *vstate = env->cur_state; in check_reg_arg()
3621 return __check_reg_arg(env, state->regs, regno, t); in check_reg_arg()
3639 static void mark_jmp_point(struct bpf_verifier_env *env, int idx) in mark_jmp_point() argument
3641 env->insn_aux_data[idx].jmp_point = true; in mark_jmp_point()
3644 static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx) in is_jmp_point() argument
3646 return env->insn_aux_data[insn_idx].jmp_point; in is_jmp_point()
3728 static int push_insn_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, in push_insn_history() argument
3735 if (env->cur_hist_ent) { in push_insn_history()
3739 WARN_ONCE((env->cur_hist_ent->flags & insn_flags) && in push_insn_history()
3740 (env->cur_hist_ent->flags & insn_flags) != insn_flags, in push_insn_history()
3742 env->insn_idx, env->cur_hist_ent->flags, insn_flags); in push_insn_history()
3743 env->cur_hist_ent->flags |= insn_flags; in push_insn_history()
3744 WARN_ONCE(env->cur_hist_ent->linked_regs != 0, in push_insn_history()
3746 env->insn_idx, env->cur_hist_ent->linked_regs); in push_insn_history()
3747 env->cur_hist_ent->linked_regs = linked_regs; in push_insn_history()
3751 if (cur->insn_hist_end + 1 > env->insn_hist_cap) { in push_insn_history()
3753 p = kvrealloc(env->insn_hist, alloc_size, GFP_USER); in push_insn_history()
3756 env->insn_hist = p; in push_insn_history()
3757 env->insn_hist_cap = alloc_size / sizeof(*p); in push_insn_history()
3760 p = &env->insn_hist[cur->insn_hist_end]; in push_insn_history()
3761 p->idx = env->insn_idx; in push_insn_history()
3762 p->prev_idx = env->prev_insn_idx; in push_insn_history()
3767 env->cur_hist_ent = p; in push_insn_history()
3772 static struct bpf_insn_hist_entry *get_insn_hist_entry(struct bpf_verifier_env *env, in get_insn_hist_entry() argument
3775 if (hist_end > hist_start && env->insn_hist[hist_end - 1].idx == insn_idx) in get_insn_hist_entry()
3776 return &env->insn_hist[hist_end - 1]; in get_insn_hist_entry()
3793 static int get_prev_insn_idx(const struct bpf_verifier_env *env, in get_prev_insn_idx() argument
3803 if (cnt == 1 && env->insn_hist[hist_start].idx == insn_idx) in get_prev_insn_idx()
3807 if (cnt && env->insn_hist[hist_end - 1].idx == insn_idx) { in get_prev_insn_idx()
3809 return env->insn_hist[hist_end - 1].prev_idx; in get_prev_insn_idx()
3838 struct bpf_verifier_env *env = bt->env; in bt_reset() local
3841 bt->env = env; in bt_reset()
3858 verbose(bt->env, "BUG subprog enter from frame %d\n", bt->frame); in bt_subprog_enter()
3869 verbose(bt->env, "BUG subprog exit from frame 0\n"); in bt_subprog_exit()
4017 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx);
4028 static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, in backtrack_insn() argument
4034 .private_data = env, in backtrack_insn()
4036 struct bpf_insn *insn = env->prog->insnsi + idx; in backtrack_insn()
4046 if (env->log.level & BPF_LOG_LEVEL2) { in backtrack_insn()
4047 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt)); in backtrack_insn()
4048 verbose(env, "mark_precise: frame%d: regs=%s ", in backtrack_insn()
4049 bt->frame, env->tmp_str_buf); in backtrack_insn()
4050 fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); in backtrack_insn()
4051 verbose(env, "stack=%s before ", env->tmp_str_buf); in backtrack_insn()
4052 verbose(env, "%d: ", idx); in backtrack_insn()
4053 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in backtrack_insn()
4142 subprog = find_subprog(env, subprog_insn_idx); in backtrack_insn()
4146 if (subprog_is_global(env, subprog)) { in backtrack_insn()
4159 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
4174 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
4182 verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt)); in backtrack_insn()
4206 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
4211 verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt)); in backtrack_insn()
4234 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
4249 if (subseq_idx >= 0 && calls_callback(env, subseq_idx)) in backtrack_insn()
4253 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
4267 bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && in backtrack_insn()
4369 static void mark_all_scalars_precise(struct bpf_verifier_env *env, in mark_all_scalars_precise() argument
4376 if (env->log.level & BPF_LOG_LEVEL2) { in mark_all_scalars_precise()
4377 verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n", in mark_all_scalars_precise()
4395 if (env->log.level & BPF_LOG_LEVEL2) { in mark_all_scalars_precise()
4396 verbose(env, "force_precise: frame%d: forcing r%d to be precise\n", in mark_all_scalars_precise()
4407 if (env->log.level & BPF_LOG_LEVEL2) { in mark_all_scalars_precise()
4408 verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n", in mark_all_scalars_precise()
4416 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in mark_all_scalars_imprecise() argument
4528 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) in __mark_chain_precision() argument
4530 struct backtrack_state *bt = &env->bt; in __mark_chain_precision()
4531 struct bpf_verifier_state *st = env->cur_state; in __mark_chain_precision()
4533 int last_idx = env->insn_idx; in __mark_chain_precision()
4540 if (!env->bpf_capable) in __mark_chain_precision()
4544 bt_init(bt, env->cur_state->curframe); in __mark_chain_precision()
4569 if (env->log.level & BPF_LOG_LEVEL2) { in __mark_chain_precision()
4570 verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n", in __mark_chain_precision()
4595 verbose(env, "BUG backtracking func entry subprog %d reg_mask %x stack_mask %llx\n", in __mark_chain_precision()
4606 hist = get_insn_hist_entry(env, hist_start, hist_end, i); in __mark_chain_precision()
4607 err = backtrack_insn(env, i, subseq_idx, hist, bt); in __mark_chain_precision()
4610 mark_all_scalars_precise(env, env->cur_state); in __mark_chain_precision()
4623 i = get_prev_insn_idx(env, st, i, hist_start, &hist_end); in __mark_chain_precision()
4626 if (i >= env->prog->len) { in __mark_chain_precision()
4633 verbose(env, "BUG backtracking idx %d\n", i); in __mark_chain_precision()
4660 verbose(env, "BUG backtracking (stack slot %d, total slots %d)\n", in __mark_chain_precision()
4676 if (env->log.level & BPF_LOG_LEVEL2) { in __mark_chain_precision()
4677 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, in __mark_chain_precision()
4679 verbose(env, "mark_precise: frame%d: parent state regs=%s ", in __mark_chain_precision()
4680 fr, env->tmp_str_buf); in __mark_chain_precision()
4681 fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, in __mark_chain_precision()
4683 verbose(env, "stack=%s: ", env->tmp_str_buf); in __mark_chain_precision()
4684 print_verifier_state(env, st, fr, true); in __mark_chain_precision()
4701 mark_all_scalars_precise(env, env->cur_state); in __mark_chain_precision()
4708 int mark_chain_precision(struct bpf_verifier_env *env, int regno) in mark_chain_precision() argument
4710 return __mark_chain_precision(env, regno); in mark_chain_precision()
4716 static int mark_chain_precision_batch(struct bpf_verifier_env *env) in mark_chain_precision_batch() argument
4718 return __mark_chain_precision(env, -1); in mark_chain_precision_batch()
4776 static void assign_scalar_id_before_mov(struct bpf_verifier_env *env, in assign_scalar_id_before_mov() argument
4797 src_reg->id = ++env->id_gen; in assign_scalar_id_before_mov()
4811 static void save_register_state(struct bpf_verifier_env *env, in save_register_state() argument
4827 mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); in save_register_state()
4841 static void check_fastcall_stack_contract(struct bpf_verifier_env *env, in check_fastcall_stack_contract() argument
4844 struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; in check_fastcall_stack_contract()
4845 struct bpf_insn_aux_data *aux = env->insn_aux_data; in check_fastcall_stack_contract()
4868 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, in check_stack_write_fixed_off() argument
4876 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_stack_write_fixed_off()
4883 if (!env->allow_ptr_leaks && in check_stack_write_fixed_off()
4887 verbose(env, "attempt to corrupt spilled pointer on stack\n"); in check_stack_write_fixed_off()
4891 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_fixed_off()
4894 if (!env->bypass_spec_v4) { in check_stack_write_fixed_off()
4907 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; in check_stack_write_fixed_off()
4910 err = destroy_if_dynptr_stack_slot(env, state, spi); in check_stack_write_fixed_off()
4914 check_fastcall_stack_contract(env, state, insn_idx, off); in check_stack_write_fixed_off()
4915 mark_stack_slot_scratched(env, spi); in check_stack_write_fixed_off()
4916 if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) { in check_stack_write_fixed_off()
4922 assign_scalar_id_before_mov(env, reg); in check_stack_write_fixed_off()
4923 save_register_state(env, state, spi, reg, size); in check_stack_write_fixed_off()
4928 env->bpf_capable) { in check_stack_write_fixed_off()
4929 struct bpf_reg_state *tmp_reg = &env->fake_reg[0]; in check_stack_write_fixed_off()
4934 save_register_state(env, state, spi, tmp_reg, size); in check_stack_write_fixed_off()
4938 verbose_linfo(env, insn_idx, "; "); in check_stack_write_fixed_off()
4939 verbose(env, "invalid size of register spill\n"); in check_stack_write_fixed_off()
4943 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); in check_stack_write_fixed_off()
4946 save_register_state(env, state, spi, reg, size); in check_stack_write_fixed_off()
4977 err = mark_chain_precision(env, value_regno); in check_stack_write_fixed_off()
4990 return push_insn_history(env, env->cur_state, insn_flags, 0); in check_stack_write_fixed_off()
5013 static int check_stack_write_var_off(struct bpf_verifier_env *env, in check_stack_write_var_off() argument
5023 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_stack_write_var_off()
5030 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_var_off()
5044 err = destroy_if_dynptr_stack_slot(env, state, spi); in check_stack_write_var_off()
5049 check_fastcall_stack_contract(env, state, insn_idx, min_off); in check_stack_write_var_off()
5058 mark_stack_slot_scratched(env, spi); in check_stack_write_var_off()
5060 if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { in check_stack_write_var_off()
5072 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", in check_stack_write_var_off()
5107 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { in check_stack_write_var_off()
5108 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", in check_stack_write_var_off()
5116 err = mark_chain_precision(env, value_regno); in check_stack_write_var_off()
5131 static void mark_reg_stack_read(struct bpf_verifier_env *env, in mark_reg_stack_read() argument
5136 struct bpf_verifier_state *vstate = env->cur_state; in mark_reg_stack_read()
5145 mark_stack_slot_scratched(env, spi); in mark_reg_stack_read()
5155 __mark_reg_const_zero(env, &state->regs[dst_regno]); in mark_reg_stack_read()
5158 mark_reg_unknown(env, state->regs, dst_regno); in mark_reg_stack_read()
5172 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, in check_stack_read_fixed_off() argument
5177 struct bpf_verifier_state *vstate = env->cur_state; in check_stack_read_fixed_off()
5187 mark_stack_slot_scratched(env, spi); in check_stack_read_fixed_off()
5188 check_fastcall_stack_contract(env, state, env->insn_idx, off); in check_stack_read_fixed_off()
5198 verbose_linfo(env, env->insn_idx, "; "); in check_stack_read_fixed_off()
5199 verbose(env, "invalid size of register fill\n"); in check_stack_read_fixed_off()
5203 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
5237 if (type == STACK_INVALID && env->allow_uninit_stack) in check_stack_read_fixed_off()
5239 verbose(env, "invalid read from stack off %d+%d size %d\n", in check_stack_read_fixed_off()
5246 __mark_reg_const_zero(env, &state->regs[dst_regno]); in check_stack_read_fixed_off()
5250 __mark_reg_const_zero(env, &state->regs[dst_regno]); in check_stack_read_fixed_off()
5253 mark_reg_unknown(env, state->regs, dst_regno); in check_stack_read_fixed_off()
5266 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { in check_stack_read_fixed_off()
5273 verbose(env, "leaking pointer from stack off %d\n", in check_stack_read_fixed_off()
5277 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
5285 if (type == STACK_INVALID && env->allow_uninit_stack) in check_stack_read_fixed_off()
5287 verbose(env, "invalid read from stack off %d+%d size %d\n", in check_stack_read_fixed_off()
5291 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
5293 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); in check_stack_read_fixed_off()
5297 return push_insn_history(env, env->cur_state, insn_flags, 0); in check_stack_read_fixed_off()
5306 static int check_stack_range_initialized(struct bpf_verifier_env *env,
5312 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) in reg_state() argument
5314 return cur_regs(env) + regno; in reg_state()
5330 static int check_stack_read_var_off(struct bpf_verifier_env *env, in check_stack_read_var_off() argument
5334 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read_var_off()
5335 struct bpf_func_state *ptr_state = func(env, reg); in check_stack_read_var_off()
5341 err = check_stack_range_initialized(env, ptr_regno, off, size, in check_stack_read_var_off()
5348 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); in check_stack_read_var_off()
5349 check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); in check_stack_read_var_off()
5362 static int check_stack_read(struct bpf_verifier_env *env, in check_stack_read() argument
5366 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read()
5367 struct bpf_func_state *state = func(env, reg); in check_stack_read()
5380 …verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=… in check_stack_read()
5395 err = check_stack_read_fixed_off(env, state, off, size, in check_stack_read()
5402 err = check_stack_read_var_off(env, ptr_regno, off, size, in check_stack_read()
5419 static int check_stack_write(struct bpf_verifier_env *env, in check_stack_write() argument
5423 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_write()
5424 struct bpf_func_state *state = func(env, reg); in check_stack_write()
5429 err = check_stack_write_fixed_off(env, state, off, size, in check_stack_write()
5435 err = check_stack_write_var_off(env, state, in check_stack_write()
5442 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, in check_map_access_type() argument
5445 struct bpf_reg_state *regs = cur_regs(env); in check_map_access_type()
5450 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", in check_map_access_type()
5456 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", in check_map_access_type()
5465 static int __check_mem_access(struct bpf_verifier_env *env, int regno, in __check_mem_access() argument
5475 reg = &cur_regs(env)[regno]; in __check_mem_access()
5478 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", in __check_mem_access()
5482 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", in __check_mem_access()
5488 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", in __check_mem_access()
5493 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", in __check_mem_access()
5501 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, in check_mem_region_access() argument
5505 struct bpf_verifier_state *vstate = env->cur_state; in check_mem_region_access()
5524 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_mem_region_access()
5528 err = __check_mem_access(env, regno, reg->smin_value + off, size, in check_mem_region_access()
5531 verbose(env, "R%d min value is outside of the allowed memory range\n", in check_mem_region_access()
5541 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", in check_mem_region_access()
5545 err = __check_mem_access(env, regno, reg->umax_value + off, size, in check_mem_region_access()
5548 verbose(env, "R%d max value is outside of the allowed memory range\n", in check_mem_region_access()
5556 static int __check_ptr_off_reg(struct bpf_verifier_env *env, in __check_ptr_off_reg() argument
5565 verbose(env, "negative offset %s ptr R%d off=%d disallowed\n", in __check_ptr_off_reg()
5566 reg_type_str(env, reg->type), regno, reg->off); in __check_ptr_off_reg()
5571 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", in __check_ptr_off_reg()
5572 reg_type_str(env, reg->type), regno, reg->off); in __check_ptr_off_reg()
5580 verbose(env, "variable %s access var_off=%s disallowed\n", in __check_ptr_off_reg()
5581 reg_type_str(env, reg->type), tn_buf); in __check_ptr_off_reg()
5588 static int check_ptr_off_reg(struct bpf_verifier_env *env, in check_ptr_off_reg() argument
5591 return __check_ptr_off_reg(env, reg, regno, false); in check_ptr_off_reg()
5594 static int map_kptr_match_type(struct bpf_verifier_env *env, in map_kptr_match_type() argument
5626 if (__check_ptr_off_reg(env, reg, regno, true)) in map_kptr_match_type()
5653 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, in map_kptr_match_type()
5659 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, in map_kptr_match_type()
5660 reg_type_str(env, reg->type), reg_name); in map_kptr_match_type()
5661 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); in map_kptr_match_type()
5663 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), in map_kptr_match_type()
5666 verbose(env, "\n"); in map_kptr_match_type()
5670 static bool in_sleepable(struct bpf_verifier_env *env) in in_sleepable() argument
5672 return env->prog->sleepable || in in_sleepable()
5673 (env->cur_state && env->cur_state->in_sleepable); in in_sleepable()
5679 static bool in_rcu_cs(struct bpf_verifier_env *env) in in_rcu_cs() argument
5681 return env->cur_state->active_rcu_lock || in in_rcu_cs()
5682 env->cur_state->active_locks || in in_rcu_cs()
5683 !in_sleepable(env); in in_rcu_cs()
5731 static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field) in btf_ld_kptr_type() argument
5737 if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) { in btf_ld_kptr_type()
5754 static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno, in mark_uptr_ld_reg() argument
5761 mark_reg_known_zero(env, cur_regs(env), regno); in mark_uptr_ld_reg()
5762 reg = reg_state(env, regno); in mark_uptr_ld_reg()
5765 reg->id = ++env->id_gen; in mark_uptr_ld_reg()
5770 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, in check_map_kptr_access() argument
5774 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_map_kptr_access()
5786 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); in check_map_kptr_access()
5795 verbose(env, "store to referenced kptr disallowed\n"); in check_map_kptr_access()
5799 verbose(env, "store to uptr disallowed\n"); in check_map_kptr_access()
5805 return mark_uptr_ld_reg(env, value_regno, kptr_field); in check_map_kptr_access()
5810 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf, in check_map_kptr_access()
5811 kptr_field->kptr.btf_id, btf_ld_kptr_type(env, kptr_field)); in check_map_kptr_access()
5813 val_reg = reg_state(env, value_regno); in check_map_kptr_access()
5815 map_kptr_match_type(env, kptr_field, val_reg, value_regno)) in check_map_kptr_access()
5819 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", in check_map_kptr_access()
5824 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); in check_map_kptr_access()
5831 static int check_map_access(struct bpf_verifier_env *env, u32 regno, in check_map_access() argument
5835 struct bpf_verifier_state *vstate = env->cur_state; in check_map_access()
5842 err = check_mem_region_access(env, regno, off, size, map->value_size, in check_map_access()
5866 verbose(env, "%s cannot be accessed indirectly by helper\n", in check_map_access()
5871 verbose(env, "%s access cannot have variable offset\n", in check_map_access()
5876 verbose(env, "%s access misaligned expected=%u off=%llu\n", in check_map_access()
5882 verbose(env, "%s access size must be BPF_DW\n", in check_map_access()
5888 verbose(env, "%s cannot be accessed directly by load/store\n", in check_map_access()
5899 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, in may_access_direct_pkt_data() argument
5903 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in may_access_direct_pkt_data()
5927 env->seen_direct_write = true; in may_access_direct_pkt_data()
5932 env->seen_direct_write = true; in may_access_direct_pkt_data()
5941 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, in check_packet_access() argument
5944 struct bpf_reg_state *regs = cur_regs(env); in check_packet_access()
5957 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_packet_access()
5963 __check_mem_access(env, regno, off, size, reg->range, in check_packet_access()
5966 verbose(env, "R%d offset is outside of the packet\n", regno); in check_packet_access()
5976 env->prog->aux->max_pkt_offset = in check_packet_access()
5977 max_t(u32, env->prog->aux->max_pkt_offset, in check_packet_access()
5984 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, in check_ctx_access() argument
5990 .log = &env->log, in check_ctx_access()
5995 if (env->ops->is_valid_access && in check_ctx_access()
5996 env->ops->is_valid_access(off, size, t, env->prog, &info)) { in check_ctx_access()
6011 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; in check_ctx_access()
6014 if (env->prog->aux->max_ctx_offset < off + size) in check_ctx_access()
6015 env->prog->aux->max_ctx_offset = off + size; in check_ctx_access()
6019 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); in check_ctx_access()
6023 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, in check_flow_keys_access() argument
6028 verbose(env, "invalid access to flow keys off=%d size=%d\n", in check_flow_keys_access()
6035 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, in check_sock_access() argument
6039 struct bpf_reg_state *regs = cur_regs(env); in check_sock_access()
6045 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_sock_access()
6069 env->insn_aux_data[insn_idx].ctx_field_size = in check_sock_access()
6074 verbose(env, "R%d invalid %s access off=%d size=%d\n", in check_sock_access()
6075 regno, reg_type_str(env, reg->type), off, size); in check_sock_access()
6080 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) in is_pointer_value() argument
6082 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); in is_pointer_value()
6085 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) in is_ctx_reg() argument
6087 const struct bpf_reg_state *reg = reg_state(env, regno); in is_ctx_reg()
6092 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) in is_sk_reg() argument
6094 const struct bpf_reg_state *reg = reg_state(env, regno); in is_sk_reg()
6099 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) in is_pkt_reg() argument
6101 const struct bpf_reg_state *reg = reg_state(env, regno); in is_pkt_reg()
6106 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) in is_flow_key_reg() argument
6108 const struct bpf_reg_state *reg = reg_state(env, regno); in is_flow_key_reg()
6114 static bool is_arena_reg(struct bpf_verifier_env *env, int regno) in is_arena_reg() argument
6116 const struct bpf_reg_state *reg = reg_state(env, regno); in is_arena_reg()
6164 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, in check_pkt_ptr_alignment() argument
6190 verbose(env, in check_pkt_ptr_alignment()
6199 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, in check_generic_ptr_alignment() argument
6215 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", in check_generic_ptr_alignment()
6223 static int check_ptr_alignment(struct bpf_verifier_env *env, in check_ptr_alignment() argument
6227 bool strict = env->strict_alignment || strict_alignment_once; in check_ptr_alignment()
6236 return check_pkt_ptr_alignment(env, reg, off, size, strict); in check_ptr_alignment()
6274 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, in check_ptr_alignment()
6306 static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth) in round_up_stack_depth() argument
6308 if (env->prog->jit_requested) in round_up_stack_depth()
6323 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, in check_max_stack_depth_subprog() argument
6326 struct bpf_subprog_info *subprog = env->subprog_info; in check_max_stack_depth_subprog()
6327 struct bpf_insn *insn = env->prog->insnsi; in check_max_stack_depth_subprog()
6358 verbose(env, in check_max_stack_depth_subprog()
6364 subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth); in check_max_stack_depth_subprog()
6377 verbose(env, "stack size of subprog %d is %d. Too large\n", in check_max_stack_depth_subprog()
6384 verbose(env, "combined stack size of %d calls is %d. Too large\n", in check_max_stack_depth_subprog()
6409 verbose(env, in check_max_stack_depth_subprog()
6423 sidx = find_subprog(env, next_insn); in check_max_stack_depth_subprog()
6431 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); in check_max_stack_depth_subprog()
6438 verbose(env, "insn %d cannot call exception cb directly\n", i); in check_max_stack_depth_subprog()
6452 verbose(env, "the call stack of %d frames is too deep !\n", in check_max_stack_depth_subprog()
6466 verbose(env, "cannot tail call within exception cb\n"); in check_max_stack_depth_subprog()
6472 env->prog->aux->tail_call_reachable = true; in check_max_stack_depth_subprog()
6480 depth -= round_up_stack_depth(env, subprog[idx].stack_depth); in check_max_stack_depth_subprog()
6487 static int check_max_stack_depth(struct bpf_verifier_env *env) in check_max_stack_depth() argument
6490 struct bpf_subprog_info *si = env->subprog_info; in check_max_stack_depth()
6494 for (int i = 0; i < env->subprog_cnt; i++) { in check_max_stack_depth()
6502 priv_stack_mode = bpf_enable_priv_stack(env->prog); in check_max_stack_depth()
6512 for (int i = env->subprog_cnt - 1; i >= 0; i--) { in check_max_stack_depth()
6515 ret = check_max_stack_depth_subprog(env, i, priv_stack_supported); in check_max_stack_depth()
6521 for (int i = 0; i < env->subprog_cnt; i++) { in check_max_stack_depth()
6523 env->prog->aux->jits_use_priv_stack = true; in check_max_stack_depth()
6532 static int get_callee_stack_depth(struct bpf_verifier_env *env, in get_callee_stack_depth() argument
6537 subprog = find_subprog(env, start); in get_callee_stack_depth()
6543 return env->subprog_info[subprog].stack_depth; in get_callee_stack_depth()
6547 static int __check_buffer_access(struct bpf_verifier_env *env, in __check_buffer_access() argument
6553 verbose(env, in __check_buffer_access()
6562 verbose(env, in __check_buffer_access()
6571 static int check_tp_buffer_access(struct bpf_verifier_env *env, in check_tp_buffer_access() argument
6577 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); in check_tp_buffer_access()
6581 if (off + size > env->prog->aux->max_tp_access) in check_tp_buffer_access()
6582 env->prog->aux->max_tp_access = off + size; in check_tp_buffer_access()
6587 static int check_buffer_access(struct bpf_verifier_env *env, in check_buffer_access() argument
6596 err = __check_buffer_access(env, buf_info, reg, regno, off, size); in check_buffer_access()
6910 static bool type_is_rcu(struct bpf_verifier_env *env, in type_is_rcu() argument
6918 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu"); in type_is_rcu()
6921 static bool type_is_rcu_or_null(struct bpf_verifier_env *env, in type_is_rcu_or_null() argument
6929 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null"); in type_is_rcu_or_null()
6932 static bool type_is_trusted(struct bpf_verifier_env *env, in type_is_trusted() argument
6942 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); in type_is_trusted()
6945 static bool type_is_trusted_or_null(struct bpf_verifier_env *env, in type_is_trusted_or_null() argument
6951 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, in type_is_trusted_or_null()
6955 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, in check_ptr_to_btf_access() argument
6969 if (!env->allow_ptr_leaks) { in check_ptr_to_btf_access()
6970 verbose(env, in check_ptr_to_btf_access()
6975 if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) { in check_ptr_to_btf_access()
6976 verbose(env, in check_ptr_to_btf_access()
6982 verbose(env, in check_ptr_to_btf_access()
6991 verbose(env, in check_ptr_to_btf_access()
6998 verbose(env, in check_ptr_to_btf_access()
7005 verbose(env, in check_ptr_to_btf_access()
7011 if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { in check_ptr_to_btf_access()
7013 verbose(env, "verifier internal error: reg->btf must be kernel btf\n"); in check_ptr_to_btf_access()
7016 ret = env->ops->btf_struct_access(&env->log, reg, off, size); in check_ptr_to_btf_access()
7023 verbose(env, "only read is supported\n"); in check_ptr_to_btf_access()
7029 verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n"); in check_ptr_to_btf_access()
7033 ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name); in check_ptr_to_btf_access()
7061 if (type_is_trusted(env, reg, field_name, btf_id)) { in check_ptr_to_btf_access()
7063 } else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) { in check_ptr_to_btf_access()
7065 } else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) { in check_ptr_to_btf_access()
7066 if (type_is_rcu(env, reg, field_name, btf_id)) { in check_ptr_to_btf_access()
7070 type_is_rcu_or_null(env, reg, field_name, btf_id)) { in check_ptr_to_btf_access()
7075 if (type_is_rcu_or_null(env, reg, field_name, btf_id) && in check_ptr_to_btf_access()
7100 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); in check_ptr_to_btf_access()
7105 static int check_ptr_to_map_access(struct bpf_verifier_env *env, in check_ptr_to_map_access() argument
7121 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); in check_ptr_to_map_access()
7126 verbose(env, "map_ptr access not supported for map type %d\n", in check_ptr_to_map_access()
7134 if (!env->allow_ptr_leaks) { in check_ptr_to_map_access()
7135 verbose(env, in check_ptr_to_map_access()
7142 verbose(env, "R%d is %s invalid negative access: off=%d\n", in check_ptr_to_map_access()
7148 verbose(env, "only read from %s is supported\n", tname); in check_ptr_to_map_access()
7154 mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0); in check_ptr_to_map_access()
7155 ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL); in check_ptr_to_map_access()
7160 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); in check_ptr_to_map_access()
7171 static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, in check_stack_slot_within_bounds() argument
7178 if (t == BPF_WRITE || env->allow_uninit_stack) in check_stack_slot_within_bounds()
7194 struct bpf_verifier_env *env, in check_stack_access_within_bounds() argument
7198 struct bpf_reg_state *regs = cur_regs(env); in check_stack_access_within_bounds()
7200 struct bpf_func_state *state = func(env, reg); in check_stack_access_within_bounds()
7216 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", in check_stack_access_within_bounds()
7224 err = check_stack_slot_within_bounds(env, min_off, state, type); in check_stack_access_within_bounds()
7235 verbose(env, "invalid%s stack R%d off=%d size=%d\n", in check_stack_access_within_bounds()
7241 verbose(env, "invalid variable-offset%s stack R%d var_off=%s off=%d size=%d\n", in check_stack_access_within_bounds()
7250 return grow_stack_state(env, state, -min_off /* size */); in check_stack_access_within_bounds()
7270 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, in check_mem_access() argument
7274 struct bpf_reg_state *regs = cur_regs(env); in check_mem_access()
7283 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); in check_mem_access()
7292 verbose(env, "write to change key R%d not allowed\n", regno); in check_mem_access()
7296 err = check_mem_region_access(env, regno, off, size, in check_mem_access()
7301 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7306 is_pointer_value(env, value_regno)) { in check_mem_access()
7307 verbose(env, "R%d leaks addr into map\n", value_regno); in check_mem_access()
7310 err = check_map_access_type(env, regno, off, size, t); in check_mem_access()
7313 err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); in check_mem_access()
7320 err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field); in check_mem_access()
7339 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7346 verbose(env, "R%d invalid mem access '%s'\n", regno, in check_mem_access()
7347 reg_type_str(env, reg->type)); in check_mem_access()
7352 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
7353 regno, reg_type_str(env, reg->type)); in check_mem_access()
7358 is_pointer_value(env, value_regno)) { in check_mem_access()
7359 verbose(env, "R%d leaks addr into mem\n", value_regno); in check_mem_access()
7363 err = check_mem_region_access(env, regno, off, size, in check_mem_access()
7366 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7375 is_pointer_value(env, value_regno)) { in check_mem_access()
7376 verbose(env, "R%d leaks addr into ctx\n", value_regno); in check_mem_access()
7380 err = check_ptr_off_reg(env, reg, regno); in check_mem_access()
7384 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, in check_mem_access()
7387 verbose_linfo(env, insn_idx, "; "); in check_mem_access()
7394 if (is_retval && get_func_retval_range(env->prog, &range)) { in check_mem_access()
7395 err = __mark_reg_s32_range(env, regs, value_regno, in check_mem_access()
7400 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7403 mark_reg_known_zero(env, regs, in check_mem_access()
7406 regs[value_regno].id = ++env->id_gen; in check_mem_access()
7423 err = check_stack_access_within_bounds(env, regno, off, size, t); in check_mem_access()
7428 err = check_stack_read(env, regno, off, size, in check_mem_access()
7431 err = check_stack_write(env, regno, off, size, in check_mem_access()
7434 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { in check_mem_access()
7435 verbose(env, "cannot write into packet\n"); in check_mem_access()
7439 is_pointer_value(env, value_regno)) { in check_mem_access()
7440 verbose(env, "R%d leaks addr into packet\n", in check_mem_access()
7444 err = check_packet_access(env, regno, off, size, false); in check_mem_access()
7446 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7449 is_pointer_value(env, value_regno)) { in check_mem_access()
7450 verbose(env, "R%d leaks addr into flow keys\n", in check_mem_access()
7455 err = check_flow_keys_access(env, off, size); in check_mem_access()
7457 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7460 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
7461 regno, reg_type_str(env, reg->type)); in check_mem_access()
7464 err = check_sock_access(env, insn_idx, regno, off, size, t); in check_mem_access()
7466 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7468 err = check_tp_buffer_access(env, reg, regno, off, size); in check_mem_access()
7470 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7473 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, in check_mem_access()
7476 err = check_ptr_to_map_access(env, regs, regno, off, size, t, in check_mem_access()
7484 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
7485 regno, reg_type_str(env, reg->type)); in check_mem_access()
7488 max_access = &env->prog->aux->max_rdonly_access; in check_mem_access()
7490 max_access = &env->prog->aux->max_rdwr_access; in check_mem_access()
7493 err = check_buffer_access(env, reg, regno, off, size, false, in check_mem_access()
7497 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7500 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7502 verbose(env, "R%d invalid mem access '%s'\n", regno, in check_mem_access()
7503 reg_type_str(env, reg->type)); in check_mem_access()
7518 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
7521 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) in check_atomic() argument
7539 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); in check_atomic()
7544 verbose(env, "invalid atomic operand size\n"); in check_atomic()
7549 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_atomic()
7554 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_atomic()
7562 err = check_reg_arg(env, aux_reg, SRC_OP); in check_atomic()
7566 if (is_pointer_value(env, aux_reg)) { in check_atomic()
7567 verbose(env, "R%d leaks addr into mem\n", aux_reg); in check_atomic()
7572 if (is_pointer_value(env, insn->src_reg)) { in check_atomic()
7573 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); in check_atomic()
7577 if (is_ctx_reg(env, insn->dst_reg) || in check_atomic()
7578 is_pkt_reg(env, insn->dst_reg) || in check_atomic()
7579 is_flow_key_reg(env, insn->dst_reg) || in check_atomic()
7580 is_sk_reg(env, insn->dst_reg) || in check_atomic()
7581 (is_arena_reg(env, insn->dst_reg) && !bpf_jit_supports_insn(insn, true))) { in check_atomic()
7582 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", in check_atomic()
7584 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic()
7595 err = check_reg_arg(env, load_reg, DST_OP); in check_atomic()
7608 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
7611 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
7617 if (is_arena_reg(env, insn->dst_reg)) { in check_atomic()
7618 err = save_aux_ptr_type(env, PTR_TO_ARENA, false); in check_atomic()
7623 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
7641 struct bpf_verifier_env *env, int regno, int off, in check_stack_range_initialized() argument
7645 struct bpf_reg_state *reg = reg_state(env, regno); in check_stack_range_initialized()
7646 struct bpf_func_state *state = func(env, reg); in check_stack_range_initialized()
7654 verbose(env, "invalid zero-sized read\n"); in check_stack_range_initialized()
7661 err = check_stack_access_within_bounds(env, regno, off, access_size, type); in check_stack_range_initialized()
7674 if (!env->bypass_spec_v1) { in check_stack_range_initialized()
7678 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", in check_stack_range_initialized()
7717 verbose(env, "potential write to dynptr at off=%d disallowed\n", i); in check_stack_range_initialized()
7732 verbose(env, "verifier bug: allocated_stack too small\n"); in check_stack_range_initialized()
7740 (*stype == STACK_INVALID && env->allow_uninit_stack)) { in check_stack_range_initialized()
7750 env->allow_ptr_leaks)) { in check_stack_range_initialized()
7752 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); in check_stack_range_initialized()
7760 verbose(env, "invalid read from stack R%d off %d+%d size %d\n", in check_stack_range_initialized()
7766 verbose(env, "invalid read from stack R%d var_off %s+%d size %d\n", in check_stack_range_initialized()
7774 mark_reg_read(env, &state->stack[spi].spilled_ptr, in check_stack_range_initialized()
7786 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, in check_helper_mem_access() argument
7791 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_helper_mem_access()
7797 return check_packet_access(env, regno, reg->off, access_size, in check_helper_mem_access()
7801 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
7802 reg_type_str(env, reg->type)); in check_helper_mem_access()
7805 return check_mem_region_access(env, regno, reg->off, access_size, in check_helper_mem_access()
7808 if (check_map_access_type(env, regno, reg->off, access_size, access_type)) in check_helper_mem_access()
7810 return check_map_access(env, regno, reg->off, access_size, in check_helper_mem_access()
7815 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
7816 reg_type_str(env, reg->type)); in check_helper_mem_access()
7820 return check_mem_region_access(env, regno, reg->off, in check_helper_mem_access()
7826 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
7827 reg_type_str(env, reg->type)); in check_helper_mem_access()
7831 max_access = &env->prog->aux->max_rdonly_access; in check_helper_mem_access()
7833 max_access = &env->prog->aux->max_rdwr_access; in check_helper_mem_access()
7835 return check_buffer_access(env, reg, regno, reg->off, in check_helper_mem_access()
7840 env, in check_helper_mem_access()
7844 return check_ptr_to_btf_access(env, regs, regno, reg->off, in check_helper_mem_access()
7852 if (!env->ops->convert_ctx_access) { in check_helper_mem_access()
7859 return check_mem_access(env, env->insn_idx, regno, offset, BPF_B, in check_helper_mem_access()
7870 verbose(env, "R%d type=%s ", regno, in check_helper_mem_access()
7871 reg_type_str(env, reg->type)); in check_helper_mem_access()
7872 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); in check_helper_mem_access()
7883 static int check_mem_size_reg(struct bpf_verifier_env *env, in check_mem_size_reg() argument
7910 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", in check_mem_size_reg()
7916 verbose(env, "R%d invalid zero-sized read: u64=[%lld,%lld]\n", in check_mem_size_reg()
7922 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", in check_mem_size_reg()
7926 err = check_helper_mem_access(env, regno - 1, reg->umax_value, in check_mem_size_reg()
7929 err = mark_chain_precision(env, regno); in check_mem_size_reg()
7933 static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in check_mem_reg() argument
7952 err = check_helper_mem_access(env, regno, mem_size, BPF_READ, true, NULL); in check_mem_reg()
7953 err = err ?: check_helper_mem_access(env, regno, mem_size, BPF_WRITE, true, NULL); in check_mem_reg()
7961 static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in check_kfunc_mem_size_reg() argument
7964 struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; in check_kfunc_mem_size_reg()
7979 err = check_mem_size_reg(env, reg, regno, BPF_READ, true, &meta); in check_kfunc_mem_size_reg()
7980 err = err ?: check_mem_size_reg(env, reg, regno, BPF_WRITE, true, &meta); in check_kfunc_mem_size_reg()
8010 static int process_spin_lock(struct bpf_verifier_env *env, int regno, in process_spin_lock() argument
8013 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_spin_lock()
8014 struct bpf_verifier_state *cur = env->cur_state; in process_spin_lock()
8023 verbose(env, in process_spin_lock()
8031 verbose(env, in process_spin_lock()
8042 verbose(env, "%s '%s' has no valid bpf_spin_lock\n", map ? "map" : "local", in process_spin_lock()
8047 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock' that is at %d\n", in process_spin_lock()
8060 verbose(env, in process_spin_lock()
8064 err = acquire_lock_state(env, env->insn_idx, REF_TYPE_LOCK, reg->id, ptr); in process_spin_lock()
8066 verbose(env, "Failed to acquire lock state\n"); in process_spin_lock()
8078 verbose(env, "bpf_spin_unlock without taking a lock\n"); in process_spin_lock()
8082 if (release_lock_state(env->cur_state, REF_TYPE_LOCK, reg->id, ptr)) { in process_spin_lock()
8083 verbose(env, "bpf_spin_unlock of different lock\n"); in process_spin_lock()
8087 invalidate_non_owning_refs(env); in process_spin_lock()
8092 static int process_timer_func(struct bpf_verifier_env *env, int regno, in process_timer_func() argument
8095 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_timer_func()
8101 verbose(env, in process_timer_func()
8107 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", in process_timer_func()
8112 verbose(env, "map '%s' has no valid bpf_timer\n", map->name); in process_timer_func()
8116 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", in process_timer_func()
8121 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); in process_timer_func()
8129 static int process_wq_func(struct bpf_verifier_env *env, int regno, in process_wq_func() argument
8132 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_wq_func()
8137 verbose(env, "off %lld doesn't point to 'struct bpf_wq' that is at %d\n", in process_wq_func()
8146 static int process_kptr_func(struct bpf_verifier_env *env, int regno, in process_kptr_func() argument
8149 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_kptr_func()
8160 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", in process_kptr_func()
8169 verbose(env, in process_kptr_func()
8176 verbose(env, "R%d has no valid kptr\n", regno); in process_kptr_func()
8183 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); in process_kptr_func()
8187 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); in process_kptr_func()
8219 static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx, in process_dynptr_func() argument
8222 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_dynptr_func()
8226 verbose(env, in process_dynptr_func()
8236 verbose(env, "verifier internal error: misconfigured dynptr helper type flags\n"); in process_dynptr_func()
8258 if (!is_dynptr_reg_valid_uninit(env, reg)) { in process_dynptr_func()
8259 verbose(env, "Dynptr has to be an uninitialized dynptr\n"); in process_dynptr_func()
8265 err = check_mem_access(env, insn_idx, regno, in process_dynptr_func()
8271 err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id); in process_dynptr_func()
8275 verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n"); in process_dynptr_func()
8279 if (!is_dynptr_reg_valid_init(env, reg)) { in process_dynptr_func()
8280 verbose(env, in process_dynptr_func()
8287 if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) { in process_dynptr_func()
8288 verbose(env, in process_dynptr_func()
8294 err = mark_dynptr_read(env, reg); in process_dynptr_func()
8299 static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi) in iter_ref_obj_id() argument
8301 struct bpf_func_state *state = func(env, reg); in iter_ref_obj_id()
8339 static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx, in process_iter_arg() argument
8342 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_iter_arg()
8347 verbose(env, "arg#%d expected pointer to an iterator on stack\n", regno - 1); in process_iter_arg()
8359 verbose(env, "expected valid iter pointer as arg #%d\n", regno - 1); in process_iter_arg()
8367 if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { in process_iter_arg()
8368 verbose(env, "expected uninitialized iter_%s as arg #%d\n", in process_iter_arg()
8374 err = check_mem_access(env, insn_idx, regno, in process_iter_arg()
8380 err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots); in process_iter_arg()
8387 err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots); in process_iter_arg()
8392 verbose(env, "expected an initialized iter_%s as arg #%d\n", in process_iter_arg()
8396 verbose(env, "expected an RCU CS when using %s\n", meta->func_name); in process_iter_arg()
8402 spi = iter_get_spi(env, reg, nr_slots); in process_iter_arg()
8406 err = mark_iter_read(env, reg, spi, nr_slots); in process_iter_arg()
8413 meta->ref_obj_id = iter_ref_obj_id(env, reg, spi); in process_iter_arg()
8416 err = unmark_stack_slots_iter(env, reg, nr_slots); in process_iter_arg()
8428 static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env, in find_prev_entry() argument
8436 sl = *explored_state(env, insn_idx); in find_prev_entry()
8450 static void reset_idmap_scratch(struct bpf_verifier_env *env);
8455 static void maybe_widen_reg(struct bpf_verifier_env *env, in maybe_widen_reg() argument
8465 __mark_reg_unknown(env, rcur); in maybe_widen_reg()
8468 static int widen_imprecise_scalars(struct bpf_verifier_env *env, in widen_imprecise_scalars() argument
8475 reset_idmap_scratch(env); in widen_imprecise_scalars()
8481 maybe_widen_reg(env, in widen_imprecise_scalars()
8484 &env->idmap_scratch); in widen_imprecise_scalars()
8491 maybe_widen_reg(env, in widen_imprecise_scalars()
8494 &env->idmap_scratch); in widen_imprecise_scalars()
8587 static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, in process_iter_next_call() argument
8590 struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; in process_iter_next_call()
8600 verbose(env, "verifier internal error: unexpected iterator state %d (%s)\n", in process_iter_next_call()
8611 verbose(env, "bug: bad parent state for iter next call"); in process_iter_next_call()
8618 prev_st = find_prev_entry(env, cur_st->parent, insn_idx); in process_iter_next_call()
8620 queued_st = push_stack(env, insn_idx + 1, insn_idx, false); in process_iter_next_call()
8628 widen_imprecise_scalars(env, prev_st, queued_st); in process_iter_next_call()
8637 __mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]); in process_iter_next_call()
8664 static int resolve_map_arg_type(struct bpf_verifier_env *env, in resolve_map_arg_type() argument
8670 verbose(env, "invalid map_ptr to access map->type\n"); in resolve_map_arg_type()
8680 verbose(env, "invalid arg_type for sockmap/sockhash\n"); in resolve_map_arg_type()
8805 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, in check_reg_type() argument
8810 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_reg_type()
8817 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); in check_reg_type()
8855 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); in check_reg_type()
8857 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); in check_reg_type()
8858 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); in check_reg_type()
8867 verbose(env, in check_reg_type()
8870 regno, reg_type_str(env, reg->type)); in check_reg_type()
8893 verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno); in check_reg_type()
8899 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); in check_reg_type()
8906 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) in check_reg_type()
8910 verbose(env, "verifier internal error:"); in check_reg_type()
8911 verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", in check_reg_type()
8916 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, in check_reg_type()
8919 verbose(env, "R%d is of type %s but %s is expected\n", in check_reg_type()
8931 verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n"); in check_reg_type()
8936 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) in check_reg_type()
8946 verbose(env, "verifier internal error: invalid PTR_TO_BTF_ID register for type match\n"); in check_reg_type()
8969 static int check_func_arg_reg_off(struct bpf_verifier_env *env, in check_func_arg_reg_off() argument
8996 verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n", in check_func_arg_reg_off()
9000 return __check_ptr_off_reg(env, reg, regno, false); in check_func_arg_reg_off()
9034 return __check_ptr_off_reg(env, reg, regno, true); in check_func_arg_reg_off()
9036 return __check_ptr_off_reg(env, reg, regno, false); in check_func_arg_reg_off()
9040 static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env, in get_dynptr_arg_reg() argument
9050 verbose(env, "verifier internal error: multiple dynptr args\n"); in get_dynptr_arg_reg()
9057 verbose(env, "verifier internal error: no dynptr arg found\n"); in get_dynptr_arg_reg()
9062 static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in dynptr_id() argument
9064 struct bpf_func_state *state = func(env, reg); in dynptr_id()
9069 spi = dynptr_get_spi(env, reg); in dynptr_id()
9075 static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in dynptr_ref_obj_id() argument
9077 struct bpf_func_state *state = func(env, reg); in dynptr_ref_obj_id()
9082 spi = dynptr_get_spi(env, reg); in dynptr_ref_obj_id()
9088 static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env, in dynptr_get_type() argument
9091 struct bpf_func_state *state = func(env, reg); in dynptr_get_type()
9099 verbose(env, "verifier internal error: invalid spi when querying dynptr type\n"); in dynptr_get_type()
9106 static int check_reg_const_str(struct bpf_verifier_env *env, in check_reg_const_str() argument
9119 verbose(env, "R%d does not point to a readonly map'\n", regno); in check_reg_const_str()
9124 verbose(env, "R%d is not a constant address'\n", regno); in check_reg_const_str()
9129 verbose(env, "no direct value access support for this map type\n"); in check_reg_const_str()
9133 err = check_map_access(env, regno, reg->off, in check_reg_const_str()
9142 verbose(env, "direct value access on string failed\n"); in check_reg_const_str()
9148 verbose(env, "string is not zero-terminated\n"); in check_reg_const_str()
9155 static int get_constant_map_key(struct bpf_verifier_env *env, in get_constant_map_key() argument
9160 struct bpf_func_state *state = func(env, key); in get_constant_map_key()
9169 if (!env->bpf_capable) in get_constant_map_key()
9206 bt_set_frame_slot(&env->bt, key->frameno, spi); in get_constant_map_key()
9207 err = mark_chain_precision_batch(env); in get_constant_map_key()
9217 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, in check_func_arg() argument
9223 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_func_arg()
9233 err = check_reg_arg(env, regno, SRC_OP); in check_func_arg()
9238 if (is_pointer_value(env, regno)) { in check_func_arg()
9239 verbose(env, "R%d leaks addr into helper function\n", in check_func_arg()
9247 !may_access_direct_pkt_data(env, meta, BPF_READ)) { in check_func_arg()
9248 verbose(env, "helper access to the packet is not allowed\n"); in check_func_arg()
9253 err = resolve_map_arg_type(env, meta, &arg_type); in check_func_arg()
9269 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); in check_func_arg()
9273 err = check_func_arg_reg_off(env, reg, regno, arg_type); in check_func_arg()
9280 struct bpf_func_state *state = func(env, reg); in check_func_arg()
9289 spi = dynptr_get_spi(env, reg); in check_func_arg()
9291 verbose(env, "arg %d is an unacquired reference\n", regno); in check_func_arg()
9295 verbose(env, "cannot release unowned const bpf_dynptr\n"); in check_func_arg()
9299 verbose(env, "R%d must be referenced when passed to release function\n", in check_func_arg()
9304 verbose(env, "verifier internal error: more than one release argument\n"); in check_func_arg()
9312 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", in check_func_arg()
9338 verbose(env, in check_func_arg()
9358 verbose(env, "invalid map_ptr to access map->key\n"); in check_func_arg()
9362 err = check_helper_mem_access(env, regno, key_size, BPF_READ, false, NULL); in check_func_arg()
9366 err = get_constant_map_key(env, reg, key_size, &meta->const_map_key); in check_func_arg()
9385 verbose(env, "invalid map_ptr to access map->value\n"); in check_func_arg()
9389 err = check_helper_mem_access(env, regno, meta->map_ptr->value_size, in check_func_arg()
9395 verbose(env, "Helper has invalid btf_id in R%d\n", regno); in check_func_arg()
9402 if (in_rbtree_lock_required_cb(env)) { in check_func_arg()
9403 verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); in check_func_arg()
9407 err = process_spin_lock(env, regno, true); in check_func_arg()
9411 err = process_spin_lock(env, regno, false); in check_func_arg()
9415 verbose(env, "verifier internal error\n"); in check_func_arg()
9420 err = process_timer_func(env, regno, meta); in check_func_arg()
9433 err = check_helper_mem_access(env, regno, fn->arg_size[arg], in check_func_arg()
9439 err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); in check_func_arg()
9443 err = check_mem_size_reg(env, reg, regno, in check_func_arg()
9449 err = check_mem_size_reg(env, reg, regno, in check_func_arg()
9455 err = process_dynptr_func(env, regno, insn_idx, arg_type, 0); in check_func_arg()
9461 verbose(env, "R%d is not a known constant'\n", in check_func_arg()
9466 err = mark_chain_precision(env, regno); in check_func_arg()
9472 err = check_reg_const_str(env, reg, regno); in check_func_arg()
9478 err = process_kptr_func(env, regno, meta); in check_func_arg()
9487 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) in may_update_sockmap() argument
9489 enum bpf_attach_type eatype = env->prog->expected_attach_type; in may_update_sockmap()
9490 enum bpf_prog_type type = resolve_prog_type(env->prog); in may_update_sockmap()
9521 verbose(env, "cannot update sockmap in this context\n"); in may_update_sockmap()
9525 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) in allow_tail_call_in_subprogs() argument
9527 return env->prog->jit_requested && in allow_tail_call_in_subprogs()
9531 static int check_map_func_compatibility(struct bpf_verifier_env *env, in check_map_func_compatibility() argument
9607 !may_update_sockmap(env, func_id)) in check_map_func_compatibility()
9616 !may_update_sockmap(env, func_id)) in check_map_func_compatibility()
9668 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { in check_map_func_compatibility()
9669 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); in check_map_func_compatibility()
9777 verbose(env, "cannot pass map_type %d into func %s#%d\n", in check_map_func_compatibility()
9869 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) in clear_all_pkt_pointers() argument
9874 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in clear_all_pkt_pointers()
9876 mark_reg_invalid(env, reg); in clear_all_pkt_pointers()
9926 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id) in release_reference() argument
9928 struct bpf_verifier_state *vstate = env->cur_state; in release_reference()
9939 mark_reg_invalid(env, reg); in release_reference()
9945 static void invalidate_non_owning_refs(struct bpf_verifier_env *env) in invalidate_non_owning_refs() argument
9950 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ in invalidate_non_owning_refs()
9952 mark_reg_invalid(env, reg); in invalidate_non_owning_refs()
9956 static void clear_caller_saved_regs(struct bpf_verifier_env *env, in clear_caller_saved_regs() argument
9963 mark_reg_not_init(env, regs, caller_saved[i]); in clear_caller_saved_regs()
9964 __check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK); in clear_caller_saved_regs()
9968 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
9973 static int set_callee_state(struct bpf_verifier_env *env,
9977 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite, in setup_func_entry() argument
9985 verbose(env, "the call stack of %d frames is too deep\n", in setup_func_entry()
9991 verbose(env, "verifier bug. Frame %d already allocated\n", in setup_func_entry()
10006 init_func_state(env, callee, in setup_func_entry()
10011 err = set_callee_state_cb(env, caller, callee, callsite); in setup_func_entry()
10026 static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, in btf_check_func_arg_match() argument
10030 struct bpf_subprog_info *sub = subprog_info(env, subprog); in btf_check_func_arg_match()
10031 struct bpf_verifier_log *log = &env->log; in btf_check_func_arg_match()
10035 ret = btf_prepare_func_args(env, subprog); in btf_check_func_arg_match()
10053 ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE); in btf_check_func_arg_match()
10064 ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE); in btf_check_func_arg_match()
10067 if (check_mem_reg(env, reg, regno, arg->mem_size)) in btf_check_func_arg_match()
10086 ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_DYNPTR); in btf_check_func_arg_match()
10090 ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0); in btf_check_func_arg_match()
10101 err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta); in btf_check_func_arg_match()
10102 err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type); in btf_check_func_arg_match()
10122 static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, in btf_check_subprog_call() argument
10125 struct bpf_prog *prog = env->prog; in btf_check_subprog_call()
10140 err = btf_check_func_arg_match(env, subprog, btf, regs); in btf_check_subprog_call()
10150 static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in push_callback_call() argument
10154 struct bpf_verifier_state *state = env->cur_state, *callback_state; in push_callback_call()
10159 err = btf_check_subprog_call(env, subprog, caller->regs); in push_callback_call()
10167 env->subprog_info[subprog].is_cb = true; in push_callback_call()
10170 verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n", in push_callback_call()
10175 verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n", in push_callback_call()
10184 env->subprog_info[subprog].is_async_cb = true; in push_callback_call()
10185 async_cb = push_async_cb(env, env->subprog_info[subprog].start, in push_callback_call()
10194 err = set_callee_state_cb(env, caller, callee, insn_idx); in push_callback_call()
10204 callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false); in push_callback_call()
10208 err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb, in push_callback_call()
10219 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_func_call() argument
10222 struct bpf_verifier_state *state = env->cur_state; in check_func_call()
10227 subprog = find_subprog(env, target_insn); in check_func_call()
10229 verbose(env, "verifier bug. No program starts at insn %d\n", target_insn); in check_func_call()
10234 err = btf_check_subprog_call(env, subprog, caller->regs); in check_func_call()
10237 if (subprog_is_global(env, subprog)) { in check_func_call()
10238 const char *sub_name = subprog_name(env, subprog); in check_func_call()
10241 if (env->cur_state->active_locks) { in check_func_call()
10242 verbose(env, "global function calls are not allowed while holding a lock,\n" in check_func_call()
10248 if (env->cur_state->active_preempt_locks) { in check_func_call()
10249 verbose(env, "global function calls are not allowed with preemption disabled,\n" in check_func_call()
10254 if (env->cur_state->active_irq_id) { in check_func_call()
10255 verbose(env, "global function calls are not allowed with IRQs disabled,\n" in check_func_call()
10261 verbose(env, "Caller passes invalid args into func#%d ('%s')\n", in check_func_call()
10266 verbose(env, "Func#%d ('%s') is global and assumed valid.\n", in check_func_call()
10268 if (env->subprog_info[subprog].changes_pkt_data) in check_func_call()
10269 clear_all_pkt_pointers(env); in check_func_call()
10271 subprog_aux(env, subprog)->called = true; in check_func_call()
10272 clear_caller_saved_regs(env, caller->regs); in check_func_call()
10275 mark_reg_unknown(env, caller->regs, BPF_REG_0); in check_func_call()
10285 err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); in check_func_call()
10289 clear_caller_saved_regs(env, caller->regs); in check_func_call()
10292 *insn_idx = env->subprog_info[subprog].start - 1; in check_func_call()
10294 if (env->log.level & BPF_LOG_LEVEL) { in check_func_call()
10295 verbose(env, "caller:\n"); in check_func_call()
10296 print_verifier_state(env, state, caller->frameno, true); in check_func_call()
10297 verbose(env, "callee:\n"); in check_func_call()
10298 print_verifier_state(env, state, state->curframe, true); in check_func_call()
10304 int map_set_for_each_callback_args(struct bpf_verifier_env *env, in map_set_for_each_callback_args() argument
10327 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in map_set_for_each_callback_args()
10331 static int set_callee_state(struct bpf_verifier_env *env, in set_callee_state() argument
10345 static int set_map_elem_callback_state(struct bpf_verifier_env *env, in set_map_elem_callback_state() argument
10350 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; in set_map_elem_callback_state()
10358 verbose(env, "callback function not allowed for map\n"); in set_map_elem_callback_state()
10362 err = map->ops->map_set_for_each_callback_args(env, caller, callee); in set_map_elem_callback_state()
10371 static int set_loop_callback_state(struct bpf_verifier_env *env, in set_loop_callback_state() argument
10384 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_loop_callback_state()
10385 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_loop_callback_state()
10386 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_loop_callback_state()
10393 static int set_timer_callback_state(struct bpf_verifier_env *env, in set_timer_callback_state() argument
10416 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_timer_callback_state()
10417 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_timer_callback_state()
10423 static int set_find_vma_callback_state(struct bpf_verifier_env *env, in set_find_vma_callback_state() argument
10444 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_find_vma_callback_state()
10445 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_find_vma_callback_state()
10451 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, in set_user_ringbuf_callback_state() argument
10460 __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); in set_user_ringbuf_callback_state()
10461 mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL); in set_user_ringbuf_callback_state()
10465 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_user_ringbuf_callback_state()
10466 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_user_ringbuf_callback_state()
10467 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_user_ringbuf_callback_state()
10474 static int set_rbtree_add_callback_state(struct bpf_verifier_env *env, in set_rbtree_add_callback_state() argument
10494 ref_set_non_owning(env, &callee->regs[BPF_REG_1]); in set_rbtree_add_callback_state()
10496 ref_set_non_owning(env, &callee->regs[BPF_REG_2]); in set_rbtree_add_callback_state()
10498 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_rbtree_add_callback_state()
10499 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_rbtree_add_callback_state()
10500 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_rbtree_add_callback_state()
10512 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) in in_rbtree_lock_required_cb() argument
10514 struct bpf_verifier_state *state = env->cur_state; in in_rbtree_lock_required_cb()
10515 struct bpf_insn *insn = env->prog->insnsi; in in_rbtree_lock_required_cb()
10540 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) in prepare_func_exit() argument
10542 struct bpf_verifier_state *state = env->cur_state, *prev_st; in prepare_func_exit()
10557 verbose(env, "cannot return stack pointer to the caller\n"); in prepare_func_exit()
10564 verbose(env, "R0 not a scalar value\n"); in prepare_func_exit()
10569 err = mark_reg_read(env, r0, r0->parent, REG_LIVE_READ64); in prepare_func_exit()
10570 err = err ?: mark_chain_precision(env, BPF_REG_0); in prepare_func_exit()
10576 verbose_invalid_scalar(env, r0, callee->callback_ret_range, in prepare_func_exit()
10580 if (!calls_callback(env, callee->callsite)) { in prepare_func_exit()
10581 verbose(env, "BUG: in callback at %d, callsite %d !calls_callback\n", in prepare_func_exit()
10600 if (env->log.level & BPF_LOG_LEVEL) { in prepare_func_exit()
10601 verbose(env, "returning from callee:\n"); in prepare_func_exit()
10602 print_verifier_state(env, state, callee->frameno, true); in prepare_func_exit()
10603 verbose(env, "to caller at %d:\n", *insn_idx); in prepare_func_exit()
10604 print_verifier_state(env, state, caller->frameno, true); in prepare_func_exit()
10622 prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL; in prepare_func_exit()
10624 err = widen_imprecise_scalars(env, prev_st, state); in prepare_func_exit()
10631 static int do_refine_retval_range(struct bpf_verifier_env *env, in do_refine_retval_range() argument
10666 return reg_bounds_sanity_check(env, ret_reg, "retval"); in do_refine_retval_range()
10670 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, in record_func_map() argument
10673 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_map()
10689 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_map()
10702 verbose(env, "write into map forbidden\n"); in record_func_map()
10716 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, in record_func_key() argument
10719 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_key()
10720 struct bpf_reg_state *regs = cur_regs(env), *reg; in record_func_key()
10728 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_key()
10741 err = mark_chain_precision(env, BPF_REG_3); in record_func_key()
10752 static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit) in check_reference_leak() argument
10754 struct bpf_verifier_state *state = env->cur_state; in check_reference_leak()
10758 if (!exception_exit && cur_func(env)->frameno) in check_reference_leak()
10764 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", in check_reference_leak()
10771 static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit, bool check_lock, … in check_resource_leak() argument
10775 if (check_lock && env->cur_state->active_locks) { in check_resource_leak()
10776 verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix); in check_resource_leak()
10780 err = check_reference_leak(env, exception_exit); in check_resource_leak()
10782 verbose(env, "%s would lead to reference leak\n", prefix); in check_resource_leak()
10786 if (check_lock && env->cur_state->active_irq_id) { in check_resource_leak()
10787 verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix); in check_resource_leak()
10791 if (check_lock && env->cur_state->active_rcu_lock) { in check_resource_leak()
10792 verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix); in check_resource_leak()
10796 if (check_lock && env->cur_state->active_preempt_locks) { in check_resource_leak()
10797 verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix); in check_resource_leak()
10804 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, in check_bpf_snprintf_call() argument
10827 verbose(env, "verifier bug\n"); in check_bpf_snprintf_call()
10837 verbose(env, "Invalid format string\n"); in check_bpf_snprintf_call()
10842 static int check_get_func_ip(struct bpf_verifier_env *env) in check_get_func_ip() argument
10844 enum bpf_prog_type type = resolve_prog_type(env->prog); in check_get_func_ip()
10848 if (!bpf_prog_has_trampoline(env->prog)) { in check_get_func_ip()
10849 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", in check_get_func_ip()
10858 verbose(env, "func %s#%d not supported for program type %d\n", in check_get_func_ip()
10863 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) in cur_aux() argument
10865 return &env->insn_aux_data[env->insn_idx]; in cur_aux()
10868 static bool loop_flag_is_zero(struct bpf_verifier_env *env) in loop_flag_is_zero() argument
10870 struct bpf_reg_state *regs = cur_regs(env); in loop_flag_is_zero()
10875 mark_chain_precision(env, BPF_REG_4); in loop_flag_is_zero()
10880 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) in update_loop_inline_state() argument
10882 struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; in update_loop_inline_state()
10886 state->fit_for_inline = loop_flag_is_zero(env); in update_loop_inline_state()
10894 state->fit_for_inline = (loop_flag_is_zero(env) && in update_loop_inline_state()
10913 static int get_helper_proto(struct bpf_verifier_env *env, int func_id, in get_helper_proto() argument
10919 if (!env->ops->get_func_proto) in get_helper_proto()
10922 *ptr = env->ops->get_func_proto(func_id, env->prog); in get_helper_proto()
10926 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_helper_call() argument
10929 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_helper_call()
10942 err = get_helper_proto(env, insn->imm, &fn); in check_helper_call()
10944 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id); in check_helper_call()
10949 verbose(env, "program of this type cannot use helper %s#%d\n", in check_helper_call()
10955 if (!env->prog->gpl_compatible && fn->gpl_only) { in check_helper_call()
10956 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); in check_helper_call()
10960 if (fn->allowed && !fn->allowed(env->prog)) { in check_helper_call()
10961 verbose(env, "helper call is not allowed in probe\n"); in check_helper_call()
10965 if (!in_sleepable(env) && fn->might_sleep) { in check_helper_call()
10966 verbose(env, "helper call might sleep in a non-sleepable prog\n"); in check_helper_call()
10973 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", in check_helper_call()
10983 verbose(env, "kernel subsystem misconfigured func %s#%d\n", in check_helper_call()
10988 if (env->cur_state->active_rcu_lock) { in check_helper_call()
10990 verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n", in check_helper_call()
10995 if (in_sleepable(env) && is_storage_get_function(func_id)) in check_helper_call()
10996 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; in check_helper_call()
10999 if (env->cur_state->active_preempt_locks) { in check_helper_call()
11001 verbose(env, "sleepable helper %s#%d in non-preemptible region\n", in check_helper_call()
11006 if (in_sleepable(env) && is_storage_get_function(func_id)) in check_helper_call()
11007 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; in check_helper_call()
11010 if (env->cur_state->active_irq_id) { in check_helper_call()
11012 verbose(env, "sleepable helper %s#%d in IRQ-disabled region\n", in check_helper_call()
11017 if (in_sleepable(env) && is_storage_get_function(func_id)) in check_helper_call()
11018 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; in check_helper_call()
11024 err = check_func_arg(env, i, &meta, fn, insn_idx); in check_helper_call()
11029 err = record_func_map(env, &meta, func_id, insn_idx); in check_helper_call()
11033 err = record_func_key(env, &meta, func_id, insn_idx); in check_helper_call()
11041 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, in check_helper_call()
11047 regs = cur_regs(env); in check_helper_call()
11057 verbose(env, "verifier internal error: CONST_PTR_TO_DYNPTR cannot be released\n"); in check_helper_call()
11060 err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); in check_helper_call()
11063 bool in_rcu = in_rcu_cs(env); in check_helper_call()
11067 err = release_reference_nomark(env->cur_state, ref_obj_id); in check_helper_call()
11069 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in check_helper_call()
11076 mark_reg_invalid(env, reg); in check_helper_call()
11082 err = release_reference(env, meta.ref_obj_id); in check_helper_call()
11090 verbose(env, "func %s#%d reference has not been acquired before\n", in check_helper_call()
11098 err = check_resource_leak(env, false, true, "tail_call"); in check_helper_call()
11107 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); in check_helper_call()
11112 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_helper_call()
11116 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_helper_call()
11120 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_helper_call()
11124 err = check_bpf_snprintf_call(env, regs); in check_helper_call()
11127 update_loop_inline_state(env, meta.subprogno); in check_helper_call()
11131 err = mark_chain_precision(env, BPF_REG_1); in check_helper_call()
11134 if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) { in check_helper_call()
11135 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_helper_call()
11138 cur_func(env)->callback_depth = 0; in check_helper_call()
11139 if (env->log.level & BPF_LOG_LEVEL2) in check_helper_call()
11140 verbose(env, "frame%d bpf_loop iteration limit reached\n", in check_helper_call()
11141 env->cur_state->curframe); in check_helper_call()
11146 verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", in check_helper_call()
11147 reg_type_str(env, regs[BPF_REG_1].type)); in check_helper_call()
11153 env->prog->expected_attach_type == BPF_LSM_CGROUP) { in check_helper_call()
11154 if (!env->prog->aux->attach_func_proto->type) { in check_helper_call()
11158 verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); in check_helper_call()
11168 reg = get_dynptr_arg_reg(env, fn, regs); in check_helper_call()
11174 verbose(env, "verifier internal error: meta.dynptr_id already set\n"); in check_helper_call()
11178 verbose(env, "verifier internal error: meta.ref_obj_id already set\n"); in check_helper_call()
11182 id = dynptr_id(env, reg); in check_helper_call()
11184 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); in check_helper_call()
11188 ref_obj_id = dynptr_ref_obj_id(env, reg); in check_helper_call()
11190 verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n"); in check_helper_call()
11204 reg = get_dynptr_arg_reg(env, fn, regs); in check_helper_call()
11208 dynptr_type = dynptr_get_type(env, reg); in check_helper_call()
11229 verbose(env, "Helper has invalid btf/btf_id in R1\n"); in check_helper_call()
11233 env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true; in check_helper_call()
11238 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_helper_call()
11248 mark_reg_not_init(env, regs, caller_saved[i]); in check_helper_call()
11249 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_helper_call()
11262 mark_reg_unknown(env, regs, BPF_REG_0); in check_helper_call()
11269 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11275 verbose(env, in check_helper_call()
11291 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
11295 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11299 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11303 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11307 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11315 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11326 verbose(env, "unable to resolve the size of type '%s': %ld\n", in check_helper_call()
11355 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11367 verbose(env, "verifier internal error:"); in check_helper_call()
11368 verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n", in check_helper_call()
11376 verbose(env, "invalid return type %u of func %s#%d\n", in check_helper_call()
11386 verbose(env, "unknown return type %u of func %s#%d\n", in check_helper_call()
11392 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
11395 verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n", in check_helper_call()
11407 int id = acquire_reference(env, insn_idx); in check_helper_call()
11417 err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta); in check_helper_call()
11421 err = check_map_func_compatibility(env, meta.map_ptr, func_id); in check_helper_call()
11427 !env->prog->has_callchain_buf) { in check_helper_call()
11438 verbose(env, err_str, func_id_name(func_id), func_id); in check_helper_call()
11442 env->prog->has_callchain_buf = true; in check_helper_call()
11446 env->prog->call_get_stack = true; in check_helper_call()
11449 if (check_get_func_ip(env)) in check_helper_call()
11451 env->prog->call_get_func_ip = true; in check_helper_call()
11455 clear_all_pkt_pointers(env); in check_helper_call()
11462 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, in mark_btf_func_reg_size() argument
11465 struct bpf_reg_state *reg = &cur_regs(env)[regno]; in mark_btf_func_reg_size()
11471 DEF_NOT_SUBREG : env->insn_idx + 1; in mark_btf_func_reg_size()
11475 mark_insn_zext(env, reg); in mark_btf_func_reg_size()
11476 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in mark_btf_func_reg_size()
11478 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); in mark_btf_func_reg_size()
11677 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf, in is_kfunc_arg_callback() argument
11690 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, in __btf_type_is_scalar_struct() argument
11707 verbose(env, "max struct nesting depth exceeded\n"); in __btf_type_is_scalar_struct()
11710 if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) in __btf_type_is_scalar_struct()
11894 get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, in get_kfunc_ptr_arg_type() argument
11901 struct bpf_reg_state *regs = cur_regs(env); in get_kfunc_ptr_arg_type()
11913 if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno)) in get_kfunc_ptr_arg_type()
11957 verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n", in get_kfunc_ptr_arg_type()
11964 if (is_kfunc_arg_callback(env, meta->btf, &args[argno])) in get_kfunc_ptr_arg_type()
11977 if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && in get_kfunc_ptr_arg_type()
11979 verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n", in get_kfunc_ptr_arg_type()
11986 static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_btf_id() argument
12034 btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) in process_kf_arg_ptr_to_btf_id()
12043 …struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, st… in process_kf_arg_ptr_to_btf_id()
12050 …verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\… in process_kf_arg_ptr_to_btf_id()
12058 static int process_irq_flag(struct bpf_verifier_env *env, int regno, in process_irq_flag() argument
12061 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_irq_flag()
12070 verbose(env, "verifier internal error: unknown irq flags kfunc\n"); in process_irq_flag()
12075 if (!is_irq_flag_reg_valid_uninit(env, reg)) { in process_irq_flag()
12076 verbose(env, "expected uninitialized irq flag as arg#%d\n", regno - 1); in process_irq_flag()
12080 err = check_mem_access(env, env->insn_idx, regno, 0, BPF_DW, BPF_WRITE, -1, false, false); in process_irq_flag()
12084 err = mark_stack_slot_irq_flag(env, meta, reg, env->insn_idx); in process_irq_flag()
12088 err = is_irq_flag_reg_valid_init(env, reg); in process_irq_flag()
12090 verbose(env, "expected an initialized irq flag as arg#%d\n", regno - 1); in process_irq_flag()
12094 err = mark_irq_flag_read(env, reg); in process_irq_flag()
12098 err = unmark_stack_slot_irq_flag(env, reg); in process_irq_flag()
12106 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in ref_set_non_owning() argument
12110 if (!env->cur_state->active_locks) { in ref_set_non_owning()
12111 verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n"); in ref_set_non_owning()
12116 verbose(env, "verifier internal error: NON_OWN_REF already set\n"); in ref_set_non_owning()
12127 static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id) in ref_convert_owning_non_owning() argument
12129 struct bpf_verifier_state *state = env->cur_state; in ref_convert_owning_non_owning()
12135 verbose(env, "verifier internal error: ref_obj_id is zero for " in ref_convert_owning_non_owning()
12147 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ in ref_convert_owning_non_owning()
12150 ref_set_non_owning(env, reg); in ref_convert_owning_non_owning()
12156 verbose(env, "verifier internal error: ref state missing for ref_obj_id\n"); in ref_convert_owning_non_owning()
12204 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in check_reg_allocation_locked() argument
12218 verbose(env, "verifier internal error: unknown reg type for lock check\n"); in check_reg_allocation_locked()
12223 if (!env->cur_state->active_locks) in check_reg_allocation_locked()
12225 s = find_lock_state(env->cur_state, REF_TYPE_LOCK, id, ptr); in check_reg_allocation_locked()
12227 verbose(env, "held lock and object are not in the same allocation\n"); in check_reg_allocation_locked()
12298 static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env, in check_kfunc_is_graph_root_api() argument
12312 verbose(env, "verifier internal error: unexpected graph root argument type %s\n", in check_kfunc_is_graph_root_api()
12318 verbose(env, "verifier internal error: %s head arg for unknown kfunc\n", in check_kfunc_is_graph_root_api()
12323 static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, in check_kfunc_is_graph_node_api() argument
12339 verbose(env, "verifier internal error: unexpected graph node argument type %s\n", in check_kfunc_is_graph_node_api()
12345 verbose(env, "verifier internal error: %s node arg for unknown kfunc\n", in check_kfunc_is_graph_node_api()
12351 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, in __process_kf_arg_ptr_to_graph_root() argument
12363 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); in __process_kf_arg_ptr_to_graph_root()
12367 if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id)) in __process_kf_arg_ptr_to_graph_root()
12372 verbose(env, in __process_kf_arg_ptr_to_graph_root()
12382 verbose(env, "%s not found at offset=%u\n", head_type_name, head_off); in __process_kf_arg_ptr_to_graph_root()
12387 if (check_reg_allocation_locked(env, reg)) { in __process_kf_arg_ptr_to_graph_root()
12388 verbose(env, "bpf_spin_lock at off=%d must be held for %s\n", in __process_kf_arg_ptr_to_graph_root()
12394 verbose(env, "verifier internal error: repeating %s arg\n", head_type_name); in __process_kf_arg_ptr_to_graph_root()
12401 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_list_head() argument
12405 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD, in process_kf_arg_ptr_to_list_head()
12409 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_rbtree_root() argument
12413 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT, in process_kf_arg_ptr_to_rbtree_root()
12418 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, in __process_kf_arg_ptr_to_graph_node() argument
12431 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); in __process_kf_arg_ptr_to_graph_node()
12435 if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id)) in __process_kf_arg_ptr_to_graph_node()
12440 verbose(env, in __process_kf_arg_ptr_to_graph_node()
12449 verbose(env, "%s not found at offset=%u\n", node_type_name, node_off); in __process_kf_arg_ptr_to_graph_node()
12457 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, in __process_kf_arg_ptr_to_graph_node()
12459 verbose(env, "operation on %s expects arg#1 %s at offset=%d " in __process_kf_arg_ptr_to_graph_node()
12472 verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n", in __process_kf_arg_ptr_to_graph_node()
12482 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_list_node() argument
12486 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, in process_kf_arg_ptr_to_list_node()
12491 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_rbtree_node() argument
12495 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, in process_kf_arg_ptr_to_rbtree_node()
12506 static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) in check_css_task_iter_allowlist() argument
12508 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_css_task_iter_allowlist()
12514 if (env->prog->expected_attach_type == BPF_TRACE_ITER) in check_css_task_iter_allowlist()
12518 return in_sleepable(env); in check_css_task_iter_allowlist()
12522 static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, in check_kfunc_args() argument
12535 verbose(env, "Function %s has %d > %d args\n", func_name, nargs, in check_kfunc_args()
12544 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[i + 1]; in check_kfunc_args()
12558 verbose(env, "R%d is not a scalar\n", regno); in check_kfunc_args()
12564 verbose(env, "verifier internal error: only one constant argument permitted\n"); in check_kfunc_args()
12568 verbose(env, "R%d must be a known constant\n", regno); in check_kfunc_args()
12571 ret = mark_chain_precision(env, regno); in check_kfunc_args()
12585 verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); in check_kfunc_args()
12590 verbose(env, "R%d is not a const\n", regno); in check_kfunc_args()
12595 ret = mark_chain_precision(env, regno); in check_kfunc_args()
12603 verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t)); in check_kfunc_args()
12610 verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i); in check_kfunc_args()
12616 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", in check_kfunc_args()
12629 kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs); in check_kfunc_args()
12638 verbose(env, "pointer in R%d isn't map pointer\n", regno); in check_kfunc_args()
12656 verbose(env, in check_kfunc_args()
12672 verbose(env, "R%d must be referenced or trusted\n", regno); in check_kfunc_args()
12676 verbose(env, "R%d must be a rcu pointer\n", regno); in check_kfunc_args()
12703 ret = check_func_arg_reg_off(env, reg, regno, arg_type); in check_kfunc_args()
12710 verbose(env, "arg#%d expected pointer to ctx, but got %s\n", in check_kfunc_args()
12711 i, reg_type_str(env, reg->type)); in check_kfunc_args()
12716 ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); in check_kfunc_args()
12725 verbose(env, "arg#%d expected for bpf_obj_drop_impl()\n", i); in check_kfunc_args()
12730 verbose(env, "arg#%d expected for bpf_percpu_obj_drop_impl()\n", i); in check_kfunc_args()
12734 verbose(env, "arg#%d expected pointer to allocated object\n", i); in check_kfunc_args()
12738 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
12766 verbose(env, "verifier internal error: no dynptr type for parent of clone\n"); in check_kfunc_args()
12773 verbose(env, "verifier internal error: missing ref obj id for parent of clone\n"); in check_kfunc_args()
12778 ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id); in check_kfunc_args()
12783 int id = dynptr_id(env, reg); in check_kfunc_args()
12786 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); in check_kfunc_args()
12790 meta->initialized_dynptr.type = dynptr_get_type(env, reg); in check_kfunc_args()
12791 meta->initialized_dynptr.ref_obj_id = dynptr_ref_obj_id(env, reg); in check_kfunc_args()
12798 if (!check_css_task_iter_allowlist(env)) { in check_kfunc_args()
12799 verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); in check_kfunc_args()
12803 ret = process_iter_arg(env, regno, insn_idx, meta); in check_kfunc_args()
12810 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); in check_kfunc_args()
12814 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
12817 ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta); in check_kfunc_args()
12824 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); in check_kfunc_args()
12828 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
12831 ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta); in check_kfunc_args()
12837 verbose(env, "arg#%d expected pointer to allocated object\n", i); in check_kfunc_args()
12841 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
12844 ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta); in check_kfunc_args()
12851 verbose(env, "rbtree_remove node input must be non-owning ref\n"); in check_kfunc_args()
12854 if (in_rbtree_lock_required_cb(env)) { in check_kfunc_args()
12855 verbose(env, "rbtree_remove not allowed in rbtree cb\n"); in check_kfunc_args()
12860 verbose(env, "arg#%d expected pointer to allocated object\n", i); in check_kfunc_args()
12864 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
12869 ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta); in check_kfunc_args()
12884 verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type)); in check_kfunc_args()
12885 verbose(env, "expected %s or socket\n", in check_kfunc_args()
12886 reg_type_str(env, base_type(reg->type) | in check_kfunc_args()
12890 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i); in check_kfunc_args()
12897 verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n", in check_kfunc_args()
12901 ret = check_mem_reg(env, reg, regno, type_size); in check_kfunc_args()
12913 ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1); in check_kfunc_args()
12915 verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1); in check_kfunc_args()
12922 verbose(env, "verifier internal error: only one constant argument permitted\n"); in check_kfunc_args()
12926 verbose(env, "R%d must be a known constant\n", regno + 1); in check_kfunc_args()
12939 verbose(env, "arg%d expected pointer to func\n", i); in check_kfunc_args()
12946 verbose(env, "arg#%d is neither owning or non-owning ref\n", i); in check_kfunc_args()
12954 verbose(env, "verifier internal error: Couldn't find btf_record\n"); in check_kfunc_args()
12959 verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i); in check_kfunc_args()
12968 verbose(env, "arg#%d doesn't point to a const string\n", i); in check_kfunc_args()
12971 ret = check_reg_const_str(env, reg, regno); in check_kfunc_args()
12977 verbose(env, "arg#%d doesn't point to a map value\n", i); in check_kfunc_args()
12980 ret = process_wq_func(env, regno, meta); in check_kfunc_args()
12986 verbose(env, "arg#%d doesn't point to an irq flag on stack\n", i); in check_kfunc_args()
12989 ret = process_irq_flag(env, regno, meta); in check_kfunc_args()
12997 verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n", in check_kfunc_args()
13005 static int fetch_kfunc_meta(struct bpf_verifier_env *env, in fetch_kfunc_meta() argument
13021 desc_btf = find_kfunc_desc_btf(env, insn->off); in fetch_kfunc_meta()
13032 kfunc_flags = btf_kfunc_id_set_contains(desc_btf, func_id, env->prog); in fetch_kfunc_meta()
13047 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
13049 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_kfunc_call() argument
13054 struct bpf_reg_state *regs = cur_regs(env); in check_kfunc_call()
13068 err = fetch_kfunc_meta(env, insn, &meta, &func_name); in check_kfunc_call()
13070 verbose(env, "calling kernel function %s is not allowed\n", func_name); in check_kfunc_call()
13074 insn_aux = &env->insn_aux_data[insn_idx]; in check_kfunc_call()
13079 verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); in check_kfunc_call()
13084 if (sleepable && !in_sleepable(env)) { in check_kfunc_call()
13085 verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); in check_kfunc_call()
13090 err = check_kfunc_args(env, &meta, insn_idx); in check_kfunc_call()
13095 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_kfunc_call()
13098 verbose(env, "kfunc %s#%d failed callback verification\n", in check_kfunc_call()
13110 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_kfunc_call()
13113 verbose(env, "kfunc %s#%d failed callback verification\n", in check_kfunc_call()
13125 if (env->cur_state->active_rcu_lock) { in check_kfunc_call()
13130 if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) { in check_kfunc_call()
13131 verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n"); in check_kfunc_call()
13136 verbose(env, "nested rcu read lock (kernel function %s)\n", func_name); in check_kfunc_call()
13139 bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({ in check_kfunc_call()
13145 env->cur_state->active_rcu_lock = false; in check_kfunc_call()
13147 verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name); in check_kfunc_call()
13151 env->cur_state->active_rcu_lock = true; in check_kfunc_call()
13153 verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); in check_kfunc_call()
13157 if (env->cur_state->active_preempt_locks) { in check_kfunc_call()
13159 env->cur_state->active_preempt_locks++; in check_kfunc_call()
13161 env->cur_state->active_preempt_locks--; in check_kfunc_call()
13163 verbose(env, "kernel func %s is sleepable within non-preemptible region\n", func_name); in check_kfunc_call()
13167 env->cur_state->active_preempt_locks++; in check_kfunc_call()
13169 verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name); in check_kfunc_call()
13173 if (env->cur_state->active_irq_id && sleepable) { in check_kfunc_call()
13174 verbose(env, "kernel func %s is sleepable within IRQ-disabled region\n", func_name); in check_kfunc_call()
13182 err = release_reference(env, regs[meta.release_regno].ref_obj_id); in check_kfunc_call()
13184 verbose(env, "kfunc %s#%d reference has not been acquired before\n", in check_kfunc_call()
13196 err = ref_convert_owning_non_owning(env, release_ref_obj_id); in check_kfunc_call()
13198 verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n", in check_kfunc_call()
13203 err = release_reference(env, release_ref_obj_id); in check_kfunc_call()
13205 verbose(env, "kfunc %s#%d reference has not been acquired before\n", in check_kfunc_call()
13213 verbose(env, "JIT does not support calling kfunc %s#%d\n", in check_kfunc_call()
13217 env->seen_exception = true; in check_kfunc_call()
13222 if (!env->exception_callback_subprog) { in check_kfunc_call()
13223 err = check_return_code(env, BPF_REG_1, "R1"); in check_kfunc_call()
13230 mark_reg_not_init(env, regs, caller_saved[i]); in check_kfunc_call()
13241 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); in check_kfunc_call()
13247 mark_reg_unknown(env, regs, BPF_REG_0); in check_kfunc_call()
13248 mark_btf_func_reg_size(env, BPF_REG_0, t->size); in check_kfunc_call()
13263 verbose(env, "local type ID argument must be in range [0, U32_MAX]\n"); in check_kfunc_call()
13267 ret_btf = env->prog->aux->btf; in check_kfunc_call()
13272 verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n"); in check_kfunc_call()
13278 verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n"); in check_kfunc_call()
13284 verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n", in check_kfunc_call()
13313 if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { in check_kfunc_call()
13314 verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); in check_kfunc_call()
13319 verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n"); in check_kfunc_call()
13324 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13334 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13353 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13360 verbose(env, in check_kfunc_call()
13365 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13373 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13376 verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n"); in check_kfunc_call()
13389 if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) { in check_kfunc_call()
13390 verbose(env, "the prog does not allow writes to packet data\n"); in check_kfunc_call()
13396 verbose(env, "verifier internal error: no dynptr id\n"); in check_kfunc_call()
13406 verbose(env, "kernel function %s unhandled dynamic return type\n", in check_kfunc_call()
13412 mark_reg_unknown(env, regs, BPF_REG_0); in check_kfunc_call()
13425 verbose(env, in check_kfunc_call()
13433 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13444 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13455 cur_iter = get_iter_from_state(env->cur_state, &meta); in check_kfunc_call()
13467 regs[BPF_REG_0].id = ++env->id_gen; in check_kfunc_call()
13469 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); in check_kfunc_call()
13471 int id = acquire_reference(env, insn_idx); in check_kfunc_call()
13479 ref_set_non_owning(env, ®s[BPF_REG_0]); in check_kfunc_call()
13483 regs[BPF_REG_0].id = ++env->id_gen; in check_kfunc_call()
13502 mark_btf_func_reg_size(env, regno, sizeof(void *)); in check_kfunc_call()
13505 mark_btf_func_reg_size(env, regno, t->size); in check_kfunc_call()
13509 err = process_iter_next_call(env, insn_idx, &meta); in check_kfunc_call()
13517 static bool check_reg_sane_offset(struct bpf_verifier_env *env, in check_reg_sane_offset() argument
13526 verbose(env, "math between %s pointer and %lld is not allowed\n", in check_reg_sane_offset()
13527 reg_type_str(env, type), val); in check_reg_sane_offset()
13532 verbose(env, "%s pointer offset %d is not allowed\n", in check_reg_sane_offset()
13533 reg_type_str(env, type), reg->off); in check_reg_sane_offset()
13538 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", in check_reg_sane_offset()
13539 reg_type_str(env, type)); in check_reg_sane_offset()
13544 verbose(env, "value %lld makes %s pointer be out of bounds\n", in check_reg_sane_offset()
13545 smin, reg_type_str(env, type)); in check_reg_sane_offset()
13591 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, in can_skip_alu_sanitation() argument
13594 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; in can_skip_alu_sanitation()
13614 static int sanitize_val_alu(struct bpf_verifier_env *env, in sanitize_val_alu() argument
13617 struct bpf_insn_aux_data *aux = cur_aux(env); in sanitize_val_alu()
13619 if (can_skip_alu_sanitation(env, insn)) in sanitize_val_alu()
13636 sanitize_speculative_path(struct bpf_verifier_env *env, in sanitize_speculative_path() argument
13643 branch = push_stack(env, next_idx, curr_idx, true); in sanitize_speculative_path()
13647 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
13649 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
13650 mark_reg_unknown(env, regs, insn->src_reg); in sanitize_speculative_path()
13656 static int sanitize_ptr_alu(struct bpf_verifier_env *env, in sanitize_ptr_alu() argument
13664 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; in sanitize_ptr_alu()
13665 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_ptr_alu()
13675 if (can_skip_alu_sanitation(env, insn)) in sanitize_ptr_alu()
13714 env->explore_alu_limits = true; in sanitize_ptr_alu()
13745 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, in sanitize_ptr_alu()
13746 env->insn_idx); in sanitize_ptr_alu()
13752 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) in sanitize_mark_insn_seen() argument
13754 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_mark_insn_seen()
13762 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; in sanitize_mark_insn_seen()
13765 static int sanitize_err(struct bpf_verifier_env *env, in sanitize_err() argument
13776 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", in sanitize_err()
13780 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", in sanitize_err()
13784 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", in sanitize_err()
13788 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", in sanitize_err()
13792 verbose(env, "R%d could not be pushed for speculative verification, %s\n", in sanitize_err()
13796 verbose(env, "verifier internal error: unknown reason (%d)\n", in sanitize_err()
13815 struct bpf_verifier_env *env, in check_stack_access_for_ptr_arithmetic() argument
13824 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", in check_stack_access_for_ptr_arithmetic()
13830 verbose(env, "R%d stack pointer arithmetic goes out of range, " in check_stack_access_for_ptr_arithmetic()
13838 static int sanitize_check_bounds(struct bpf_verifier_env *env, in sanitize_check_bounds() argument
13847 if (env->bypass_spec_v1) in sanitize_check_bounds()
13852 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, in sanitize_check_bounds()
13857 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { in sanitize_check_bounds()
13858 verbose(env, "R%d pointer arithmetic of map value goes out of range, " in sanitize_check_bounds()
13875 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, in adjust_ptr_min_max_vals() argument
13880 struct bpf_verifier_state *vstate = env->cur_state; in adjust_ptr_min_max_vals()
13900 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
13906 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_ptr_min_max_vals()
13907 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
13911 verbose(env, in adjust_ptr_min_max_vals()
13918 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", in adjust_ptr_min_max_vals()
13919 dst, reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
13947 verbose(env, "R%d pointer arithmetic on %s prohibited\n", in adjust_ptr_min_max_vals()
13948 dst, reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
13958 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || in adjust_ptr_min_max_vals()
13959 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
13966 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
13969 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
14012 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
14020 verbose(env, "R%d tried to subtract pointer from scalar\n", in adjust_ptr_min_max_vals()
14029 verbose(env, "R%d subtraction from stack pointer prohibited\n", in adjust_ptr_min_max_vals()
14068 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
14078 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", in adjust_ptr_min_max_vals()
14083 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", in adjust_ptr_min_max_vals()
14088 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
14091 if (sanitize_check_bounds(env, insn, dst_reg) < 0) in adjust_ptr_min_max_vals()
14094 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
14097 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
14674 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, in adjust_scalar_min_max_vals() argument
14684 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
14689 ret = sanitize_val_alu(env, insn); in adjust_scalar_min_max_vals()
14691 return sanitize_err(env, insn, ret, NULL, NULL); in adjust_scalar_min_max_vals()
14771 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, in adjust_reg_min_max_vals() argument
14774 struct bpf_verifier_state *vstate = env->cur_state; in adjust_reg_min_max_vals()
14786 struct bpf_insn_aux_data *aux = cur_aux(env); in adjust_reg_min_max_vals()
14810 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_reg_min_max_vals()
14811 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
14814 verbose(env, "R%d pointer %s pointer prohibited\n", in adjust_reg_min_max_vals()
14823 err = mark_chain_precision(env, insn->dst_reg); in adjust_reg_min_max_vals()
14826 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
14831 err = mark_chain_precision(env, insn->src_reg); in adjust_reg_min_max_vals()
14834 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
14838 err = mark_chain_precision(env, insn->src_reg); in adjust_reg_min_max_vals()
14850 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
14856 print_verifier_state(env, vstate, vstate->curframe, true); in adjust_reg_min_max_vals()
14857 verbose(env, "verifier internal error: unexpected ptr_reg\n"); in adjust_reg_min_max_vals()
14861 print_verifier_state(env, vstate, vstate->curframe, true); in adjust_reg_min_max_vals()
14862 verbose(env, "verifier internal error: no src_reg\n"); in adjust_reg_min_max_vals()
14865 err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
14877 if (env->bpf_capable && in adjust_reg_min_max_vals()
14906 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_alu_op() argument
14908 struct bpf_reg_state *regs = cur_regs(env); in check_alu_op()
14917 verbose(env, "BPF_NEG uses reserved fields\n"); in check_alu_op()
14925 verbose(env, "BPF_END uses reserved fields\n"); in check_alu_op()
14931 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
14935 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
14936 verbose(env, "R%d pointer arithmetic prohibited\n", in check_alu_op()
14942 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
14952 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
14957 verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n"); in check_alu_op()
14960 if (!env->prog->aux->arena) { in check_alu_op()
14961 … verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n"); in check_alu_op()
14967 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
14973 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
14978 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
14984 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
14995 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
14999 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
15005 assign_scalar_id_before_mov(env, src_reg); in check_alu_op()
15011 if (is_pointer_value(env, insn->src_reg)) { in check_alu_op()
15012 verbose(env, in check_alu_op()
15021 assign_scalar_id_before_mov(env, src_reg); in check_alu_op()
15029 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
15034 if (is_pointer_value(env, insn->src_reg)) { in check_alu_op()
15035 verbose(env, in check_alu_op()
15044 assign_scalar_id_before_mov(env, src_reg); in check_alu_op()
15053 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
15059 assign_scalar_id_before_mov(env, src_reg); in check_alu_op()
15064 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
15068 mark_reg_unknown(env, regs, in check_alu_op()
15079 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
15091 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); in check_alu_op()
15099 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
15103 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
15109 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
15115 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
15121 verbose(env, "div by zero\n"); in check_alu_op()
15130 verbose(env, "invalid shift %d\n", insn->imm); in check_alu_op()
15136 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
15137 err = err ?: adjust_reg_min_max_vals(env, insn); in check_alu_op()
15142 return reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); in check_alu_op()
15663 static int reg_set_min_max(struct bpf_verifier_env *env, in reg_set_min_max() argument
15689 err = reg_bounds_sanity_check(env, true_reg1, "true_reg1"); in reg_set_min_max()
15690 err = err ?: reg_bounds_sanity_check(env, true_reg2, "true_reg2"); in reg_set_min_max()
15691 err = err ?: reg_bounds_sanity_check(env, false_reg1, "false_reg1"); in reg_set_min_max()
15692 err = err ?: reg_bounds_sanity_check(env, false_reg2, "false_reg2"); in reg_set_min_max()
15964 static int check_cond_jmp_op(struct bpf_verifier_env *env, in check_cond_jmp_op() argument
15967 struct bpf_verifier_state *this_branch = env->cur_state; in check_cond_jmp_op()
15980 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); in check_cond_jmp_op()
15985 struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; in check_cond_jmp_op()
15991 verbose(env, "invalid may_goto imm %d\n", insn->imm); in check_cond_jmp_op()
15994 prev_st = find_prev_entry(env, cur_st->parent, idx); in check_cond_jmp_op()
15997 queued_st = push_stack(env, idx + 1, idx, false); in check_cond_jmp_op()
16003 widen_imprecise_scalars(env, prev_st, queued_st); in check_cond_jmp_op()
16009 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
16016 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
16021 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_cond_jmp_op()
16027 is_pointer_value(env, insn->src_reg)) { in check_cond_jmp_op()
16028 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
16034 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
16037 src_reg = &env->fake_reg[0]; in check_cond_jmp_op()
16050 err = mark_chain_precision(env, insn->dst_reg); in check_cond_jmp_op()
16053 err = mark_chain_precision(env, insn->src_reg); in check_cond_jmp_op()
16063 if (!env->bypass_spec_v1 && in check_cond_jmp_op()
16064 !sanitize_speculative_path(env, insn, *insn_idx + 1, in check_cond_jmp_op()
16067 if (env->log.level & BPF_LOG_LEVEL) in check_cond_jmp_op()
16068 print_insn_state(env, this_branch, this_branch->curframe); in check_cond_jmp_op()
16076 if (!env->bypass_spec_v1 && in check_cond_jmp_op()
16077 !sanitize_speculative_path(env, insn, in check_cond_jmp_op()
16081 if (env->log.level & BPF_LOG_LEVEL) in check_cond_jmp_op()
16082 print_insn_state(env, this_branch, this_branch->curframe); in check_cond_jmp_op()
16096 err = push_insn_history(env, this_branch, 0, linked_regs_pack(&linked_regs)); in check_cond_jmp_op()
16101 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, in check_cond_jmp_op()
16108 err = reg_set_min_max(env, in check_cond_jmp_op()
16117 memcpy(&env->fake_reg[1], &env->fake_reg[0], in check_cond_jmp_op()
16118 sizeof(env->fake_reg[0])); in check_cond_jmp_op()
16119 err = reg_set_min_max(env, in check_cond_jmp_op()
16121 &env->fake_reg[0], in check_cond_jmp_op()
16122 dst_reg, &env->fake_reg[1], in check_cond_jmp_op()
16193 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
16194 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
16198 if (env->log.level & BPF_LOG_LEVEL) in check_cond_jmp_op()
16199 print_insn_state(env, this_branch, this_branch->curframe); in check_cond_jmp_op()
16204 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_imm() argument
16206 struct bpf_insn_aux_data *aux = cur_aux(env); in check_ld_imm()
16207 struct bpf_reg_state *regs = cur_regs(env); in check_ld_imm()
16213 verbose(env, "invalid BPF_LD_IMM insn\n"); in check_ld_imm()
16217 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); in check_ld_imm()
16221 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
16238 mark_reg_known_zero(env, regs, insn->dst_reg); in check_ld_imm()
16251 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
16258 struct bpf_prog_aux *aux = env->prog->aux; in check_ld_imm()
16259 u32 subprogno = find_subprog(env, in check_ld_imm()
16260 env->insn_idx + insn->imm + 1); in check_ld_imm()
16263 verbose(env, "missing btf func_info\n"); in check_ld_imm()
16267 verbose(env, "callback function not static\n"); in check_ld_imm()
16276 map = env->used_maps[aux->map_index]; in check_ld_imm()
16282 __mark_reg_unknown(env, dst_reg); in check_ld_imm()
16293 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
16327 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_abs() argument
16329 struct bpf_reg_state *regs = cur_regs(env); in check_ld_abs()
16334 if (!may_access_skb(resolve_prog_type(env->prog))) { in check_ld_abs()
16335 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); in check_ld_abs()
16339 if (!env->ops->gen_ld_abs) { in check_ld_abs()
16340 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_abs()
16347 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); in check_ld_abs()
16352 err = check_reg_arg(env, ctx_reg, SRC_OP); in check_ld_abs()
16360 err = check_resource_leak(env, false, true, "BPF_LD_[ABS|IND]"); in check_ld_abs()
16365 verbose(env, in check_ld_abs()
16372 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_ld_abs()
16377 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); in check_ld_abs()
16383 mark_reg_not_init(env, regs, caller_saved[i]); in check_ld_abs()
16384 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_ld_abs()
16391 mark_reg_unknown(env, regs, BPF_REG_0); in check_ld_abs()
16393 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; in check_ld_abs()
16397 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name) in check_return_code() argument
16401 const struct bpf_prog *prog = env->prog; in check_return_code()
16404 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_return_code()
16406 struct bpf_func_state *frame = env->cur_state->frame[0]; in check_return_code()
16433 err = check_reg_arg(env, regno, SRC_OP); in check_return_code()
16437 if (is_pointer_value(env, regno)) { in check_return_code()
16438 verbose(env, "R%d leaks addr as return value\n", regno); in check_return_code()
16442 reg = cur_regs(env) + regno; in check_return_code()
16453 verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", in check_return_code()
16454 regno, reg_type_str(env, reg->type)); in check_return_code()
16462 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || in check_return_code()
16463 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || in check_return_code()
16464 env->prog->expected_attach_type == BPF_CGROUP_UNIX_RECVMSG || in check_return_code()
16465 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || in check_return_code()
16466 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || in check_return_code()
16467 env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETPEERNAME || in check_return_code()
16468 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || in check_return_code()
16469 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME || in check_return_code()
16470 env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETSOCKNAME) in check_return_code()
16472 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || in check_return_code()
16473 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) in check_return_code()
16477 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { in check_return_code()
16489 if (!env->prog->aux->attach_btf_id) in check_return_code()
16494 switch (env->prog->expected_attach_type) { in check_return_code()
16509 switch (env->prog->expected_attach_type) { in check_return_code()
16523 if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { in check_return_code()
16525 if (!get_func_retval_range(env->prog, &range)) in check_return_code()
16531 } else if (!env->prog->aux->attach_func_proto->type) { in check_return_code()
16552 verbose(env, "%s the register R%d is not a known value (%s)\n", in check_return_code()
16553 exit_ctx, regno, reg_type_str(env, reg->type)); in check_return_code()
16557 err = mark_chain_precision(env, regno); in check_return_code()
16562 verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name); in check_return_code()
16567 verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); in check_return_code()
16573 env->prog->enforce_expected_attach_type = 1; in check_return_code()
16577 static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off) in mark_subprog_changes_pkt_data() argument
16581 subprog = find_containing_subprog(env, off); in mark_subprog_changes_pkt_data()
16591 static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) in merge_callee_effects() argument
16595 caller = find_containing_subprog(env, t); in merge_callee_effects()
16596 callee = find_containing_subprog(env, w); in merge_callee_effects()
16640 static void mark_prune_point(struct bpf_verifier_env *env, int idx) in mark_prune_point() argument
16642 env->insn_aux_data[idx].prune_point = true; in mark_prune_point()
16645 static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx) in is_prune_point() argument
16647 return env->insn_aux_data[insn_idx].prune_point; in is_prune_point()
16650 static void mark_force_checkpoint(struct bpf_verifier_env *env, int idx) in mark_force_checkpoint() argument
16652 env->insn_aux_data[idx].force_checkpoint = true; in mark_force_checkpoint()
16655 static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx) in is_force_checkpoint() argument
16657 return env->insn_aux_data[insn_idx].force_checkpoint; in is_force_checkpoint()
16660 static void mark_calls_callback(struct bpf_verifier_env *env, int idx) in mark_calls_callback() argument
16662 env->insn_aux_data[idx].calls_callback = true; in mark_calls_callback()
16665 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx) in calls_callback() argument
16667 return env->insn_aux_data[insn_idx].calls_callback; in calls_callback()
16680 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) in push_insn() argument
16682 int *insn_stack = env->cfg.insn_stack; in push_insn()
16683 int *insn_state = env->cfg.insn_state; in push_insn()
16691 if (w < 0 || w >= env->prog->len) { in push_insn()
16692 verbose_linfo(env, t, "%d: ", t); in push_insn()
16693 verbose(env, "jump out of range from insn %d to %d\n", t, w); in push_insn()
16699 mark_prune_point(env, w); in push_insn()
16700 mark_jmp_point(env, w); in push_insn()
16707 if (env->cfg.cur_stack >= env->prog->len) in push_insn()
16709 insn_stack[env->cfg.cur_stack++] = w; in push_insn()
16712 if (env->bpf_capable) in push_insn()
16714 verbose_linfo(env, t, "%d: ", t); in push_insn()
16715 verbose_linfo(env, w, "%d: ", w); in push_insn()
16716 verbose(env, "back-edge from insn %d to %d\n", t, w); in push_insn()
16722 verbose(env, "insn state internal bug\n"); in push_insn()
16729 struct bpf_verifier_env *env, in visit_func_call_insn() argument
16736 ret = push_insn(t, t + insn_sz, FALLTHROUGH, env); in visit_func_call_insn()
16740 mark_prune_point(env, t + insn_sz); in visit_func_call_insn()
16742 mark_jmp_point(env, t + insn_sz); in visit_func_call_insn()
16746 mark_prune_point(env, t); in visit_func_call_insn()
16747 merge_callee_effects(env, t, w); in visit_func_call_insn()
16748 ret = push_insn(t, w, BRANCH, env); in visit_func_call_insn()
16781 static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) in verifier_inlines_helper_call() argument
16786 return env->prog->jit_requested && bpf_jit_supports_percpu_insn(); in verifier_inlines_helper_call()
16887 static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, in mark_fastcall_pattern_for_call() argument
16891 struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx; in mark_fastcall_pattern_for_call()
16892 struct bpf_insn *call = &env->prog->insnsi[insn_idx]; in mark_fastcall_pattern_for_call()
16901 if (get_helper_proto(env, call->imm, &fn) < 0) in mark_fastcall_pattern_for_call()
16906 (verifier_inlines_helper_call(env, call->imm) || in mark_fastcall_pattern_for_call()
16914 err = fetch_kfunc_meta(env, call, &meta, NULL); in mark_fastcall_pattern_for_call()
16938 if (insn_idx - i < 0 || insn_idx + i >= env->prog->len) in mark_fastcall_pattern_for_call()
16961 env->insn_aux_data[insn_idx - i].fastcall_pattern = 1; in mark_fastcall_pattern_for_call()
16962 env->insn_aux_data[insn_idx + i].fastcall_pattern = 1; in mark_fastcall_pattern_for_call()
16984 env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1; in mark_fastcall_pattern_for_call()
16990 static int mark_fastcall_patterns(struct bpf_verifier_env *env) in mark_fastcall_patterns() argument
16992 struct bpf_subprog_info *subprog = env->subprog_info; in mark_fastcall_patterns()
16997 for (s = 0; s < env->subprog_cnt; ++s, ++subprog) { in mark_fastcall_patterns()
17001 insn = env->prog->insnsi + i; in mark_fastcall_patterns()
17009 insn = env->prog->insnsi + i; in mark_fastcall_patterns()
17012 mark_fastcall_pattern_for_call(env, subprog, i, lowest_off); in mark_fastcall_patterns()
17023 static int visit_insn(int t, struct bpf_verifier_env *env) in visit_insn() argument
17025 struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t]; in visit_insn()
17029 return visit_func_call_insn(t, insns, env, true); in visit_insn()
17035 return push_insn(t, t + insn_sz, FALLTHROUGH, env); in visit_insn()
17049 mark_prune_point(env, t); in visit_insn()
17060 mark_calls_callback(env, t); in visit_insn()
17061 mark_force_checkpoint(env, t); in visit_insn()
17062 mark_prune_point(env, t); in visit_insn()
17063 mark_jmp_point(env, t); in visit_insn()
17066 mark_subprog_changes_pkt_data(env, t); in visit_insn()
17070 ret = fetch_kfunc_meta(env, insn, &meta, NULL); in visit_insn()
17072 mark_prune_point(env, t); in visit_insn()
17084 mark_force_checkpoint(env, t); in visit_insn()
17087 return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); in visit_insn()
17099 ret = push_insn(t, t + off + 1, FALLTHROUGH, env); in visit_insn()
17103 mark_prune_point(env, t + off + 1); in visit_insn()
17104 mark_jmp_point(env, t + off + 1); in visit_insn()
17110 mark_prune_point(env, t); in visit_insn()
17112 mark_force_checkpoint(env, t); in visit_insn()
17114 ret = push_insn(t, t + 1, FALLTHROUGH, env); in visit_insn()
17118 return push_insn(t, t + insn->off + 1, BRANCH, env); in visit_insn()
17125 static int check_cfg(struct bpf_verifier_env *env) in check_cfg() argument
17127 int insn_cnt = env->prog->len; in check_cfg()
17132 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
17136 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
17144 env->cfg.cur_stack = 1; in check_cfg()
17147 while (env->cfg.cur_stack > 0) { in check_cfg()
17148 int t = insn_stack[env->cfg.cur_stack - 1]; in check_cfg()
17150 ret = visit_insn(t, env); in check_cfg()
17154 env->cfg.cur_stack--; in check_cfg()
17160 verbose(env, "visit_insn internal bug\n"); in check_cfg()
17167 if (env->cfg.cur_stack < 0) { in check_cfg()
17168 verbose(env, "pop stack internal bug\n"); in check_cfg()
17173 if (env->exception_callback_subprog && !ex_done) { in check_cfg()
17174 ex_insn_beg = env->subprog_info[env->exception_callback_subprog].start; in check_cfg()
17178 env->cfg.cur_stack = 1; in check_cfg()
17184 struct bpf_insn *insn = &env->prog->insnsi[i]; in check_cfg()
17187 verbose(env, "unreachable insn %d\n", i); in check_cfg()
17193 verbose(env, "jump into the middle of ldimm64 insn %d\n", i); in check_cfg()
17201 env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data; in check_cfg()
17206 env->cfg.insn_state = env->cfg.insn_stack = NULL; in check_cfg()
17210 static int check_abnormal_return(struct bpf_verifier_env *env) in check_abnormal_return() argument
17214 for (i = 1; i < env->subprog_cnt; i++) { in check_abnormal_return()
17215 if (env->subprog_info[i].has_ld_abs) { in check_abnormal_return()
17216 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); in check_abnormal_return()
17219 if (env->subprog_info[i].has_tail_call) { in check_abnormal_return()
17220 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); in check_abnormal_return()
17231 static int check_btf_func_early(struct bpf_verifier_env *env, in check_btf_func_early() argument
17247 if (check_abnormal_return(env)) in check_btf_func_early()
17256 verbose(env, "invalid func info rec size %u\n", urec_size); in check_btf_func_early()
17260 prog = env->prog; in check_btf_func_early()
17274 verbose(env, "nonzero tailing record in func info"); in check_btf_func_early()
17295 verbose(env, in check_btf_func_early()
17301 verbose(env, in check_btf_func_early()
17310 verbose(env, "invalid type id %d in func info", in check_btf_func_early()
17333 static int check_btf_func(struct bpf_verifier_env *env, in check_btf_func() argument
17349 if (check_abnormal_return(env)) in check_btf_func()
17353 if (nfuncs != env->subprog_cnt) { in check_btf_func()
17354 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); in check_btf_func()
17360 prog = env->prog; in check_btf_func()
17374 if (env->subprog_info[i].start != krecord[i].insn_off) { in check_btf_func()
17375 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); in check_btf_func()
17388 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { in check_btf_func()
17389 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); in check_btf_func()
17392 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { in check_btf_func()
17393 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); in check_btf_func()
17408 static void adjust_btf_func(struct bpf_verifier_env *env) in adjust_btf_func() argument
17410 struct bpf_prog_aux *aux = env->prog->aux; in adjust_btf_func()
17417 for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++) in adjust_btf_func()
17418 aux->func_info[i].insn_off = env->subprog_info[i].start; in adjust_btf_func()
17424 static int check_btf_line(struct bpf_verifier_env *env, in check_btf_line() argument
17456 prog = env->prog; in check_btf_line()
17460 sub = env->subprog_info; in check_btf_line()
17468 verbose(env, "nonzero tailing record in line_info"); in check_btf_line()
17495 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", in check_btf_line()
17503 verbose(env, in check_btf_line()
17512 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); in check_btf_line()
17517 if (s != env->subprog_cnt) { in check_btf_line()
17522 verbose(env, "missing bpf_line_info for func#%u\n", s); in check_btf_line()
17532 if (s != env->subprog_cnt) { in check_btf_line()
17533 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", in check_btf_line()
17534 env->subprog_cnt - s, s); in check_btf_line()
17552 static int check_core_relo(struct bpf_verifier_env *env, in check_core_relo() argument
17558 struct bpf_prog *prog = env->prog; in check_core_relo()
17561 .log = &env->log, in check_core_relo()
17591 verbose(env, "nonzero tailing record in core_relo"); in check_core_relo()
17606 verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", in check_core_relo()
17621 static int check_btf_info_early(struct bpf_verifier_env *env, in check_btf_info_early() argument
17629 if (check_abnormal_return(env)) in check_btf_info_early()
17641 env->prog->aux->btf = btf; in check_btf_info_early()
17643 err = check_btf_func_early(env, attr, uattr); in check_btf_info_early()
17649 static int check_btf_info(struct bpf_verifier_env *env, in check_btf_info() argument
17656 if (check_abnormal_return(env)) in check_btf_info()
17661 err = check_btf_func(env, attr, uattr); in check_btf_info()
17665 err = check_btf_line(env, attr, uattr); in check_btf_info()
17669 err = check_core_relo(env, attr, uattr); in check_btf_info()
17741 static void clean_func_state(struct bpf_verifier_env *env, in clean_func_state() argument
17755 __mark_reg_not_init(env, &st->regs[i]); in clean_func_state()
17763 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); in clean_func_state()
17770 static void clean_verifier_state(struct bpf_verifier_env *env, in clean_verifier_state() argument
17780 clean_func_state(env, st->frame[i]); in clean_verifier_state()
17815 static void clean_live_states(struct bpf_verifier_env *env, int insn, in clean_live_states() argument
17820 sl = *explored_state(env, insn); in clean_live_states()
17827 clean_verifier_state(env, &sl->state); in clean_live_states()
17849 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, in regsafe() argument
17891 if (env->explore_alu_limits) { in regsafe()
17987 static bool is_stack_all_misc(struct bpf_verifier_env *env, in is_stack_all_misc() argument
17994 (stack->slot_type[i] == STACK_INVALID && env->allow_uninit_stack)) in is_stack_all_misc()
18002 static struct bpf_reg_state *scalar_reg_for_stack(struct bpf_verifier_env *env, in scalar_reg_for_stack() argument
18008 if (is_stack_all_misc(env, stack)) in scalar_reg_for_stack()
18014 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, in stacksafe() argument
18045 if (env->allow_uninit_stack && in stacksafe()
18060 old_reg = scalar_reg_for_stack(env, &old->stack[spi]); in stacksafe()
18061 cur_reg = scalar_reg_for_stack(env, &cur->stack[spi]); in stacksafe()
18063 if (!regsafe(env, old_reg, cur_reg, idmap, exact)) in stacksafe()
18099 if (!regsafe(env, &old->stack[spi].spilled_ptr, in stacksafe()
18212 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, in func_states_equal() argument
18221 if (!regsafe(env, &old->regs[i], &cur->regs[i], in func_states_equal()
18222 &env->idmap_scratch, exact)) in func_states_equal()
18225 if (!stacksafe(env, old, cur, &env->idmap_scratch, exact)) in func_states_equal()
18231 static void reset_idmap_scratch(struct bpf_verifier_env *env) in reset_idmap_scratch() argument
18233 env->idmap_scratch.tmp_id_gen = env->id_gen; in reset_idmap_scratch()
18234 memset(&env->idmap_scratch.map, 0, sizeof(env->idmap_scratch.map)); in reset_idmap_scratch()
18237 static bool states_equal(struct bpf_verifier_env *env, in states_equal() argument
18247 reset_idmap_scratch(env); in states_equal()
18258 if (!refsafe(old, cur, &env->idmap_scratch)) in states_equal()
18267 if (!func_states_equal(env, old->frame[i], cur->frame[i], exact)) in states_equal()
18276 static int propagate_liveness_reg(struct bpf_verifier_env *env, in propagate_liveness_reg() argument
18295 err = mark_reg_read(env, reg, parent_reg, flag); in propagate_liveness_reg()
18309 static int propagate_liveness(struct bpf_verifier_env *env, in propagate_liveness() argument
18331 err = propagate_liveness_reg(env, &state_reg[i], in propagate_liveness()
18336 mark_insn_zext(env, &parent_reg[i]); in propagate_liveness()
18344 err = propagate_liveness_reg(env, state_reg, in propagate_liveness()
18356 static int propagate_precision(struct bpf_verifier_env *env, in propagate_precision() argument
18373 if (env->log.level & BPF_LOG_LEVEL2) { in propagate_precision()
18375 verbose(env, "frame %d: propagating r%d", fr, i); in propagate_precision()
18377 verbose(env, ",r%d", i); in propagate_precision()
18379 bt_set_frame_reg(&env->bt, fr, i); in propagate_precision()
18391 if (env->log.level & BPF_LOG_LEVEL2) { in propagate_precision()
18393 verbose(env, "frame %d: propagating fp%d", in propagate_precision()
18396 verbose(env, ",fp%d", (-i - 1) * BPF_REG_SIZE); in propagate_precision()
18398 bt_set_frame_slot(&env->bt, fr, i); in propagate_precision()
18402 verbose(env, "\n"); in propagate_precision()
18405 err = mark_chain_precision_batch(env); in propagate_precision()
18430 static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx) in is_iter_next_insn() argument
18432 return env->insn_aux_data[insn_idx].is_iter_next; in is_iter_next_insn()
18517 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) in is_state_visited() argument
18521 struct bpf_verifier_state *cur = env->cur_state, *new, *loop_entry; in is_state_visited()
18525 force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx) || in is_state_visited()
18538 if (env->jmps_processed - env->prev_jmps_processed >= 2 && in is_state_visited()
18539 env->insn_processed - env->prev_insn_processed >= 8) in is_state_visited()
18542 pprev = explored_state(env, insn_idx); in is_state_visited()
18545 clean_live_states(env, insn_idx, cur); in is_state_visited()
18606 if (is_iter_next_insn(env, insn_idx)) { in is_state_visited()
18607 if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { in is_state_visited()
18622 iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr; in is_state_visited()
18630 if (is_may_goto_insn_at(env, insn_idx)) { in is_state_visited()
18632 states_equal(env, &sl->state, cur, RANGE_WITHIN)) { in is_state_visited()
18637 if (calls_callback(env, insn_idx)) { in is_state_visited()
18638 if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) in is_state_visited()
18644 states_equal(env, &sl->state, cur, EXACT) && in is_state_visited()
18648 verbose_linfo(env, insn_idx, "; "); in is_state_visited()
18649 verbose(env, "infinite loop detected at insn %d\n", insn_idx); in is_state_visited()
18650 verbose(env, "cur state:"); in is_state_visited()
18651 print_verifier_state(env, cur, cur->curframe, true); in is_state_visited()
18652 verbose(env, "old state:"); in is_state_visited()
18653 print_verifier_state(env, &sl->state, cur->curframe, true); in is_state_visited()
18670 env->jmps_processed - env->prev_jmps_processed < 20 && in is_state_visited()
18671 env->insn_processed - env->prev_insn_processed < 100) in is_state_visited()
18702 if (states_equal(env, &sl->state, cur, force_exact ? RANGE_WITHIN : NOT_EXACT)) { in is_state_visited()
18717 err = propagate_liveness(env, &sl->state, cur); in is_state_visited()
18724 if (is_jmp_point(env, env->insn_idx)) in is_state_visited()
18725 err = err ? : push_insn_history(env, cur, 0, 0); in is_state_visited()
18726 err = err ? : propagate_precision(env, &sl->state); in is_state_visited()
18748 n = is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3; in is_state_visited()
18763 env->peak_states--; in is_state_visited()
18769 sl->next = env->free_list; in is_state_visited()
18770 env->free_list = sl; in is_state_visited()
18780 if (env->max_states_per_insn < states_cnt) in is_state_visited()
18781 env->max_states_per_insn = states_cnt; in is_state_visited()
18783 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) in is_state_visited()
18801 env->total_states++; in is_state_visited()
18802 env->peak_states++; in is_state_visited()
18803 env->prev_jmps_processed = env->jmps_processed; in is_state_visited()
18804 env->prev_insn_processed = env->insn_processed; in is_state_visited()
18807 if (env->bpf_capable) in is_state_visited()
18808 mark_all_scalars_imprecise(env, cur); in is_state_visited()
18826 new_sl->next = *explored_state(env, insn_idx); in is_state_visited()
18827 *explored_state(env, insn_idx) = new_sl; in is_state_visited()
18897 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, in save_aux_ptr_type() argument
18900 enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type; in save_aux_ptr_type()
18927 verbose(env, "same insn cannot be used with different pointers\n"); in save_aux_ptr_type()
18935 static int do_check(struct bpf_verifier_env *env) in do_check() argument
18937 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check()
18938 struct bpf_verifier_state *state = env->cur_state; in do_check()
18939 struct bpf_insn *insns = env->prog->insnsi; in do_check()
18941 int insn_cnt = env->prog->len; in do_check()
18952 env->cur_hist_ent = NULL; in do_check()
18954 env->prev_insn_idx = prev_insn_idx; in do_check()
18955 if (env->insn_idx >= insn_cnt) { in do_check()
18956 verbose(env, "invalid insn idx %d insn_cnt %d\n", in do_check()
18957 env->insn_idx, insn_cnt); in do_check()
18961 insn = &insns[env->insn_idx]; in do_check()
18964 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { in do_check()
18965 verbose(env, in do_check()
18967 env->insn_processed); in do_check()
18971 state->last_insn_idx = env->prev_insn_idx; in do_check()
18973 if (is_prune_point(env, env->insn_idx)) { in do_check()
18974 err = is_state_visited(env, env->insn_idx); in do_check()
18979 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
18981 verbose(env, "\nfrom %d to %d%s: safe\n", in do_check()
18982 env->prev_insn_idx, env->insn_idx, in do_check()
18983 env->cur_state->speculative ? in do_check()
18986 verbose(env, "%d: safe\n", env->insn_idx); in do_check()
18992 if (is_jmp_point(env, env->insn_idx)) { in do_check()
18993 err = push_insn_history(env, state, 0, 0); in do_check()
19004 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { in do_check()
19005 verbose(env, "\nfrom %d to %d%s:", in do_check()
19006 env->prev_insn_idx, env->insn_idx, in do_check()
19007 env->cur_state->speculative ? in do_check()
19009 print_verifier_state(env, state, state->curframe, true); in do_check()
19013 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
19017 .private_data = env, in do_check()
19020 if (verifier_state_scratched(env)) in do_check()
19021 print_insn_state(env, state, state->curframe); in do_check()
19023 verbose_linfo(env, env->insn_idx, "; "); in do_check()
19024 env->prev_log_pos = env->log.end_pos; in do_check()
19025 verbose(env, "%d: ", env->insn_idx); in do_check()
19026 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in do_check()
19027 env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos; in do_check()
19028 env->prev_log_pos = env->log.end_pos; in do_check()
19031 if (bpf_prog_is_offloaded(env->prog->aux)) { in do_check()
19032 err = bpf_prog_offload_verify_insn(env, env->insn_idx, in do_check()
19033 env->prev_insn_idx); in do_check()
19038 regs = cur_regs(env); in do_check()
19039 sanitize_mark_insn_seen(env); in do_check()
19040 prev_insn_idx = env->insn_idx; in do_check()
19043 err = check_alu_op(env, insn); in do_check()
19053 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
19057 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in do_check()
19066 err = check_mem_access(env, env->insn_idx, insn->src_reg, in do_check()
19070 err = err ?: save_aux_ptr_type(env, src_reg_type, true); in do_check()
19071 err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], "ldx"); in do_check()
19078 err = check_atomic(env, env->insn_idx, insn); in do_check()
19081 env->insn_idx++; in do_check()
19086 verbose(env, "BPF_STX uses reserved fields\n"); in do_check()
19091 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
19095 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
19102 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
19108 err = save_aux_ptr_type(env, dst_reg_type, false); in do_check()
19116 verbose(env, "BPF_ST uses reserved fields\n"); in do_check()
19120 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
19127 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
19133 err = save_aux_ptr_type(env, dst_reg_type, false); in do_check()
19139 env->jmps_processed++; in do_check()
19149 verbose(env, "BPF_CALL uses reserved fields\n"); in do_check()
19153 if (env->cur_state->active_locks) { in do_check()
19157 verbose(env, "function calls are not allowed while holding a lock\n"); in do_check()
19162 err = check_func_call(env, insn, &env->insn_idx); in do_check()
19164 err = check_kfunc_call(env, insn, &env->insn_idx); in do_check()
19170 err = check_helper_call(env, insn, &env->insn_idx); in do_check()
19175 mark_reg_scratched(env, BPF_REG_0); in do_check()
19182 verbose(env, "BPF_JA uses reserved fields\n"); in do_check()
19187 env->insn_idx += insn->off + 1; in do_check()
19189 env->insn_idx += insn->imm + 1; in do_check()
19198 verbose(env, "BPF_EXIT uses reserved fields\n"); in do_check()
19208 err = check_resource_leak(env, exception_exit, !env->cur_state->curframe, in do_check()
19228 err = prepare_func_exit(env, &env->insn_idx); in do_check()
19235 err = check_return_code(env, BPF_REG_0, "R0"); in do_check()
19239 mark_verifier_state_scratched(env); in do_check()
19240 update_branch_counts(env, env->cur_state); in do_check()
19241 err = pop_stack(env, &prev_insn_idx, in do_check()
19242 &env->insn_idx, pop_log); in do_check()
19252 err = check_cond_jmp_op(env, insn, &env->insn_idx); in do_check()
19260 err = check_ld_abs(env, insn); in do_check()
19265 err = check_ld_imm(env, insn); in do_check()
19269 env->insn_idx++; in do_check()
19270 sanitize_mark_insn_seen(env); in do_check()
19272 verbose(env, "invalid BPF_LD mode\n"); in do_check()
19276 verbose(env, "unknown insn class %d\n", class); in do_check()
19280 env->insn_idx++; in do_check()
19322 static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) in __add_used_btf() argument
19328 for (i = 0; i < env->used_btf_cnt; i++) in __add_used_btf()
19329 if (env->used_btfs[i].btf == btf) in __add_used_btf()
19332 if (env->used_btf_cnt >= MAX_USED_BTFS) in __add_used_btf()
19337 btf_mod = &env->used_btfs[env->used_btf_cnt]; in __add_used_btf()
19350 return env->used_btf_cnt++; in __add_used_btf()
19354 static int __check_pseudo_btf_id(struct bpf_verifier_env *env, in __check_pseudo_btf_id() argument
19371 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); in __check_pseudo_btf_id()
19376 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id); in __check_pseudo_btf_id()
19383 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", in __check_pseudo_btf_id()
19422 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", in __check_pseudo_btf_id()
19437 static int check_pseudo_btf_id(struct bpf_verifier_env *env, in check_pseudo_btf_id() argument
19451 verbose(env, "invalid module BTF object FD specified.\n"); in check_pseudo_btf_id()
19456 …verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"… in check_pseudo_btf_id()
19462 err = __check_pseudo_btf_id(env, insn, aux, btf); in check_pseudo_btf_id()
19466 err = __add_used_btf(env, btf); in check_pseudo_btf_id()
19492 static int check_map_prog_compatibility(struct bpf_verifier_env *env, in check_map_prog_compatibility() argument
19502 verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n"); in check_map_prog_compatibility()
19509 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
19514 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
19521 verbose(env, "tracing progs cannot use bpf_timer yet\n"); in check_map_prog_compatibility()
19528 verbose(env, "tracing progs cannot use bpf_wq yet\n"); in check_map_prog_compatibility()
19535 verbose(env, "offload device mismatch between prog and map\n"); in check_map_prog_compatibility()
19540 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); in check_map_prog_compatibility()
19565 verbose(env, in check_map_prog_compatibility()
19571 bpf_cgroup_storage_assign(env->prog->aux, map)) { in check_map_prog_compatibility()
19572 verbose(env, "only one cgroup storage of each type is allowed\n"); in check_map_prog_compatibility()
19577 if (env->prog->aux->arena) { in check_map_prog_compatibility()
19578 verbose(env, "Only one arena per program\n"); in check_map_prog_compatibility()
19581 if (!env->allow_ptr_leaks || !env->bpf_capable) { in check_map_prog_compatibility()
19582 verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n"); in check_map_prog_compatibility()
19585 if (!env->prog->jit_requested) { in check_map_prog_compatibility()
19586 verbose(env, "JIT is required to use arena\n"); in check_map_prog_compatibility()
19590 verbose(env, "JIT doesn't support arena\n"); in check_map_prog_compatibility()
19593 env->prog->aux->arena = (void *)map; in check_map_prog_compatibility()
19594 if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { in check_map_prog_compatibility()
19595 verbose(env, "arena's user address must be set via map_extra or mmap()\n"); in check_map_prog_compatibility()
19603 static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) in __add_used_map() argument
19608 for (i = 0; i < env->used_map_cnt; i++) in __add_used_map()
19609 if (env->used_maps[i] == map) in __add_used_map()
19612 if (env->used_map_cnt >= MAX_USED_MAPS) { in __add_used_map()
19613 verbose(env, "The total number of maps per program has reached the limit of %u\n", in __add_used_map()
19618 err = check_map_prog_compatibility(env, map, env->prog); in __add_used_map()
19622 if (env->prog->sleepable) in __add_used_map()
19632 env->used_maps[env->used_map_cnt++] = map; in __add_used_map()
19634 return env->used_map_cnt - 1; in __add_used_map()
19641 static int add_used_map(struct bpf_verifier_env *env, int fd) in add_used_map() argument
19648 verbose(env, "fd %d is not pointing to valid bpf_map\n", fd); in add_used_map()
19652 return __add_used_map(env, map); in add_used_map()
19662 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) in resolve_pseudo_ldimm64() argument
19664 struct bpf_insn *insn = env->prog->insnsi; in resolve_pseudo_ldimm64()
19665 int insn_cnt = env->prog->len; in resolve_pseudo_ldimm64()
19668 err = bpf_prog_calc_tag(env->prog); in resolve_pseudo_ldimm64()
19676 verbose(env, "BPF_LDX uses reserved fields\n"); in resolve_pseudo_ldimm64()
19690 verbose(env, "invalid bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
19699 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
19700 err = check_pseudo_btf_id(env, insn, aux); in resolve_pseudo_ldimm64()
19707 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
19725 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
19732 if (bpfptr_is_null(env->fd_array)) { in resolve_pseudo_ldimm64()
19733 verbose(env, "fd_idx without fd_array is invalid\n"); in resolve_pseudo_ldimm64()
19736 if (copy_from_bpfptr_offset(&fd, env->fd_array, in resolve_pseudo_ldimm64()
19746 map_idx = add_used_map(env, fd); in resolve_pseudo_ldimm64()
19749 map = env->used_maps[map_idx]; in resolve_pseudo_ldimm64()
19751 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
19761 verbose(env, "direct value offset of %u is not allowed\n", off); in resolve_pseudo_ldimm64()
19766 verbose(env, "no direct value access support for this map type\n"); in resolve_pseudo_ldimm64()
19772 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", in resolve_pseudo_ldimm64()
19792 verbose(env, "unknown opcode %02x\n", insn->code); in resolve_pseudo_ldimm64()
19805 static void release_maps(struct bpf_verifier_env *env) in release_maps() argument
19807 __bpf_free_used_maps(env->prog->aux, env->used_maps, in release_maps()
19808 env->used_map_cnt); in release_maps()
19812 static void release_btfs(struct bpf_verifier_env *env) in release_btfs() argument
19814 __bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt); in release_btfs()
19818 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) in convert_pseudo_ld_imm64() argument
19820 struct bpf_insn *insn = env->prog->insnsi; in convert_pseudo_ld_imm64()
19821 int insn_cnt = env->prog->len; in convert_pseudo_ld_imm64()
19837 static void adjust_insn_aux_data(struct bpf_verifier_env *env, in adjust_insn_aux_data() argument
19841 struct bpf_insn_aux_data *old_data = env->insn_aux_data; in adjust_insn_aux_data()
19851 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); in adjust_insn_aux_data()
19863 new_data[i].zext_dst = insn_has_def32(env, insn + i); in adjust_insn_aux_data()
19865 env->insn_aux_data = new_data; in adjust_insn_aux_data()
19869 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) in adjust_subprog_starts() argument
19876 for (i = 0; i <= env->subprog_cnt; i++) { in adjust_subprog_starts()
19877 if (env->subprog_info[i].start <= off) in adjust_subprog_starts()
19879 env->subprog_info[i].start += len - 1; in adjust_subprog_starts()
19897 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, in bpf_patch_insn_data() argument
19904 new_data = vzalloc(array_size(env->prog->len + len - 1, in bpf_patch_insn_data()
19910 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); in bpf_patch_insn_data()
19913 verbose(env, in bpf_patch_insn_data()
19915 env->insn_aux_data[off].orig_idx); in bpf_patch_insn_data()
19919 adjust_insn_aux_data(env, new_data, new_prog, off, len); in bpf_patch_insn_data()
19920 adjust_subprog_starts(env, off, len); in bpf_patch_insn_data()
19963 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, in adjust_subprog_starts_after_remove() argument
19969 for (i = 0; i < env->subprog_cnt; i++) in adjust_subprog_starts_after_remove()
19970 if (env->subprog_info[i].start >= off) in adjust_subprog_starts_after_remove()
19973 for (j = i; j < env->subprog_cnt; j++) in adjust_subprog_starts_after_remove()
19974 if (env->subprog_info[j].start >= off + cnt) in adjust_subprog_starts_after_remove()
19979 if (env->subprog_info[j].start != off + cnt) in adjust_subprog_starts_after_remove()
19983 struct bpf_prog_aux *aux = env->prog->aux; in adjust_subprog_starts_after_remove()
19987 move = env->subprog_cnt + 1 - j; in adjust_subprog_starts_after_remove()
19989 memmove(env->subprog_info + i, in adjust_subprog_starts_after_remove()
19990 env->subprog_info + j, in adjust_subprog_starts_after_remove()
19991 sizeof(*env->subprog_info) * move); in adjust_subprog_starts_after_remove()
19992 env->subprog_cnt -= j - i; in adjust_subprog_starts_after_remove()
20008 if (env->subprog_info[i].start == off) in adjust_subprog_starts_after_remove()
20013 for (; i <= env->subprog_cnt; i++) in adjust_subprog_starts_after_remove()
20014 env->subprog_info[i].start -= cnt; in adjust_subprog_starts_after_remove()
20019 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, in bpf_adj_linfo_after_remove() argument
20022 struct bpf_prog *prog = env->prog; in bpf_adj_linfo_after_remove()
20069 for (i = 0; i <= env->subprog_cnt; i++) in bpf_adj_linfo_after_remove()
20070 if (env->subprog_info[i].linfo_idx > l_off) { in bpf_adj_linfo_after_remove()
20074 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) in bpf_adj_linfo_after_remove()
20075 env->subprog_info[i].linfo_idx -= l_cnt; in bpf_adj_linfo_after_remove()
20077 env->subprog_info[i].linfo_idx = l_off; in bpf_adj_linfo_after_remove()
20083 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) in verifier_remove_insns() argument
20085 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in verifier_remove_insns()
20086 unsigned int orig_prog_len = env->prog->len; in verifier_remove_insns()
20089 if (bpf_prog_is_offloaded(env->prog->aux)) in verifier_remove_insns()
20090 bpf_prog_offload_remove_insns(env, off, cnt); in verifier_remove_insns()
20092 err = bpf_remove_insns(env->prog, off, cnt); in verifier_remove_insns()
20096 err = adjust_subprog_starts_after_remove(env, off, cnt); in verifier_remove_insns()
20100 err = bpf_adj_linfo_after_remove(env, off, cnt); in verifier_remove_insns()
20121 static void sanitize_dead_code(struct bpf_verifier_env *env) in sanitize_dead_code() argument
20123 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in sanitize_dead_code()
20125 struct bpf_insn *insn = env->prog->insnsi; in sanitize_dead_code()
20126 const int insn_cnt = env->prog->len; in sanitize_dead_code()
20151 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) in opt_hard_wire_dead_code_branches() argument
20153 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_hard_wire_dead_code_branches()
20155 struct bpf_insn *insn = env->prog->insnsi; in opt_hard_wire_dead_code_branches()
20156 const int insn_cnt = env->prog->len; in opt_hard_wire_dead_code_branches()
20170 if (bpf_prog_is_offloaded(env->prog->aux)) in opt_hard_wire_dead_code_branches()
20171 bpf_prog_offload_replace_insn(env, i, &ja); in opt_hard_wire_dead_code_branches()
20177 static int opt_remove_dead_code(struct bpf_verifier_env *env) in opt_remove_dead_code() argument
20179 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_remove_dead_code()
20180 int insn_cnt = env->prog->len; in opt_remove_dead_code()
20192 err = verifier_remove_insns(env, i, j); in opt_remove_dead_code()
20195 insn_cnt = env->prog->len; in opt_remove_dead_code()
20204 static int opt_remove_nops(struct bpf_verifier_env *env) in opt_remove_nops() argument
20206 struct bpf_insn *insn = env->prog->insnsi; in opt_remove_nops()
20207 int insn_cnt = env->prog->len; in opt_remove_nops()
20218 err = verifier_remove_insns(env, i, 1); in opt_remove_nops()
20229 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, in opt_subreg_zext_lo32_rnd_hi32() argument
20233 struct bpf_insn_aux_data *aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
20234 int i, patch_len, delta = 0, len = env->prog->len; in opt_subreg_zext_lo32_rnd_hi32()
20235 struct bpf_insn *insns = env->prog->insnsi; in opt_subreg_zext_lo32_rnd_hi32()
20267 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { in opt_subreg_zext_lo32_rnd_hi32()
20306 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); in opt_subreg_zext_lo32_rnd_hi32()
20316 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); in opt_subreg_zext_lo32_rnd_hi32()
20319 env->prog = new_prog; in opt_subreg_zext_lo32_rnd_hi32()
20321 aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
20333 static int convert_ctx_accesses(struct bpf_verifier_env *env) in convert_ctx_accesses() argument
20335 struct bpf_subprog_info *subprogs = env->subprog_info; in convert_ctx_accesses()
20336 const struct bpf_verifier_ops *ops = env->ops; in convert_ctx_accesses()
20338 const int insn_cnt = env->prog->len; in convert_ctx_accesses()
20339 struct bpf_insn *epilogue_buf = env->epilogue_buf; in convert_ctx_accesses()
20340 struct bpf_insn *insn_buf = env->insn_buf; in convert_ctx_accesses()
20349 epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog, in convert_ctx_accesses()
20352 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
20360 insn_buf[cnt++] = env->prog->insnsi[0]; in convert_ctx_accesses()
20361 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in convert_ctx_accesses()
20364 env->prog = new_prog; in convert_ctx_accesses()
20369 if (ops->gen_prologue || env->seen_direct_write) { in convert_ctx_accesses()
20371 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
20374 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, in convert_ctx_accesses()
20375 env->prog); in convert_ctx_accesses()
20377 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
20380 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in convert_ctx_accesses()
20384 env->prog = new_prog; in convert_ctx_accesses()
20390 WARN_ON(adjust_jmp_off(env->prog, 0, delta)); in convert_ctx_accesses()
20392 if (bpf_prog_is_offloaded(env->prog->aux)) in convert_ctx_accesses()
20395 insn = env->prog->insnsi + delta; in convert_ctx_accesses()
20420 env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) { in convert_ctx_accesses()
20422 env->prog->aux->num_exentries++; in convert_ctx_accesses()
20448 env->insn_aux_data[i + delta].sanitize_stack_spill) { in convert_ctx_accesses()
20455 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); in convert_ctx_accesses()
20460 env->prog = new_prog; in convert_ctx_accesses()
20465 switch ((int)env->insn_aux_data[i + delta].ptr_type) { in convert_ctx_accesses()
20497 env->prog->aux->num_exentries++; in convert_ctx_accesses()
20502 verbose(env, "sign extending loads from arena are not supported yet\n"); in convert_ctx_accesses()
20506 env->prog->aux->num_exentries++; in convert_ctx_accesses()
20512 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; in convert_ctx_accesses()
20528 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); in convert_ctx_accesses()
20543 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, in convert_ctx_accesses()
20547 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
20555 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); in convert_ctx_accesses()
20580 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in convert_ctx_accesses()
20587 env->prog = new_prog; in convert_ctx_accesses()
20594 static int jit_subprogs(struct bpf_verifier_env *env) in jit_subprogs() argument
20596 struct bpf_prog *prog = env->prog, **func, *tmp; in jit_subprogs()
20603 if (env->subprog_cnt <= 1) in jit_subprogs()
20614 subprog = find_subprog(env, i + insn->imm + 1); in jit_subprogs()
20627 env->insn_aux_data[i].call_imm = insn->imm; in jit_subprogs()
20650 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); in jit_subprogs()
20654 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
20656 subprog_end = env->subprog_info[i + 1].start; in jit_subprogs()
20693 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; in jit_subprogs()
20694 if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) in jit_subprogs()
20704 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; in jit_subprogs()
20723 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; in jit_subprogs()
20724 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; in jit_subprogs()
20725 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; in jit_subprogs()
20727 func[i]->aux->exception_boundary = env->seen_exception; in jit_subprogs()
20740 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
20767 func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; in jit_subprogs()
20768 func[i]->aux->real_func_cnt = env->subprog_cnt; in jit_subprogs()
20770 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
20774 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); in jit_subprogs()
20785 for (i = 1; i < env->subprog_cnt; i++) { in jit_subprogs()
20791 for (i = 1; i < env->subprog_cnt; i++) in jit_subprogs()
20800 insn[0].imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
20807 insn->off = env->insn_aux_data[i].call_imm; in jit_subprogs()
20808 subprog = find_subprog(env, i + insn->off + 1); in jit_subprogs()
20818 prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; in jit_subprogs()
20819 prog->aux->real_func_cnt = env->subprog_cnt; in jit_subprogs()
20820 prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; in jit_subprogs()
20837 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
20852 insn->imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
20858 static int fixup_call_args(struct bpf_verifier_env *env) in fixup_call_args() argument
20861 struct bpf_prog *prog = env->prog; in fixup_call_args()
20868 if (env->prog->jit_requested && in fixup_call_args()
20869 !bpf_prog_is_offloaded(env->prog->aux)) { in fixup_call_args()
20870 err = jit_subprogs(env); in fixup_call_args()
20878 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); in fixup_call_args()
20881 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { in fixup_call_args()
20885 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); in fixup_call_args()
20893 verbose(env, "callbacks are not allowed in non-JITed programs\n"); in fixup_call_args()
20899 depth = get_callee_stack_depth(env, insn, i); in fixup_call_args()
20910 static void specialize_kfunc(struct bpf_verifier_env *env, in specialize_kfunc() argument
20913 struct bpf_prog *prog = env->prog; in specialize_kfunc()
20931 seen_direct_write = env->seen_direct_write; in specialize_kfunc()
20932 is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE); in specialize_kfunc()
20940 env->seen_direct_write = seen_direct_write; in specialize_kfunc()
20961 static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in fixup_kfunc_call() argument
20967 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); in fixup_kfunc_call()
20977 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); in fixup_kfunc_call()
20979 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", in fixup_kfunc_call()
20990 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; in fixup_kfunc_call()
20992 u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size; in fixup_kfunc_call()
20995 verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n", in fixup_kfunc_call()
21008 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; in fixup_kfunc_call()
21012 verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n", in fixup_kfunc_call()
21019 verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n", in fixup_kfunc_call()
21031 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; in fixup_kfunc_call()
21042 verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n", in fixup_kfunc_call()
21047 __fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg, in fixup_kfunc_call()
21054 struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(BPF_REG_4, (long)env->prog->aux) }; in fixup_kfunc_call()
21065 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) in add_hidden_subprog() argument
21067 struct bpf_subprog_info *info = env->subprog_info; in add_hidden_subprog()
21068 int cnt = env->subprog_cnt; in add_hidden_subprog()
21072 if (env->hidden_subprog_cnt) { in add_hidden_subprog()
21073 verbose(env, "verifier internal error: only one hidden subprog supported\n"); in add_hidden_subprog()
21080 prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); in add_hidden_subprog()
21083 env->prog = prog; in add_hidden_subprog()
21086 env->subprog_cnt++; in add_hidden_subprog()
21087 env->hidden_subprog_cnt++; in add_hidden_subprog()
21094 static int do_misc_fixups(struct bpf_verifier_env *env) in do_misc_fixups() argument
21096 struct bpf_prog *prog = env->prog; in do_misc_fixups()
21104 struct bpf_insn *insn_buf = env->insn_buf; in do_misc_fixups()
21108 struct bpf_subprog_info *subprogs = env->subprog_info; in do_misc_fixups()
21112 if (env->seen_exception && !env->exception_callback_subprog) { in do_misc_fixups()
21114 env->prog->insnsi[insn_cnt - 1], in do_misc_fixups()
21119 ret = add_hidden_subprog(env, patch, ARRAY_SIZE(patch)); in do_misc_fixups()
21122 prog = env->prog; in do_misc_fixups()
21125 env->exception_callback_subprog = env->subprog_cnt - 1; in do_misc_fixups()
21127 mark_subprog_exc_cb(env, env->exception_callback_subprog); in do_misc_fixups()
21133 (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) { in do_misc_fixups()
21143 if (env->insn_aux_data[i + delta].needs_zext) in do_misc_fixups()
21168 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); in do_misc_fixups()
21173 env->prog = prog = new_prog; in do_misc_fixups()
21263 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); in do_misc_fixups()
21268 env->prog = prog = new_prog; in do_misc_fixups()
21293 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21298 env->prog = prog = new_prog; in do_misc_fixups()
21307 cnt = env->ops->gen_ld_abs(insn, insn_buf); in do_misc_fixups()
21309 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
21313 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21318 env->prog = prog = new_prog; in do_misc_fixups()
21332 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
21366 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21371 env->prog = prog = new_prog; in do_misc_fixups()
21389 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21394 env->prog = prog = new_prog; in do_misc_fixups()
21404 ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); in do_misc_fixups()
21410 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21415 env->prog = prog = new_prog; in do_misc_fixups()
21437 if (!allow_tail_call_in_subprogs(env)) in do_misc_fixups()
21449 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
21450 if (env->bpf_capable && !prog->blinding_requested && in do_misc_fixups()
21464 verbose(env, "adding tail call poke descriptor failed\n"); in do_misc_fixups()
21482 verbose(env, "tail_call abusing map_ptr\n"); in do_misc_fixups()
21495 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21500 env->prog = prog = new_prog; in do_misc_fixups()
21528 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21533 env->prog = prog = new_prog; in do_misc_fixups()
21539 if (!in_sleepable(env) || in do_misc_fixups()
21540 env->insn_aux_data[i + delta].storage_get_func_atomic) in do_misc_fixups()
21547 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21552 env->prog = prog = new_prog; in do_misc_fixups()
21558 if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) { in do_misc_fixups()
21566 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21571 env->prog = prog = new_prog; in do_misc_fixups()
21590 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
21602 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
21606 new_prog = bpf_patch_insn_data(env, i + delta, in do_misc_fixups()
21612 env->prog = prog = new_prog; in do_misc_fixups()
21689 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, in do_misc_fixups()
21695 env->prog = prog = new_prog; in do_misc_fixups()
21703 verifier_inlines_helper_call(env, insn->imm)) { in do_misc_fixups()
21718 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21723 env->prog = prog = new_prog; in do_misc_fixups()
21743 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21748 env->prog = prog = new_prog; in do_misc_fixups()
21771 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21776 env->prog = prog = new_prog; in do_misc_fixups()
21787 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); in do_misc_fixups()
21791 env->prog = prog = new_prog; in do_misc_fixups()
21802 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); in do_misc_fixups()
21806 env->prog = prog = new_prog; in do_misc_fixups()
21857 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21862 env->prog = prog = new_prog; in do_misc_fixups()
21875 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21880 env->prog = prog = new_prog; in do_misc_fixups()
21885 fn = env->ops->get_func_proto(insn->imm, env->prog); in do_misc_fixups()
21890 verbose(env, in do_misc_fixups()
21908 env->prog->aux->stack_depth = subprogs[0].stack_depth; in do_misc_fixups()
21909 for (i = 0; i < env->subprog_cnt; i++) { in do_misc_fixups()
21916 verbose(env, "verifier bug: stack_slots supports may_goto only\n"); in do_misc_fixups()
21924 insn_buf[1] = env->prog->insnsi[subprog_start]; in do_misc_fixups()
21926 new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, 2); in do_misc_fixups()
21929 env->prog = prog = new_prog; in do_misc_fixups()
21936 WARN_ON(adjust_jmp_off(env->prog, subprog_start, 1)); in do_misc_fixups()
21945 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
21951 verbose(env, "tracking tail call prog failed\n"); in do_misc_fixups()
21956 sort_kfunc_descs_by_imm_off(env->prog); in do_misc_fixups()
21961 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, in inline_bpf_loop() argument
21974 struct bpf_insn *insn_buf = env->insn_buf; in inline_bpf_loop()
22023 new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); in inline_bpf_loop()
22028 callback_start = env->subprog_info[callback_subprogno].start; in inline_bpf_loop()
22053 static int optimize_bpf_loop(struct bpf_verifier_env *env) in optimize_bpf_loop() argument
22055 struct bpf_subprog_info *subprogs = env->subprog_info; in optimize_bpf_loop()
22057 struct bpf_insn *insn = env->prog->insnsi; in optimize_bpf_loop()
22058 int insn_cnt = env->prog->len; in optimize_bpf_loop()
22065 &env->insn_aux_data[i + delta].loop_inline_state; in optimize_bpf_loop()
22071 new_prog = inline_bpf_loop(env, in optimize_bpf_loop()
22080 env->prog = new_prog; in optimize_bpf_loop()
22093 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in optimize_bpf_loop()
22101 static int remove_fastcall_spills_fills(struct bpf_verifier_env *env) in remove_fastcall_spills_fills() argument
22103 struct bpf_subprog_info *subprog = env->subprog_info; in remove_fastcall_spills_fills()
22104 struct bpf_insn_aux_data *aux = env->insn_aux_data; in remove_fastcall_spills_fills()
22105 struct bpf_insn *insn = env->prog->insnsi; in remove_fastcall_spills_fills()
22106 int insn_cnt = env->prog->len; in remove_fastcall_spills_fills()
22132 static void free_states(struct bpf_verifier_env *env) in free_states() argument
22137 sl = env->free_list; in free_states()
22144 env->free_list = NULL; in free_states()
22146 if (!env->explored_states) in free_states()
22149 for (i = 0; i < state_htab_size(env); i++) { in free_states()
22150 sl = env->explored_states[i]; in free_states()
22158 env->explored_states[i] = NULL; in free_states()
22162 static int do_check_common(struct bpf_verifier_env *env, int subprog) in do_check_common() argument
22164 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check_common()
22165 struct bpf_subprog_info *sub = subprog_info(env, subprog); in do_check_common()
22170 env->prev_linfo = NULL; in do_check_common()
22171 env->pass_cnt++; in do_check_common()
22184 env->cur_state = state; in do_check_common()
22185 init_func_state(env, state->frame[0], in do_check_common()
22189 state->first_insn_idx = env->subprog_info[subprog].start; in do_check_common()
22193 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { in do_check_common()
22194 const char *sub_name = subprog_name(env, subprog); in do_check_common()
22198 verbose(env, "Validating %s() func#%d...\n", sub_name, subprog); in do_check_common()
22199 ret = btf_prepare_func_args(env, subprog); in do_check_common()
22203 if (subprog_is_exc_cb(env, subprog)) { in do_check_common()
22210 verbose(env, "exception cb only supports single integer argument\n"); in do_check_common()
22221 mark_reg_known_zero(env, regs, i); in do_check_common()
22224 mark_reg_unknown(env, regs, i); in do_check_common()
22227 __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen); in do_check_common()
22232 mark_reg_known_zero(env, regs, i); in do_check_common()
22234 reg->id = ++env->id_gen; in do_check_common()
22243 mark_reg_known_zero(env, regs, i); in do_check_common()
22246 reg->id = ++env->id_gen; in do_check_common()
22249 mark_reg_unknown(env, regs, i); in do_check_common()
22262 if (env->prog->aux->func_info_aux) { in do_check_common()
22263 ret = btf_prepare_func_args(env, 0); in do_check_common()
22265 env->prog->aux->func_info_aux[0].unreliable = true; in do_check_common()
22270 mark_reg_known_zero(env, regs, BPF_REG_1); in do_check_common()
22273 ret = do_check(env); in do_check_common()
22278 if (env->cur_state) { in do_check_common()
22279 free_verifier_state(env->cur_state, true); in do_check_common()
22280 env->cur_state = NULL; in do_check_common()
22282 while (!pop_stack(env, NULL, NULL, false)); in do_check_common()
22284 bpf_vlog_reset(&env->log, 0); in do_check_common()
22285 free_states(env); in do_check_common()
22309 static int do_check_subprogs(struct bpf_verifier_env *env) in do_check_subprogs() argument
22311 struct bpf_prog_aux *aux = env->prog->aux; in do_check_subprogs()
22319 if (env->exception_callback_subprog) in do_check_subprogs()
22320 subprog_aux(env, env->exception_callback_subprog)->called = true; in do_check_subprogs()
22324 for (i = 1; i < env->subprog_cnt; i++) { in do_check_subprogs()
22325 if (!subprog_is_global(env, i)) in do_check_subprogs()
22328 sub_aux = subprog_aux(env, i); in do_check_subprogs()
22332 env->insn_idx = env->subprog_info[i].start; in do_check_subprogs()
22333 WARN_ON_ONCE(env->insn_idx == 0); in do_check_subprogs()
22334 ret = do_check_common(env, i); in do_check_subprogs()
22337 } else if (env->log.level & BPF_LOG_LEVEL) { in do_check_subprogs()
22338 verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", in do_check_subprogs()
22339 i, subprog_name(env, i)); in do_check_subprogs()
22359 static int do_check_main(struct bpf_verifier_env *env) in do_check_main() argument
22363 env->insn_idx = 0; in do_check_main()
22364 ret = do_check_common(env, 0); in do_check_main()
22366 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in do_check_main()
22371 static void print_verification_stats(struct bpf_verifier_env *env) in print_verification_stats() argument
22375 if (env->log.level & BPF_LOG_STATS) { in print_verification_stats()
22376 verbose(env, "verification time %lld usec\n", in print_verification_stats()
22377 div_u64(env->verification_time, 1000)); in print_verification_stats()
22378 verbose(env, "stack depth "); in print_verification_stats()
22379 for (i = 0; i < env->subprog_cnt; i++) { in print_verification_stats()
22380 u32 depth = env->subprog_info[i].stack_depth; in print_verification_stats()
22382 verbose(env, "%d", depth); in print_verification_stats()
22383 if (i + 1 < env->subprog_cnt) in print_verification_stats()
22384 verbose(env, "+"); in print_verification_stats()
22386 verbose(env, "\n"); in print_verification_stats()
22388 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " in print_verification_stats()
22390 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, in print_verification_stats()
22391 env->max_states_per_insn, env->total_states, in print_verification_stats()
22392 env->peak_states, env->longest_mark_read_walk); in print_verification_stats()
22395 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) in check_struct_ops_btf_id() argument
22401 struct bpf_prog *prog = env->prog; in check_struct_ops_btf_id()
22408 verbose(env, "struct ops programs must have a GPL compatible license\n"); in check_struct_ops_btf_id()
22418 env->attach_btf_mod = btf_try_get_module(btf); in check_struct_ops_btf_id()
22419 if (!env->attach_btf_mod) { in check_struct_ops_btf_id()
22420 verbose(env, "struct_ops module %s is not found\n", in check_struct_ops_btf_id()
22429 verbose(env, "attach_btf_id %u is not a supported struct\n", in check_struct_ops_btf_id()
22438 verbose(env, "attach to invalid member idx %u of struct %s\n", in check_struct_ops_btf_id()
22448 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", in check_struct_ops_btf_id()
22455 verbose(env, "attach to unsupported member %s of struct %s\n", in check_struct_ops_btf_id()
22464 verbose(env, "attach to unsupported member %s of struct %s\n", in check_struct_ops_btf_id()
22471 verbose(env, "Private stack not supported by jit\n"); in check_struct_ops_btf_id()
22483 env->ops = st_ops->verifier_ops; in check_struct_ops_btf_id()
22877 static int check_attach_btf_id(struct bpf_verifier_env *env) in check_attach_btf_id() argument
22879 struct bpf_prog *prog = env->prog; in check_attach_btf_id()
22891 verbose(env, "Syscall programs can only be sleepable\n"); in check_attach_btf_id()
22896 …verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepa… in check_attach_btf_id()
22901 return check_struct_ops_btf_id(env); in check_attach_btf_id()
22908 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); in check_attach_btf_id()
22917 env->ops = bpf_verifier_ops[tgt_prog->type]; in check_attach_btf_id()
22941 ret = bpf_lsm_verify_prog(&env->log, prog); in check_attach_btf_id()
22977 static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) in add_fd_from_fd_array() argument
22986 err = __add_used_map(env, map); in add_fd_from_fd_array()
22994 err = __add_used_btf(env, btf); in add_fd_from_fd_array()
23000 verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); in add_fd_from_fd_array()
23004 static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr) in process_fd_array() argument
23011 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); in process_fd_array()
23023 verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt); in process_fd_array()
23028 if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size)) in process_fd_array()
23031 ret = add_fd_from_fd_array(env, fd); in process_fd_array()
23042 struct bpf_verifier_env *env; in bpf_check() local
23054 env = kvzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); in bpf_check()
23055 if (!env) in bpf_check()
23058 env->bt.env = env; in bpf_check()
23061 env->insn_aux_data = in bpf_check()
23064 if (!env->insn_aux_data) in bpf_check()
23067 env->insn_aux_data[i].orig_idx = i; in bpf_check()
23068 env->prog = *prog; in bpf_check()
23069 env->ops = bpf_verifier_ops[env->prog->type]; in bpf_check()
23071 env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); in bpf_check()
23072 env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); in bpf_check()
23073 env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); in bpf_check()
23074 env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); in bpf_check()
23075 env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); in bpf_check()
23086 ret = bpf_vlog_init(&env->log, attr->log_level, in bpf_check()
23092 ret = process_fd_array(env, attr, uattr); in bpf_check()
23096 mark_verifier_state_clean(env); in bpf_check()
23100 verbose(env, "in-kernel BTF is malformed\n"); in bpf_check()
23105 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); in bpf_check()
23107 env->strict_alignment = true; in bpf_check()
23109 env->strict_alignment = false; in bpf_check()
23112 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; in bpf_check()
23113 env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; in bpf_check()
23115 env->explored_states = kvcalloc(state_htab_size(env), in bpf_check()
23119 if (!env->explored_states) in bpf_check()
23122 ret = check_btf_info_early(env, attr, uattr); in bpf_check()
23126 ret = add_subprog_and_kfunc(env); in bpf_check()
23130 ret = check_subprogs(env); in bpf_check()
23134 ret = check_btf_info(env, attr, uattr); in bpf_check()
23138 ret = resolve_pseudo_ldimm64(env); in bpf_check()
23142 if (bpf_prog_is_offloaded(env->prog->aux)) { in bpf_check()
23143 ret = bpf_prog_offload_verifier_prep(env->prog); in bpf_check()
23148 ret = check_cfg(env); in bpf_check()
23152 ret = check_attach_btf_id(env); in bpf_check()
23156 ret = mark_fastcall_patterns(env); in bpf_check()
23160 ret = do_check_main(env); in bpf_check()
23161 ret = ret ?: do_check_subprogs(env); in bpf_check()
23163 if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) in bpf_check()
23164 ret = bpf_prog_offload_finalize(env); in bpf_check()
23167 kvfree(env->explored_states); in bpf_check()
23173 ret = remove_fastcall_spills_fills(env); in bpf_check()
23176 ret = check_max_stack_depth(env); in bpf_check()
23180 ret = optimize_bpf_loop(env); in bpf_check()
23184 opt_hard_wire_dead_code_branches(env); in bpf_check()
23186 ret = opt_remove_dead_code(env); in bpf_check()
23188 ret = opt_remove_nops(env); in bpf_check()
23191 sanitize_dead_code(env); in bpf_check()
23196 ret = convert_ctx_accesses(env); in bpf_check()
23199 ret = do_misc_fixups(env); in bpf_check()
23204 if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) { in bpf_check()
23205 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); in bpf_check()
23206 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret in bpf_check()
23211 ret = fixup_call_args(env); in bpf_check()
23213 env->verification_time = ktime_get_ns() - start_time; in bpf_check()
23214 print_verification_stats(env); in bpf_check()
23215 env->prog->aux->verified_insns = env->insn_processed; in bpf_check()
23218 err = bpf_vlog_finalize(&env->log, &log_true_size); in bpf_check()
23232 if (env->used_map_cnt) { in bpf_check()
23234 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, in bpf_check()
23235 sizeof(env->used_maps[0]), in bpf_check()
23238 if (!env->prog->aux->used_maps) { in bpf_check()
23243 memcpy(env->prog->aux->used_maps, env->used_maps, in bpf_check()
23244 sizeof(env->used_maps[0]) * env->used_map_cnt); in bpf_check()
23245 env->prog->aux->used_map_cnt = env->used_map_cnt; in bpf_check()
23247 if (env->used_btf_cnt) { in bpf_check()
23249 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, in bpf_check()
23250 sizeof(env->used_btfs[0]), in bpf_check()
23252 if (!env->prog->aux->used_btfs) { in bpf_check()
23257 memcpy(env->prog->aux->used_btfs, env->used_btfs, in bpf_check()
23258 sizeof(env->used_btfs[0]) * env->used_btf_cnt); in bpf_check()
23259 env->prog->aux->used_btf_cnt = env->used_btf_cnt; in bpf_check()
23261 if (env->used_map_cnt || env->used_btf_cnt) { in bpf_check()
23265 convert_pseudo_ld_imm64(env); in bpf_check()
23268 adjust_btf_func(env); in bpf_check()
23271 if (!env->prog->aux->used_maps) in bpf_check()
23275 release_maps(env); in bpf_check()
23276 if (!env->prog->aux->used_btfs) in bpf_check()
23277 release_btfs(env); in bpf_check()
23282 if (env->prog->type == BPF_PROG_TYPE_EXT) in bpf_check()
23283 env->prog->expected_attach_type = 0; in bpf_check()
23285 *prog = env->prog; in bpf_check()
23287 module_put(env->attach_btf_mod); in bpf_check()
23291 vfree(env->insn_aux_data); in bpf_check()
23292 kvfree(env->insn_hist); in bpf_check()
23294 kvfree(env); in bpf_check()