8 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)  argument
11 int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,  in bpf_push_jmp_history()  argument
19 	if (env->cur_hist_ent) {  in bpf_push_jmp_history()
23 		verifier_bug_if((env->cur_hist_ent->flags & insn_flags) &&  in bpf_push_jmp_history()
24 				(env->cur_hist_ent->flags & insn_flags) != insn_flags,  in bpf_push_jmp_history()
25 				env, "insn history: insn_idx %d cur flags %x new flags %x",  in bpf_push_jmp_history()
26 				env->insn_idx, env->cur_hist_ent->flags, insn_flags);  in bpf_push_jmp_history()
27 		env->cur_hist_ent->flags |= insn_flags;  in bpf_push_jmp_history()
28 		env->cur_hist_ent->spi = spi;  in bpf_push_jmp_history()
29 		env->cur_hist_ent->frame = frame;  in bpf_push_jmp_history()
30 		verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,  in bpf_push_jmp_history()
32 				env->insn_idx, env->cur_hist_ent->linked_regs);  in bpf_push_jmp_history()
33 		env->cur_hist_ent->linked_regs = linked_regs;  in bpf_push_jmp_history()
45 	p->idx = env->insn_idx;  in bpf_push_jmp_history()
46 	p->prev_idx = env->prev_insn_idx;  in bpf_push_jmp_history()
52 	env->cur_hist_ent = p;  in bpf_push_jmp_history()
120 	struct bpf_verifier_env *env = bt->env;  in bt_reset()  local
123 	bt->env = env;  in bt_reset()
150 		verifier_bug(bt->env, "subprog enter from frame %d", bt->frame);  in bt_subprog_enter()
160 		verifier_bug(bt->env, "subprog exit from frame 0");  in bt_subprog_exit()
265 static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,  in backtrack_insn()  argument
268 	struct bpf_insn *insn = env->prog->insnsi + idx;  in backtrack_insn()
278 	if (env->log.level & BPF_LOG_LEVEL2) {  in backtrack_insn()
279 		fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));  in backtrack_insn()
280 		verbose(env, "mark_precise: frame%d: regs=%s ",  in backtrack_insn()
281 			bt->frame, env->tmp_str_buf);  in backtrack_insn()
282 		bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));  in backtrack_insn()
283 		verbose(env, "stack=%s before ", env->tmp_str_buf);  in backtrack_insn()
284 		verbose(env, "%d: ", idx);  in backtrack_insn()
285 		bpf_verbose_insn(env, insn);  in backtrack_insn()
286 		verbose(env, "\n");  in backtrack_insn()
413 			subprog = bpf_find_subprog(env, subprog_insn_idx);  in backtrack_insn()
417 			if (bpf_subprog_is_global(env, subprog)) {  in backtrack_insn()
424 				verifier_bug_if(idx + 1 != subseq_idx, env,  in backtrack_insn()
431 					verifier_bug(env, "global subprog unexpected regs %x",  in backtrack_insn()
446 					verifier_bug(env, "static subprog unexpected regs %x",  in backtrack_insn()
454 					verifier_bug(env,  in backtrack_insn()
467 					verifier_bug(env,  in backtrack_insn()
485 				verifier_bug(env, "callback unexpected regs %x",  in backtrack_insn()
490 				verifier_bug(env, "callback leftover stack slots %llx",  in backtrack_insn()
513 				verifier_bug(env, "backtracking call unexpected regs %x",  in backtrack_insn()
534 					    bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]);  in backtrack_insn()
546 				verifier_bug(env, "backtracking exit unexpected regs %x",  in backtrack_insn()
665 void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env,  in bpf_mark_all_scalars_precise()  argument
672 	if (env->log.level & BPF_LOG_LEVEL2) {  in bpf_mark_all_scalars_precise()
673 		verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",  in bpf_mark_all_scalars_precise()
691 				if (env->log.level & BPF_LOG_LEVEL2) {  in bpf_mark_all_scalars_precise()
692 					verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",  in bpf_mark_all_scalars_precise()
703 				if (env->log.level & BPF_LOG_LEVEL2) {  in bpf_mark_all_scalars_precise()
704 					verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",  in bpf_mark_all_scalars_precise()
799 int bpf_mark_chain_precision(struct bpf_verifier_env *env,  in bpf_mark_chain_precision()  argument
805 	struct backtrack_state *bt = &env->bt;  in bpf_mark_chain_precision()
814 	if (!env->bpf_capable)  in bpf_mark_chain_precision()
829 			verifier_bug(env, "backtracking misuse");  in bpf_mark_chain_precision()
843 		if (env->log.level & BPF_LOG_LEVEL2) {  in bpf_mark_chain_precision()
844 			verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",  in bpf_mark_chain_precision()
871 			verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx",  in bpf_mark_chain_precision()
882 				err = backtrack_insn(env, i, subseq_idx, hist, bt);  in bpf_mark_chain_precision()
885 				bpf_mark_all_scalars_precise(env, starting_state);  in bpf_mark_chain_precision()
901 			if (i >= env->prog->len) {  in bpf_mark_chain_precision()
908 				verifier_bug(env, "backtracking idx %d", i);  in bpf_mark_chain_precision()
936 						    env, "stack slot %d, total slots %d",  in bpf_mark_chain_precision()
963 			if (env->log.level & BPF_LOG_LEVEL2) {  in bpf_mark_chain_precision()
964 				fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,  in bpf_mark_chain_precision()
966 				verbose(env, "mark_precise: frame%d: parent state regs=%s ",  in bpf_mark_chain_precision()
967 					fr, env->tmp_str_buf);  in bpf_mark_chain_precision()
968 				bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,  in bpf_mark_chain_precision()
970 				verbose(env, "stack=%s: ", env->tmp_str_buf);  in bpf_mark_chain_precision()
971 				print_verifier_state(env, st, fr, true);  in bpf_mark_chain_precision()
988 		bpf_mark_all_scalars_precise(env, starting_state);  in bpf_mark_chain_precision()