16 static bool bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log)  in bpf_verifier_log_attr_valid()  argument
19 	if (!!log->ubuf != !!log->len_total)  in bpf_verifier_log_attr_valid()
21 	/* log buf without log_level is meaningless */  in bpf_verifier_log_attr_valid()
22 	if (log->ubuf && log->level == 0)  in bpf_verifier_log_attr_valid()
24 	if (log->level & ~BPF_LOG_MASK)  in bpf_verifier_log_attr_valid()
26 	if (log->len_total > UINT_MAX >> 2)  in bpf_verifier_log_attr_valid()
31 int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,  in bpf_vlog_init()  argument
34 	log->level = log_level;  in bpf_vlog_init()
35 	log->ubuf = log_buf;  in bpf_vlog_init()
36 	log->len_total = log_size;  in bpf_vlog_init()
38 	/* log attributes have to be sane */  in bpf_vlog_init()
39 	if (!bpf_verifier_log_attr_valid(log))  in bpf_vlog_init()
45 static void bpf_vlog_update_len_max(struct bpf_verifier_log *log, u32 add_len)  in bpf_vlog_update_len_max()  argument
48 	u64 len = log->end_pos + add_len;  in bpf_vlog_update_len_max()
50 	/* log->len_max could be larger than our current len due to  in bpf_vlog_update_len_max()
55 		log->len_max = UINT_MAX;  in bpf_vlog_update_len_max()
56 	else if (len > log->len_max)  in bpf_vlog_update_len_max()
57 		log->len_max = len;  in bpf_vlog_update_len_max()
60 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,  in bpf_verifier_vlog()  argument
66 	n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);  in bpf_verifier_vlog()
68 	if (log->level == BPF_LOG_KERNEL) {  in bpf_verifier_vlog()
69 		bool newline = n > 0 && log->kbuf[n - 1] == '\n';  in bpf_verifier_vlog()
71 		pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n");  in bpf_verifier_vlog()
76 	bpf_vlog_update_len_max(log, n);  in bpf_verifier_vlog()
78 	if (log->level & BPF_LOG_FIXED) {  in bpf_verifier_vlog()
81 		if (log->end_pos < log->len_total) {  in bpf_verifier_vlog()
82 			new_n = min_t(u32, log->len_total - log->end_pos, n);  in bpf_verifier_vlog()
83 			log->kbuf[new_n - 1] = '\0';  in bpf_verifier_vlog()
86 		cur_pos = log->end_pos;  in bpf_verifier_vlog()
87 		log->end_pos += n - 1; /* don't count terminating '\0' */  in bpf_verifier_vlog()
89 		if (log->ubuf && new_n &&  in bpf_verifier_vlog()
90 		    copy_to_user(log->ubuf + cur_pos, log->kbuf, new_n))  in bpf_verifier_vlog()
96 		new_end = log->end_pos + n;  in bpf_verifier_vlog()
97 		if (new_end - log->start_pos >= log->len_total)  in bpf_verifier_vlog()
98 			new_start = new_end - log->len_total;  in bpf_verifier_vlog()
100 			new_start = log->start_pos;  in bpf_verifier_vlog()
102 		log->start_pos = new_start;  in bpf_verifier_vlog()
103 		log->end_pos = new_end - 1; /* don't count terminating '\0' */  in bpf_verifier_vlog()
105 		if (!log->ubuf)  in bpf_verifier_vlog()
108 		new_n = min(n, log->len_total);  in bpf_verifier_vlog()
110 		div_u64_rem(cur_pos, log->len_total, &buf_start);  in bpf_verifier_vlog()
111 		div_u64_rem(new_end, log->len_total, &buf_end);  in bpf_verifier_vlog()
117 			buf_end = log->len_total;  in bpf_verifier_vlog()
127 			if (copy_to_user(log->ubuf + buf_start,  in bpf_verifier_vlog()
128 					 log->kbuf + n - new_n,  in bpf_verifier_vlog()
133 			if (copy_to_user(log->ubuf + buf_start,  in bpf_verifier_vlog()
134 					 log->kbuf + n - new_n,  in bpf_verifier_vlog()
135 					 log->len_total - buf_start))  in bpf_verifier_vlog()
137 			if (copy_to_user(log->ubuf,  in bpf_verifier_vlog()
138 					 log->kbuf + n - buf_end,  in bpf_verifier_vlog()
146 	log->ubuf = NULL;  in bpf_verifier_vlog()
149 void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos)  in bpf_vlog_reset()  argument
154 	if (WARN_ON_ONCE(new_pos > log->end_pos))  in bpf_vlog_reset()
157 	if (!bpf_verifier_log_needed(log) || log->level == BPF_LOG_KERNEL)  in bpf_vlog_reset()
160 	/* if position to which we reset is beyond current log window,  in bpf_vlog_reset()
162 	 * start_pos to end up with an empty log (start_pos == end_pos)  in bpf_vlog_reset()
164 	log->end_pos = new_pos;  in bpf_vlog_reset()
165 	if (log->end_pos < log->start_pos)  in bpf_vlog_reset()
166 		log->start_pos = log->end_pos;  in bpf_vlog_reset()
168 	if (!log->ubuf)  in bpf_vlog_reset()
171 	if (log->level & BPF_LOG_FIXED)  in bpf_vlog_reset()
172 		pos = log->end_pos + 1;  in bpf_vlog_reset()
174 		div_u64_rem(new_pos, log->len_total, &pos);  in bpf_vlog_reset()
176 	if (pos < log->len_total && put_user(zero, log->ubuf + pos))  in bpf_vlog_reset()
177 		log->ubuf = NULL;  in bpf_vlog_reset()
188 static int bpf_vlog_reverse_ubuf(struct bpf_verifier_log *log, int start, int end)  in bpf_vlog_reverse_ubuf()  argument
190 	/* we split log->kbuf into two equal parts for both ends of array */  in bpf_vlog_reverse_ubuf()
191 	int n = sizeof(log->kbuf) / 2, nn;  in bpf_vlog_reverse_ubuf()
192 	char *lbuf = log->kbuf, *rbuf = log->kbuf + n;  in bpf_vlog_reverse_ubuf()
203 		if (copy_from_user(lbuf, log->ubuf + start, nn))  in bpf_vlog_reverse_ubuf()
205 		if (copy_from_user(rbuf, log->ubuf + end - nn, nn))  in bpf_vlog_reverse_ubuf()
214 		if (copy_to_user(log->ubuf + start, rbuf, nn))  in bpf_vlog_reverse_ubuf()
216 		if (copy_to_user(log->ubuf + end - nn, lbuf, nn))  in bpf_vlog_reverse_ubuf()
226 int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual)  in bpf_vlog_finalize()  argument
232 	if (!log || log->level == 0 || log->level == BPF_LOG_KERNEL)  in bpf_vlog_finalize()
235 	if (!log->ubuf)  in bpf_vlog_finalize()
237 	/* If we never truncated log, there is nothing to move around. */  in bpf_vlog_finalize()
238 	if (log->start_pos == 0)  in bpf_vlog_finalize()
241 	/* Otherwise we need to rotate log contents to make it start from the  in bpf_vlog_finalize()
243 	 * that if log->start_pos != 0 then we definitely filled up entire log  in bpf_vlog_finalize()
245 	 * the left by (log->start_pos % log->len_total) bytes.  in bpf_vlog_finalize()
253 	 * Let's say we have log buffer that has to be shifted left by 7 bytes  in bpf_vlog_finalize()
264 	 * We'll utilize log->kbuf to read user memory chunk by chunk, swap  in bpf_vlog_finalize()
274 	div_u64_rem(log->start_pos, log->len_total, &sublen);  in bpf_vlog_finalize()
275 	sublen = log->len_total - sublen;  in bpf_vlog_finalize()
277 	err = bpf_vlog_reverse_ubuf(log, 0, log->len_total);  in bpf_vlog_finalize()
278 	err = err ?: bpf_vlog_reverse_ubuf(log, 0, sublen);  in bpf_vlog_finalize()
279 	err = err ?: bpf_vlog_reverse_ubuf(log, sublen, log->len_total);  in bpf_vlog_finalize()
281 		log->ubuf = NULL;  in bpf_vlog_finalize()
284 	*log_size_actual = log->len_max;  in bpf_vlog_finalize()
286 	/* properly initialized log has either both ubuf!=NULL and len_total>0  in bpf_vlog_finalize()
290 	if (!!log->ubuf != !!log->len_total)  in bpf_vlog_finalize()
294 	if (log->ubuf && log->len_max > log->len_total)  in bpf_vlog_finalize()
301  * bpf_verifier_log_write() is used to dump the verification trace to the log,
309 	if (!bpf_verifier_log_needed(&env->log))  in bpf_verifier_log_write()
313 	bpf_verifier_vlog(&env->log, fmt, args);  in bpf_verifier_log_write()
318 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,  in bpf_log()  argument
323 	if (!bpf_verifier_log_needed(log))  in bpf_log()
327 	bpf_verifier_vlog(log, fmt, args);  in bpf_log()
389 	if (!bpf_verifier_log_needed(&env->log))  in verbose_linfo()
399 	 * log doesn't emit column information, from user perspective we just  in verbose_linfo()
413 		bpf_verifier_vlog(&env->log, prefix_fmt, args);  in verbose_linfo()
870 	if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) {  in print_insn_state()
872 		bpf_vlog_reset(&env->log, env->prev_log_pos - 1);  in print_insn_state()