1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 * Copyright (c) 2016 Facebook
4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
5 */
6 #include <uapi/linux/btf.h>
7 #include <linux/kernel.h>
8 #include <linux/types.h>
9 #include <linux/bpf.h>
10 #include <linux/bpf_verifier.h>
11 #include <linux/math64.h>
12 #include <linux/string.h>
13
14 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
15
bpf_verifier_log_attr_valid(u32 log_level,char __user * log_buf,u32 log_size)16 static bool bpf_verifier_log_attr_valid(u32 log_level, char __user *log_buf, u32 log_size)
17 {
18 /* ubuf and len_total should both be specified (or not) together */
19 if (!!log_buf != !!log_size)
20 return false;
21 /* log buf without log_level is meaningless */
22 if (log_buf && log_level == 0)
23 return false;
24 if (log_level & ~BPF_LOG_MASK)
25 return false;
26 if (log_size > UINT_MAX >> 2)
27 return false;
28 return true;
29 }
30
bpf_vlog_init(struct bpf_verifier_log * log,u32 log_level,char __user * log_buf,u32 log_size)31 int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,
32 char __user *log_buf, u32 log_size)
33 {
34 log->level = log_level;
35 log->ubuf = log_buf;
36 log->len_total = log_size;
37
38 /* log attributes have to be sane */
39 if (!bpf_verifier_log_attr_valid(log_level, log_buf, log_size))
40 return -EINVAL;
41
42 return 0;
43 }
44
bpf_vlog_update_len_max(struct bpf_verifier_log * log,u32 add_len)45 static void bpf_vlog_update_len_max(struct bpf_verifier_log *log, u32 add_len)
46 {
47 /* add_len includes terminal \0, so no need for +1. */
48 u64 len = log->end_pos + add_len;
49
50 /* log->len_max could be larger than our current len due to
51 * bpf_vlog_reset() calls, so we maintain the max of any length at any
52 * previous point
53 */
54 if (len > UINT_MAX)
55 log->len_max = UINT_MAX;
56 else if (len > log->len_max)
57 log->len_max = len;
58 }
59
bpf_verifier_vlog(struct bpf_verifier_log * log,const char * fmt,va_list args)60 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,
61 va_list args)
62 {
63 u64 cur_pos;
64 u32 new_n, n;
65
66 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);
67
68 if (log->level == BPF_LOG_KERNEL) {
69 bool newline = n > 0 && log->kbuf[n - 1] == '\n';
70
71 pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n");
72 return;
73 }
74
75 n += 1; /* include terminating zero */
76 bpf_vlog_update_len_max(log, n);
77
78 if (log->level & BPF_LOG_FIXED) {
79 /* check if we have at least something to put into user buf */
80 new_n = 0;
81 if (log->end_pos < log->len_total) {
82 new_n = min_t(u32, log->len_total - log->end_pos, n);
83 log->kbuf[new_n - 1] = '\0';
84 }
85
86 cur_pos = log->end_pos;
87 log->end_pos += n - 1; /* don't count terminating '\0' */
88
89 if (log->ubuf && new_n &&
90 copy_to_user(log->ubuf + cur_pos, log->kbuf, new_n))
91 goto fail;
92 } else {
93 u64 new_end, new_start;
94 u32 buf_start, buf_end;
95
96 new_end = log->end_pos + n;
97 if (new_end - log->start_pos >= log->len_total)
98 new_start = new_end - log->len_total;
99 else
100 new_start = log->start_pos;
101
102 log->start_pos = new_start;
103 log->end_pos = new_end - 1; /* don't count terminating '\0' */
104
105 if (!log->ubuf)
106 return;
107
108 new_n = min(n, log->len_total);
109 cur_pos = new_end - new_n;
110 div_u64_rem(cur_pos, log->len_total, &buf_start);
111 div_u64_rem(new_end, log->len_total, &buf_end);
112 /* new_end and buf_end are exclusive indices, so if buf_end is
113 * exactly zero, then it actually points right to the end of
114 * ubuf and there is no wrap around
115 */
116 if (buf_end == 0)
117 buf_end = log->len_total;
118
119 /* if buf_start > buf_end, we wrapped around;
120 * if buf_start == buf_end, then we fill ubuf completely; we
121 * can't have buf_start == buf_end to mean that there is
122 * nothing to write, because we always write at least
123 * something, even if terminal '\0'
124 */
125 if (buf_start < buf_end) {
126 /* message fits within contiguous chunk of ubuf */
127 if (copy_to_user(log->ubuf + buf_start,
128 log->kbuf + n - new_n,
129 buf_end - buf_start))
130 goto fail;
131 } else {
132 /* message wraps around the end of ubuf, copy in two chunks */
133 if (copy_to_user(log->ubuf + buf_start,
134 log->kbuf + n - new_n,
135 log->len_total - buf_start))
136 goto fail;
137 if (copy_to_user(log->ubuf,
138 log->kbuf + n - buf_end,
139 buf_end))
140 goto fail;
141 }
142 }
143
144 return;
145 fail:
146 log->ubuf = NULL;
147 }
148
bpf_vlog_reset(struct bpf_verifier_log * log,u64 new_pos)149 void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos)
150 {
151 char zero = 0;
152 u32 pos;
153
154 if (WARN_ON_ONCE(new_pos > log->end_pos))
155 return;
156
157 if (!bpf_verifier_log_needed(log) || log->level == BPF_LOG_KERNEL)
158 return;
159
160 /* if position to which we reset is beyond current log window,
161 * then we didn't preserve any useful content and should adjust
162 * start_pos to end up with an empty log (start_pos == end_pos)
163 */
164 log->end_pos = new_pos;
165 if (log->end_pos < log->start_pos)
166 log->start_pos = log->end_pos;
167
168 if (!log->ubuf)
169 return;
170
171 if (log->level & BPF_LOG_FIXED)
172 pos = log->end_pos + 1;
173 else
174 div_u64_rem(new_pos, log->len_total, &pos);
175
176 if (pos < log->len_total && put_user(zero, log->ubuf + pos))
177 log->ubuf = NULL;
178 }
179
bpf_vlog_reverse_kbuf(char * buf,int len)180 static void bpf_vlog_reverse_kbuf(char *buf, int len)
181 {
182 int i, j;
183
184 for (i = 0, j = len - 1; i < j; i++, j--)
185 swap(buf[i], buf[j]);
186 }
187
bpf_vlog_reverse_ubuf(struct bpf_verifier_log * log,int start,int end)188 static int bpf_vlog_reverse_ubuf(struct bpf_verifier_log *log, int start, int end)
189 {
190 /* we split log->kbuf into two equal parts for both ends of array */
191 int n = sizeof(log->kbuf) / 2, nn;
192 char *lbuf = log->kbuf, *rbuf = log->kbuf + n;
193
194 /* Read ubuf's section [start, end) two chunks at a time, from left
195 * and right side; within each chunk, swap all the bytes; after that
196 * reverse the order of lbuf and rbuf and write result back to ubuf.
197 * This way we'll end up with swapped contents of specified
198 * [start, end) ubuf segment.
199 */
200 while (end - start > 1) {
201 nn = min(n, (end - start ) / 2);
202
203 if (copy_from_user(lbuf, log->ubuf + start, nn))
204 return -EFAULT;
205 if (copy_from_user(rbuf, log->ubuf + end - nn, nn))
206 return -EFAULT;
207
208 bpf_vlog_reverse_kbuf(lbuf, nn);
209 bpf_vlog_reverse_kbuf(rbuf, nn);
210
211 /* we write lbuf to the right end of ubuf, while rbuf to the
212 * left one to end up with properly reversed overall ubuf
213 */
214 if (copy_to_user(log->ubuf + start, rbuf, nn))
215 return -EFAULT;
216 if (copy_to_user(log->ubuf + end - nn, lbuf, nn))
217 return -EFAULT;
218
219 start += nn;
220 end -= nn;
221 }
222
223 return 0;
224 }
225
bpf_vlog_finalize(struct bpf_verifier_log * log,u32 * log_size_actual)226 int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual)
227 {
228 u32 sublen;
229 int err;
230
231 *log_size_actual = 0;
232 if (!log || log->level == 0 || log->level == BPF_LOG_KERNEL)
233 return 0;
234
235 if (!log->ubuf)
236 goto skip_log_rotate;
237 /* If we never truncated log, there is nothing to move around. */
238 if (log->start_pos == 0)
239 goto skip_log_rotate;
240
241 /* Otherwise we need to rotate log contents to make it start from the
242 * buffer beginning and be a continuous zero-terminated string. Note
243 * that if log->start_pos != 0 then we definitely filled up entire log
244 * buffer with no gaps, and we just need to shift buffer contents to
245 * the left by (log->start_pos % log->len_total) bytes.
246 *
247 * Unfortunately, user buffer could be huge and we don't want to
248 * allocate temporary kernel memory of the same size just to shift
249 * contents in a straightforward fashion. Instead, we'll be clever and
250 * do in-place array rotation. This is a leetcode-style problem, which
251 * could be solved by three rotations.
252 *
253 * Let's say we have log buffer that has to be shifted left by 7 bytes
254 * (spaces and vertical bar is just for demonstrative purposes):
255 * E F G H I J K | A B C D
256 *
257 * First, we reverse entire array:
258 * D C B A | K J I H G F E
259 *
260 * Then we rotate first 4 bytes (DCBA) and separately last 7 bytes
261 * (KJIHGFE), resulting in a properly rotated array:
262 * A B C D | E F G H I J K
263 *
264 * We'll utilize log->kbuf to read user memory chunk by chunk, swap
265 * bytes, and write them back. Doing it byte-by-byte would be
266 * unnecessarily inefficient. Altogether we are going to read and
267 * write each byte twice, for total 4 memory copies between kernel and
268 * user space.
269 */
270
271 /* length of the chopped off part that will be the beginning;
272 * len(ABCD) in the example above
273 */
274 div_u64_rem(log->start_pos, log->len_total, &sublen);
275 sublen = log->len_total - sublen;
276
277 err = bpf_vlog_reverse_ubuf(log, 0, log->len_total);
278 err = err ?: bpf_vlog_reverse_ubuf(log, 0, sublen);
279 err = err ?: bpf_vlog_reverse_ubuf(log, sublen, log->len_total);
280 if (err)
281 log->ubuf = NULL;
282
283 skip_log_rotate:
284 *log_size_actual = log->len_max;
285
286 /* properly initialized log has either both ubuf!=NULL and len_total>0
287 * or ubuf==NULL and len_total==0, so if this condition doesn't hold,
288 * we got a fault somewhere along the way, so report it back
289 */
290 if (!!log->ubuf != !!log->len_total)
291 return -EFAULT;
292
293 /* did truncation actually happen? */
294 if (log->ubuf && log->len_max > log->len_total)
295 return -ENOSPC;
296
297 return 0;
298 }
299
300 /* log_level controls verbosity level of eBPF verifier.
301 * bpf_verifier_log_write() is used to dump the verification trace to the log,
302 * so the user can figure out what's wrong with the program
303 */
bpf_verifier_log_write(struct bpf_verifier_env * env,const char * fmt,...)304 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
305 const char *fmt, ...)
306 {
307 va_list args;
308
309 if (!bpf_verifier_log_needed(&env->log))
310 return;
311
312 va_start(args, fmt);
313 bpf_verifier_vlog(&env->log, fmt, args);
314 va_end(args);
315 }
316 EXPORT_SYMBOL_GPL(bpf_verifier_log_write);
317
bpf_log(struct bpf_verifier_log * log,const char * fmt,...)318 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
319 const char *fmt, ...)
320 {
321 va_list args;
322
323 if (!bpf_verifier_log_needed(log))
324 return;
325
326 va_start(args, fmt);
327 bpf_verifier_vlog(log, fmt, args);
328 va_end(args);
329 }
330 EXPORT_SYMBOL_GPL(bpf_log);
331
ltrim(const char * s)332 static const char *ltrim(const char *s)
333 {
334 while (isspace(*s))
335 s++;
336
337 return s;
338 }
339
verbose_linfo(struct bpf_verifier_env * env,u32 insn_off,const char * prefix_fmt,...)340 __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
341 u32 insn_off,
342 const char *prefix_fmt, ...)
343 {
344 const struct bpf_line_info *linfo, *prev_linfo;
345 const struct btf *btf;
346 const char *s, *fname;
347
348 if (!bpf_verifier_log_needed(&env->log))
349 return;
350
351 prev_linfo = env->prev_linfo;
352 linfo = bpf_find_linfo(env->prog, insn_off);
353 if (!linfo || linfo == prev_linfo)
354 return;
355
356 /* It often happens that two separate linfo records point to the same
357 * source code line, but have differing column numbers. Given verifier
358 * log doesn't emit column information, from user perspective we just
359 * end up emitting the same source code line twice unnecessarily.
360 * So instead check that previous and current linfo record point to
361 * the same file (file_name_offs match) and the same line number, and
362 * avoid emitting duplicated source code line in such case.
363 */
364 if (prev_linfo && linfo->file_name_off == prev_linfo->file_name_off &&
365 BPF_LINE_INFO_LINE_NUM(linfo->line_col) == BPF_LINE_INFO_LINE_NUM(prev_linfo->line_col))
366 return;
367
368 if (prefix_fmt) {
369 va_list args;
370
371 va_start(args, prefix_fmt);
372 bpf_verifier_vlog(&env->log, prefix_fmt, args);
373 va_end(args);
374 }
375
376 btf = env->prog->aux->btf;
377 s = ltrim(btf_name_by_offset(btf, linfo->line_off));
378 verbose(env, "%s", s); /* source code line */
379
380 s = btf_name_by_offset(btf, linfo->file_name_off);
381 /* leave only file name */
382 fname = strrchr(s, '/');
383 fname = fname ? fname + 1 : s;
384 verbose(env, " @ %s:%u\n", fname, BPF_LINE_INFO_LINE_NUM(linfo->line_col));
385
386 env->prev_linfo = linfo;
387 }
388
btf_type_name(const struct btf * btf,u32 id)389 static const char *btf_type_name(const struct btf *btf, u32 id)
390 {
391 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
392 }
393
394 /* string representation of 'enum bpf_reg_type'
395 *
396 * Note that reg_type_str() can not appear more than once in a single verbose()
397 * statement.
398 */
reg_type_str(struct bpf_verifier_env * env,enum bpf_reg_type type)399 const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type)
400 {
401 char postfix[16] = {0}, prefix[64] = {0};
402 static const char * const str[] = {
403 [NOT_INIT] = "?",
404 [SCALAR_VALUE] = "scalar",
405 [PTR_TO_CTX] = "ctx",
406 [CONST_PTR_TO_MAP] = "map_ptr",
407 [PTR_TO_MAP_VALUE] = "map_value",
408 [PTR_TO_STACK] = "fp",
409 [PTR_TO_PACKET] = "pkt",
410 [PTR_TO_PACKET_META] = "pkt_meta",
411 [PTR_TO_PACKET_END] = "pkt_end",
412 [PTR_TO_FLOW_KEYS] = "flow_keys",
413 [PTR_TO_SOCKET] = "sock",
414 [PTR_TO_SOCK_COMMON] = "sock_common",
415 [PTR_TO_TCP_SOCK] = "tcp_sock",
416 [PTR_TO_TP_BUFFER] = "tp_buffer",
417 [PTR_TO_XDP_SOCK] = "xdp_sock",
418 [PTR_TO_BTF_ID] = "ptr_",
419 [PTR_TO_MEM] = "mem",
420 [PTR_TO_ARENA] = "arena",
421 [PTR_TO_BUF] = "buf",
422 [PTR_TO_FUNC] = "func",
423 [PTR_TO_INSN] = "insn",
424 [PTR_TO_MAP_KEY] = "map_key",
425 [CONST_PTR_TO_DYNPTR] = "dynptr_ptr",
426 };
427
428 if (type & PTR_MAYBE_NULL) {
429 if (base_type(type) == PTR_TO_BTF_ID)
430 strscpy(postfix, "or_null_");
431 else
432 strscpy(postfix, "_or_null");
433 }
434
435 snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s",
436 type & MEM_RDONLY ? "rdonly_" : "",
437 type & MEM_RINGBUF ? "ringbuf_" : "",
438 type & MEM_USER ? "user_" : "",
439 type & MEM_PERCPU ? "percpu_" : "",
440 type & MEM_RCU ? "rcu_" : "",
441 type & PTR_UNTRUSTED ? "untrusted_" : "",
442 type & PTR_TRUSTED ? "trusted_" : ""
443 );
444
445 snprintf(env->tmp_str_buf, TMP_STR_BUF_LEN, "%s%s%s",
446 prefix, str[base_type(type)], postfix);
447 return env->tmp_str_buf;
448 }
449
dynptr_type_str(enum bpf_dynptr_type type)450 const char *dynptr_type_str(enum bpf_dynptr_type type)
451 {
452 switch (type) {
453 case BPF_DYNPTR_TYPE_LOCAL:
454 return "local";
455 case BPF_DYNPTR_TYPE_RINGBUF:
456 return "ringbuf";
457 case BPF_DYNPTR_TYPE_SKB:
458 return "skb";
459 case BPF_DYNPTR_TYPE_XDP:
460 return "xdp";
461 case BPF_DYNPTR_TYPE_SKB_META:
462 return "skb_meta";
463 case BPF_DYNPTR_TYPE_FILE:
464 return "file";
465 case BPF_DYNPTR_TYPE_INVALID:
466 return "<invalid>";
467 default:
468 WARN_ONCE(1, "unknown dynptr type %d\n", type);
469 return "<unknown>";
470 }
471 }
472
iter_type_str(const struct btf * btf,u32 btf_id)473 const char *iter_type_str(const struct btf *btf, u32 btf_id)
474 {
475 if (!btf || btf_id == 0)
476 return "<invalid>";
477
478 /* we already validated that type is valid and has conforming name */
479 return btf_type_name(btf, btf_id) + sizeof(ITER_PREFIX) - 1;
480 }
481
iter_state_str(enum bpf_iter_state state)482 const char *iter_state_str(enum bpf_iter_state state)
483 {
484 switch (state) {
485 case BPF_ITER_STATE_ACTIVE:
486 return "active";
487 case BPF_ITER_STATE_DRAINED:
488 return "drained";
489 case BPF_ITER_STATE_INVALID:
490 return "<invalid>";
491 default:
492 WARN_ONCE(1, "unknown iter state %d\n", state);
493 return "<unknown>";
494 }
495 }
496
497 static char slot_type_char[] = {
498 [STACK_INVALID] = '?',
499 [STACK_SPILL] = 'r',
500 [STACK_MISC] = 'm',
501 [STACK_ZERO] = '0',
502 [STACK_DYNPTR] = 'd',
503 [STACK_ITER] = 'i',
504 [STACK_IRQ_FLAG] = 'f',
505 [STACK_POISON] = 'p',
506 };
507
508 #define UNUM_MAX_DECIMAL U16_MAX
509 #define SNUM_MAX_DECIMAL S16_MAX
510 #define SNUM_MIN_DECIMAL S16_MIN
511
is_unum_decimal(u64 num)512 static bool is_unum_decimal(u64 num)
513 {
514 return num <= UNUM_MAX_DECIMAL;
515 }
516
is_snum_decimal(s64 num)517 static bool is_snum_decimal(s64 num)
518 {
519 return num >= SNUM_MIN_DECIMAL && num <= SNUM_MAX_DECIMAL;
520 }
521
verbose_unum(struct bpf_verifier_env * env,u64 num)522 static void verbose_unum(struct bpf_verifier_env *env, u64 num)
523 {
524 if (is_unum_decimal(num))
525 verbose(env, "%llu", num);
526 else
527 verbose(env, "%#llx", num);
528 }
529
verbose_snum(struct bpf_verifier_env * env,s64 num)530 static void verbose_snum(struct bpf_verifier_env *env, s64 num)
531 {
532 if (is_snum_decimal(num))
533 verbose(env, "%lld", num);
534 else
535 verbose(env, "%#llx", num);
536 }
537
tnum_strn(char * str,size_t size,struct tnum a)538 int tnum_strn(char *str, size_t size, struct tnum a)
539 {
540 /* print as a constant, if tnum is fully known */
541 if (a.mask == 0) {
542 if (is_unum_decimal(a.value))
543 return snprintf(str, size, "%llu", a.value);
544 if (is_snum_decimal(a.value))
545 return snprintf(str, size, "%lld", a.value);
546 else
547 return snprintf(str, size, "%#llx", a.value);
548 }
549 return snprintf(str, size, "(%#llx; %#llx)", a.value, a.mask);
550 }
551 EXPORT_SYMBOL_GPL(tnum_strn);
552
print_scalar_ranges(struct bpf_verifier_env * env,const struct bpf_reg_state * reg,const char ** sep)553 static void print_scalar_ranges(struct bpf_verifier_env *env,
554 const struct bpf_reg_state *reg,
555 const char **sep)
556 {
557 /* For signed ranges, we want to unify 64-bit and 32-bit values in the
558 * output as much as possible, but there is a bit of a complication.
559 * If we choose to print values as decimals, this is natural to do,
560 * because negative 64-bit and 32-bit values >= -S32_MIN have the same
561 * representation due to sign extension. But if we choose to print
562 * them in hex format (see is_snum_decimal()), then sign extension is
563 * misleading.
564 * E.g., smin=-2 and smin32=-2 are exactly the same in decimal, but in
565 * hex they will be smin=0xfffffffffffffffe and smin32=0xfffffffe, two
566 * very different numbers.
567 * So we avoid sign extension if we choose to print values in hex.
568 */
569 struct {
570 const char *name;
571 u64 val;
572 bool omit;
573 } minmaxs[] = {
574 {"smin", reg_smin(reg), reg_smin(reg) == S64_MIN},
575 {"smax", reg_smax(reg), reg_smax(reg) == S64_MAX},
576 {"umin", reg_umin(reg), reg_umin(reg) == 0},
577 {"umax", reg_umax(reg), reg_umax(reg) == U64_MAX},
578 {"smin32",
579 is_snum_decimal((s64)reg_s32_min(reg))
580 ? (s64)reg_s32_min(reg)
581 : (u32)reg_s32_min(reg), reg_s32_min(reg) == S32_MIN},
582 {"smax32",
583 is_snum_decimal((s64)reg_s32_max(reg))
584 ? (s64)reg_s32_max(reg)
585 : (u32)reg_s32_max(reg), reg_s32_max(reg) == S32_MAX},
586 {"umin32", reg_u32_min(reg), reg_u32_min(reg) == 0},
587 {"umax32", reg_u32_max(reg), reg_u32_max(reg) == U32_MAX},
588 }, *m1, *m2, *mend = &minmaxs[ARRAY_SIZE(minmaxs)];
589 bool neg1, neg2;
590
591 for (m1 = &minmaxs[0]; m1 < mend; m1++) {
592 if (m1->omit)
593 continue;
594
595 neg1 = m1->name[0] == 's' && (s64)m1->val < 0;
596
597 verbose(env, "%s%s=", *sep, m1->name);
598 *sep = ",";
599
600 for (m2 = m1 + 2; m2 < mend; m2 += 2) {
601 if (m2->omit || m2->val != m1->val)
602 continue;
603 /* don't mix negatives with positives */
604 neg2 = m2->name[0] == 's' && (s64)m2->val < 0;
605 if (neg2 != neg1)
606 continue;
607 m2->omit = true;
608 verbose(env, "%s=", m2->name);
609 }
610
611 if (m1->name[0] == 's')
612 verbose_snum(env, m1->val);
613 else
614 verbose_unum(env, m1->val);
615 }
616 }
617
618 /*
619 * _a stands for append, was shortened to avoid multiline statements below.
620 * This macro is used to output a comma separated list of attributes.
621 */
622 #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, ##__VA_ARGS__); sep = ","; })
623
print_reg_state(struct bpf_verifier_env * env,const struct bpf_func_state * state,const struct bpf_reg_state * reg)624 static void print_reg_state(struct bpf_verifier_env *env,
625 const struct bpf_func_state *state,
626 const struct bpf_reg_state *reg)
627 {
628 enum bpf_reg_type t;
629 const char *sep = "";
630
631 t = reg->type;
632 if (t == SCALAR_VALUE && reg->precise)
633 verbose(env, "P");
634 if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) {
635 verbose_snum(env, reg->var_off.value);
636 return;
637 }
638
639 verbose(env, "%s", reg_type_str(env, t));
640 if (t == PTR_TO_ARENA)
641 return;
642 if (t == PTR_TO_STACK) {
643 if (state->frameno != reg->frameno)
644 verbose(env, "[%d]", reg->frameno);
645 if (tnum_is_const(reg->var_off)) {
646 verbose_snum(env, reg->var_off.value + reg->delta);
647 return;
648 }
649 }
650 if (base_type(t) == PTR_TO_BTF_ID)
651 verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id));
652 verbose(env, "(");
653 if (reg->id)
654 verbose_a("id=%d", reg->id & ~BPF_ADD_CONST);
655 if (reg->id & BPF_ADD_CONST)
656 verbose(env, "%+d", reg->delta);
657 if (reg->parent_id)
658 verbose_a("parent_id=%d", reg->parent_id);
659 if (type_is_non_owning_ref(reg->type))
660 verbose_a("%s", "non_own_ref");
661 if (type_is_map_ptr(t)) {
662 if (reg->map_ptr->name[0])
663 verbose_a("map=%s", reg->map_ptr->name);
664 verbose_a("ks=%d,vs=%d",
665 reg->map_ptr->key_size,
666 reg->map_ptr->value_size);
667 }
668 if (t != SCALAR_VALUE && reg->delta) {
669 verbose_a("off=");
670 verbose_snum(env, reg->delta);
671 }
672 if (type_is_pkt_pointer(t)) {
673 verbose_a("r=");
674 verbose_unum(env, reg->range);
675 }
676 if (base_type(t) == PTR_TO_MEM) {
677 verbose_a("sz=");
678 verbose_unum(env, reg->mem_size);
679 }
680 if (t == CONST_PTR_TO_DYNPTR)
681 verbose_a("type=%s", dynptr_type_str(reg->dynptr.type));
682 if (tnum_is_const(reg->var_off)) {
683 /* a pointer register with fixed offset */
684 if (reg->var_off.value) {
685 verbose_a("imm=");
686 verbose_snum(env, reg->var_off.value);
687 }
688 } else {
689 print_scalar_ranges(env, reg, &sep);
690 if (!tnum_is_unknown(reg->var_off)) {
691 char tn_buf[48];
692
693 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
694 verbose_a("var_off=%s", tn_buf);
695 }
696 }
697 verbose(env, ")");
698 }
699
print_verifier_state(struct bpf_verifier_env * env,const struct bpf_verifier_state * vstate,u32 frameno,bool print_all)700 void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,
701 u32 frameno, bool print_all)
702 {
703 const struct bpf_func_state *state = vstate->frame[frameno];
704 const struct bpf_reg_state *reg;
705 int i;
706
707 if (state->frameno)
708 verbose(env, " frame%d:", state->frameno);
709 for (i = 0; i < MAX_BPF_REG; i++) {
710 reg = &state->regs[i];
711 if (reg->type == NOT_INIT)
712 continue;
713 if (!print_all && !reg_scratched(env, i))
714 continue;
715 verbose(env, " R%d", i);
716 verbose(env, "=");
717 print_reg_state(env, state, reg);
718 }
719 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
720 char types_buf[BPF_REG_SIZE + 1];
721 const char *sep = "";
722 bool valid = false;
723 u8 slot_type;
724 int j;
725
726 if (!print_all && !stack_slot_scratched(env, i))
727 continue;
728
729 for (j = 0; j < BPF_REG_SIZE; j++) {
730 slot_type = state->stack[i].slot_type[j];
731 if (slot_type != STACK_INVALID && slot_type != STACK_POISON)
732 valid = true;
733 types_buf[j] = slot_type_char[slot_type];
734 }
735 types_buf[BPF_REG_SIZE] = 0;
736 if (!valid)
737 continue;
738
739 reg = &state->stack[i].spilled_ptr;
740 switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) {
741 case STACK_SPILL:
742 /* print MISC/ZERO/INVALID slots above subreg spill */
743 for (j = 0; j < BPF_REG_SIZE; j++)
744 if (state->stack[i].slot_type[j] == STACK_SPILL)
745 break;
746 types_buf[j] = '\0';
747
748 verbose(env, " fp%d=%s", (-i - 1) * BPF_REG_SIZE, types_buf);
749 print_reg_state(env, state, reg);
750 break;
751 case STACK_DYNPTR:
752 /* skip to main dynptr slot */
753 i += BPF_DYNPTR_NR_SLOTS - 1;
754 reg = &state->stack[i].spilled_ptr;
755
756 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
757 verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type));
758 if (reg->id)
759 verbose_a("id=%d", reg->id);
760 if (reg->parent_id)
761 verbose_a("parent_id=%d", reg->parent_id);
762 verbose(env, ")");
763 break;
764 case STACK_ITER:
765 /* only main slot has id set; skip others */
766 if (!reg->id)
767 continue;
768
769 verbose(env, " fp%d=iter_%s(id=%d,state=%s,depth=%u)",
770 (-i - 1) * BPF_REG_SIZE,
771 iter_type_str(reg->iter.btf, reg->iter.btf_id),
772 reg->id, iter_state_str(reg->iter.state),
773 reg->iter.depth);
774 break;
775 case STACK_MISC:
776 case STACK_ZERO:
777 default:
778 verbose(env, " fp%d=%s", (-i - 1) * BPF_REG_SIZE, types_buf);
779 break;
780 }
781 }
782 if (vstate->acquired_refs && vstate->refs[0].id) {
783 verbose(env, " refs=%d", vstate->refs[0].id);
784 for (i = 1; i < vstate->acquired_refs; i++)
785 if (vstate->refs[i].id)
786 verbose(env, ",%d", vstate->refs[i].id);
787 }
788 if (state->in_callback_fn)
789 verbose(env, " cb");
790 if (state->in_async_callback_fn)
791 verbose(env, " async_cb");
792 verbose(env, "\n");
793 if (!print_all)
794 mark_verifier_state_clean(env);
795 }
796
bpf_vlog_alignment(u32 pos)797 u32 bpf_vlog_alignment(u32 pos)
798 {
799 return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT),
800 BPF_LOG_MIN_ALIGNMENT) - pos - 1;
801 }
802
print_insn_state(struct bpf_verifier_env * env,const struct bpf_verifier_state * vstate,u32 frameno)803 void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,
804 u32 frameno)
805 {
806 if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) {
807 /* remove new line character */
808 bpf_vlog_reset(&env->log, env->prev_log_pos - 1);
809 verbose(env, "%*c;", bpf_vlog_alignment(env->prev_insn_print_pos), ' ');
810 } else {
811 verbose(env, "%d:", env->insn_idx);
812 }
813 print_verifier_state(env, vstate, frameno, false);
814 }
815
bpf_log_attr_init(struct bpf_log_attr * log,u64 log_buf,u32 log_size,u32 log_level,u32 offsetof_log_true_size,bpfptr_t uattr,struct bpf_common_attr * common,bpfptr_t uattr_common,u32 size_common)816 int bpf_log_attr_init(struct bpf_log_attr *log, u64 log_buf, u32 log_size, u32 log_level,
817 u32 offsetof_log_true_size, bpfptr_t uattr, struct bpf_common_attr *common,
818 bpfptr_t uattr_common, u32 size_common)
819 {
820 char __user *ubuf_common = u64_to_user_ptr(common->log_buf);
821 char __user *ubuf = u64_to_user_ptr(log_buf);
822
823 if (!bpf_verifier_log_attr_valid(common->log_level, ubuf_common, common->log_size) ||
824 !bpf_verifier_log_attr_valid(log_level, ubuf, log_size))
825 return -EINVAL;
826
827 if (ubuf && ubuf_common && (ubuf != ubuf_common || log_size != common->log_size ||
828 log_level != common->log_level))
829 return -EINVAL;
830
831 memset(log, 0, sizeof(*log));
832 log->ubuf = ubuf;
833 log->size = log_size;
834 log->level = log_level;
835 log->offsetof_true_size = offsetof_log_true_size;
836 log->uattr = uattr;
837
838 if (!ubuf && ubuf_common) {
839 log->ubuf = ubuf_common;
840 log->size = common->log_size;
841 log->level = common->log_level;
842 log->uattr = uattr_common;
843 log->offsetof_true_size = 0;
844 if (size_common >= offsetofend(struct bpf_common_attr, log_true_size))
845 log->offsetof_true_size = offsetof(struct bpf_common_attr, log_true_size);
846 }
847 return 0;
848 }
849
bpf_log_attr_create_vlog(struct bpf_log_attr * attr_log,struct bpf_common_attr * common,bpfptr_t uattr,u32 size)850 struct bpf_verifier_log *bpf_log_attr_create_vlog(struct bpf_log_attr *attr_log,
851 struct bpf_common_attr *common, bpfptr_t uattr,
852 u32 size)
853 {
854 struct bpf_verifier_log *log;
855 int err;
856
857 memset(attr_log, 0, sizeof(*attr_log));
858 attr_log->uattr = uattr;
859 if (size >= offsetofend(struct bpf_common_attr, log_true_size))
860 attr_log->offsetof_true_size = offsetof(struct bpf_common_attr, log_true_size);
861
862 if (!size)
863 return NULL;
864
865 log = kzalloc_obj(*log, GFP_KERNEL);
866 if (!log)
867 return ERR_PTR(-ENOMEM);
868
869 err = bpf_vlog_init(log, common->log_level, u64_to_user_ptr(common->log_buf),
870 common->log_size);
871 if (err) {
872 kfree(log);
873 return ERR_PTR(err);
874 }
875
876 return log;
877 }
878
bpf_log_attr_finalize(struct bpf_log_attr * attr,struct bpf_verifier_log * log)879 int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *log)
880 {
881 u32 log_true_size;
882 int err;
883
884 err = bpf_vlog_finalize(log, &log_true_size);
885
886 if (attr->offsetof_true_size &&
887 copy_to_bpfptr_offset(attr->uattr, attr->offsetof_true_size, &log_true_size,
888 sizeof(log_true_size)))
889 return -EFAULT;
890
891 return err;
892 }
893