xref: /linux/kernel/bpf/verifier.c (revision 9410645520e9b820069761f3450ef6661418e279)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3  * Copyright (c) 2016 Facebook
4  * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
5  */
6 #include <uapi/linux/btf.h>
7 #include <linux/bpf-cgroup.h>
8 #include <linux/kernel.h>
9 #include <linux/types.h>
10 #include <linux/slab.h>
11 #include <linux/bpf.h>
12 #include <linux/btf.h>
13 #include <linux/bpf_verifier.h>
14 #include <linux/filter.h>
15 #include <net/netlink.h>
16 #include <linux/file.h>
17 #include <linux/vmalloc.h>
18 #include <linux/stringify.h>
19 #include <linux/bsearch.h>
20 #include <linux/sort.h>
21 #include <linux/perf_event.h>
22 #include <linux/ctype.h>
23 #include <linux/error-injection.h>
24 #include <linux/bpf_lsm.h>
25 #include <linux/btf_ids.h>
26 #include <linux/poison.h>
27 #include <linux/module.h>
28 #include <linux/cpumask.h>
29 #include <linux/bpf_mem_alloc.h>
30 #include <net/xdp.h>
31 #include <linux/trace_events.h>
32 #include <linux/kallsyms.h>
33 
34 #include "disasm.h"
35 
36 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
37 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
38 	[_id] = & _name ## _verifier_ops,
39 #define BPF_MAP_TYPE(_id, _ops)
40 #define BPF_LINK_TYPE(_id, _name)
41 #include <linux/bpf_types.h>
42 #undef BPF_PROG_TYPE
43 #undef BPF_MAP_TYPE
44 #undef BPF_LINK_TYPE
45 };
46 
47 struct bpf_mem_alloc bpf_global_percpu_ma;
48 static bool bpf_global_percpu_ma_set;
49 
50 /* bpf_check() is a static code analyzer that walks eBPF program
51  * instruction by instruction and updates register/stack state.
52  * All paths of conditional branches are analyzed until 'bpf_exit' insn.
53  *
54  * The first pass is depth-first-search to check that the program is a DAG.
55  * It rejects the following programs:
56  * - larger than BPF_MAXINSNS insns
57  * - if loop is present (detected via back-edge)
58  * - unreachable insns exist (shouldn't be a forest. program = one function)
59  * - out of bounds or malformed jumps
60  * The second pass is all possible path descent from the 1st insn.
61  * Since it's analyzing all paths through the program, the length of the
62  * analysis is limited to 64k insn, which may be hit even if total number of
63  * insn is less then 4K, but there are too many branches that change stack/regs.
64  * Number of 'branches to be analyzed' is limited to 1k
65  *
66  * On entry to each instruction, each register has a type, and the instruction
67  * changes the types of the registers depending on instruction semantics.
68  * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is
69  * copied to R1.
70  *
71  * All registers are 64-bit.
72  * R0 - return register
73  * R1-R5 argument passing registers
74  * R6-R9 callee saved registers
75  * R10 - frame pointer read-only
76  *
77  * At the start of BPF program the register R1 contains a pointer to bpf_context
78  * and has type PTR_TO_CTX.
79  *
80  * Verifier tracks arithmetic operations on pointers in case:
81  *    BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
82  *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20),
83  * 1st insn copies R10 (which has FRAME_PTR) type into R1
84  * and 2nd arithmetic instruction is pattern matched to recognize
85  * that it wants to construct a pointer to some element within stack.
86  * So after 2nd insn, the register R1 has type PTR_TO_STACK
87  * (and -20 constant is saved for further stack bounds checking).
88  * Meaning that this reg is a pointer to stack plus known immediate constant.
89  *
90  * Most of the time the registers have SCALAR_VALUE type, which
91  * means the register has some value, but it's not a valid pointer.
92  * (like pointer plus pointer becomes SCALAR_VALUE type)
93  *
94  * When verifier sees load or store instructions the type of base register
95  * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are
96  * four pointer types recognized by check_mem_access() function.
97  *
98  * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value'
99  * and the range of [ptr, ptr + map's value_size) is accessible.
100  *
101  * registers used to pass values to function calls are checked against
102  * function argument constraints.
103  *
104  * ARG_PTR_TO_MAP_KEY is one of such argument constraints.
105  * It means that the register type passed to this function must be
106  * PTR_TO_STACK and it will be used inside the function as
107  * 'pointer to map element key'
108  *
109  * For example the argument constraints for bpf_map_lookup_elem():
110  *   .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
111  *   .arg1_type = ARG_CONST_MAP_PTR,
112  *   .arg2_type = ARG_PTR_TO_MAP_KEY,
113  *
114  * ret_type says that this function returns 'pointer to map elem value or null'
115  * function expects 1st argument to be a const pointer to 'struct bpf_map' and
116  * 2nd argument should be a pointer to stack, which will be used inside
117  * the helper function as a pointer to map element key.
118  *
119  * On the kernel side the helper function looks like:
120  * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
121  * {
122  *    struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;
123  *    void *key = (void *) (unsigned long) r2;
124  *    void *value;
125  *
126  *    here kernel can access 'key' and 'map' pointers safely, knowing that
127  *    [key, key + map->key_size) bytes are valid and were initialized on
128  *    the stack of eBPF program.
129  * }
130  *
131  * Corresponding eBPF program may look like:
132  *    BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),  // after this insn R2 type is FRAME_PTR
133  *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK
134  *    BPF_LD_MAP_FD(BPF_REG_1, map_fd),      // after this insn R1 type is CONST_PTR_TO_MAP
135  *    BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
136  * here verifier looks at prototype of map_lookup_elem() and sees:
137  * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok,
138  * Now verifier knows that this map has key of R1->map_ptr->key_size bytes
139  *
140  * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far,
141  * Now verifier checks that [R2, R2 + map's key_size) are within stack limits
142  * and were initialized prior to this call.
143  * If it's ok, then verifier allows this BPF_CALL insn and looks at
144  * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets
145  * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function
146  * returns either pointer to map value or NULL.
147  *
148  * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off'
149  * insn, the register holding that pointer in the true branch changes state to
150  * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false
151  * branch. See check_cond_jmp_op().
152  *
153  * After the call R0 is set to return type of the function and registers R1-R5
154  * are set to NOT_INIT to indicate that they are no longer readable.
155  *
156  * The following reference types represent a potential reference to a kernel
157  * resource which, after first being allocated, must be checked and freed by
158  * the BPF program:
159  * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET
160  *
161  * When the verifier sees a helper call return a reference type, it allocates a
162  * pointer id for the reference and stores it in the current function state.
163  * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into
164  * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type
165  * passes through a NULL-check conditional. For the branch wherein the state is
166  * changed to CONST_IMM, the verifier releases the reference.
167  *
168  * For each helper function that allocates a reference, such as
169  * bpf_sk_lookup_tcp(), there is a corresponding release function, such as
170  * bpf_sk_release(). When a reference type passes into the release function,
171  * the verifier also releases the reference. If any unchecked or unreleased
172  * reference remains at the end of the program, the verifier rejects it.
173  */
174 
175 /* verifier_state + insn_idx are pushed to stack when branch is encountered */
176 struct bpf_verifier_stack_elem {
177 	/* verifier state is 'st'
178 	 * before processing instruction 'insn_idx'
179 	 * and after processing instruction 'prev_insn_idx'
180 	 */
181 	struct bpf_verifier_state st;
182 	int insn_idx;
183 	int prev_insn_idx;
184 	struct bpf_verifier_stack_elem *next;
185 	/* length of verifier log at the time this state was pushed on stack */
186 	u32 log_pos;
187 };
188 
189 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ	8192
190 #define BPF_COMPLEXITY_LIMIT_STATES	64
191 
192 #define BPF_MAP_KEY_POISON	(1ULL << 63)
193 #define BPF_MAP_KEY_SEEN	(1ULL << 62)
194 
195 #define BPF_GLOBAL_PERCPU_MA_MAX_SIZE  512
196 
197 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);
198 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
199 static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
200 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);
201 static int ref_set_non_owning(struct bpf_verifier_env *env,
202 			      struct bpf_reg_state *reg);
203 static void specialize_kfunc(struct bpf_verifier_env *env,
204 			     u32 func_id, u16 offset, unsigned long *addr);
205 static bool is_trusted_reg(const struct bpf_reg_state *reg);
206 
207 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
208 {
209 	return aux->map_ptr_state.poison;
210 }
211 
212 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
213 {
214 	return aux->map_ptr_state.unpriv;
215 }
216 
217 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
218 			      struct bpf_map *map,
219 			      bool unpriv, bool poison)
220 {
221 	unpriv |= bpf_map_ptr_unpriv(aux);
222 	aux->map_ptr_state.unpriv = unpriv;
223 	aux->map_ptr_state.poison = poison;
224 	aux->map_ptr_state.map_ptr = map;
225 }
226 
227 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux)
228 {
229 	return aux->map_key_state & BPF_MAP_KEY_POISON;
230 }
231 
232 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux)
233 {
234 	return !(aux->map_key_state & BPF_MAP_KEY_SEEN);
235 }
236 
237 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)
238 {
239 	return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);
240 }
241 
242 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state)
243 {
244 	bool poisoned = bpf_map_key_poisoned(aux);
245 
246 	aux->map_key_state = state | BPF_MAP_KEY_SEEN |
247 			     (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
248 }
249 
250 static bool bpf_helper_call(const struct bpf_insn *insn)
251 {
252 	return insn->code == (BPF_JMP | BPF_CALL) &&
253 	       insn->src_reg == 0;
254 }
255 
256 static bool bpf_pseudo_call(const struct bpf_insn *insn)
257 {
258 	return insn->code == (BPF_JMP | BPF_CALL) &&
259 	       insn->src_reg == BPF_PSEUDO_CALL;
260 }
261 
262 static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
263 {
264 	return insn->code == (BPF_JMP | BPF_CALL) &&
265 	       insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
266 }
267 
268 struct bpf_call_arg_meta {
269 	struct bpf_map *map_ptr;
270 	bool raw_mode;
271 	bool pkt_access;
272 	u8 release_regno;
273 	int regno;
274 	int access_size;
275 	int mem_size;
276 	u64 msize_max_value;
277 	int ref_obj_id;
278 	int dynptr_id;
279 	int map_uid;
280 	int func_id;
281 	struct btf *btf;
282 	u32 btf_id;
283 	struct btf *ret_btf;
284 	u32 ret_btf_id;
285 	u32 subprogno;
286 	struct btf_field *kptr_field;
287 };
288 
289 struct bpf_kfunc_call_arg_meta {
290 	/* In parameters */
291 	struct btf *btf;
292 	u32 func_id;
293 	u32 kfunc_flags;
294 	const struct btf_type *func_proto;
295 	const char *func_name;
296 	/* Out parameters */
297 	u32 ref_obj_id;
298 	u8 release_regno;
299 	bool r0_rdonly;
300 	u32 ret_btf_id;
301 	u64 r0_size;
302 	u32 subprogno;
303 	struct {
304 		u64 value;
305 		bool found;
306 	} arg_constant;
307 
308 	/* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling,
309 	 * generally to pass info about user-defined local kptr types to later
310 	 * verification logic
311 	 *   bpf_obj_drop/bpf_percpu_obj_drop
312 	 *     Record the local kptr type to be drop'd
313 	 *   bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type)
314 	 *     Record the local kptr type to be refcount_incr'd and use
315 	 *     arg_owning_ref to determine whether refcount_acquire should be
316 	 *     fallible
317 	 */
318 	struct btf *arg_btf;
319 	u32 arg_btf_id;
320 	bool arg_owning_ref;
321 
322 	struct {
323 		struct btf_field *field;
324 	} arg_list_head;
325 	struct {
326 		struct btf_field *field;
327 	} arg_rbtree_root;
328 	struct {
329 		enum bpf_dynptr_type type;
330 		u32 id;
331 		u32 ref_obj_id;
332 	} initialized_dynptr;
333 	struct {
334 		u8 spi;
335 		u8 frameno;
336 	} iter;
337 	struct {
338 		struct bpf_map *ptr;
339 		int uid;
340 	} map;
341 	u64 mem_size;
342 };
343 
344 struct btf *btf_vmlinux;
345 
346 static const char *btf_type_name(const struct btf *btf, u32 id)
347 {
348 	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
349 }
350 
351 static DEFINE_MUTEX(bpf_verifier_lock);
352 static DEFINE_MUTEX(bpf_percpu_ma_lock);
353 
354 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
355 {
356 	struct bpf_verifier_env *env = private_data;
357 	va_list args;
358 
359 	if (!bpf_verifier_log_needed(&env->log))
360 		return;
361 
362 	va_start(args, fmt);
363 	bpf_verifier_vlog(&env->log, fmt, args);
364 	va_end(args);
365 }
366 
367 static void verbose_invalid_scalar(struct bpf_verifier_env *env,
368 				   struct bpf_reg_state *reg,
369 				   struct bpf_retval_range range, const char *ctx,
370 				   const char *reg_name)
371 {
372 	bool unknown = true;
373 
374 	verbose(env, "%s the register %s has", ctx, reg_name);
375 	if (reg->smin_value > S64_MIN) {
376 		verbose(env, " smin=%lld", reg->smin_value);
377 		unknown = false;
378 	}
379 	if (reg->smax_value < S64_MAX) {
380 		verbose(env, " smax=%lld", reg->smax_value);
381 		unknown = false;
382 	}
383 	if (unknown)
384 		verbose(env, " unknown scalar value");
385 	verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval);
386 }
387 
388 static bool type_may_be_null(u32 type)
389 {
390 	return type & PTR_MAYBE_NULL;
391 }
392 
393 static bool reg_not_null(const struct bpf_reg_state *reg)
394 {
395 	enum bpf_reg_type type;
396 
397 	type = reg->type;
398 	if (type_may_be_null(type))
399 		return false;
400 
401 	type = base_type(type);
402 	return type == PTR_TO_SOCKET ||
403 		type == PTR_TO_TCP_SOCK ||
404 		type == PTR_TO_MAP_VALUE ||
405 		type == PTR_TO_MAP_KEY ||
406 		type == PTR_TO_SOCK_COMMON ||
407 		(type == PTR_TO_BTF_ID && is_trusted_reg(reg)) ||
408 		type == PTR_TO_MEM;
409 }
410 
411 static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg)
412 {
413 	struct btf_record *rec = NULL;
414 	struct btf_struct_meta *meta;
415 
416 	if (reg->type == PTR_TO_MAP_VALUE) {
417 		rec = reg->map_ptr->record;
418 	} else if (type_is_ptr_alloc_obj(reg->type)) {
419 		meta = btf_find_struct_meta(reg->btf, reg->btf_id);
420 		if (meta)
421 			rec = meta->record;
422 	}
423 	return rec;
424 }
425 
426 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)
427 {
428 	struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;
429 
430 	return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;
431 }
432 
433 static const char *subprog_name(const struct bpf_verifier_env *env, int subprog)
434 {
435 	struct bpf_func_info *info;
436 
437 	if (!env->prog->aux->func_info)
438 		return "";
439 
440 	info = &env->prog->aux->func_info[subprog];
441 	return btf_type_name(env->prog->aux->btf, info->type_id);
442 }
443 
444 static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog)
445 {
446 	struct bpf_subprog_info *info = subprog_info(env, subprog);
447 
448 	info->is_cb = true;
449 	info->is_async_cb = true;
450 	info->is_exception_cb = true;
451 }
452 
453 static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog)
454 {
455 	return subprog_info(env, subprog)->is_exception_cb;
456 }
457 
458 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
459 {
460 	return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK);
461 }
462 
463 static bool type_is_rdonly_mem(u32 type)
464 {
465 	return type & MEM_RDONLY;
466 }
467 
468 static bool is_acquire_function(enum bpf_func_id func_id,
469 				const struct bpf_map *map)
470 {
471 	enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC;
472 
473 	if (func_id == BPF_FUNC_sk_lookup_tcp ||
474 	    func_id == BPF_FUNC_sk_lookup_udp ||
475 	    func_id == BPF_FUNC_skc_lookup_tcp ||
476 	    func_id == BPF_FUNC_ringbuf_reserve ||
477 	    func_id == BPF_FUNC_kptr_xchg)
478 		return true;
479 
480 	if (func_id == BPF_FUNC_map_lookup_elem &&
481 	    (map_type == BPF_MAP_TYPE_SOCKMAP ||
482 	     map_type == BPF_MAP_TYPE_SOCKHASH))
483 		return true;
484 
485 	return false;
486 }
487 
488 static bool is_ptr_cast_function(enum bpf_func_id func_id)
489 {
490 	return func_id == BPF_FUNC_tcp_sock ||
491 		func_id == BPF_FUNC_sk_fullsock ||
492 		func_id == BPF_FUNC_skc_to_tcp_sock ||
493 		func_id == BPF_FUNC_skc_to_tcp6_sock ||
494 		func_id == BPF_FUNC_skc_to_udp6_sock ||
495 		func_id == BPF_FUNC_skc_to_mptcp_sock ||
496 		func_id == BPF_FUNC_skc_to_tcp_timewait_sock ||
497 		func_id == BPF_FUNC_skc_to_tcp_request_sock;
498 }
499 
500 static bool is_dynptr_ref_function(enum bpf_func_id func_id)
501 {
502 	return func_id == BPF_FUNC_dynptr_data;
503 }
504 
505 static bool is_sync_callback_calling_kfunc(u32 btf_id);
506 static bool is_async_callback_calling_kfunc(u32 btf_id);
507 static bool is_callback_calling_kfunc(u32 btf_id);
508 static bool is_bpf_throw_kfunc(struct bpf_insn *insn);
509 
510 static bool is_bpf_wq_set_callback_impl_kfunc(u32 btf_id);
511 
512 static bool is_sync_callback_calling_function(enum bpf_func_id func_id)
513 {
514 	return func_id == BPF_FUNC_for_each_map_elem ||
515 	       func_id == BPF_FUNC_find_vma ||
516 	       func_id == BPF_FUNC_loop ||
517 	       func_id == BPF_FUNC_user_ringbuf_drain;
518 }
519 
520 static bool is_async_callback_calling_function(enum bpf_func_id func_id)
521 {
522 	return func_id == BPF_FUNC_timer_set_callback;
523 }
524 
525 static bool is_callback_calling_function(enum bpf_func_id func_id)
526 {
527 	return is_sync_callback_calling_function(func_id) ||
528 	       is_async_callback_calling_function(func_id);
529 }
530 
531 static bool is_sync_callback_calling_insn(struct bpf_insn *insn)
532 {
533 	return (bpf_helper_call(insn) && is_sync_callback_calling_function(insn->imm)) ||
534 	       (bpf_pseudo_kfunc_call(insn) && is_sync_callback_calling_kfunc(insn->imm));
535 }
536 
537 static bool is_async_callback_calling_insn(struct bpf_insn *insn)
538 {
539 	return (bpf_helper_call(insn) && is_async_callback_calling_function(insn->imm)) ||
540 	       (bpf_pseudo_kfunc_call(insn) && is_async_callback_calling_kfunc(insn->imm));
541 }
542 
543 static bool is_may_goto_insn(struct bpf_insn *insn)
544 {
545 	return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO;
546 }
547 
548 static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx)
549 {
550 	return is_may_goto_insn(&env->prog->insnsi[insn_idx]);
551 }
552 
553 static bool is_storage_get_function(enum bpf_func_id func_id)
554 {
555 	return func_id == BPF_FUNC_sk_storage_get ||
556 	       func_id == BPF_FUNC_inode_storage_get ||
557 	       func_id == BPF_FUNC_task_storage_get ||
558 	       func_id == BPF_FUNC_cgrp_storage_get;
559 }
560 
561 static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id,
562 					const struct bpf_map *map)
563 {
564 	int ref_obj_uses = 0;
565 
566 	if (is_ptr_cast_function(func_id))
567 		ref_obj_uses++;
568 	if (is_acquire_function(func_id, map))
569 		ref_obj_uses++;
570 	if (is_dynptr_ref_function(func_id))
571 		ref_obj_uses++;
572 
573 	return ref_obj_uses > 1;
574 }
575 
576 static bool is_cmpxchg_insn(const struct bpf_insn *insn)
577 {
578 	return BPF_CLASS(insn->code) == BPF_STX &&
579 	       BPF_MODE(insn->code) == BPF_ATOMIC &&
580 	       insn->imm == BPF_CMPXCHG;
581 }
582 
583 static int __get_spi(s32 off)
584 {
585 	return (-off - 1) / BPF_REG_SIZE;
586 }
587 
588 static struct bpf_func_state *func(struct bpf_verifier_env *env,
589 				   const struct bpf_reg_state *reg)
590 {
591 	struct bpf_verifier_state *cur = env->cur_state;
592 
593 	return cur->frame[reg->frameno];
594 }
595 
596 static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots)
597 {
598        int allocated_slots = state->allocated_stack / BPF_REG_SIZE;
599 
600        /* We need to check that slots between [spi - nr_slots + 1, spi] are
601 	* within [0, allocated_stack).
602 	*
603 	* Please note that the spi grows downwards. For example, a dynptr
604 	* takes the size of two stack slots; the first slot will be at
605 	* spi and the second slot will be at spi - 1.
606 	*/
607        return spi - nr_slots + 1 >= 0 && spi < allocated_slots;
608 }
609 
610 static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
611 			          const char *obj_kind, int nr_slots)
612 {
613 	int off, spi;
614 
615 	if (!tnum_is_const(reg->var_off)) {
616 		verbose(env, "%s has to be at a constant offset\n", obj_kind);
617 		return -EINVAL;
618 	}
619 
620 	off = reg->off + reg->var_off.value;
621 	if (off % BPF_REG_SIZE) {
622 		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
623 		return -EINVAL;
624 	}
625 
626 	spi = __get_spi(off);
627 	if (spi + 1 < nr_slots) {
628 		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
629 		return -EINVAL;
630 	}
631 
632 	if (!is_spi_bounds_valid(func(env, reg), spi, nr_slots))
633 		return -ERANGE;
634 	return spi;
635 }
636 
637 static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
638 {
639 	return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS);
640 }
641 
642 static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots)
643 {
644 	return stack_slot_obj_get_spi(env, reg, "iter", nr_slots);
645 }
646 
647 static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
648 {
649 	switch (arg_type & DYNPTR_TYPE_FLAG_MASK) {
650 	case DYNPTR_TYPE_LOCAL:
651 		return BPF_DYNPTR_TYPE_LOCAL;
652 	case DYNPTR_TYPE_RINGBUF:
653 		return BPF_DYNPTR_TYPE_RINGBUF;
654 	case DYNPTR_TYPE_SKB:
655 		return BPF_DYNPTR_TYPE_SKB;
656 	case DYNPTR_TYPE_XDP:
657 		return BPF_DYNPTR_TYPE_XDP;
658 	default:
659 		return BPF_DYNPTR_TYPE_INVALID;
660 	}
661 }
662 
663 static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type)
664 {
665 	switch (type) {
666 	case BPF_DYNPTR_TYPE_LOCAL:
667 		return DYNPTR_TYPE_LOCAL;
668 	case BPF_DYNPTR_TYPE_RINGBUF:
669 		return DYNPTR_TYPE_RINGBUF;
670 	case BPF_DYNPTR_TYPE_SKB:
671 		return DYNPTR_TYPE_SKB;
672 	case BPF_DYNPTR_TYPE_XDP:
673 		return DYNPTR_TYPE_XDP;
674 	default:
675 		return 0;
676 	}
677 }
678 
679 static bool dynptr_type_refcounted(enum bpf_dynptr_type type)
680 {
681 	return type == BPF_DYNPTR_TYPE_RINGBUF;
682 }
683 
684 static void __mark_dynptr_reg(struct bpf_reg_state *reg,
685 			      enum bpf_dynptr_type type,
686 			      bool first_slot, int dynptr_id);
687 
688 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
689 				struct bpf_reg_state *reg);
690 
691 static void mark_dynptr_stack_regs(struct bpf_verifier_env *env,
692 				   struct bpf_reg_state *sreg1,
693 				   struct bpf_reg_state *sreg2,
694 				   enum bpf_dynptr_type type)
695 {
696 	int id = ++env->id_gen;
697 
698 	__mark_dynptr_reg(sreg1, type, true, id);
699 	__mark_dynptr_reg(sreg2, type, false, id);
700 }
701 
702 static void mark_dynptr_cb_reg(struct bpf_verifier_env *env,
703 			       struct bpf_reg_state *reg,
704 			       enum bpf_dynptr_type type)
705 {
706 	__mark_dynptr_reg(reg, type, true, ++env->id_gen);
707 }
708 
709 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
710 				        struct bpf_func_state *state, int spi);
711 
712 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
713 				   enum bpf_arg_type arg_type, int insn_idx, int clone_ref_obj_id)
714 {
715 	struct bpf_func_state *state = func(env, reg);
716 	enum bpf_dynptr_type type;
717 	int spi, i, err;
718 
719 	spi = dynptr_get_spi(env, reg);
720 	if (spi < 0)
721 		return spi;
722 
723 	/* We cannot assume both spi and spi - 1 belong to the same dynptr,
724 	 * hence we need to call destroy_if_dynptr_stack_slot twice for both,
725 	 * to ensure that for the following example:
726 	 *	[d1][d1][d2][d2]
727 	 * spi    3   2   1   0
728 	 * So marking spi = 2 should lead to destruction of both d1 and d2. In
729 	 * case they do belong to same dynptr, second call won't see slot_type
730 	 * as STACK_DYNPTR and will simply skip destruction.
731 	 */
732 	err = destroy_if_dynptr_stack_slot(env, state, spi);
733 	if (err)
734 		return err;
735 	err = destroy_if_dynptr_stack_slot(env, state, spi - 1);
736 	if (err)
737 		return err;
738 
739 	for (i = 0; i < BPF_REG_SIZE; i++) {
740 		state->stack[spi].slot_type[i] = STACK_DYNPTR;
741 		state->stack[spi - 1].slot_type[i] = STACK_DYNPTR;
742 	}
743 
744 	type = arg_to_dynptr_type(arg_type);
745 	if (type == BPF_DYNPTR_TYPE_INVALID)
746 		return -EINVAL;
747 
748 	mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr,
749 			       &state->stack[spi - 1].spilled_ptr, type);
750 
751 	if (dynptr_type_refcounted(type)) {
752 		/* The id is used to track proper releasing */
753 		int id;
754 
755 		if (clone_ref_obj_id)
756 			id = clone_ref_obj_id;
757 		else
758 			id = acquire_reference_state(env, insn_idx);
759 
760 		if (id < 0)
761 			return id;
762 
763 		state->stack[spi].spilled_ptr.ref_obj_id = id;
764 		state->stack[spi - 1].spilled_ptr.ref_obj_id = id;
765 	}
766 
767 	state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
768 	state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN;
769 
770 	return 0;
771 }
772 
773 static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi)
774 {
775 	int i;
776 
777 	for (i = 0; i < BPF_REG_SIZE; i++) {
778 		state->stack[spi].slot_type[i] = STACK_INVALID;
779 		state->stack[spi - 1].slot_type[i] = STACK_INVALID;
780 	}
781 
782 	__mark_reg_not_init(env, &state->stack[spi].spilled_ptr);
783 	__mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);
784 
785 	/* Why do we need to set REG_LIVE_WRITTEN for STACK_INVALID slot?
786 	 *
787 	 * While we don't allow reading STACK_INVALID, it is still possible to
788 	 * do <8 byte writes marking some but not all slots as STACK_MISC. Then,
789 	 * helpers or insns can do partial read of that part without failing,
790 	 * but check_stack_range_initialized, check_stack_read_var_off, and
791 	 * check_stack_read_fixed_off will do mark_reg_read for all 8-bytes of
792 	 * the slot conservatively. Hence we need to prevent those liveness
793 	 * marking walks.
794 	 *
795 	 * This was not a problem before because STACK_INVALID is only set by
796 	 * default (where the default reg state has its reg->parent as NULL), or
797 	 * in clean_live_states after REG_LIVE_DONE (at which point
798 	 * mark_reg_read won't walk reg->parent chain), but not randomly during
799 	 * verifier state exploration (like we did above). Hence, for our case
800 	 * parentage chain will still be live (i.e. reg->parent may be
801 	 * non-NULL), while earlier reg->parent was NULL, so we need
802 	 * REG_LIVE_WRITTEN to screen off read marker propagation when it is
803 	 * done later on reads or by mark_dynptr_read as well to unnecessary
804 	 * mark registers in verifier state.
805 	 */
806 	state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
807 	state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN;
808 }
809 
810 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
811 {
812 	struct bpf_func_state *state = func(env, reg);
813 	int spi, ref_obj_id, i;
814 
815 	spi = dynptr_get_spi(env, reg);
816 	if (spi < 0)
817 		return spi;
818 
819 	if (!dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) {
820 		invalidate_dynptr(env, state, spi);
821 		return 0;
822 	}
823 
824 	ref_obj_id = state->stack[spi].spilled_ptr.ref_obj_id;
825 
826 	/* If the dynptr has a ref_obj_id, then we need to invalidate
827 	 * two things:
828 	 *
829 	 * 1) Any dynptrs with a matching ref_obj_id (clones)
830 	 * 2) Any slices derived from this dynptr.
831 	 */
832 
833 	/* Invalidate any slices associated with this dynptr */
834 	WARN_ON_ONCE(release_reference(env, ref_obj_id));
835 
836 	/* Invalidate any dynptr clones */
837 	for (i = 1; i < state->allocated_stack / BPF_REG_SIZE; i++) {
838 		if (state->stack[i].spilled_ptr.ref_obj_id != ref_obj_id)
839 			continue;
840 
841 		/* it should always be the case that if the ref obj id
842 		 * matches then the stack slot also belongs to a
843 		 * dynptr
844 		 */
845 		if (state->stack[i].slot_type[0] != STACK_DYNPTR) {
846 			verbose(env, "verifier internal error: misconfigured ref_obj_id\n");
847 			return -EFAULT;
848 		}
849 		if (state->stack[i].spilled_ptr.dynptr.first_slot)
850 			invalidate_dynptr(env, state, i);
851 	}
852 
853 	return 0;
854 }
855 
856 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
857 			       struct bpf_reg_state *reg);
858 
859 static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
860 {
861 	if (!env->allow_ptr_leaks)
862 		__mark_reg_not_init(env, reg);
863 	else
864 		__mark_reg_unknown(env, reg);
865 }
866 
867 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
868 				        struct bpf_func_state *state, int spi)
869 {
870 	struct bpf_func_state *fstate;
871 	struct bpf_reg_state *dreg;
872 	int i, dynptr_id;
873 
874 	/* We always ensure that STACK_DYNPTR is never set partially,
875 	 * hence just checking for slot_type[0] is enough. This is
876 	 * different for STACK_SPILL, where it may be only set for
877 	 * 1 byte, so code has to use is_spilled_reg.
878 	 */
879 	if (state->stack[spi].slot_type[0] != STACK_DYNPTR)
880 		return 0;
881 
882 	/* Reposition spi to first slot */
883 	if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
884 		spi = spi + 1;
885 
886 	if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) {
887 		verbose(env, "cannot overwrite referenced dynptr\n");
888 		return -EINVAL;
889 	}
890 
891 	mark_stack_slot_scratched(env, spi);
892 	mark_stack_slot_scratched(env, spi - 1);
893 
894 	/* Writing partially to one dynptr stack slot destroys both. */
895 	for (i = 0; i < BPF_REG_SIZE; i++) {
896 		state->stack[spi].slot_type[i] = STACK_INVALID;
897 		state->stack[spi - 1].slot_type[i] = STACK_INVALID;
898 	}
899 
900 	dynptr_id = state->stack[spi].spilled_ptr.id;
901 	/* Invalidate any slices associated with this dynptr */
902 	bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({
903 		/* Dynptr slices are only PTR_TO_MEM_OR_NULL and PTR_TO_MEM */
904 		if (dreg->type != (PTR_TO_MEM | PTR_MAYBE_NULL) && dreg->type != PTR_TO_MEM)
905 			continue;
906 		if (dreg->dynptr_id == dynptr_id)
907 			mark_reg_invalid(env, dreg);
908 	}));
909 
910 	/* Do not release reference state, we are destroying dynptr on stack,
911 	 * not using some helper to release it. Just reset register.
912 	 */
913 	__mark_reg_not_init(env, &state->stack[spi].spilled_ptr);
914 	__mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);
915 
916 	/* Same reason as unmark_stack_slots_dynptr above */
917 	state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
918 	state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN;
919 
920 	return 0;
921 }
922 
923 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
924 {
925 	int spi;
926 
927 	if (reg->type == CONST_PTR_TO_DYNPTR)
928 		return false;
929 
930 	spi = dynptr_get_spi(env, reg);
931 
932 	/* -ERANGE (i.e. spi not falling into allocated stack slots) isn't an
933 	 * error because this just means the stack state hasn't been updated yet.
934 	 * We will do check_mem_access to check and update stack bounds later.
935 	 */
936 	if (spi < 0 && spi != -ERANGE)
937 		return false;
938 
939 	/* We don't need to check if the stack slots are marked by previous
940 	 * dynptr initializations because we allow overwriting existing unreferenced
941 	 * STACK_DYNPTR slots, see mark_stack_slots_dynptr which calls
942 	 * destroy_if_dynptr_stack_slot to ensure dynptr objects at the slots we are
943 	 * touching are completely destructed before we reinitialize them for a new
944 	 * one. For referenced ones, destroy_if_dynptr_stack_slot returns an error early
945 	 * instead of delaying it until the end where the user will get "Unreleased
946 	 * reference" error.
947 	 */
948 	return true;
949 }
950 
951 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
952 {
953 	struct bpf_func_state *state = func(env, reg);
954 	int i, spi;
955 
956 	/* This already represents first slot of initialized bpf_dynptr.
957 	 *
958 	 * CONST_PTR_TO_DYNPTR already has fixed and var_off as 0 due to
959 	 * check_func_arg_reg_off's logic, so we don't need to check its
960 	 * offset and alignment.
961 	 */
962 	if (reg->type == CONST_PTR_TO_DYNPTR)
963 		return true;
964 
965 	spi = dynptr_get_spi(env, reg);
966 	if (spi < 0)
967 		return false;
968 	if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
969 		return false;
970 
971 	for (i = 0; i < BPF_REG_SIZE; i++) {
972 		if (state->stack[spi].slot_type[i] != STACK_DYNPTR ||
973 		    state->stack[spi - 1].slot_type[i] != STACK_DYNPTR)
974 			return false;
975 	}
976 
977 	return true;
978 }
979 
980 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
981 				    enum bpf_arg_type arg_type)
982 {
983 	struct bpf_func_state *state = func(env, reg);
984 	enum bpf_dynptr_type dynptr_type;
985 	int spi;
986 
987 	/* ARG_PTR_TO_DYNPTR takes any type of dynptr */
988 	if (arg_type == ARG_PTR_TO_DYNPTR)
989 		return true;
990 
991 	dynptr_type = arg_to_dynptr_type(arg_type);
992 	if (reg->type == CONST_PTR_TO_DYNPTR) {
993 		return reg->dynptr.type == dynptr_type;
994 	} else {
995 		spi = dynptr_get_spi(env, reg);
996 		if (spi < 0)
997 			return false;
998 		return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type;
999 	}
1000 }
1001 
1002 static void __mark_reg_known_zero(struct bpf_reg_state *reg);
1003 
1004 static bool in_rcu_cs(struct bpf_verifier_env *env);
1005 
1006 static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta);
1007 
1008 static int mark_stack_slots_iter(struct bpf_verifier_env *env,
1009 				 struct bpf_kfunc_call_arg_meta *meta,
1010 				 struct bpf_reg_state *reg, int insn_idx,
1011 				 struct btf *btf, u32 btf_id, int nr_slots)
1012 {
1013 	struct bpf_func_state *state = func(env, reg);
1014 	int spi, i, j, id;
1015 
1016 	spi = iter_get_spi(env, reg, nr_slots);
1017 	if (spi < 0)
1018 		return spi;
1019 
1020 	id = acquire_reference_state(env, insn_idx);
1021 	if (id < 0)
1022 		return id;
1023 
1024 	for (i = 0; i < nr_slots; i++) {
1025 		struct bpf_stack_state *slot = &state->stack[spi - i];
1026 		struct bpf_reg_state *st = &slot->spilled_ptr;
1027 
1028 		__mark_reg_known_zero(st);
1029 		st->type = PTR_TO_STACK; /* we don't have dedicated reg type */
1030 		if (is_kfunc_rcu_protected(meta)) {
1031 			if (in_rcu_cs(env))
1032 				st->type |= MEM_RCU;
1033 			else
1034 				st->type |= PTR_UNTRUSTED;
1035 		}
1036 		st->live |= REG_LIVE_WRITTEN;
1037 		st->ref_obj_id = i == 0 ? id : 0;
1038 		st->iter.btf = btf;
1039 		st->iter.btf_id = btf_id;
1040 		st->iter.state = BPF_ITER_STATE_ACTIVE;
1041 		st->iter.depth = 0;
1042 
1043 		for (j = 0; j < BPF_REG_SIZE; j++)
1044 			slot->slot_type[j] = STACK_ITER;
1045 
1046 		mark_stack_slot_scratched(env, spi - i);
1047 	}
1048 
1049 	return 0;
1050 }
1051 
1052 static int unmark_stack_slots_iter(struct bpf_verifier_env *env,
1053 				   struct bpf_reg_state *reg, int nr_slots)
1054 {
1055 	struct bpf_func_state *state = func(env, reg);
1056 	int spi, i, j;
1057 
1058 	spi = iter_get_spi(env, reg, nr_slots);
1059 	if (spi < 0)
1060 		return spi;
1061 
1062 	for (i = 0; i < nr_slots; i++) {
1063 		struct bpf_stack_state *slot = &state->stack[spi - i];
1064 		struct bpf_reg_state *st = &slot->spilled_ptr;
1065 
1066 		if (i == 0)
1067 			WARN_ON_ONCE(release_reference(env, st->ref_obj_id));
1068 
1069 		__mark_reg_not_init(env, st);
1070 
1071 		/* see unmark_stack_slots_dynptr() for why we need to set REG_LIVE_WRITTEN */
1072 		st->live |= REG_LIVE_WRITTEN;
1073 
1074 		for (j = 0; j < BPF_REG_SIZE; j++)
1075 			slot->slot_type[j] = STACK_INVALID;
1076 
1077 		mark_stack_slot_scratched(env, spi - i);
1078 	}
1079 
1080 	return 0;
1081 }
1082 
1083 static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env,
1084 				     struct bpf_reg_state *reg, int nr_slots)
1085 {
1086 	struct bpf_func_state *state = func(env, reg);
1087 	int spi, i, j;
1088 
1089 	/* For -ERANGE (i.e. spi not falling into allocated stack slots), we
1090 	 * will do check_mem_access to check and update stack bounds later, so
1091 	 * return true for that case.
1092 	 */
1093 	spi = iter_get_spi(env, reg, nr_slots);
1094 	if (spi == -ERANGE)
1095 		return true;
1096 	if (spi < 0)
1097 		return false;
1098 
1099 	for (i = 0; i < nr_slots; i++) {
1100 		struct bpf_stack_state *slot = &state->stack[spi - i];
1101 
1102 		for (j = 0; j < BPF_REG_SIZE; j++)
1103 			if (slot->slot_type[j] == STACK_ITER)
1104 				return false;
1105 	}
1106 
1107 	return true;
1108 }
1109 
1110 static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
1111 				   struct btf *btf, u32 btf_id, int nr_slots)
1112 {
1113 	struct bpf_func_state *state = func(env, reg);
1114 	int spi, i, j;
1115 
1116 	spi = iter_get_spi(env, reg, nr_slots);
1117 	if (spi < 0)
1118 		return -EINVAL;
1119 
1120 	for (i = 0; i < nr_slots; i++) {
1121 		struct bpf_stack_state *slot = &state->stack[spi - i];
1122 		struct bpf_reg_state *st = &slot->spilled_ptr;
1123 
1124 		if (st->type & PTR_UNTRUSTED)
1125 			return -EPROTO;
1126 		/* only main (first) slot has ref_obj_id set */
1127 		if (i == 0 && !st->ref_obj_id)
1128 			return -EINVAL;
1129 		if (i != 0 && st->ref_obj_id)
1130 			return -EINVAL;
1131 		if (st->iter.btf != btf || st->iter.btf_id != btf_id)
1132 			return -EINVAL;
1133 
1134 		for (j = 0; j < BPF_REG_SIZE; j++)
1135 			if (slot->slot_type[j] != STACK_ITER)
1136 				return -EINVAL;
1137 	}
1138 
1139 	return 0;
1140 }
1141 
1142 /* Check if given stack slot is "special":
1143  *   - spilled register state (STACK_SPILL);
1144  *   - dynptr state (STACK_DYNPTR);
1145  *   - iter state (STACK_ITER).
1146  */
1147 static bool is_stack_slot_special(const struct bpf_stack_state *stack)
1148 {
1149 	enum bpf_stack_slot_type type = stack->slot_type[BPF_REG_SIZE - 1];
1150 
1151 	switch (type) {
1152 	case STACK_SPILL:
1153 	case STACK_DYNPTR:
1154 	case STACK_ITER:
1155 		return true;
1156 	case STACK_INVALID:
1157 	case STACK_MISC:
1158 	case STACK_ZERO:
1159 		return false;
1160 	default:
1161 		WARN_ONCE(1, "unknown stack slot type %d\n", type);
1162 		return true;
1163 	}
1164 }
1165 
1166 /* The reg state of a pointer or a bounded scalar was saved when
1167  * it was spilled to the stack.
1168  */
1169 static bool is_spilled_reg(const struct bpf_stack_state *stack)
1170 {
1171 	return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL;
1172 }
1173 
1174 static bool is_spilled_scalar_reg(const struct bpf_stack_state *stack)
1175 {
1176 	return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL &&
1177 	       stack->spilled_ptr.type == SCALAR_VALUE;
1178 }
1179 
1180 static bool is_spilled_scalar_reg64(const struct bpf_stack_state *stack)
1181 {
1182 	return stack->slot_type[0] == STACK_SPILL &&
1183 	       stack->spilled_ptr.type == SCALAR_VALUE;
1184 }
1185 
1186 /* Mark stack slot as STACK_MISC, unless it is already STACK_INVALID, in which
1187  * case they are equivalent, or it's STACK_ZERO, in which case we preserve
1188  * more precise STACK_ZERO.
1189  * Note, in uprivileged mode leaving STACK_INVALID is wrong, so we take
1190  * env->allow_ptr_leaks into account and force STACK_MISC, if necessary.
1191  */
1192 static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype)
1193 {
1194 	if (*stype == STACK_ZERO)
1195 		return;
1196 	if (env->allow_ptr_leaks && *stype == STACK_INVALID)
1197 		return;
1198 	*stype = STACK_MISC;
1199 }
1200 
1201 static void scrub_spilled_slot(u8 *stype)
1202 {
1203 	if (*stype != STACK_INVALID)
1204 		*stype = STACK_MISC;
1205 }
1206 
1207 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too
1208  * small to hold src. This is different from krealloc since we don't want to preserve
1209  * the contents of dst.
1210  *
1211  * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could
1212  * not be allocated.
1213  */
1214 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags)
1215 {
1216 	size_t alloc_bytes;
1217 	void *orig = dst;
1218 	size_t bytes;
1219 
1220 	if (ZERO_OR_NULL_PTR(src))
1221 		goto out;
1222 
1223 	if (unlikely(check_mul_overflow(n, size, &bytes)))
1224 		return NULL;
1225 
1226 	alloc_bytes = max(ksize(orig), kmalloc_size_roundup(bytes));
1227 	dst = krealloc(orig, alloc_bytes, flags);
1228 	if (!dst) {
1229 		kfree(orig);
1230 		return NULL;
1231 	}
1232 
1233 	memcpy(dst, src, bytes);
1234 out:
1235 	return dst ? dst : ZERO_SIZE_PTR;
1236 }
1237 
1238 /* resize an array from old_n items to new_n items. the array is reallocated if it's too
1239  * small to hold new_n items. new items are zeroed out if the array grows.
1240  *
1241  * Contrary to krealloc_array, does not free arr if new_n is zero.
1242  */
1243 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size)
1244 {
1245 	size_t alloc_size;
1246 	void *new_arr;
1247 
1248 	if (!new_n || old_n == new_n)
1249 		goto out;
1250 
1251 	alloc_size = kmalloc_size_roundup(size_mul(new_n, size));
1252 	new_arr = krealloc(arr, alloc_size, GFP_KERNEL);
1253 	if (!new_arr) {
1254 		kfree(arr);
1255 		return NULL;
1256 	}
1257 	arr = new_arr;
1258 
1259 	if (new_n > old_n)
1260 		memset(arr + old_n * size, 0, (new_n - old_n) * size);
1261 
1262 out:
1263 	return arr ? arr : ZERO_SIZE_PTR;
1264 }
1265 
1266 static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
1267 {
1268 	dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs,
1269 			       sizeof(struct bpf_reference_state), GFP_KERNEL);
1270 	if (!dst->refs)
1271 		return -ENOMEM;
1272 
1273 	dst->acquired_refs = src->acquired_refs;
1274 	return 0;
1275 }
1276 
1277 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
1278 {
1279 	size_t n = src->allocated_stack / BPF_REG_SIZE;
1280 
1281 	dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state),
1282 				GFP_KERNEL);
1283 	if (!dst->stack)
1284 		return -ENOMEM;
1285 
1286 	dst->allocated_stack = src->allocated_stack;
1287 	return 0;
1288 }
1289 
1290 static int resize_reference_state(struct bpf_func_state *state, size_t n)
1291 {
1292 	state->refs = realloc_array(state->refs, state->acquired_refs, n,
1293 				    sizeof(struct bpf_reference_state));
1294 	if (!state->refs)
1295 		return -ENOMEM;
1296 
1297 	state->acquired_refs = n;
1298 	return 0;
1299 }
1300 
1301 /* Possibly update state->allocated_stack to be at least size bytes. Also
1302  * possibly update the function's high-water mark in its bpf_subprog_info.
1303  */
1304 static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size)
1305 {
1306 	size_t old_n = state->allocated_stack / BPF_REG_SIZE, n;
1307 
1308 	/* The stack size is always a multiple of BPF_REG_SIZE. */
1309 	size = round_up(size, BPF_REG_SIZE);
1310 	n = size / BPF_REG_SIZE;
1311 
1312 	if (old_n >= n)
1313 		return 0;
1314 
1315 	state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state));
1316 	if (!state->stack)
1317 		return -ENOMEM;
1318 
1319 	state->allocated_stack = size;
1320 
1321 	/* update known max for given subprogram */
1322 	if (env->subprog_info[state->subprogno].stack_depth < size)
1323 		env->subprog_info[state->subprogno].stack_depth = size;
1324 
1325 	return 0;
1326 }
1327 
1328 /* Acquire a pointer id from the env and update the state->refs to include
1329  * this new pointer reference.
1330  * On success, returns a valid pointer id to associate with the register
1331  * On failure, returns a negative errno.
1332  */
1333 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
1334 {
1335 	struct bpf_func_state *state = cur_func(env);
1336 	int new_ofs = state->acquired_refs;
1337 	int id, err;
1338 
1339 	err = resize_reference_state(state, state->acquired_refs + 1);
1340 	if (err)
1341 		return err;
1342 	id = ++env->id_gen;
1343 	state->refs[new_ofs].id = id;
1344 	state->refs[new_ofs].insn_idx = insn_idx;
1345 	state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0;
1346 
1347 	return id;
1348 }
1349 
1350 /* release function corresponding to acquire_reference_state(). Idempotent. */
1351 static int release_reference_state(struct bpf_func_state *state, int ptr_id)
1352 {
1353 	int i, last_idx;
1354 
1355 	last_idx = state->acquired_refs - 1;
1356 	for (i = 0; i < state->acquired_refs; i++) {
1357 		if (state->refs[i].id == ptr_id) {
1358 			/* Cannot release caller references in callbacks */
1359 			if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
1360 				return -EINVAL;
1361 			if (last_idx && i != last_idx)
1362 				memcpy(&state->refs[i], &state->refs[last_idx],
1363 				       sizeof(*state->refs));
1364 			memset(&state->refs[last_idx], 0, sizeof(*state->refs));
1365 			state->acquired_refs--;
1366 			return 0;
1367 		}
1368 	}
1369 	return -EINVAL;
1370 }
1371 
1372 static void free_func_state(struct bpf_func_state *state)
1373 {
1374 	if (!state)
1375 		return;
1376 	kfree(state->refs);
1377 	kfree(state->stack);
1378 	kfree(state);
1379 }
1380 
1381 static void clear_jmp_history(struct bpf_verifier_state *state)
1382 {
1383 	kfree(state->jmp_history);
1384 	state->jmp_history = NULL;
1385 	state->jmp_history_cnt = 0;
1386 }
1387 
1388 static void free_verifier_state(struct bpf_verifier_state *state,
1389 				bool free_self)
1390 {
1391 	int i;
1392 
1393 	for (i = 0; i <= state->curframe; i++) {
1394 		free_func_state(state->frame[i]);
1395 		state->frame[i] = NULL;
1396 	}
1397 	clear_jmp_history(state);
1398 	if (free_self)
1399 		kfree(state);
1400 }
1401 
1402 /* copy verifier state from src to dst growing dst stack space
1403  * when necessary to accommodate larger src stack
1404  */
1405 static int copy_func_state(struct bpf_func_state *dst,
1406 			   const struct bpf_func_state *src)
1407 {
1408 	int err;
1409 
1410 	memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs));
1411 	err = copy_reference_state(dst, src);
1412 	if (err)
1413 		return err;
1414 	return copy_stack_state(dst, src);
1415 }
1416 
1417 static int copy_verifier_state(struct bpf_verifier_state *dst_state,
1418 			       const struct bpf_verifier_state *src)
1419 {
1420 	struct bpf_func_state *dst;
1421 	int i, err;
1422 
1423 	dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history,
1424 					  src->jmp_history_cnt, sizeof(*dst_state->jmp_history),
1425 					  GFP_USER);
1426 	if (!dst_state->jmp_history)
1427 		return -ENOMEM;
1428 	dst_state->jmp_history_cnt = src->jmp_history_cnt;
1429 
1430 	/* if dst has more stack frames then src frame, free them, this is also
1431 	 * necessary in case of exceptional exits using bpf_throw.
1432 	 */
1433 	for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
1434 		free_func_state(dst_state->frame[i]);
1435 		dst_state->frame[i] = NULL;
1436 	}
1437 	dst_state->speculative = src->speculative;
1438 	dst_state->active_rcu_lock = src->active_rcu_lock;
1439 	dst_state->active_preempt_lock = src->active_preempt_lock;
1440 	dst_state->in_sleepable = src->in_sleepable;
1441 	dst_state->curframe = src->curframe;
1442 	dst_state->active_lock.ptr = src->active_lock.ptr;
1443 	dst_state->active_lock.id = src->active_lock.id;
1444 	dst_state->branches = src->branches;
1445 	dst_state->parent = src->parent;
1446 	dst_state->first_insn_idx = src->first_insn_idx;
1447 	dst_state->last_insn_idx = src->last_insn_idx;
1448 	dst_state->dfs_depth = src->dfs_depth;
1449 	dst_state->callback_unroll_depth = src->callback_unroll_depth;
1450 	dst_state->used_as_loop_entry = src->used_as_loop_entry;
1451 	dst_state->may_goto_depth = src->may_goto_depth;
1452 	for (i = 0; i <= src->curframe; i++) {
1453 		dst = dst_state->frame[i];
1454 		if (!dst) {
1455 			dst = kzalloc(sizeof(*dst), GFP_KERNEL);
1456 			if (!dst)
1457 				return -ENOMEM;
1458 			dst_state->frame[i] = dst;
1459 		}
1460 		err = copy_func_state(dst, src->frame[i]);
1461 		if (err)
1462 			return err;
1463 	}
1464 	return 0;
1465 }
1466 
1467 static u32 state_htab_size(struct bpf_verifier_env *env)
1468 {
1469 	return env->prog->len;
1470 }
1471 
1472 static struct bpf_verifier_state_list **explored_state(struct bpf_verifier_env *env, int idx)
1473 {
1474 	struct bpf_verifier_state *cur = env->cur_state;
1475 	struct bpf_func_state *state = cur->frame[cur->curframe];
1476 
1477 	return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];
1478 }
1479 
1480 static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_state *b)
1481 {
1482 	int fr;
1483 
1484 	if (a->curframe != b->curframe)
1485 		return false;
1486 
1487 	for (fr = a->curframe; fr >= 0; fr--)
1488 		if (a->frame[fr]->callsite != b->frame[fr]->callsite)
1489 			return false;
1490 
1491 	return true;
1492 }
1493 
1494 /* Open coded iterators allow back-edges in the state graph in order to
1495  * check unbounded loops that iterators.
1496  *
1497  * In is_state_visited() it is necessary to know if explored states are
1498  * part of some loops in order to decide whether non-exact states
1499  * comparison could be used:
1500  * - non-exact states comparison establishes sub-state relation and uses
1501  *   read and precision marks to do so, these marks are propagated from
1502  *   children states and thus are not guaranteed to be final in a loop;
1503  * - exact states comparison just checks if current and explored states
1504  *   are identical (and thus form a back-edge).
1505  *
1506  * Paper "A New Algorithm for Identifying Loops in Decompilation"
1507  * by Tao Wei, Jian Mao, Wei Zou and Yu Chen [1] presents a convenient
1508  * algorithm for loop structure detection and gives an overview of
1509  * relevant terminology. It also has helpful illustrations.
1510  *
1511  * [1] https://api.semanticscholar.org/CorpusID:15784067
1512  *
1513  * We use a similar algorithm but because loop nested structure is
1514  * irrelevant for verifier ours is significantly simpler and resembles
1515  * strongly connected components algorithm from Sedgewick's textbook.
1516  *
1517  * Define topmost loop entry as a first node of the loop traversed in a
1518  * depth first search starting from initial state. The goal of the loop
1519  * tracking algorithm is to associate topmost loop entries with states
1520  * derived from these entries.
1521  *
1522  * For each step in the DFS states traversal algorithm needs to identify
1523  * the following situations:
1524  *
1525  *          initial                     initial                   initial
1526  *            |                           |                         |
1527  *            V                           V                         V
1528  *           ...                         ...           .---------> hdr
1529  *            |                           |            |            |
1530  *            V                           V            |            V
1531  *           cur                     .-> succ          |    .------...
1532  *            |                      |    |            |    |       |
1533  *            V                      |    V            |    V       V
1534  *           succ                    '-- cur           |   ...     ...
1535  *                                                     |    |       |
1536  *                                                     |    V       V
1537  *                                                     |   succ <- cur
1538  *                                                     |    |
1539  *                                                     |    V
1540  *                                                     |   ...
1541  *                                                     |    |
1542  *                                                     '----'
1543  *
1544  *  (A) successor state of cur   (B) successor state of cur or it's entry
1545  *      not yet traversed            are in current DFS path, thus cur and succ
1546  *                                   are members of the same outermost loop
1547  *
1548  *                      initial                  initial
1549  *                        |                        |
1550  *                        V                        V
1551  *                       ...                      ...
1552  *                        |                        |
1553  *                        V                        V
1554  *                .------...               .------...
1555  *                |       |                |       |
1556  *                V       V                V       V
1557  *           .-> hdr     ...              ...     ...
1558  *           |    |       |                |       |
1559  *           |    V       V                V       V
1560  *           |   succ <- cur              succ <- cur
1561  *           |    |                        |
1562  *           |    V                        V
1563  *           |   ...                      ...
1564  *           |    |                        |
1565  *           '----'                       exit
1566  *
1567  * (C) successor state of cur is a part of some loop but this loop
1568  *     does not include cur or successor state is not in a loop at all.
1569  *
1570  * Algorithm could be described as the following python code:
1571  *
1572  *     traversed = set()   # Set of traversed nodes
1573  *     entries = {}        # Mapping from node to loop entry
1574  *     depths = {}         # Depth level assigned to graph node
1575  *     path = set()        # Current DFS path
1576  *
1577  *     # Find outermost loop entry known for n
1578  *     def get_loop_entry(n):
1579  *         h = entries.get(n, None)
1580  *         while h in entries and entries[h] != h:
1581  *             h = entries[h]
1582  *         return h
1583  *
1584  *     # Update n's loop entry if h's outermost entry comes
1585  *     # before n's outermost entry in current DFS path.
1586  *     def update_loop_entry(n, h):
1587  *         n1 = get_loop_entry(n) or n
1588  *         h1 = get_loop_entry(h) or h
1589  *         if h1 in path and depths[h1] <= depths[n1]:
1590  *             entries[n] = h1
1591  *
1592  *     def dfs(n, depth):
1593  *         traversed.add(n)
1594  *         path.add(n)
1595  *         depths[n] = depth
1596  *         for succ in G.successors(n):
1597  *             if succ not in traversed:
1598  *                 # Case A: explore succ and update cur's loop entry
1599  *                 #         only if succ's entry is in current DFS path.
1600  *                 dfs(succ, depth + 1)
1601  *                 h = get_loop_entry(succ)
1602  *                 update_loop_entry(n, h)
1603  *             else:
1604  *                 # Case B or C depending on `h1 in path` check in update_loop_entry().
1605  *                 update_loop_entry(n, succ)
1606  *         path.remove(n)
1607  *
1608  * To adapt this algorithm for use with verifier:
1609  * - use st->branch == 0 as a signal that DFS of succ had been finished
1610  *   and cur's loop entry has to be updated (case A), handle this in
1611  *   update_branch_counts();
1612  * - use st->branch > 0 as a signal that st is in the current DFS path;
1613  * - handle cases B and C in is_state_visited();
1614  * - update topmost loop entry for intermediate states in get_loop_entry().
1615  */
1616 static struct bpf_verifier_state *get_loop_entry(struct bpf_verifier_state *st)
1617 {
1618 	struct bpf_verifier_state *topmost = st->loop_entry, *old;
1619 
1620 	while (topmost && topmost->loop_entry && topmost != topmost->loop_entry)
1621 		topmost = topmost->loop_entry;
1622 	/* Update loop entries for intermediate states to avoid this
1623 	 * traversal in future get_loop_entry() calls.
1624 	 */
1625 	while (st && st->loop_entry != topmost) {
1626 		old = st->loop_entry;
1627 		st->loop_entry = topmost;
1628 		st = old;
1629 	}
1630 	return topmost;
1631 }
1632 
1633 static void update_loop_entry(struct bpf_verifier_state *cur, struct bpf_verifier_state *hdr)
1634 {
1635 	struct bpf_verifier_state *cur1, *hdr1;
1636 
1637 	cur1 = get_loop_entry(cur) ?: cur;
1638 	hdr1 = get_loop_entry(hdr) ?: hdr;
1639 	/* The head1->branches check decides between cases B and C in
1640 	 * comment for get_loop_entry(). If hdr1->branches == 0 then
1641 	 * head's topmost loop entry is not in current DFS path,
1642 	 * hence 'cur' and 'hdr' are not in the same loop and there is
1643 	 * no need to update cur->loop_entry.
1644 	 */
1645 	if (hdr1->branches && hdr1->dfs_depth <= cur1->dfs_depth) {
1646 		cur->loop_entry = hdr;
1647 		hdr->used_as_loop_entry = true;
1648 	}
1649 }
1650 
1651 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
1652 {
1653 	while (st) {
1654 		u32 br = --st->branches;
1655 
1656 		/* br == 0 signals that DFS exploration for 'st' is finished,
1657 		 * thus it is necessary to update parent's loop entry if it
1658 		 * turned out that st is a part of some loop.
1659 		 * This is a part of 'case A' in get_loop_entry() comment.
1660 		 */
1661 		if (br == 0 && st->parent && st->loop_entry)
1662 			update_loop_entry(st->parent, st->loop_entry);
1663 
1664 		/* WARN_ON(br > 1) technically makes sense here,
1665 		 * but see comment in push_stack(), hence:
1666 		 */
1667 		WARN_ONCE((int)br < 0,
1668 			  "BUG update_branch_counts:branches_to_explore=%d\n",
1669 			  br);
1670 		if (br)
1671 			break;
1672 		st = st->parent;
1673 	}
1674 }
1675 
1676 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
1677 		     int *insn_idx, bool pop_log)
1678 {
1679 	struct bpf_verifier_state *cur = env->cur_state;
1680 	struct bpf_verifier_stack_elem *elem, *head = env->head;
1681 	int err;
1682 
1683 	if (env->head == NULL)
1684 		return -ENOENT;
1685 
1686 	if (cur) {
1687 		err = copy_verifier_state(cur, &head->st);
1688 		if (err)
1689 			return err;
1690 	}
1691 	if (pop_log)
1692 		bpf_vlog_reset(&env->log, head->log_pos);
1693 	if (insn_idx)
1694 		*insn_idx = head->insn_idx;
1695 	if (prev_insn_idx)
1696 		*prev_insn_idx = head->prev_insn_idx;
1697 	elem = head->next;
1698 	free_verifier_state(&head->st, false);
1699 	kfree(head);
1700 	env->head = elem;
1701 	env->stack_size--;
1702 	return 0;
1703 }
1704 
1705 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
1706 					     int insn_idx, int prev_insn_idx,
1707 					     bool speculative)
1708 {
1709 	struct bpf_verifier_state *cur = env->cur_state;
1710 	struct bpf_verifier_stack_elem *elem;
1711 	int err;
1712 
1713 	elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
1714 	if (!elem)
1715 		goto err;
1716 
1717 	elem->insn_idx = insn_idx;
1718 	elem->prev_insn_idx = prev_insn_idx;
1719 	elem->next = env->head;
1720 	elem->log_pos = env->log.end_pos;
1721 	env->head = elem;
1722 	env->stack_size++;
1723 	err = copy_verifier_state(&elem->st, cur);
1724 	if (err)
1725 		goto err;
1726 	elem->st.speculative |= speculative;
1727 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
1728 		verbose(env, "The sequence of %d jumps is too complex.\n",
1729 			env->stack_size);
1730 		goto err;
1731 	}
1732 	if (elem->st.parent) {
1733 		++elem->st.parent->branches;
1734 		/* WARN_ON(branches > 2) technically makes sense here,
1735 		 * but
1736 		 * 1. speculative states will bump 'branches' for non-branch
1737 		 * instructions
1738 		 * 2. is_state_visited() heuristics may decide not to create
1739 		 * a new state for a sequence of branches and all such current
1740 		 * and cloned states will be pointing to a single parent state
1741 		 * which might have large 'branches' count.
1742 		 */
1743 	}
1744 	return &elem->st;
1745 err:
1746 	free_verifier_state(env->cur_state, true);
1747 	env->cur_state = NULL;
1748 	/* pop all elements and return */
1749 	while (!pop_stack(env, NULL, NULL, false));
1750 	return NULL;
1751 }
1752 
1753 #define CALLER_SAVED_REGS 6
1754 static const int caller_saved[CALLER_SAVED_REGS] = {
1755 	BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
1756 };
1757 
1758 /* This helper doesn't clear reg->id */
1759 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)
1760 {
1761 	reg->var_off = tnum_const(imm);
1762 	reg->smin_value = (s64)imm;
1763 	reg->smax_value = (s64)imm;
1764 	reg->umin_value = imm;
1765 	reg->umax_value = imm;
1766 
1767 	reg->s32_min_value = (s32)imm;
1768 	reg->s32_max_value = (s32)imm;
1769 	reg->u32_min_value = (u32)imm;
1770 	reg->u32_max_value = (u32)imm;
1771 }
1772 
1773 /* Mark the unknown part of a register (variable offset or scalar value) as
1774  * known to have the value @imm.
1775  */
1776 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
1777 {
1778 	/* Clear off and union(map_ptr, range) */
1779 	memset(((u8 *)reg) + sizeof(reg->type), 0,
1780 	       offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
1781 	reg->id = 0;
1782 	reg->ref_obj_id = 0;
1783 	___mark_reg_known(reg, imm);
1784 }
1785 
1786 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)
1787 {
1788 	reg->var_off = tnum_const_subreg(reg->var_off, imm);
1789 	reg->s32_min_value = (s32)imm;
1790 	reg->s32_max_value = (s32)imm;
1791 	reg->u32_min_value = (u32)imm;
1792 	reg->u32_max_value = (u32)imm;
1793 }
1794 
1795 /* Mark the 'variable offset' part of a register as zero.  This should be
1796  * used only on registers holding a pointer type.
1797  */
1798 static void __mark_reg_known_zero(struct bpf_reg_state *reg)
1799 {
1800 	__mark_reg_known(reg, 0);
1801 }
1802 
1803 static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
1804 {
1805 	__mark_reg_known(reg, 0);
1806 	reg->type = SCALAR_VALUE;
1807 	/* all scalars are assumed imprecise initially (unless unprivileged,
1808 	 * in which case everything is forced to be precise)
1809 	 */
1810 	reg->precise = !env->bpf_capable;
1811 }
1812 
1813 static void mark_reg_known_zero(struct bpf_verifier_env *env,
1814 				struct bpf_reg_state *regs, u32 regno)
1815 {
1816 	if (WARN_ON(regno >= MAX_BPF_REG)) {
1817 		verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
1818 		/* Something bad happened, let's kill all regs */
1819 		for (regno = 0; regno < MAX_BPF_REG; regno++)
1820 			__mark_reg_not_init(env, regs + regno);
1821 		return;
1822 	}
1823 	__mark_reg_known_zero(regs + regno);
1824 }
1825 
1826 static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type,
1827 			      bool first_slot, int dynptr_id)
1828 {
1829 	/* reg->type has no meaning for STACK_DYNPTR, but when we set reg for
1830 	 * callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply
1831 	 * set it unconditionally as it is ignored for STACK_DYNPTR anyway.
1832 	 */
1833 	__mark_reg_known_zero(reg);
1834 	reg->type = CONST_PTR_TO_DYNPTR;
1835 	/* Give each dynptr a unique id to uniquely associate slices to it. */
1836 	reg->id = dynptr_id;
1837 	reg->dynptr.type = type;
1838 	reg->dynptr.first_slot = first_slot;
1839 }
1840 
1841 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
1842 {
1843 	if (base_type(reg->type) == PTR_TO_MAP_VALUE) {
1844 		const struct bpf_map *map = reg->map_ptr;
1845 
1846 		if (map->inner_map_meta) {
1847 			reg->type = CONST_PTR_TO_MAP;
1848 			reg->map_ptr = map->inner_map_meta;
1849 			/* transfer reg's id which is unique for every map_lookup_elem
1850 			 * as UID of the inner map.
1851 			 */
1852 			if (btf_record_has_field(map->inner_map_meta->record, BPF_TIMER))
1853 				reg->map_uid = reg->id;
1854 			if (btf_record_has_field(map->inner_map_meta->record, BPF_WORKQUEUE))
1855 				reg->map_uid = reg->id;
1856 		} else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
1857 			reg->type = PTR_TO_XDP_SOCK;
1858 		} else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
1859 			   map->map_type == BPF_MAP_TYPE_SOCKHASH) {
1860 			reg->type = PTR_TO_SOCKET;
1861 		} else {
1862 			reg->type = PTR_TO_MAP_VALUE;
1863 		}
1864 		return;
1865 	}
1866 
1867 	reg->type &= ~PTR_MAYBE_NULL;
1868 }
1869 
1870 static void mark_reg_graph_node(struct bpf_reg_state *regs, u32 regno,
1871 				struct btf_field_graph_root *ds_head)
1872 {
1873 	__mark_reg_known_zero(&regs[regno]);
1874 	regs[regno].type = PTR_TO_BTF_ID | MEM_ALLOC;
1875 	regs[regno].btf = ds_head->btf;
1876 	regs[regno].btf_id = ds_head->value_btf_id;
1877 	regs[regno].off = ds_head->node_offset;
1878 }
1879 
1880 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
1881 {
1882 	return type_is_pkt_pointer(reg->type);
1883 }
1884 
1885 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
1886 {
1887 	return reg_is_pkt_pointer(reg) ||
1888 	       reg->type == PTR_TO_PACKET_END;
1889 }
1890 
1891 static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg)
1892 {
1893 	return base_type(reg->type) == PTR_TO_MEM &&
1894 		(reg->type & DYNPTR_TYPE_SKB || reg->type & DYNPTR_TYPE_XDP);
1895 }
1896 
1897 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
1898 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
1899 				    enum bpf_reg_type which)
1900 {
1901 	/* The register can already have a range from prior markings.
1902 	 * This is fine as long as it hasn't been advanced from its
1903 	 * origin.
1904 	 */
1905 	return reg->type == which &&
1906 	       reg->id == 0 &&
1907 	       reg->off == 0 &&
1908 	       tnum_equals_const(reg->var_off, 0);
1909 }
1910 
1911 /* Reset the min/max bounds of a register */
1912 static void __mark_reg_unbounded(struct bpf_reg_state *reg)
1913 {
1914 	reg->smin_value = S64_MIN;
1915 	reg->smax_value = S64_MAX;
1916 	reg->umin_value = 0;
1917 	reg->umax_value = U64_MAX;
1918 
1919 	reg->s32_min_value = S32_MIN;
1920 	reg->s32_max_value = S32_MAX;
1921 	reg->u32_min_value = 0;
1922 	reg->u32_max_value = U32_MAX;
1923 }
1924 
1925 static void __mark_reg64_unbounded(struct bpf_reg_state *reg)
1926 {
1927 	reg->smin_value = S64_MIN;
1928 	reg->smax_value = S64_MAX;
1929 	reg->umin_value = 0;
1930 	reg->umax_value = U64_MAX;
1931 }
1932 
1933 static void __mark_reg32_unbounded(struct bpf_reg_state *reg)
1934 {
1935 	reg->s32_min_value = S32_MIN;
1936 	reg->s32_max_value = S32_MAX;
1937 	reg->u32_min_value = 0;
1938 	reg->u32_max_value = U32_MAX;
1939 }
1940 
1941 static void __update_reg32_bounds(struct bpf_reg_state *reg)
1942 {
1943 	struct tnum var32_off = tnum_subreg(reg->var_off);
1944 
1945 	/* min signed is max(sign bit) | min(other bits) */
1946 	reg->s32_min_value = max_t(s32, reg->s32_min_value,
1947 			var32_off.value | (var32_off.mask & S32_MIN));
1948 	/* max signed is min(sign bit) | max(other bits) */
1949 	reg->s32_max_value = min_t(s32, reg->s32_max_value,
1950 			var32_off.value | (var32_off.mask & S32_MAX));
1951 	reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value);
1952 	reg->u32_max_value = min(reg->u32_max_value,
1953 				 (u32)(var32_off.value | var32_off.mask));
1954 }
1955 
1956 static void __update_reg64_bounds(struct bpf_reg_state *reg)
1957 {
1958 	/* min signed is max(sign bit) | min(other bits) */
1959 	reg->smin_value = max_t(s64, reg->smin_value,
1960 				reg->var_off.value | (reg->var_off.mask & S64_MIN));
1961 	/* max signed is min(sign bit) | max(other bits) */
1962 	reg->smax_value = min_t(s64, reg->smax_value,
1963 				reg->var_off.value | (reg->var_off.mask & S64_MAX));
1964 	reg->umin_value = max(reg->umin_value, reg->var_off.value);
1965 	reg->umax_value = min(reg->umax_value,
1966 			      reg->var_off.value | reg->var_off.mask);
1967 }
1968 
1969 static void __update_reg_bounds(struct bpf_reg_state *reg)
1970 {
1971 	__update_reg32_bounds(reg);
1972 	__update_reg64_bounds(reg);
1973 }
1974 
1975 /* Uses signed min/max values to inform unsigned, and vice-versa */
1976 static void __reg32_deduce_bounds(struct bpf_reg_state *reg)
1977 {
1978 	/* If upper 32 bits of u64/s64 range don't change, we can use lower 32
1979 	 * bits to improve our u32/s32 boundaries.
1980 	 *
1981 	 * E.g., the case where we have upper 32 bits as zero ([10, 20] in
1982 	 * u64) is pretty trivial, it's obvious that in u32 we'll also have
1983 	 * [10, 20] range. But this property holds for any 64-bit range as
1984 	 * long as upper 32 bits in that entire range of values stay the same.
1985 	 *
1986 	 * E.g., u64 range [0x10000000A, 0x10000000F] ([4294967306, 4294967311]
1987 	 * in decimal) has the same upper 32 bits throughout all the values in
1988 	 * that range. As such, lower 32 bits form a valid [0xA, 0xF] ([10, 15])
1989 	 * range.
1990 	 *
1991 	 * Note also, that [0xA, 0xF] is a valid range both in u32 and in s32,
1992 	 * following the rules outlined below about u64/s64 correspondence
1993 	 * (which equally applies to u32 vs s32 correspondence). In general it
1994 	 * depends on actual hexadecimal values of 32-bit range. They can form
1995 	 * only valid u32, or only valid s32 ranges in some cases.
1996 	 *
1997 	 * So we use all these insights to derive bounds for subregisters here.
1998 	 */
1999 	if ((reg->umin_value >> 32) == (reg->umax_value >> 32)) {
2000 		/* u64 to u32 casting preserves validity of low 32 bits as
2001 		 * a range, if upper 32 bits are the same
2002 		 */
2003 		reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->umin_value);
2004 		reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->umax_value);
2005 
2006 		if ((s32)reg->umin_value <= (s32)reg->umax_value) {
2007 			reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value);
2008 			reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value);
2009 		}
2010 	}
2011 	if ((reg->smin_value >> 32) == (reg->smax_value >> 32)) {
2012 		/* low 32 bits should form a proper u32 range */
2013 		if ((u32)reg->smin_value <= (u32)reg->smax_value) {
2014 			reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->smin_value);
2015 			reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->smax_value);
2016 		}
2017 		/* low 32 bits should form a proper s32 range */
2018 		if ((s32)reg->smin_value <= (s32)reg->smax_value) {
2019 			reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value);
2020 			reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value);
2021 		}
2022 	}
2023 	/* Special case where upper bits form a small sequence of two
2024 	 * sequential numbers (in 32-bit unsigned space, so 0xffffffff to
2025 	 * 0x00000000 is also valid), while lower bits form a proper s32 range
2026 	 * going from negative numbers to positive numbers. E.g., let's say we
2027 	 * have s64 range [-1, 1] ([0xffffffffffffffff, 0x0000000000000001]).
2028 	 * Possible s64 values are {-1, 0, 1} ({0xffffffffffffffff,
2029 	 * 0x0000000000000000, 0x00000000000001}). Ignoring upper 32 bits,
2030 	 * we still get a valid s32 range [-1, 1] ([0xffffffff, 0x00000001]).
2031 	 * Note that it doesn't have to be 0xffffffff going to 0x00000000 in
2032 	 * upper 32 bits. As a random example, s64 range
2033 	 * [0xfffffff0fffffff0; 0xfffffff100000010], forms a valid s32 range
2034 	 * [-16, 16] ([0xfffffff0; 0x00000010]) in its 32 bit subregister.
2035 	 */
2036 	if ((u32)(reg->umin_value >> 32) + 1 == (u32)(reg->umax_value >> 32) &&
2037 	    (s32)reg->umin_value < 0 && (s32)reg->umax_value >= 0) {
2038 		reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value);
2039 		reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value);
2040 	}
2041 	if ((u32)(reg->smin_value >> 32) + 1 == (u32)(reg->smax_value >> 32) &&
2042 	    (s32)reg->smin_value < 0 && (s32)reg->smax_value >= 0) {
2043 		reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value);
2044 		reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value);
2045 	}
2046 	/* if u32 range forms a valid s32 range (due to matching sign bit),
2047 	 * try to learn from that
2048 	 */
2049 	if ((s32)reg->u32_min_value <= (s32)reg->u32_max_value) {
2050 		reg->s32_min_value = max_t(s32, reg->s32_min_value, reg->u32_min_value);
2051 		reg->s32_max_value = min_t(s32, reg->s32_max_value, reg->u32_max_value);
2052 	}
2053 	/* If we cannot cross the sign boundary, then signed and unsigned bounds
2054 	 * are the same, so combine.  This works even in the negative case, e.g.
2055 	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
2056 	 */
2057 	if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) {
2058 		reg->u32_min_value = max_t(u32, reg->s32_min_value, reg->u32_min_value);
2059 		reg->u32_max_value = min_t(u32, reg->s32_max_value, reg->u32_max_value);
2060 	}
2061 }
2062 
2063 static void __reg64_deduce_bounds(struct bpf_reg_state *reg)
2064 {
2065 	/* If u64 range forms a valid s64 range (due to matching sign bit),
2066 	 * try to learn from that. Let's do a bit of ASCII art to see when
2067 	 * this is happening. Let's take u64 range first:
2068 	 *
2069 	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
2070 	 * |-------------------------------|--------------------------------|
2071 	 *
2072 	 * Valid u64 range is formed when umin and umax are anywhere in the
2073 	 * range [0, U64_MAX], and umin <= umax. u64 case is simple and
2074 	 * straightforward. Let's see how s64 range maps onto the same range
2075 	 * of values, annotated below the line for comparison:
2076 	 *
2077 	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
2078 	 * |-------------------------------|--------------------------------|
2079 	 * 0                        S64_MAX S64_MIN                        -1
2080 	 *
2081 	 * So s64 values basically start in the middle and they are logically
2082 	 * contiguous to the right of it, wrapping around from -1 to 0, and
2083 	 * then finishing as S64_MAX (0x7fffffffffffffff) right before
2084 	 * S64_MIN. We can try drawing the continuity of u64 vs s64 values
2085 	 * more visually as mapped to sign-agnostic range of hex values.
2086 	 *
2087 	 *  u64 start                                               u64 end
2088 	 *  _______________________________________________________________
2089 	 * /                                                               \
2090 	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
2091 	 * |-------------------------------|--------------------------------|
2092 	 * 0                        S64_MAX S64_MIN                        -1
2093 	 *                                / \
2094 	 * >------------------------------   ------------------------------->
2095 	 * s64 continues...        s64 end   s64 start          s64 "midpoint"
2096 	 *
2097 	 * What this means is that, in general, we can't always derive
2098 	 * something new about u64 from any random s64 range, and vice versa.
2099 	 *
2100 	 * But we can do that in two particular cases. One is when entire
2101 	 * u64/s64 range is *entirely* contained within left half of the above
2102 	 * diagram or when it is *entirely* contained in the right half. I.e.:
2103 	 *
2104 	 * |-------------------------------|--------------------------------|
2105 	 *     ^                   ^            ^                 ^
2106 	 *     A                   B            C                 D
2107 	 *
2108 	 * [A, B] and [C, D] are contained entirely in their respective halves
2109 	 * and form valid contiguous ranges as both u64 and s64 values. [A, B]
2110 	 * will be non-negative both as u64 and s64 (and in fact it will be
2111 	 * identical ranges no matter the signedness). [C, D] treated as s64
2112 	 * will be a range of negative values, while in u64 it will be
2113 	 * non-negative range of values larger than 0x8000000000000000.
2114 	 *
2115 	 * Now, any other range here can't be represented in both u64 and s64
2116 	 * simultaneously. E.g., [A, C], [A, D], [B, C], [B, D] are valid
2117 	 * contiguous u64 ranges, but they are discontinuous in s64. [B, C]
2118 	 * in s64 would be properly presented as [S64_MIN, C] and [B, S64_MAX],
2119 	 * for example. Similarly, valid s64 range [D, A] (going from negative
2120 	 * to positive values), would be two separate [D, U64_MAX] and [0, A]
2121 	 * ranges as u64. Currently reg_state can't represent two segments per
2122 	 * numeric domain, so in such situations we can only derive maximal
2123 	 * possible range ([0, U64_MAX] for u64, and [S64_MIN, S64_MAX] for s64).
2124 	 *
2125 	 * So we use these facts to derive umin/umax from smin/smax and vice
2126 	 * versa only if they stay within the same "half". This is equivalent
2127 	 * to checking sign bit: lower half will have sign bit as zero, upper
2128 	 * half have sign bit 1. Below in code we simplify this by just
2129 	 * casting umin/umax as smin/smax and checking if they form valid
2130 	 * range, and vice versa. Those are equivalent checks.
2131 	 */
2132 	if ((s64)reg->umin_value <= (s64)reg->umax_value) {
2133 		reg->smin_value = max_t(s64, reg->smin_value, reg->umin_value);
2134 		reg->smax_value = min_t(s64, reg->smax_value, reg->umax_value);
2135 	}
2136 	/* If we cannot cross the sign boundary, then signed and unsigned bounds
2137 	 * are the same, so combine.  This works even in the negative case, e.g.
2138 	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
2139 	 */
2140 	if ((u64)reg->smin_value <= (u64)reg->smax_value) {
2141 		reg->umin_value = max_t(u64, reg->smin_value, reg->umin_value);
2142 		reg->umax_value = min_t(u64, reg->smax_value, reg->umax_value);
2143 	}
2144 }
2145 
2146 static void __reg_deduce_mixed_bounds(struct bpf_reg_state *reg)
2147 {
2148 	/* Try to tighten 64-bit bounds from 32-bit knowledge, using 32-bit
2149 	 * values on both sides of 64-bit range in hope to have tighter range.
2150 	 * E.g., if r1 is [0x1'00000000, 0x3'80000000], and we learn from
2151 	 * 32-bit signed > 0 operation that s32 bounds are now [1; 0x7fffffff].
2152 	 * With this, we can substitute 1 as low 32-bits of _low_ 64-bit bound
2153 	 * (0x100000000 -> 0x100000001) and 0x7fffffff as low 32-bits of
2154 	 * _high_ 64-bit bound (0x380000000 -> 0x37fffffff) and arrive at a
2155 	 * better overall bounds for r1 as [0x1'000000001; 0x3'7fffffff].
2156 	 * We just need to make sure that derived bounds we are intersecting
2157 	 * with are well-formed ranges in respective s64 or u64 domain, just
2158 	 * like we do with similar kinds of 32-to-64 or 64-to-32 adjustments.
2159 	 */
2160 	__u64 new_umin, new_umax;
2161 	__s64 new_smin, new_smax;
2162 
2163 	/* u32 -> u64 tightening, it's always well-formed */
2164 	new_umin = (reg->umin_value & ~0xffffffffULL) | reg->u32_min_value;
2165 	new_umax = (reg->umax_value & ~0xffffffffULL) | reg->u32_max_value;
2166 	reg->umin_value = max_t(u64, reg->umin_value, new_umin);
2167 	reg->umax_value = min_t(u64, reg->umax_value, new_umax);
2168 	/* u32 -> s64 tightening, u32 range embedded into s64 preserves range validity */
2169 	new_smin = (reg->smin_value & ~0xffffffffULL) | reg->u32_min_value;
2170 	new_smax = (reg->smax_value & ~0xffffffffULL) | reg->u32_max_value;
2171 	reg->smin_value = max_t(s64, reg->smin_value, new_smin);
2172 	reg->smax_value = min_t(s64, reg->smax_value, new_smax);
2173 
2174 	/* if s32 can be treated as valid u32 range, we can use it as well */
2175 	if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) {
2176 		/* s32 -> u64 tightening */
2177 		new_umin = (reg->umin_value & ~0xffffffffULL) | (u32)reg->s32_min_value;
2178 		new_umax = (reg->umax_value & ~0xffffffffULL) | (u32)reg->s32_max_value;
2179 		reg->umin_value = max_t(u64, reg->umin_value, new_umin);
2180 		reg->umax_value = min_t(u64, reg->umax_value, new_umax);
2181 		/* s32 -> s64 tightening */
2182 		new_smin = (reg->smin_value & ~0xffffffffULL) | (u32)reg->s32_min_value;
2183 		new_smax = (reg->smax_value & ~0xffffffffULL) | (u32)reg->s32_max_value;
2184 		reg->smin_value = max_t(s64, reg->smin_value, new_smin);
2185 		reg->smax_value = min_t(s64, reg->smax_value, new_smax);
2186 	}
2187 }
2188 
2189 static void __reg_deduce_bounds(struct bpf_reg_state *reg)
2190 {
2191 	__reg32_deduce_bounds(reg);
2192 	__reg64_deduce_bounds(reg);
2193 	__reg_deduce_mixed_bounds(reg);
2194 }
2195 
2196 /* Attempts to improve var_off based on unsigned min/max information */
2197 static void __reg_bound_offset(struct bpf_reg_state *reg)
2198 {
2199 	struct tnum var64_off = tnum_intersect(reg->var_off,
2200 					       tnum_range(reg->umin_value,
2201 							  reg->umax_value));
2202 	struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off),
2203 					       tnum_range(reg->u32_min_value,
2204 							  reg->u32_max_value));
2205 
2206 	reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off);
2207 }
2208 
2209 static void reg_bounds_sync(struct bpf_reg_state *reg)
2210 {
2211 	/* We might have learned new bounds from the var_off. */
2212 	__update_reg_bounds(reg);
2213 	/* We might have learned something about the sign bit. */
2214 	__reg_deduce_bounds(reg);
2215 	__reg_deduce_bounds(reg);
2216 	/* We might have learned some bits from the bounds. */
2217 	__reg_bound_offset(reg);
2218 	/* Intersecting with the old var_off might have improved our bounds
2219 	 * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
2220 	 * then new var_off is (0; 0x7f...fc) which improves our umax.
2221 	 */
2222 	__update_reg_bounds(reg);
2223 }
2224 
2225 static int reg_bounds_sanity_check(struct bpf_verifier_env *env,
2226 				   struct bpf_reg_state *reg, const char *ctx)
2227 {
2228 	const char *msg;
2229 
2230 	if (reg->umin_value > reg->umax_value ||
2231 	    reg->smin_value > reg->smax_value ||
2232 	    reg->u32_min_value > reg->u32_max_value ||
2233 	    reg->s32_min_value > reg->s32_max_value) {
2234 		    msg = "range bounds violation";
2235 		    goto out;
2236 	}
2237 
2238 	if (tnum_is_const(reg->var_off)) {
2239 		u64 uval = reg->var_off.value;
2240 		s64 sval = (s64)uval;
2241 
2242 		if (reg->umin_value != uval || reg->umax_value != uval ||
2243 		    reg->smin_value != sval || reg->smax_value != sval) {
2244 			msg = "const tnum out of sync with range bounds";
2245 			goto out;
2246 		}
2247 	}
2248 
2249 	if (tnum_subreg_is_const(reg->var_off)) {
2250 		u32 uval32 = tnum_subreg(reg->var_off).value;
2251 		s32 sval32 = (s32)uval32;
2252 
2253 		if (reg->u32_min_value != uval32 || reg->u32_max_value != uval32 ||
2254 		    reg->s32_min_value != sval32 || reg->s32_max_value != sval32) {
2255 			msg = "const subreg tnum out of sync with range bounds";
2256 			goto out;
2257 		}
2258 	}
2259 
2260 	return 0;
2261 out:
2262 	verbose(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] "
2263 		"s64=[%#llx, %#llx] u32=[%#x, %#x] s32=[%#x, %#x] var_off=(%#llx, %#llx)\n",
2264 		ctx, msg, reg->umin_value, reg->umax_value,
2265 		reg->smin_value, reg->smax_value,
2266 		reg->u32_min_value, reg->u32_max_value,
2267 		reg->s32_min_value, reg->s32_max_value,
2268 		reg->var_off.value, reg->var_off.mask);
2269 	if (env->test_reg_invariants)
2270 		return -EFAULT;
2271 	__mark_reg_unbounded(reg);
2272 	return 0;
2273 }
2274 
2275 static bool __reg32_bound_s64(s32 a)
2276 {
2277 	return a >= 0 && a <= S32_MAX;
2278 }
2279 
2280 static void __reg_assign_32_into_64(struct bpf_reg_state *reg)
2281 {
2282 	reg->umin_value = reg->u32_min_value;
2283 	reg->umax_value = reg->u32_max_value;
2284 
2285 	/* Attempt to pull 32-bit signed bounds into 64-bit bounds but must
2286 	 * be positive otherwise set to worse case bounds and refine later
2287 	 * from tnum.
2288 	 */
2289 	if (__reg32_bound_s64(reg->s32_min_value) &&
2290 	    __reg32_bound_s64(reg->s32_max_value)) {
2291 		reg->smin_value = reg->s32_min_value;
2292 		reg->smax_value = reg->s32_max_value;
2293 	} else {
2294 		reg->smin_value = 0;
2295 		reg->smax_value = U32_MAX;
2296 	}
2297 }
2298 
2299 /* Mark a register as having a completely unknown (scalar) value. */
2300 static void __mark_reg_unknown_imprecise(struct bpf_reg_state *reg)
2301 {
2302 	/*
2303 	 * Clear type, off, and union(map_ptr, range) and
2304 	 * padding between 'type' and union
2305 	 */
2306 	memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
2307 	reg->type = SCALAR_VALUE;
2308 	reg->id = 0;
2309 	reg->ref_obj_id = 0;
2310 	reg->var_off = tnum_unknown;
2311 	reg->frameno = 0;
2312 	reg->precise = false;
2313 	__mark_reg_unbounded(reg);
2314 }
2315 
2316 /* Mark a register as having a completely unknown (scalar) value,
2317  * initialize .precise as true when not bpf capable.
2318  */
2319 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
2320 			       struct bpf_reg_state *reg)
2321 {
2322 	__mark_reg_unknown_imprecise(reg);
2323 	reg->precise = !env->bpf_capable;
2324 }
2325 
2326 static void mark_reg_unknown(struct bpf_verifier_env *env,
2327 			     struct bpf_reg_state *regs, u32 regno)
2328 {
2329 	if (WARN_ON(regno >= MAX_BPF_REG)) {
2330 		verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
2331 		/* Something bad happened, let's kill all regs except FP */
2332 		for (regno = 0; regno < BPF_REG_FP; regno++)
2333 			__mark_reg_not_init(env, regs + regno);
2334 		return;
2335 	}
2336 	__mark_reg_unknown(env, regs + regno);
2337 }
2338 
2339 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
2340 				struct bpf_reg_state *reg)
2341 {
2342 	__mark_reg_unknown(env, reg);
2343 	reg->type = NOT_INIT;
2344 }
2345 
2346 static void mark_reg_not_init(struct bpf_verifier_env *env,
2347 			      struct bpf_reg_state *regs, u32 regno)
2348 {
2349 	if (WARN_ON(regno >= MAX_BPF_REG)) {
2350 		verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
2351 		/* Something bad happened, let's kill all regs except FP */
2352 		for (regno = 0; regno < BPF_REG_FP; regno++)
2353 			__mark_reg_not_init(env, regs + regno);
2354 		return;
2355 	}
2356 	__mark_reg_not_init(env, regs + regno);
2357 }
2358 
2359 static void mark_btf_ld_reg(struct bpf_verifier_env *env,
2360 			    struct bpf_reg_state *regs, u32 regno,
2361 			    enum bpf_reg_type reg_type,
2362 			    struct btf *btf, u32 btf_id,
2363 			    enum bpf_type_flag flag)
2364 {
2365 	if (reg_type == SCALAR_VALUE) {
2366 		mark_reg_unknown(env, regs, regno);
2367 		return;
2368 	}
2369 	mark_reg_known_zero(env, regs, regno);
2370 	regs[regno].type = PTR_TO_BTF_ID | flag;
2371 	regs[regno].btf = btf;
2372 	regs[regno].btf_id = btf_id;
2373 	if (type_may_be_null(flag))
2374 		regs[regno].id = ++env->id_gen;
2375 }
2376 
2377 #define DEF_NOT_SUBREG	(0)
2378 static void init_reg_state(struct bpf_verifier_env *env,
2379 			   struct bpf_func_state *state)
2380 {
2381 	struct bpf_reg_state *regs = state->regs;
2382 	int i;
2383 
2384 	for (i = 0; i < MAX_BPF_REG; i++) {
2385 		mark_reg_not_init(env, regs, i);
2386 		regs[i].live = REG_LIVE_NONE;
2387 		regs[i].parent = NULL;
2388 		regs[i].subreg_def = DEF_NOT_SUBREG;
2389 	}
2390 
2391 	/* frame pointer */
2392 	regs[BPF_REG_FP].type = PTR_TO_STACK;
2393 	mark_reg_known_zero(env, regs, BPF_REG_FP);
2394 	regs[BPF_REG_FP].frameno = state->frameno;
2395 }
2396 
2397 static struct bpf_retval_range retval_range(s32 minval, s32 maxval)
2398 {
2399 	return (struct bpf_retval_range){ minval, maxval };
2400 }
2401 
2402 #define BPF_MAIN_FUNC (-1)
2403 static void init_func_state(struct bpf_verifier_env *env,
2404 			    struct bpf_func_state *state,
2405 			    int callsite, int frameno, int subprogno)
2406 {
2407 	state->callsite = callsite;
2408 	state->frameno = frameno;
2409 	state->subprogno = subprogno;
2410 	state->callback_ret_range = retval_range(0, 0);
2411 	init_reg_state(env, state);
2412 	mark_verifier_state_scratched(env);
2413 }
2414 
2415 /* Similar to push_stack(), but for async callbacks */
2416 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
2417 						int insn_idx, int prev_insn_idx,
2418 						int subprog, bool is_sleepable)
2419 {
2420 	struct bpf_verifier_stack_elem *elem;
2421 	struct bpf_func_state *frame;
2422 
2423 	elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
2424 	if (!elem)
2425 		goto err;
2426 
2427 	elem->insn_idx = insn_idx;
2428 	elem->prev_insn_idx = prev_insn_idx;
2429 	elem->next = env->head;
2430 	elem->log_pos = env->log.end_pos;
2431 	env->head = elem;
2432 	env->stack_size++;
2433 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
2434 		verbose(env,
2435 			"The sequence of %d jumps is too complex for async cb.\n",
2436 			env->stack_size);
2437 		goto err;
2438 	}
2439 	/* Unlike push_stack() do not copy_verifier_state().
2440 	 * The caller state doesn't matter.
2441 	 * This is async callback. It starts in a fresh stack.
2442 	 * Initialize it similar to do_check_common().
2443 	 */
2444 	elem->st.branches = 1;
2445 	elem->st.in_sleepable = is_sleepable;
2446 	frame = kzalloc(sizeof(*frame), GFP_KERNEL);
2447 	if (!frame)
2448 		goto err;
2449 	init_func_state(env, frame,
2450 			BPF_MAIN_FUNC /* callsite */,
2451 			0 /* frameno within this callchain */,
2452 			subprog /* subprog number within this prog */);
2453 	elem->st.frame[0] = frame;
2454 	return &elem->st;
2455 err:
2456 	free_verifier_state(env->cur_state, true);
2457 	env->cur_state = NULL;
2458 	/* pop all elements and return */
2459 	while (!pop_stack(env, NULL, NULL, false));
2460 	return NULL;
2461 }
2462 
2463 
2464 enum reg_arg_type {
2465 	SRC_OP,		/* register is used as source operand */
2466 	DST_OP,		/* register is used as destination operand */
2467 	DST_OP_NO_MARK	/* same as above, check only, don't mark */
2468 };
2469 
2470 static int cmp_subprogs(const void *a, const void *b)
2471 {
2472 	return ((struct bpf_subprog_info *)a)->start -
2473 	       ((struct bpf_subprog_info *)b)->start;
2474 }
2475 
2476 static int find_subprog(struct bpf_verifier_env *env, int off)
2477 {
2478 	struct bpf_subprog_info *p;
2479 
2480 	p = bsearch(&off, env->subprog_info, env->subprog_cnt,
2481 		    sizeof(env->subprog_info[0]), cmp_subprogs);
2482 	if (!p)
2483 		return -ENOENT;
2484 	return p - env->subprog_info;
2485 
2486 }
2487 
2488 static int add_subprog(struct bpf_verifier_env *env, int off)
2489 {
2490 	int insn_cnt = env->prog->len;
2491 	int ret;
2492 
2493 	if (off >= insn_cnt || off < 0) {
2494 		verbose(env, "call to invalid destination\n");
2495 		return -EINVAL;
2496 	}
2497 	ret = find_subprog(env, off);
2498 	if (ret >= 0)
2499 		return ret;
2500 	if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
2501 		verbose(env, "too many subprograms\n");
2502 		return -E2BIG;
2503 	}
2504 	/* determine subprog starts. The end is one before the next starts */
2505 	env->subprog_info[env->subprog_cnt++].start = off;
2506 	sort(env->subprog_info, env->subprog_cnt,
2507 	     sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
2508 	return env->subprog_cnt - 1;
2509 }
2510 
2511 static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env)
2512 {
2513 	struct bpf_prog_aux *aux = env->prog->aux;
2514 	struct btf *btf = aux->btf;
2515 	const struct btf_type *t;
2516 	u32 main_btf_id, id;
2517 	const char *name;
2518 	int ret, i;
2519 
2520 	/* Non-zero func_info_cnt implies valid btf */
2521 	if (!aux->func_info_cnt)
2522 		return 0;
2523 	main_btf_id = aux->func_info[0].type_id;
2524 
2525 	t = btf_type_by_id(btf, main_btf_id);
2526 	if (!t) {
2527 		verbose(env, "invalid btf id for main subprog in func_info\n");
2528 		return -EINVAL;
2529 	}
2530 
2531 	name = btf_find_decl_tag_value(btf, t, -1, "exception_callback:");
2532 	if (IS_ERR(name)) {
2533 		ret = PTR_ERR(name);
2534 		/* If there is no tag present, there is no exception callback */
2535 		if (ret == -ENOENT)
2536 			ret = 0;
2537 		else if (ret == -EEXIST)
2538 			verbose(env, "multiple exception callback tags for main subprog\n");
2539 		return ret;
2540 	}
2541 
2542 	ret = btf_find_by_name_kind(btf, name, BTF_KIND_FUNC);
2543 	if (ret < 0) {
2544 		verbose(env, "exception callback '%s' could not be found in BTF\n", name);
2545 		return ret;
2546 	}
2547 	id = ret;
2548 	t = btf_type_by_id(btf, id);
2549 	if (btf_func_linkage(t) != BTF_FUNC_GLOBAL) {
2550 		verbose(env, "exception callback '%s' must have global linkage\n", name);
2551 		return -EINVAL;
2552 	}
2553 	ret = 0;
2554 	for (i = 0; i < aux->func_info_cnt; i++) {
2555 		if (aux->func_info[i].type_id != id)
2556 			continue;
2557 		ret = aux->func_info[i].insn_off;
2558 		/* Further func_info and subprog checks will also happen
2559 		 * later, so assume this is the right insn_off for now.
2560 		 */
2561 		if (!ret) {
2562 			verbose(env, "invalid exception callback insn_off in func_info: 0\n");
2563 			ret = -EINVAL;
2564 		}
2565 	}
2566 	if (!ret) {
2567 		verbose(env, "exception callback type id not found in func_info\n");
2568 		ret = -EINVAL;
2569 	}
2570 	return ret;
2571 }
2572 
2573 #define MAX_KFUNC_DESCS 256
2574 #define MAX_KFUNC_BTFS	256
2575 
2576 struct bpf_kfunc_desc {
2577 	struct btf_func_model func_model;
2578 	u32 func_id;
2579 	s32 imm;
2580 	u16 offset;
2581 	unsigned long addr;
2582 };
2583 
2584 struct bpf_kfunc_btf {
2585 	struct btf *btf;
2586 	struct module *module;
2587 	u16 offset;
2588 };
2589 
2590 struct bpf_kfunc_desc_tab {
2591 	/* Sorted by func_id (BTF ID) and offset (fd_array offset) during
2592 	 * verification. JITs do lookups by bpf_insn, where func_id may not be
2593 	 * available, therefore at the end of verification do_misc_fixups()
2594 	 * sorts this by imm and offset.
2595 	 */
2596 	struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
2597 	u32 nr_descs;
2598 };
2599 
2600 struct bpf_kfunc_btf_tab {
2601 	struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS];
2602 	u32 nr_descs;
2603 };
2604 
2605 static int kfunc_desc_cmp_by_id_off(const void *a, const void *b)
2606 {
2607 	const struct bpf_kfunc_desc *d0 = a;
2608 	const struct bpf_kfunc_desc *d1 = b;
2609 
2610 	/* func_id is not greater than BTF_MAX_TYPE */
2611 	return d0->func_id - d1->func_id ?: d0->offset - d1->offset;
2612 }
2613 
2614 static int kfunc_btf_cmp_by_off(const void *a, const void *b)
2615 {
2616 	const struct bpf_kfunc_btf *d0 = a;
2617 	const struct bpf_kfunc_btf *d1 = b;
2618 
2619 	return d0->offset - d1->offset;
2620 }
2621 
2622 static const struct bpf_kfunc_desc *
2623 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
2624 {
2625 	struct bpf_kfunc_desc desc = {
2626 		.func_id = func_id,
2627 		.offset = offset,
2628 	};
2629 	struct bpf_kfunc_desc_tab *tab;
2630 
2631 	tab = prog->aux->kfunc_tab;
2632 	return bsearch(&desc, tab->descs, tab->nr_descs,
2633 		       sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
2634 }
2635 
2636 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2637 		       u16 btf_fd_idx, u8 **func_addr)
2638 {
2639 	const struct bpf_kfunc_desc *desc;
2640 
2641 	desc = find_kfunc_desc(prog, func_id, btf_fd_idx);
2642 	if (!desc)
2643 		return -EFAULT;
2644 
2645 	*func_addr = (u8 *)desc->addr;
2646 	return 0;
2647 }
2648 
2649 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
2650 					 s16 offset)
2651 {
2652 	struct bpf_kfunc_btf kf_btf = { .offset = offset };
2653 	struct bpf_kfunc_btf_tab *tab;
2654 	struct bpf_kfunc_btf *b;
2655 	struct module *mod;
2656 	struct btf *btf;
2657 	int btf_fd;
2658 
2659 	tab = env->prog->aux->kfunc_btf_tab;
2660 	b = bsearch(&kf_btf, tab->descs, tab->nr_descs,
2661 		    sizeof(tab->descs[0]), kfunc_btf_cmp_by_off);
2662 	if (!b) {
2663 		if (tab->nr_descs == MAX_KFUNC_BTFS) {
2664 			verbose(env, "too many different module BTFs\n");
2665 			return ERR_PTR(-E2BIG);
2666 		}
2667 
2668 		if (bpfptr_is_null(env->fd_array)) {
2669 			verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
2670 			return ERR_PTR(-EPROTO);
2671 		}
2672 
2673 		if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,
2674 					    offset * sizeof(btf_fd),
2675 					    sizeof(btf_fd)))
2676 			return ERR_PTR(-EFAULT);
2677 
2678 		btf = btf_get_by_fd(btf_fd);
2679 		if (IS_ERR(btf)) {
2680 			verbose(env, "invalid module BTF fd specified\n");
2681 			return btf;
2682 		}
2683 
2684 		if (!btf_is_module(btf)) {
2685 			verbose(env, "BTF fd for kfunc is not a module BTF\n");
2686 			btf_put(btf);
2687 			return ERR_PTR(-EINVAL);
2688 		}
2689 
2690 		mod = btf_try_get_module(btf);
2691 		if (!mod) {
2692 			btf_put(btf);
2693 			return ERR_PTR(-ENXIO);
2694 		}
2695 
2696 		b = &tab->descs[tab->nr_descs++];
2697 		b->btf = btf;
2698 		b->module = mod;
2699 		b->offset = offset;
2700 
2701 		sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
2702 		     kfunc_btf_cmp_by_off, NULL);
2703 	}
2704 	return b->btf;
2705 }
2706 
2707 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab)
2708 {
2709 	if (!tab)
2710 		return;
2711 
2712 	while (tab->nr_descs--) {
2713 		module_put(tab->descs[tab->nr_descs].module);
2714 		btf_put(tab->descs[tab->nr_descs].btf);
2715 	}
2716 	kfree(tab);
2717 }
2718 
2719 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)
2720 {
2721 	if (offset) {
2722 		if (offset < 0) {
2723 			/* In the future, this can be allowed to increase limit
2724 			 * of fd index into fd_array, interpreted as u16.
2725 			 */
2726 			verbose(env, "negative offset disallowed for kernel module function call\n");
2727 			return ERR_PTR(-EINVAL);
2728 		}
2729 
2730 		return __find_kfunc_desc_btf(env, offset);
2731 	}
2732 	return btf_vmlinux ?: ERR_PTR(-ENOENT);
2733 }
2734 
2735 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
2736 {
2737 	const struct btf_type *func, *func_proto;
2738 	struct bpf_kfunc_btf_tab *btf_tab;
2739 	struct bpf_kfunc_desc_tab *tab;
2740 	struct bpf_prog_aux *prog_aux;
2741 	struct bpf_kfunc_desc *desc;
2742 	const char *func_name;
2743 	struct btf *desc_btf;
2744 	unsigned long call_imm;
2745 	unsigned long addr;
2746 	int err;
2747 
2748 	prog_aux = env->prog->aux;
2749 	tab = prog_aux->kfunc_tab;
2750 	btf_tab = prog_aux->kfunc_btf_tab;
2751 	if (!tab) {
2752 		if (!btf_vmlinux) {
2753 			verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
2754 			return -ENOTSUPP;
2755 		}
2756 
2757 		if (!env->prog->jit_requested) {
2758 			verbose(env, "JIT is required for calling kernel function\n");
2759 			return -ENOTSUPP;
2760 		}
2761 
2762 		if (!bpf_jit_supports_kfunc_call()) {
2763 			verbose(env, "JIT does not support calling kernel function\n");
2764 			return -ENOTSUPP;
2765 		}
2766 
2767 		if (!env->prog->gpl_compatible) {
2768 			verbose(env, "cannot call kernel function from non-GPL compatible program\n");
2769 			return -EINVAL;
2770 		}
2771 
2772 		tab = kzalloc(sizeof(*tab), GFP_KERNEL);
2773 		if (!tab)
2774 			return -ENOMEM;
2775 		prog_aux->kfunc_tab = tab;
2776 	}
2777 
2778 	/* func_id == 0 is always invalid, but instead of returning an error, be
2779 	 * conservative and wait until the code elimination pass before returning
2780 	 * error, so that invalid calls that get pruned out can be in BPF programs
2781 	 * loaded from userspace.  It is also required that offset be untouched
2782 	 * for such calls.
2783 	 */
2784 	if (!func_id && !offset)
2785 		return 0;
2786 
2787 	if (!btf_tab && offset) {
2788 		btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL);
2789 		if (!btf_tab)
2790 			return -ENOMEM;
2791 		prog_aux->kfunc_btf_tab = btf_tab;
2792 	}
2793 
2794 	desc_btf = find_kfunc_desc_btf(env, offset);
2795 	if (IS_ERR(desc_btf)) {
2796 		verbose(env, "failed to find BTF for kernel function\n");
2797 		return PTR_ERR(desc_btf);
2798 	}
2799 
2800 	if (find_kfunc_desc(env->prog, func_id, offset))
2801 		return 0;
2802 
2803 	if (tab->nr_descs == MAX_KFUNC_DESCS) {
2804 		verbose(env, "too many different kernel function calls\n");
2805 		return -E2BIG;
2806 	}
2807 
2808 	func = btf_type_by_id(desc_btf, func_id);
2809 	if (!func || !btf_type_is_func(func)) {
2810 		verbose(env, "kernel btf_id %u is not a function\n",
2811 			func_id);
2812 		return -EINVAL;
2813 	}
2814 	func_proto = btf_type_by_id(desc_btf, func->type);
2815 	if (!func_proto || !btf_type_is_func_proto(func_proto)) {
2816 		verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
2817 			func_id);
2818 		return -EINVAL;
2819 	}
2820 
2821 	func_name = btf_name_by_offset(desc_btf, func->name_off);
2822 	addr = kallsyms_lookup_name(func_name);
2823 	if (!addr) {
2824 		verbose(env, "cannot find address for kernel function %s\n",
2825 			func_name);
2826 		return -EINVAL;
2827 	}
2828 	specialize_kfunc(env, func_id, offset, &addr);
2829 
2830 	if (bpf_jit_supports_far_kfunc_call()) {
2831 		call_imm = func_id;
2832 	} else {
2833 		call_imm = BPF_CALL_IMM(addr);
2834 		/* Check whether the relative offset overflows desc->imm */
2835 		if ((unsigned long)(s32)call_imm != call_imm) {
2836 			verbose(env, "address of kernel function %s is out of range\n",
2837 				func_name);
2838 			return -EINVAL;
2839 		}
2840 	}
2841 
2842 	if (bpf_dev_bound_kfunc_id(func_id)) {
2843 		err = bpf_dev_bound_kfunc_check(&env->log, prog_aux);
2844 		if (err)
2845 			return err;
2846 	}
2847 
2848 	desc = &tab->descs[tab->nr_descs++];
2849 	desc->func_id = func_id;
2850 	desc->imm = call_imm;
2851 	desc->offset = offset;
2852 	desc->addr = addr;
2853 	err = btf_distill_func_proto(&env->log, desc_btf,
2854 				     func_proto, func_name,
2855 				     &desc->func_model);
2856 	if (!err)
2857 		sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
2858 		     kfunc_desc_cmp_by_id_off, NULL);
2859 	return err;
2860 }
2861 
2862 static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b)
2863 {
2864 	const struct bpf_kfunc_desc *d0 = a;
2865 	const struct bpf_kfunc_desc *d1 = b;
2866 
2867 	if (d0->imm != d1->imm)
2868 		return d0->imm < d1->imm ? -1 : 1;
2869 	if (d0->offset != d1->offset)
2870 		return d0->offset < d1->offset ? -1 : 1;
2871 	return 0;
2872 }
2873 
2874 static void sort_kfunc_descs_by_imm_off(struct bpf_prog *prog)
2875 {
2876 	struct bpf_kfunc_desc_tab *tab;
2877 
2878 	tab = prog->aux->kfunc_tab;
2879 	if (!tab)
2880 		return;
2881 
2882 	sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
2883 	     kfunc_desc_cmp_by_imm_off, NULL);
2884 }
2885 
2886 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
2887 {
2888 	return !!prog->aux->kfunc_tab;
2889 }
2890 
2891 const struct btf_func_model *
2892 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2893 			 const struct bpf_insn *insn)
2894 {
2895 	const struct bpf_kfunc_desc desc = {
2896 		.imm = insn->imm,
2897 		.offset = insn->off,
2898 	};
2899 	const struct bpf_kfunc_desc *res;
2900 	struct bpf_kfunc_desc_tab *tab;
2901 
2902 	tab = prog->aux->kfunc_tab;
2903 	res = bsearch(&desc, tab->descs, tab->nr_descs,
2904 		      sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off);
2905 
2906 	return res ? &res->func_model : NULL;
2907 }
2908 
2909 static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
2910 {
2911 	struct bpf_subprog_info *subprog = env->subprog_info;
2912 	int i, ret, insn_cnt = env->prog->len, ex_cb_insn;
2913 	struct bpf_insn *insn = env->prog->insnsi;
2914 
2915 	/* Add entry function. */
2916 	ret = add_subprog(env, 0);
2917 	if (ret)
2918 		return ret;
2919 
2920 	for (i = 0; i < insn_cnt; i++, insn++) {
2921 		if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
2922 		    !bpf_pseudo_kfunc_call(insn))
2923 			continue;
2924 
2925 		if (!env->bpf_capable) {
2926 			verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
2927 			return -EPERM;
2928 		}
2929 
2930 		if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn))
2931 			ret = add_subprog(env, i + insn->imm + 1);
2932 		else
2933 			ret = add_kfunc_call(env, insn->imm, insn->off);
2934 
2935 		if (ret < 0)
2936 			return ret;
2937 	}
2938 
2939 	ret = bpf_find_exception_callback_insn_off(env);
2940 	if (ret < 0)
2941 		return ret;
2942 	ex_cb_insn = ret;
2943 
2944 	/* If ex_cb_insn > 0, this means that the main program has a subprog
2945 	 * marked using BTF decl tag to serve as the exception callback.
2946 	 */
2947 	if (ex_cb_insn) {
2948 		ret = add_subprog(env, ex_cb_insn);
2949 		if (ret < 0)
2950 			return ret;
2951 		for (i = 1; i < env->subprog_cnt; i++) {
2952 			if (env->subprog_info[i].start != ex_cb_insn)
2953 				continue;
2954 			env->exception_callback_subprog = i;
2955 			mark_subprog_exc_cb(env, i);
2956 			break;
2957 		}
2958 	}
2959 
2960 	/* Add a fake 'exit' subprog which could simplify subprog iteration
2961 	 * logic. 'subprog_cnt' should not be increased.
2962 	 */
2963 	subprog[env->subprog_cnt].start = insn_cnt;
2964 
2965 	if (env->log.level & BPF_LOG_LEVEL2)
2966 		for (i = 0; i < env->subprog_cnt; i++)
2967 			verbose(env, "func#%d @%d\n", i, subprog[i].start);
2968 
2969 	return 0;
2970 }
2971 
2972 static int check_subprogs(struct bpf_verifier_env *env)
2973 {
2974 	int i, subprog_start, subprog_end, off, cur_subprog = 0;
2975 	struct bpf_subprog_info *subprog = env->subprog_info;
2976 	struct bpf_insn *insn = env->prog->insnsi;
2977 	int insn_cnt = env->prog->len;
2978 
2979 	/* now check that all jumps are within the same subprog */
2980 	subprog_start = subprog[cur_subprog].start;
2981 	subprog_end = subprog[cur_subprog + 1].start;
2982 	for (i = 0; i < insn_cnt; i++) {
2983 		u8 code = insn[i].code;
2984 
2985 		if (code == (BPF_JMP | BPF_CALL) &&
2986 		    insn[i].src_reg == 0 &&
2987 		    insn[i].imm == BPF_FUNC_tail_call) {
2988 			subprog[cur_subprog].has_tail_call = true;
2989 			subprog[cur_subprog].tail_call_reachable = true;
2990 		}
2991 		if (BPF_CLASS(code) == BPF_LD &&
2992 		    (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND))
2993 			subprog[cur_subprog].has_ld_abs = true;
2994 		if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
2995 			goto next;
2996 		if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
2997 			goto next;
2998 		if (code == (BPF_JMP32 | BPF_JA))
2999 			off = i + insn[i].imm + 1;
3000 		else
3001 			off = i + insn[i].off + 1;
3002 		if (off < subprog_start || off >= subprog_end) {
3003 			verbose(env, "jump out of range from insn %d to %d\n", i, off);
3004 			return -EINVAL;
3005 		}
3006 next:
3007 		if (i == subprog_end - 1) {
3008 			/* to avoid fall-through from one subprog into another
3009 			 * the last insn of the subprog should be either exit
3010 			 * or unconditional jump back or bpf_throw call
3011 			 */
3012 			if (code != (BPF_JMP | BPF_EXIT) &&
3013 			    code != (BPF_JMP32 | BPF_JA) &&
3014 			    code != (BPF_JMP | BPF_JA)) {
3015 				verbose(env, "last insn is not an exit or jmp\n");
3016 				return -EINVAL;
3017 			}
3018 			subprog_start = subprog_end;
3019 			cur_subprog++;
3020 			if (cur_subprog < env->subprog_cnt)
3021 				subprog_end = subprog[cur_subprog + 1].start;
3022 		}
3023 	}
3024 	return 0;
3025 }
3026 
3027 /* Parentage chain of this register (or stack slot) should take care of all
3028  * issues like callee-saved registers, stack slot allocation time, etc.
3029  */
3030 static int mark_reg_read(struct bpf_verifier_env *env,
3031 			 const struct bpf_reg_state *state,
3032 			 struct bpf_reg_state *parent, u8 flag)
3033 {
3034 	bool writes = parent == state->parent; /* Observe write marks */
3035 	int cnt = 0;
3036 
3037 	while (parent) {
3038 		/* if read wasn't screened by an earlier write ... */
3039 		if (writes && state->live & REG_LIVE_WRITTEN)
3040 			break;
3041 		if (parent->live & REG_LIVE_DONE) {
3042 			verbose(env, "verifier BUG type %s var_off %lld off %d\n",
3043 				reg_type_str(env, parent->type),
3044 				parent->var_off.value, parent->off);
3045 			return -EFAULT;
3046 		}
3047 		/* The first condition is more likely to be true than the
3048 		 * second, checked it first.
3049 		 */
3050 		if ((parent->live & REG_LIVE_READ) == flag ||
3051 		    parent->live & REG_LIVE_READ64)
3052 			/* The parentage chain never changes and
3053 			 * this parent was already marked as LIVE_READ.
3054 			 * There is no need to keep walking the chain again and
3055 			 * keep re-marking all parents as LIVE_READ.
3056 			 * This case happens when the same register is read
3057 			 * multiple times without writes into it in-between.
3058 			 * Also, if parent has the stronger REG_LIVE_READ64 set,
3059 			 * then no need to set the weak REG_LIVE_READ32.
3060 			 */
3061 			break;
3062 		/* ... then we depend on parent's value */
3063 		parent->live |= flag;
3064 		/* REG_LIVE_READ64 overrides REG_LIVE_READ32. */
3065 		if (flag == REG_LIVE_READ64)
3066 			parent->live &= ~REG_LIVE_READ32;
3067 		state = parent;
3068 		parent = state->parent;
3069 		writes = true;
3070 		cnt++;
3071 	}
3072 
3073 	if (env->longest_mark_read_walk < cnt)
3074 		env->longest_mark_read_walk = cnt;
3075 	return 0;
3076 }
3077 
3078 static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
3079 {
3080 	struct bpf_func_state *state = func(env, reg);
3081 	int spi, ret;
3082 
3083 	/* For CONST_PTR_TO_DYNPTR, it must have already been done by
3084 	 * check_reg_arg in check_helper_call and mark_btf_func_reg_size in
3085 	 * check_kfunc_call.
3086 	 */
3087 	if (reg->type == CONST_PTR_TO_DYNPTR)
3088 		return 0;
3089 	spi = dynptr_get_spi(env, reg);
3090 	if (spi < 0)
3091 		return spi;
3092 	/* Caller ensures dynptr is valid and initialized, which means spi is in
3093 	 * bounds and spi is the first dynptr slot. Simply mark stack slot as
3094 	 * read.
3095 	 */
3096 	ret = mark_reg_read(env, &state->stack[spi].spilled_ptr,
3097 			    state->stack[spi].spilled_ptr.parent, REG_LIVE_READ64);
3098 	if (ret)
3099 		return ret;
3100 	return mark_reg_read(env, &state->stack[spi - 1].spilled_ptr,
3101 			     state->stack[spi - 1].spilled_ptr.parent, REG_LIVE_READ64);
3102 }
3103 
3104 static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
3105 			  int spi, int nr_slots)
3106 {
3107 	struct bpf_func_state *state = func(env, reg);
3108 	int err, i;
3109 
3110 	for (i = 0; i < nr_slots; i++) {
3111 		struct bpf_reg_state *st = &state->stack[spi - i].spilled_ptr;
3112 
3113 		err = mark_reg_read(env, st, st->parent, REG_LIVE_READ64);
3114 		if (err)
3115 			return err;
3116 
3117 		mark_stack_slot_scratched(env, spi - i);
3118 	}
3119 
3120 	return 0;
3121 }
3122 
3123 /* This function is supposed to be used by the following 32-bit optimization
3124  * code only. It returns TRUE if the source or destination register operates
3125  * on 64-bit, otherwise return FALSE.
3126  */
3127 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,
3128 		     u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t)
3129 {
3130 	u8 code, class, op;
3131 
3132 	code = insn->code;
3133 	class = BPF_CLASS(code);
3134 	op = BPF_OP(code);
3135 	if (class == BPF_JMP) {
3136 		/* BPF_EXIT for "main" will reach here. Return TRUE
3137 		 * conservatively.
3138 		 */
3139 		if (op == BPF_EXIT)
3140 			return true;
3141 		if (op == BPF_CALL) {
3142 			/* BPF to BPF call will reach here because of marking
3143 			 * caller saved clobber with DST_OP_NO_MARK for which we
3144 			 * don't care the register def because they are anyway
3145 			 * marked as NOT_INIT already.
3146 			 */
3147 			if (insn->src_reg == BPF_PSEUDO_CALL)
3148 				return false;
3149 			/* Helper call will reach here because of arg type
3150 			 * check, conservatively return TRUE.
3151 			 */
3152 			if (t == SRC_OP)
3153 				return true;
3154 
3155 			return false;
3156 		}
3157 	}
3158 
3159 	if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32))
3160 		return false;
3161 
3162 	if (class == BPF_ALU64 || class == BPF_JMP ||
3163 	    (class == BPF_ALU && op == BPF_END && insn->imm == 64))
3164 		return true;
3165 
3166 	if (class == BPF_ALU || class == BPF_JMP32)
3167 		return false;
3168 
3169 	if (class == BPF_LDX) {
3170 		if (t != SRC_OP)
3171 			return BPF_SIZE(code) == BPF_DW || BPF_MODE(code) == BPF_MEMSX;
3172 		/* LDX source must be ptr. */
3173 		return true;
3174 	}
3175 
3176 	if (class == BPF_STX) {
3177 		/* BPF_STX (including atomic variants) has multiple source
3178 		 * operands, one of which is a ptr. Check whether the caller is
3179 		 * asking about it.
3180 		 */
3181 		if (t == SRC_OP && reg->type != SCALAR_VALUE)
3182 			return true;
3183 		return BPF_SIZE(code) == BPF_DW;
3184 	}
3185 
3186 	if (class == BPF_LD) {
3187 		u8 mode = BPF_MODE(code);
3188 
3189 		/* LD_IMM64 */
3190 		if (mode == BPF_IMM)
3191 			return true;
3192 
3193 		/* Both LD_IND and LD_ABS return 32-bit data. */
3194 		if (t != SRC_OP)
3195 			return  false;
3196 
3197 		/* Implicit ctx ptr. */
3198 		if (regno == BPF_REG_6)
3199 			return true;
3200 
3201 		/* Explicit source could be any width. */
3202 		return true;
3203 	}
3204 
3205 	if (class == BPF_ST)
3206 		/* The only source register for BPF_ST is a ptr. */
3207 		return true;
3208 
3209 	/* Conservatively return true at default. */
3210 	return true;
3211 }
3212 
3213 /* Return the regno defined by the insn, or -1. */
3214 static int insn_def_regno(const struct bpf_insn *insn)
3215 {
3216 	switch (BPF_CLASS(insn->code)) {
3217 	case BPF_JMP:
3218 	case BPF_JMP32:
3219 	case BPF_ST:
3220 		return -1;
3221 	case BPF_STX:
3222 		if ((BPF_MODE(insn->code) == BPF_ATOMIC ||
3223 		     BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) &&
3224 		    (insn->imm & BPF_FETCH)) {
3225 			if (insn->imm == BPF_CMPXCHG)
3226 				return BPF_REG_0;
3227 			else
3228 				return insn->src_reg;
3229 		} else {
3230 			return -1;
3231 		}
3232 	default:
3233 		return insn->dst_reg;
3234 	}
3235 }
3236 
3237 /* Return TRUE if INSN has defined any 32-bit value explicitly. */
3238 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)
3239 {
3240 	int dst_reg = insn_def_regno(insn);
3241 
3242 	if (dst_reg == -1)
3243 		return false;
3244 
3245 	return !is_reg64(env, insn, dst_reg, NULL, DST_OP);
3246 }
3247 
3248 static void mark_insn_zext(struct bpf_verifier_env *env,
3249 			   struct bpf_reg_state *reg)
3250 {
3251 	s32 def_idx = reg->subreg_def;
3252 
3253 	if (def_idx == DEF_NOT_SUBREG)
3254 		return;
3255 
3256 	env->insn_aux_data[def_idx - 1].zext_dst = true;
3257 	/* The dst will be zero extended, so won't be sub-register anymore. */
3258 	reg->subreg_def = DEF_NOT_SUBREG;
3259 }
3260 
3261 static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno,
3262 			   enum reg_arg_type t)
3263 {
3264 	struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
3265 	struct bpf_reg_state *reg;
3266 	bool rw64;
3267 
3268 	if (regno >= MAX_BPF_REG) {
3269 		verbose(env, "R%d is invalid\n", regno);
3270 		return -EINVAL;
3271 	}
3272 
3273 	mark_reg_scratched(env, regno);
3274 
3275 	reg = &regs[regno];
3276 	rw64 = is_reg64(env, insn, regno, reg, t);
3277 	if (t == SRC_OP) {
3278 		/* check whether register used as source operand can be read */
3279 		if (reg->type == NOT_INIT) {
3280 			verbose(env, "R%d !read_ok\n", regno);
3281 			return -EACCES;
3282 		}
3283 		/* We don't need to worry about FP liveness because it's read-only */
3284 		if (regno == BPF_REG_FP)
3285 			return 0;
3286 
3287 		if (rw64)
3288 			mark_insn_zext(env, reg);
3289 
3290 		return mark_reg_read(env, reg, reg->parent,
3291 				     rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32);
3292 	} else {
3293 		/* check whether register used as dest operand can be written to */
3294 		if (regno == BPF_REG_FP) {
3295 			verbose(env, "frame pointer is read only\n");
3296 			return -EACCES;
3297 		}
3298 		reg->live |= REG_LIVE_WRITTEN;
3299 		reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;
3300 		if (t == DST_OP)
3301 			mark_reg_unknown(env, regs, regno);
3302 	}
3303 	return 0;
3304 }
3305 
3306 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
3307 			 enum reg_arg_type t)
3308 {
3309 	struct bpf_verifier_state *vstate = env->cur_state;
3310 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
3311 
3312 	return __check_reg_arg(env, state->regs, regno, t);
3313 }
3314 
3315 static int insn_stack_access_flags(int frameno, int spi)
3316 {
3317 	return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno;
3318 }
3319 
3320 static int insn_stack_access_spi(int insn_flags)
3321 {
3322 	return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK;
3323 }
3324 
3325 static int insn_stack_access_frameno(int insn_flags)
3326 {
3327 	return insn_flags & INSN_F_FRAMENO_MASK;
3328 }
3329 
3330 static void mark_jmp_point(struct bpf_verifier_env *env, int idx)
3331 {
3332 	env->insn_aux_data[idx].jmp_point = true;
3333 }
3334 
3335 static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx)
3336 {
3337 	return env->insn_aux_data[insn_idx].jmp_point;
3338 }
3339 
3340 /* for any branch, call, exit record the history of jmps in the given state */
3341 static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
3342 			    int insn_flags)
3343 {
3344 	u32 cnt = cur->jmp_history_cnt;
3345 	struct bpf_jmp_history_entry *p;
3346 	size_t alloc_size;
3347 
3348 	/* combine instruction flags if we already recorded this instruction */
3349 	if (env->cur_hist_ent) {
3350 		/* atomic instructions push insn_flags twice, for READ and
3351 		 * WRITE sides, but they should agree on stack slot
3352 		 */
3353 		WARN_ONCE((env->cur_hist_ent->flags & insn_flags) &&
3354 			  (env->cur_hist_ent->flags & insn_flags) != insn_flags,
3355 			  "verifier insn history bug: insn_idx %d cur flags %x new flags %x\n",
3356 			  env->insn_idx, env->cur_hist_ent->flags, insn_flags);
3357 		env->cur_hist_ent->flags |= insn_flags;
3358 		return 0;
3359 	}
3360 
3361 	cnt++;
3362 	alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p)));
3363 	p = krealloc(cur->jmp_history, alloc_size, GFP_USER);
3364 	if (!p)
3365 		return -ENOMEM;
3366 	cur->jmp_history = p;
3367 
3368 	p = &cur->jmp_history[cnt - 1];
3369 	p->idx = env->insn_idx;
3370 	p->prev_idx = env->prev_insn_idx;
3371 	p->flags = insn_flags;
3372 	cur->jmp_history_cnt = cnt;
3373 	env->cur_hist_ent = p;
3374 
3375 	return 0;
3376 }
3377 
3378 static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st,
3379 						        u32 hist_end, int insn_idx)
3380 {
3381 	if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx)
3382 		return &st->jmp_history[hist_end - 1];
3383 	return NULL;
3384 }
3385 
3386 /* Backtrack one insn at a time. If idx is not at the top of recorded
3387  * history then previous instruction came from straight line execution.
3388  * Return -ENOENT if we exhausted all instructions within given state.
3389  *
3390  * It's legal to have a bit of a looping with the same starting and ending
3391  * insn index within the same state, e.g.: 3->4->5->3, so just because current
3392  * instruction index is the same as state's first_idx doesn't mean we are
3393  * done. If there is still some jump history left, we should keep going. We
3394  * need to take into account that we might have a jump history between given
3395  * state's parent and itself, due to checkpointing. In this case, we'll have
3396  * history entry recording a jump from last instruction of parent state and
3397  * first instruction of given state.
3398  */
3399 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
3400 			     u32 *history)
3401 {
3402 	u32 cnt = *history;
3403 
3404 	if (i == st->first_insn_idx) {
3405 		if (cnt == 0)
3406 			return -ENOENT;
3407 		if (cnt == 1 && st->jmp_history[0].idx == i)
3408 			return -ENOENT;
3409 	}
3410 
3411 	if (cnt && st->jmp_history[cnt - 1].idx == i) {
3412 		i = st->jmp_history[cnt - 1].prev_idx;
3413 		(*history)--;
3414 	} else {
3415 		i--;
3416 	}
3417 	return i;
3418 }
3419 
3420 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
3421 {
3422 	const struct btf_type *func;
3423 	struct btf *desc_btf;
3424 
3425 	if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
3426 		return NULL;
3427 
3428 	desc_btf = find_kfunc_desc_btf(data, insn->off);
3429 	if (IS_ERR(desc_btf))
3430 		return "<error>";
3431 
3432 	func = btf_type_by_id(desc_btf, insn->imm);
3433 	return btf_name_by_offset(desc_btf, func->name_off);
3434 }
3435 
3436 static inline void bt_init(struct backtrack_state *bt, u32 frame)
3437 {
3438 	bt->frame = frame;
3439 }
3440 
3441 static inline void bt_reset(struct backtrack_state *bt)
3442 {
3443 	struct bpf_verifier_env *env = bt->env;
3444 
3445 	memset(bt, 0, sizeof(*bt));
3446 	bt->env = env;
3447 }
3448 
3449 static inline u32 bt_empty(struct backtrack_state *bt)
3450 {
3451 	u64 mask = 0;
3452 	int i;
3453 
3454 	for (i = 0; i <= bt->frame; i++)
3455 		mask |= bt->reg_masks[i] | bt->stack_masks[i];
3456 
3457 	return mask == 0;
3458 }
3459 
3460 static inline int bt_subprog_enter(struct backtrack_state *bt)
3461 {
3462 	if (bt->frame == MAX_CALL_FRAMES - 1) {
3463 		verbose(bt->env, "BUG subprog enter from frame %d\n", bt->frame);
3464 		WARN_ONCE(1, "verifier backtracking bug");
3465 		return -EFAULT;
3466 	}
3467 	bt->frame++;
3468 	return 0;
3469 }
3470 
3471 static inline int bt_subprog_exit(struct backtrack_state *bt)
3472 {
3473 	if (bt->frame == 0) {
3474 		verbose(bt->env, "BUG subprog exit from frame 0\n");
3475 		WARN_ONCE(1, "verifier backtracking bug");
3476 		return -EFAULT;
3477 	}
3478 	bt->frame--;
3479 	return 0;
3480 }
3481 
3482 static inline void bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
3483 {
3484 	bt->reg_masks[frame] |= 1 << reg;
3485 }
3486 
3487 static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
3488 {
3489 	bt->reg_masks[frame] &= ~(1 << reg);
3490 }
3491 
3492 static inline void bt_set_reg(struct backtrack_state *bt, u32 reg)
3493 {
3494 	bt_set_frame_reg(bt, bt->frame, reg);
3495 }
3496 
3497 static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg)
3498 {
3499 	bt_clear_frame_reg(bt, bt->frame, reg);
3500 }
3501 
3502 static inline void bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
3503 {
3504 	bt->stack_masks[frame] |= 1ull << slot;
3505 }
3506 
3507 static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
3508 {
3509 	bt->stack_masks[frame] &= ~(1ull << slot);
3510 }
3511 
3512 static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame)
3513 {
3514 	return bt->reg_masks[frame];
3515 }
3516 
3517 static inline u32 bt_reg_mask(struct backtrack_state *bt)
3518 {
3519 	return bt->reg_masks[bt->frame];
3520 }
3521 
3522 static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame)
3523 {
3524 	return bt->stack_masks[frame];
3525 }
3526 
3527 static inline u64 bt_stack_mask(struct backtrack_state *bt)
3528 {
3529 	return bt->stack_masks[bt->frame];
3530 }
3531 
3532 static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg)
3533 {
3534 	return bt->reg_masks[bt->frame] & (1 << reg);
3535 }
3536 
3537 static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot)
3538 {
3539 	return bt->stack_masks[frame] & (1ull << slot);
3540 }
3541 
3542 /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */
3543 static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask)
3544 {
3545 	DECLARE_BITMAP(mask, 64);
3546 	bool first = true;
3547 	int i, n;
3548 
3549 	buf[0] = '\0';
3550 
3551 	bitmap_from_u64(mask, reg_mask);
3552 	for_each_set_bit(i, mask, 32) {
3553 		n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i);
3554 		first = false;
3555 		buf += n;
3556 		buf_sz -= n;
3557 		if (buf_sz < 0)
3558 			break;
3559 	}
3560 }
3561 /* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */
3562 static void fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask)
3563 {
3564 	DECLARE_BITMAP(mask, 64);
3565 	bool first = true;
3566 	int i, n;
3567 
3568 	buf[0] = '\0';
3569 
3570 	bitmap_from_u64(mask, stack_mask);
3571 	for_each_set_bit(i, mask, 64) {
3572 		n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8);
3573 		first = false;
3574 		buf += n;
3575 		buf_sz -= n;
3576 		if (buf_sz < 0)
3577 			break;
3578 	}
3579 }
3580 
3581 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx);
3582 
3583 /* For given verifier state backtrack_insn() is called from the last insn to
3584  * the first insn. Its purpose is to compute a bitmask of registers and
3585  * stack slots that needs precision in the parent verifier state.
3586  *
3587  * @idx is an index of the instruction we are currently processing;
3588  * @subseq_idx is an index of the subsequent instruction that:
3589  *   - *would be* executed next, if jump history is viewed in forward order;
3590  *   - *was* processed previously during backtracking.
3591  */
3592 static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
3593 			  struct bpf_jmp_history_entry *hist, struct backtrack_state *bt)
3594 {
3595 	const struct bpf_insn_cbs cbs = {
3596 		.cb_call	= disasm_kfunc_name,
3597 		.cb_print	= verbose,
3598 		.private_data	= env,
3599 	};
3600 	struct bpf_insn *insn = env->prog->insnsi + idx;
3601 	u8 class = BPF_CLASS(insn->code);
3602 	u8 opcode = BPF_OP(insn->code);
3603 	u8 mode = BPF_MODE(insn->code);
3604 	u32 dreg = insn->dst_reg;
3605 	u32 sreg = insn->src_reg;
3606 	u32 spi, i, fr;
3607 
3608 	if (insn->code == 0)
3609 		return 0;
3610 	if (env->log.level & BPF_LOG_LEVEL2) {
3611 		fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));
3612 		verbose(env, "mark_precise: frame%d: regs=%s ",
3613 			bt->frame, env->tmp_str_buf);
3614 		fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
3615 		verbose(env, "stack=%s before ", env->tmp_str_buf);
3616 		verbose(env, "%d: ", idx);
3617 		print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
3618 	}
3619 
3620 	if (class == BPF_ALU || class == BPF_ALU64) {
3621 		if (!bt_is_reg_set(bt, dreg))
3622 			return 0;
3623 		if (opcode == BPF_END || opcode == BPF_NEG) {
3624 			/* sreg is reserved and unused
3625 			 * dreg still need precision before this insn
3626 			 */
3627 			return 0;
3628 		} else if (opcode == BPF_MOV) {
3629 			if (BPF_SRC(insn->code) == BPF_X) {
3630 				/* dreg = sreg or dreg = (s8, s16, s32)sreg
3631 				 * dreg needs precision after this insn
3632 				 * sreg needs precision before this insn
3633 				 */
3634 				bt_clear_reg(bt, dreg);
3635 				if (sreg != BPF_REG_FP)
3636 					bt_set_reg(bt, sreg);
3637 			} else {
3638 				/* dreg = K
3639 				 * dreg needs precision after this insn.
3640 				 * Corresponding register is already marked
3641 				 * as precise=true in this verifier state.
3642 				 * No further markings in parent are necessary
3643 				 */
3644 				bt_clear_reg(bt, dreg);
3645 			}
3646 		} else {
3647 			if (BPF_SRC(insn->code) == BPF_X) {
3648 				/* dreg += sreg
3649 				 * both dreg and sreg need precision
3650 				 * before this insn
3651 				 */
3652 				if (sreg != BPF_REG_FP)
3653 					bt_set_reg(bt, sreg);
3654 			} /* else dreg += K
3655 			   * dreg still needs precision before this insn
3656 			   */
3657 		}
3658 	} else if (class == BPF_LDX) {
3659 		if (!bt_is_reg_set(bt, dreg))
3660 			return 0;
3661 		bt_clear_reg(bt, dreg);
3662 
3663 		/* scalars can only be spilled into stack w/o losing precision.
3664 		 * Load from any other memory can be zero extended.
3665 		 * The desire to keep that precision is already indicated
3666 		 * by 'precise' mark in corresponding register of this state.
3667 		 * No further tracking necessary.
3668 		 */
3669 		if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
3670 			return 0;
3671 		/* dreg = *(u64 *)[fp - off] was a fill from the stack.
3672 		 * that [fp - off] slot contains scalar that needs to be
3673 		 * tracked with precision
3674 		 */
3675 		spi = insn_stack_access_spi(hist->flags);
3676 		fr = insn_stack_access_frameno(hist->flags);
3677 		bt_set_frame_slot(bt, fr, spi);
3678 	} else if (class == BPF_STX || class == BPF_ST) {
3679 		if (bt_is_reg_set(bt, dreg))
3680 			/* stx & st shouldn't be using _scalar_ dst_reg
3681 			 * to access memory. It means backtracking
3682 			 * encountered a case of pointer subtraction.
3683 			 */
3684 			return -ENOTSUPP;
3685 		/* scalars can only be spilled into stack */
3686 		if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
3687 			return 0;
3688 		spi = insn_stack_access_spi(hist->flags);
3689 		fr = insn_stack_access_frameno(hist->flags);
3690 		if (!bt_is_frame_slot_set(bt, fr, spi))
3691 			return 0;
3692 		bt_clear_frame_slot(bt, fr, spi);
3693 		if (class == BPF_STX)
3694 			bt_set_reg(bt, sreg);
3695 	} else if (class == BPF_JMP || class == BPF_JMP32) {
3696 		if (bpf_pseudo_call(insn)) {
3697 			int subprog_insn_idx, subprog;
3698 
3699 			subprog_insn_idx = idx + insn->imm + 1;
3700 			subprog = find_subprog(env, subprog_insn_idx);
3701 			if (subprog < 0)
3702 				return -EFAULT;
3703 
3704 			if (subprog_is_global(env, subprog)) {
3705 				/* check that jump history doesn't have any
3706 				 * extra instructions from subprog; the next
3707 				 * instruction after call to global subprog
3708 				 * should be literally next instruction in
3709 				 * caller program
3710 				 */
3711 				WARN_ONCE(idx + 1 != subseq_idx, "verifier backtracking bug");
3712 				/* r1-r5 are invalidated after subprog call,
3713 				 * so for global func call it shouldn't be set
3714 				 * anymore
3715 				 */
3716 				if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
3717 					verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
3718 					WARN_ONCE(1, "verifier backtracking bug");
3719 					return -EFAULT;
3720 				}
3721 				/* global subprog always sets R0 */
3722 				bt_clear_reg(bt, BPF_REG_0);
3723 				return 0;
3724 			} else {
3725 				/* static subprog call instruction, which
3726 				 * means that we are exiting current subprog,
3727 				 * so only r1-r5 could be still requested as
3728 				 * precise, r0 and r6-r10 or any stack slot in
3729 				 * the current frame should be zero by now
3730 				 */
3731 				if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
3732 					verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
3733 					WARN_ONCE(1, "verifier backtracking bug");
3734 					return -EFAULT;
3735 				}
3736 				/* we are now tracking register spills correctly,
3737 				 * so any instance of leftover slots is a bug
3738 				 */
3739 				if (bt_stack_mask(bt) != 0) {
3740 					verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt));
3741 					WARN_ONCE(1, "verifier backtracking bug (subprog leftover stack slots)");
3742 					return -EFAULT;
3743 				}
3744 				/* propagate r1-r5 to the caller */
3745 				for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
3746 					if (bt_is_reg_set(bt, i)) {
3747 						bt_clear_reg(bt, i);
3748 						bt_set_frame_reg(bt, bt->frame - 1, i);
3749 					}
3750 				}
3751 				if (bt_subprog_exit(bt))
3752 					return -EFAULT;
3753 				return 0;
3754 			}
3755 		} else if (is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) {
3756 			/* exit from callback subprog to callback-calling helper or
3757 			 * kfunc call. Use idx/subseq_idx check to discern it from
3758 			 * straight line code backtracking.
3759 			 * Unlike the subprog call handling above, we shouldn't
3760 			 * propagate precision of r1-r5 (if any requested), as they are
3761 			 * not actually arguments passed directly to callback subprogs
3762 			 */
3763 			if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
3764 				verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
3765 				WARN_ONCE(1, "verifier backtracking bug");
3766 				return -EFAULT;
3767 			}
3768 			if (bt_stack_mask(bt) != 0) {
3769 				verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt));
3770 				WARN_ONCE(1, "verifier backtracking bug (callback leftover stack slots)");
3771 				return -EFAULT;
3772 			}
3773 			/* clear r1-r5 in callback subprog's mask */
3774 			for (i = BPF_REG_1; i <= BPF_REG_5; i++)
3775 				bt_clear_reg(bt, i);
3776 			if (bt_subprog_exit(bt))
3777 				return -EFAULT;
3778 			return 0;
3779 		} else if (opcode == BPF_CALL) {
3780 			/* kfunc with imm==0 is invalid and fixup_kfunc_call will
3781 			 * catch this error later. Make backtracking conservative
3782 			 * with ENOTSUPP.
3783 			 */
3784 			if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0)
3785 				return -ENOTSUPP;
3786 			/* regular helper call sets R0 */
3787 			bt_clear_reg(bt, BPF_REG_0);
3788 			if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
3789 				/* if backtracing was looking for registers R1-R5
3790 				 * they should have been found already.
3791 				 */
3792 				verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
3793 				WARN_ONCE(1, "verifier backtracking bug");
3794 				return -EFAULT;
3795 			}
3796 		} else if (opcode == BPF_EXIT) {
3797 			bool r0_precise;
3798 
3799 			/* Backtracking to a nested function call, 'idx' is a part of
3800 			 * the inner frame 'subseq_idx' is a part of the outer frame.
3801 			 * In case of a regular function call, instructions giving
3802 			 * precision to registers R1-R5 should have been found already.
3803 			 * In case of a callback, it is ok to have R1-R5 marked for
3804 			 * backtracking, as these registers are set by the function
3805 			 * invoking callback.
3806 			 */
3807 			if (subseq_idx >= 0 && calls_callback(env, subseq_idx))
3808 				for (i = BPF_REG_1; i <= BPF_REG_5; i++)
3809 					bt_clear_reg(bt, i);
3810 			if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
3811 				verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
3812 				WARN_ONCE(1, "verifier backtracking bug");
3813 				return -EFAULT;
3814 			}
3815 
3816 			/* BPF_EXIT in subprog or callback always returns
3817 			 * right after the call instruction, so by checking
3818 			 * whether the instruction at subseq_idx-1 is subprog
3819 			 * call or not we can distinguish actual exit from
3820 			 * *subprog* from exit from *callback*. In the former
3821 			 * case, we need to propagate r0 precision, if
3822 			 * necessary. In the former we never do that.
3823 			 */
3824 			r0_precise = subseq_idx - 1 >= 0 &&
3825 				     bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&
3826 				     bt_is_reg_set(bt, BPF_REG_0);
3827 
3828 			bt_clear_reg(bt, BPF_REG_0);
3829 			if (bt_subprog_enter(bt))
3830 				return -EFAULT;
3831 
3832 			if (r0_precise)
3833 				bt_set_reg(bt, BPF_REG_0);
3834 			/* r6-r9 and stack slots will stay set in caller frame
3835 			 * bitmasks until we return back from callee(s)
3836 			 */
3837 			return 0;
3838 		} else if (BPF_SRC(insn->code) == BPF_X) {
3839 			if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg))
3840 				return 0;
3841 			/* dreg <cond> sreg
3842 			 * Both dreg and sreg need precision before
3843 			 * this insn. If only sreg was marked precise
3844 			 * before it would be equally necessary to
3845 			 * propagate it to dreg.
3846 			 */
3847 			bt_set_reg(bt, dreg);
3848 			bt_set_reg(bt, sreg);
3849 			 /* else dreg <cond> K
3850 			  * Only dreg still needs precision before
3851 			  * this insn, so for the K-based conditional
3852 			  * there is nothing new to be marked.
3853 			  */
3854 		}
3855 	} else if (class == BPF_LD) {
3856 		if (!bt_is_reg_set(bt, dreg))
3857 			return 0;
3858 		bt_clear_reg(bt, dreg);
3859 		/* It's ld_imm64 or ld_abs or ld_ind.
3860 		 * For ld_imm64 no further tracking of precision
3861 		 * into parent is necessary
3862 		 */
3863 		if (mode == BPF_IND || mode == BPF_ABS)
3864 			/* to be analyzed */
3865 			return -ENOTSUPP;
3866 	}
3867 	return 0;
3868 }
3869 
3870 /* the scalar precision tracking algorithm:
3871  * . at the start all registers have precise=false.
3872  * . scalar ranges are tracked as normal through alu and jmp insns.
3873  * . once precise value of the scalar register is used in:
3874  *   .  ptr + scalar alu
3875  *   . if (scalar cond K|scalar)
3876  *   .  helper_call(.., scalar, ...) where ARG_CONST is expected
3877  *   backtrack through the verifier states and mark all registers and
3878  *   stack slots with spilled constants that these scalar regisers
3879  *   should be precise.
3880  * . during state pruning two registers (or spilled stack slots)
3881  *   are equivalent if both are not precise.
3882  *
3883  * Note the verifier cannot simply walk register parentage chain,
3884  * since many different registers and stack slots could have been
3885  * used to compute single precise scalar.
3886  *
3887  * The approach of starting with precise=true for all registers and then
3888  * backtrack to mark a register as not precise when the verifier detects
3889  * that program doesn't care about specific value (e.g., when helper
3890  * takes register as ARG_ANYTHING parameter) is not safe.
3891  *
3892  * It's ok to walk single parentage chain of the verifier states.
3893  * It's possible that this backtracking will go all the way till 1st insn.
3894  * All other branches will be explored for needing precision later.
3895  *
3896  * The backtracking needs to deal with cases like:
3897  *   R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0)
3898  * r9 -= r8
3899  * r5 = r9
3900  * if r5 > 0x79f goto pc+7
3901  *    R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff))
3902  * r5 += 1
3903  * ...
3904  * call bpf_perf_event_output#25
3905  *   where .arg5_type = ARG_CONST_SIZE_OR_ZERO
3906  *
3907  * and this case:
3908  * r6 = 1
3909  * call foo // uses callee's r6 inside to compute r0
3910  * r0 += r6
3911  * if r0 == 0 goto
3912  *
3913  * to track above reg_mask/stack_mask needs to be independent for each frame.
3914  *
3915  * Also if parent's curframe > frame where backtracking started,
3916  * the verifier need to mark registers in both frames, otherwise callees
3917  * may incorrectly prune callers. This is similar to
3918  * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences")
3919  *
3920  * For now backtracking falls back into conservative marking.
3921  */
3922 static void mark_all_scalars_precise(struct bpf_verifier_env *env,
3923 				     struct bpf_verifier_state *st)
3924 {
3925 	struct bpf_func_state *func;
3926 	struct bpf_reg_state *reg;
3927 	int i, j;
3928 
3929 	if (env->log.level & BPF_LOG_LEVEL2) {
3930 		verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",
3931 			st->curframe);
3932 	}
3933 
3934 	/* big hammer: mark all scalars precise in this path.
3935 	 * pop_stack may still get !precise scalars.
3936 	 * We also skip current state and go straight to first parent state,
3937 	 * because precision markings in current non-checkpointed state are
3938 	 * not needed. See why in the comment in __mark_chain_precision below.
3939 	 */
3940 	for (st = st->parent; st; st = st->parent) {
3941 		for (i = 0; i <= st->curframe; i++) {
3942 			func = st->frame[i];
3943 			for (j = 0; j < BPF_REG_FP; j++) {
3944 				reg = &func->regs[j];
3945 				if (reg->type != SCALAR_VALUE || reg->precise)
3946 					continue;
3947 				reg->precise = true;
3948 				if (env->log.level & BPF_LOG_LEVEL2) {
3949 					verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",
3950 						i, j);
3951 				}
3952 			}
3953 			for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
3954 				if (!is_spilled_reg(&func->stack[j]))
3955 					continue;
3956 				reg = &func->stack[j].spilled_ptr;
3957 				if (reg->type != SCALAR_VALUE || reg->precise)
3958 					continue;
3959 				reg->precise = true;
3960 				if (env->log.level & BPF_LOG_LEVEL2) {
3961 					verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",
3962 						i, -(j + 1) * 8);
3963 				}
3964 			}
3965 		}
3966 	}
3967 }
3968 
3969 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
3970 {
3971 	struct bpf_func_state *func;
3972 	struct bpf_reg_state *reg;
3973 	int i, j;
3974 
3975 	for (i = 0; i <= st->curframe; i++) {
3976 		func = st->frame[i];
3977 		for (j = 0; j < BPF_REG_FP; j++) {
3978 			reg = &func->regs[j];
3979 			if (reg->type != SCALAR_VALUE)
3980 				continue;
3981 			reg->precise = false;
3982 		}
3983 		for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
3984 			if (!is_spilled_reg(&func->stack[j]))
3985 				continue;
3986 			reg = &func->stack[j].spilled_ptr;
3987 			if (reg->type != SCALAR_VALUE)
3988 				continue;
3989 			reg->precise = false;
3990 		}
3991 	}
3992 }
3993 
3994 static bool idset_contains(struct bpf_idset *s, u32 id)
3995 {
3996 	u32 i;
3997 
3998 	for (i = 0; i < s->count; ++i)
3999 		if (s->ids[i] == (id & ~BPF_ADD_CONST))
4000 			return true;
4001 
4002 	return false;
4003 }
4004 
4005 static int idset_push(struct bpf_idset *s, u32 id)
4006 {
4007 	if (WARN_ON_ONCE(s->count >= ARRAY_SIZE(s->ids)))
4008 		return -EFAULT;
4009 	s->ids[s->count++] = id & ~BPF_ADD_CONST;
4010 	return 0;
4011 }
4012 
4013 static void idset_reset(struct bpf_idset *s)
4014 {
4015 	s->count = 0;
4016 }
4017 
4018 /* Collect a set of IDs for all registers currently marked as precise in env->bt.
4019  * Mark all registers with these IDs as precise.
4020  */
4021 static int mark_precise_scalar_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
4022 {
4023 	struct bpf_idset *precise_ids = &env->idset_scratch;
4024 	struct backtrack_state *bt = &env->bt;
4025 	struct bpf_func_state *func;
4026 	struct bpf_reg_state *reg;
4027 	DECLARE_BITMAP(mask, 64);
4028 	int i, fr;
4029 
4030 	idset_reset(precise_ids);
4031 
4032 	for (fr = bt->frame; fr >= 0; fr--) {
4033 		func = st->frame[fr];
4034 
4035 		bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr));
4036 		for_each_set_bit(i, mask, 32) {
4037 			reg = &func->regs[i];
4038 			if (!reg->id || reg->type != SCALAR_VALUE)
4039 				continue;
4040 			if (idset_push(precise_ids, reg->id))
4041 				return -EFAULT;
4042 		}
4043 
4044 		bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
4045 		for_each_set_bit(i, mask, 64) {
4046 			if (i >= func->allocated_stack / BPF_REG_SIZE)
4047 				break;
4048 			if (!is_spilled_scalar_reg(&func->stack[i]))
4049 				continue;
4050 			reg = &func->stack[i].spilled_ptr;
4051 			if (!reg->id)
4052 				continue;
4053 			if (idset_push(precise_ids, reg->id))
4054 				return -EFAULT;
4055 		}
4056 	}
4057 
4058 	for (fr = 0; fr <= st->curframe; ++fr) {
4059 		func = st->frame[fr];
4060 
4061 		for (i = BPF_REG_0; i < BPF_REG_10; ++i) {
4062 			reg = &func->regs[i];
4063 			if (!reg->id)
4064 				continue;
4065 			if (!idset_contains(precise_ids, reg->id))
4066 				continue;
4067 			bt_set_frame_reg(bt, fr, i);
4068 		}
4069 		for (i = 0; i < func->allocated_stack / BPF_REG_SIZE; ++i) {
4070 			if (!is_spilled_scalar_reg(&func->stack[i]))
4071 				continue;
4072 			reg = &func->stack[i].spilled_ptr;
4073 			if (!reg->id)
4074 				continue;
4075 			if (!idset_contains(precise_ids, reg->id))
4076 				continue;
4077 			bt_set_frame_slot(bt, fr, i);
4078 		}
4079 	}
4080 
4081 	return 0;
4082 }
4083 
4084 /*
4085  * __mark_chain_precision() backtracks BPF program instruction sequence and
4086  * chain of verifier states making sure that register *regno* (if regno >= 0)
4087  * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked
4088  * SCALARS, as well as any other registers and slots that contribute to
4089  * a tracked state of given registers/stack slots, depending on specific BPF
4090  * assembly instructions (see backtrack_insns() for exact instruction handling
4091  * logic). This backtracking relies on recorded jmp_history and is able to
4092  * traverse entire chain of parent states. This process ends only when all the
4093  * necessary registers/slots and their transitive dependencies are marked as
4094  * precise.
4095  *
4096  * One important and subtle aspect is that precise marks *do not matter* in
4097  * the currently verified state (current state). It is important to understand
4098  * why this is the case.
4099  *
4100  * First, note that current state is the state that is not yet "checkpointed",
4101  * i.e., it is not yet put into env->explored_states, and it has no children
4102  * states as well. It's ephemeral, and can end up either a) being discarded if
4103  * compatible explored state is found at some point or BPF_EXIT instruction is
4104  * reached or b) checkpointed and put into env->explored_states, branching out
4105  * into one or more children states.
4106  *
4107  * In the former case, precise markings in current state are completely
4108  * ignored by state comparison code (see regsafe() for details). Only
4109  * checkpointed ("old") state precise markings are important, and if old
4110  * state's register/slot is precise, regsafe() assumes current state's
4111  * register/slot as precise and checks value ranges exactly and precisely. If
4112  * states turn out to be compatible, current state's necessary precise
4113  * markings and any required parent states' precise markings are enforced
4114  * after the fact with propagate_precision() logic, after the fact. But it's
4115  * important to realize that in this case, even after marking current state
4116  * registers/slots as precise, we immediately discard current state. So what
4117  * actually matters is any of the precise markings propagated into current
4118  * state's parent states, which are always checkpointed (due to b) case above).
4119  * As such, for scenario a) it doesn't matter if current state has precise
4120  * markings set or not.
4121  *
4122  * Now, for the scenario b), checkpointing and forking into child(ren)
4123  * state(s). Note that before current state gets to checkpointing step, any
4124  * processed instruction always assumes precise SCALAR register/slot
4125  * knowledge: if precise value or range is useful to prune jump branch, BPF
4126  * verifier takes this opportunity enthusiastically. Similarly, when
4127  * register's value is used to calculate offset or memory address, exact
4128  * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to
4129  * what we mentioned above about state comparison ignoring precise markings
4130  * during state comparison, BPF verifier ignores and also assumes precise
4131  * markings *at will* during instruction verification process. But as verifier
4132  * assumes precision, it also propagates any precision dependencies across
4133  * parent states, which are not yet finalized, so can be further restricted
4134  * based on new knowledge gained from restrictions enforced by their children
4135  * states. This is so that once those parent states are finalized, i.e., when
4136  * they have no more active children state, state comparison logic in
4137  * is_state_visited() would enforce strict and precise SCALAR ranges, if
4138  * required for correctness.
4139  *
4140  * To build a bit more intuition, note also that once a state is checkpointed,
4141  * the path we took to get to that state is not important. This is crucial
4142  * property for state pruning. When state is checkpointed and finalized at
4143  * some instruction index, it can be correctly and safely used to "short
4144  * circuit" any *compatible* state that reaches exactly the same instruction
4145  * index. I.e., if we jumped to that instruction from a completely different
4146  * code path than original finalized state was derived from, it doesn't
4147  * matter, current state can be discarded because from that instruction
4148  * forward having a compatible state will ensure we will safely reach the
4149  * exit. States describe preconditions for further exploration, but completely
4150  * forget the history of how we got here.
4151  *
4152  * This also means that even if we needed precise SCALAR range to get to
4153  * finalized state, but from that point forward *that same* SCALAR register is
4154  * never used in a precise context (i.e., it's precise value is not needed for
4155  * correctness), it's correct and safe to mark such register as "imprecise"
4156  * (i.e., precise marking set to false). This is what we rely on when we do
4157  * not set precise marking in current state. If no child state requires
4158  * precision for any given SCALAR register, it's safe to dictate that it can
4159  * be imprecise. If any child state does require this register to be precise,
4160  * we'll mark it precise later retroactively during precise markings
4161  * propagation from child state to parent states.
4162  *
4163  * Skipping precise marking setting in current state is a mild version of
4164  * relying on the above observation. But we can utilize this property even
4165  * more aggressively by proactively forgetting any precise marking in the
4166  * current state (which we inherited from the parent state), right before we
4167  * checkpoint it and branch off into new child state. This is done by
4168  * mark_all_scalars_imprecise() to hopefully get more permissive and generic
4169  * finalized states which help in short circuiting more future states.
4170  */
4171 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno)
4172 {
4173 	struct backtrack_state *bt = &env->bt;
4174 	struct bpf_verifier_state *st = env->cur_state;
4175 	int first_idx = st->first_insn_idx;
4176 	int last_idx = env->insn_idx;
4177 	int subseq_idx = -1;
4178 	struct bpf_func_state *func;
4179 	struct bpf_reg_state *reg;
4180 	bool skip_first = true;
4181 	int i, fr, err;
4182 
4183 	if (!env->bpf_capable)
4184 		return 0;
4185 
4186 	/* set frame number from which we are starting to backtrack */
4187 	bt_init(bt, env->cur_state->curframe);
4188 
4189 	/* Do sanity checks against current state of register and/or stack
4190 	 * slot, but don't set precise flag in current state, as precision
4191 	 * tracking in the current state is unnecessary.
4192 	 */
4193 	func = st->frame[bt->frame];
4194 	if (regno >= 0) {
4195 		reg = &func->regs[regno];
4196 		if (reg->type != SCALAR_VALUE) {
4197 			WARN_ONCE(1, "backtracing misuse");
4198 			return -EFAULT;
4199 		}
4200 		bt_set_reg(bt, regno);
4201 	}
4202 
4203 	if (bt_empty(bt))
4204 		return 0;
4205 
4206 	for (;;) {
4207 		DECLARE_BITMAP(mask, 64);
4208 		u32 history = st->jmp_history_cnt;
4209 		struct bpf_jmp_history_entry *hist;
4210 
4211 		if (env->log.level & BPF_LOG_LEVEL2) {
4212 			verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",
4213 				bt->frame, last_idx, first_idx, subseq_idx);
4214 		}
4215 
4216 		/* If some register with scalar ID is marked as precise,
4217 		 * make sure that all registers sharing this ID are also precise.
4218 		 * This is needed to estimate effect of find_equal_scalars().
4219 		 * Do this at the last instruction of each state,
4220 		 * bpf_reg_state::id fields are valid for these instructions.
4221 		 *
4222 		 * Allows to track precision in situation like below:
4223 		 *
4224 		 *     r2 = unknown value
4225 		 *     ...
4226 		 *   --- state #0 ---
4227 		 *     ...
4228 		 *     r1 = r2                 // r1 and r2 now share the same ID
4229 		 *     ...
4230 		 *   --- state #1 {r1.id = A, r2.id = A} ---
4231 		 *     ...
4232 		 *     if (r2 > 10) goto exit; // find_equal_scalars() assigns range to r1
4233 		 *     ...
4234 		 *   --- state #2 {r1.id = A, r2.id = A} ---
4235 		 *     r3 = r10
4236 		 *     r3 += r1                // need to mark both r1 and r2
4237 		 */
4238 		if (mark_precise_scalar_ids(env, st))
4239 			return -EFAULT;
4240 
4241 		if (last_idx < 0) {
4242 			/* we are at the entry into subprog, which
4243 			 * is expected for global funcs, but only if
4244 			 * requested precise registers are R1-R5
4245 			 * (which are global func's input arguments)
4246 			 */
4247 			if (st->curframe == 0 &&
4248 			    st->frame[0]->subprogno > 0 &&
4249 			    st->frame[0]->callsite == BPF_MAIN_FUNC &&
4250 			    bt_stack_mask(bt) == 0 &&
4251 			    (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) {
4252 				bitmap_from_u64(mask, bt_reg_mask(bt));
4253 				for_each_set_bit(i, mask, 32) {
4254 					reg = &st->frame[0]->regs[i];
4255 					bt_clear_reg(bt, i);
4256 					if (reg->type == SCALAR_VALUE)
4257 						reg->precise = true;
4258 				}
4259 				return 0;
4260 			}
4261 
4262 			verbose(env, "BUG backtracking func entry subprog %d reg_mask %x stack_mask %llx\n",
4263 				st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt));
4264 			WARN_ONCE(1, "verifier backtracking bug");
4265 			return -EFAULT;
4266 		}
4267 
4268 		for (i = last_idx;;) {
4269 			if (skip_first) {
4270 				err = 0;
4271 				skip_first = false;
4272 			} else {
4273 				hist = get_jmp_hist_entry(st, history, i);
4274 				err = backtrack_insn(env, i, subseq_idx, hist, bt);
4275 			}
4276 			if (err == -ENOTSUPP) {
4277 				mark_all_scalars_precise(env, env->cur_state);
4278 				bt_reset(bt);
4279 				return 0;
4280 			} else if (err) {
4281 				return err;
4282 			}
4283 			if (bt_empty(bt))
4284 				/* Found assignment(s) into tracked register in this state.
4285 				 * Since this state is already marked, just return.
4286 				 * Nothing to be tracked further in the parent state.
4287 				 */
4288 				return 0;
4289 			subseq_idx = i;
4290 			i = get_prev_insn_idx(st, i, &history);
4291 			if (i == -ENOENT)
4292 				break;
4293 			if (i >= env->prog->len) {
4294 				/* This can happen if backtracking reached insn 0
4295 				 * and there are still reg_mask or stack_mask
4296 				 * to backtrack.
4297 				 * It means the backtracking missed the spot where
4298 				 * particular register was initialized with a constant.
4299 				 */
4300 				verbose(env, "BUG backtracking idx %d\n", i);
4301 				WARN_ONCE(1, "verifier backtracking bug");
4302 				return -EFAULT;
4303 			}
4304 		}
4305 		st = st->parent;
4306 		if (!st)
4307 			break;
4308 
4309 		for (fr = bt->frame; fr >= 0; fr--) {
4310 			func = st->frame[fr];
4311 			bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr));
4312 			for_each_set_bit(i, mask, 32) {
4313 				reg = &func->regs[i];
4314 				if (reg->type != SCALAR_VALUE) {
4315 					bt_clear_frame_reg(bt, fr, i);
4316 					continue;
4317 				}
4318 				if (reg->precise)
4319 					bt_clear_frame_reg(bt, fr, i);
4320 				else
4321 					reg->precise = true;
4322 			}
4323 
4324 			bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
4325 			for_each_set_bit(i, mask, 64) {
4326 				if (i >= func->allocated_stack / BPF_REG_SIZE) {
4327 					verbose(env, "BUG backtracking (stack slot %d, total slots %d)\n",
4328 						i, func->allocated_stack / BPF_REG_SIZE);
4329 					WARN_ONCE(1, "verifier backtracking bug (stack slot out of bounds)");
4330 					return -EFAULT;
4331 				}
4332 
4333 				if (!is_spilled_scalar_reg(&func->stack[i])) {
4334 					bt_clear_frame_slot(bt, fr, i);
4335 					continue;
4336 				}
4337 				reg = &func->stack[i].spilled_ptr;
4338 				if (reg->precise)
4339 					bt_clear_frame_slot(bt, fr, i);
4340 				else
4341 					reg->precise = true;
4342 			}
4343 			if (env->log.level & BPF_LOG_LEVEL2) {
4344 				fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
4345 					     bt_frame_reg_mask(bt, fr));
4346 				verbose(env, "mark_precise: frame%d: parent state regs=%s ",
4347 					fr, env->tmp_str_buf);
4348 				fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
4349 					       bt_frame_stack_mask(bt, fr));
4350 				verbose(env, "stack=%s: ", env->tmp_str_buf);
4351 				print_verifier_state(env, func, true);
4352 			}
4353 		}
4354 
4355 		if (bt_empty(bt))
4356 			return 0;
4357 
4358 		subseq_idx = first_idx;
4359 		last_idx = st->last_insn_idx;
4360 		first_idx = st->first_insn_idx;
4361 	}
4362 
4363 	/* if we still have requested precise regs or slots, we missed
4364 	 * something (e.g., stack access through non-r10 register), so
4365 	 * fallback to marking all precise
4366 	 */
4367 	if (!bt_empty(bt)) {
4368 		mark_all_scalars_precise(env, env->cur_state);
4369 		bt_reset(bt);
4370 	}
4371 
4372 	return 0;
4373 }
4374 
4375 int mark_chain_precision(struct bpf_verifier_env *env, int regno)
4376 {
4377 	return __mark_chain_precision(env, regno);
4378 }
4379 
4380 /* mark_chain_precision_batch() assumes that env->bt is set in the caller to
4381  * desired reg and stack masks across all relevant frames
4382  */
4383 static int mark_chain_precision_batch(struct bpf_verifier_env *env)
4384 {
4385 	return __mark_chain_precision(env, -1);
4386 }
4387 
4388 static bool is_spillable_regtype(enum bpf_reg_type type)
4389 {
4390 	switch (base_type(type)) {
4391 	case PTR_TO_MAP_VALUE:
4392 	case PTR_TO_STACK:
4393 	case PTR_TO_CTX:
4394 	case PTR_TO_PACKET:
4395 	case PTR_TO_PACKET_META:
4396 	case PTR_TO_PACKET_END:
4397 	case PTR_TO_FLOW_KEYS:
4398 	case CONST_PTR_TO_MAP:
4399 	case PTR_TO_SOCKET:
4400 	case PTR_TO_SOCK_COMMON:
4401 	case PTR_TO_TCP_SOCK:
4402 	case PTR_TO_XDP_SOCK:
4403 	case PTR_TO_BTF_ID:
4404 	case PTR_TO_BUF:
4405 	case PTR_TO_MEM:
4406 	case PTR_TO_FUNC:
4407 	case PTR_TO_MAP_KEY:
4408 	case PTR_TO_ARENA:
4409 		return true;
4410 	default:
4411 		return false;
4412 	}
4413 }
4414 
4415 /* Does this register contain a constant zero? */
4416 static bool register_is_null(struct bpf_reg_state *reg)
4417 {
4418 	return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
4419 }
4420 
4421 /* check if register is a constant scalar value */
4422 static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32)
4423 {
4424 	return reg->type == SCALAR_VALUE &&
4425 	       tnum_is_const(subreg32 ? tnum_subreg(reg->var_off) : reg->var_off);
4426 }
4427 
4428 /* assuming is_reg_const() is true, return constant value of a register */
4429 static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32)
4430 {
4431 	return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value;
4432 }
4433 
4434 static bool __is_pointer_value(bool allow_ptr_leaks,
4435 			       const struct bpf_reg_state *reg)
4436 {
4437 	if (allow_ptr_leaks)
4438 		return false;
4439 
4440 	return reg->type != SCALAR_VALUE;
4441 }
4442 
4443 static void assign_scalar_id_before_mov(struct bpf_verifier_env *env,
4444 					struct bpf_reg_state *src_reg)
4445 {
4446 	if (src_reg->type != SCALAR_VALUE)
4447 		return;
4448 
4449 	if (src_reg->id & BPF_ADD_CONST) {
4450 		/*
4451 		 * The verifier is processing rX = rY insn and
4452 		 * rY->id has special linked register already.
4453 		 * Cleared it, since multiple rX += const are not supported.
4454 		 */
4455 		src_reg->id = 0;
4456 		src_reg->off = 0;
4457 	}
4458 
4459 	if (!src_reg->id && !tnum_is_const(src_reg->var_off))
4460 		/* Ensure that src_reg has a valid ID that will be copied to
4461 		 * dst_reg and then will be used by find_equal_scalars() to
4462 		 * propagate min/max range.
4463 		 */
4464 		src_reg->id = ++env->id_gen;
4465 }
4466 
4467 /* Copy src state preserving dst->parent and dst->live fields */
4468 static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_state *src)
4469 {
4470 	struct bpf_reg_state *parent = dst->parent;
4471 	enum bpf_reg_liveness live = dst->live;
4472 
4473 	*dst = *src;
4474 	dst->parent = parent;
4475 	dst->live = live;
4476 }
4477 
4478 static void save_register_state(struct bpf_verifier_env *env,
4479 				struct bpf_func_state *state,
4480 				int spi, struct bpf_reg_state *reg,
4481 				int size)
4482 {
4483 	int i;
4484 
4485 	copy_register_state(&state->stack[spi].spilled_ptr, reg);
4486 	if (size == BPF_REG_SIZE)
4487 		state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
4488 
4489 	for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--)
4490 		state->stack[spi].slot_type[i - 1] = STACK_SPILL;
4491 
4492 	/* size < 8 bytes spill */
4493 	for (; i; i--)
4494 		mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]);
4495 }
4496 
4497 static bool is_bpf_st_mem(struct bpf_insn *insn)
4498 {
4499 	return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM;
4500 }
4501 
4502 static int get_reg_width(struct bpf_reg_state *reg)
4503 {
4504 	return fls64(reg->umax_value);
4505 }
4506 
4507 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
4508  * stack boundary and alignment are checked in check_mem_access()
4509  */
4510 static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
4511 				       /* stack frame we're writing to */
4512 				       struct bpf_func_state *state,
4513 				       int off, int size, int value_regno,
4514 				       int insn_idx)
4515 {
4516 	struct bpf_func_state *cur; /* state of the current function */
4517 	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
4518 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
4519 	struct bpf_reg_state *reg = NULL;
4520 	int insn_flags = insn_stack_access_flags(state->frameno, spi);
4521 
4522 	/* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
4523 	 * so it's aligned access and [off, off + size) are within stack limits
4524 	 */
4525 	if (!env->allow_ptr_leaks &&
4526 	    is_spilled_reg(&state->stack[spi]) &&
4527 	    size != BPF_REG_SIZE) {
4528 		verbose(env, "attempt to corrupt spilled pointer on stack\n");
4529 		return -EACCES;
4530 	}
4531 
4532 	cur = env->cur_state->frame[env->cur_state->curframe];
4533 	if (value_regno >= 0)
4534 		reg = &cur->regs[value_regno];
4535 	if (!env->bypass_spec_v4) {
4536 		bool sanitize = reg && is_spillable_regtype(reg->type);
4537 
4538 		for (i = 0; i < size; i++) {
4539 			u8 type = state->stack[spi].slot_type[i];
4540 
4541 			if (type != STACK_MISC && type != STACK_ZERO) {
4542 				sanitize = true;
4543 				break;
4544 			}
4545 		}
4546 
4547 		if (sanitize)
4548 			env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
4549 	}
4550 
4551 	err = destroy_if_dynptr_stack_slot(env, state, spi);
4552 	if (err)
4553 		return err;
4554 
4555 	mark_stack_slot_scratched(env, spi);
4556 	if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {
4557 		bool reg_value_fits;
4558 
4559 		reg_value_fits = get_reg_width(reg) <= BITS_PER_BYTE * size;
4560 		/* Make sure that reg had an ID to build a relation on spill. */
4561 		if (reg_value_fits)
4562 			assign_scalar_id_before_mov(env, reg);
4563 		save_register_state(env, state, spi, reg, size);
4564 		/* Break the relation on a narrowing spill. */
4565 		if (!reg_value_fits)
4566 			state->stack[spi].spilled_ptr.id = 0;
4567 	} else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) &&
4568 		   env->bpf_capable) {
4569 		struct bpf_reg_state *tmp_reg = &env->fake_reg[0];
4570 
4571 		memset(tmp_reg, 0, sizeof(*tmp_reg));
4572 		__mark_reg_known(tmp_reg, insn->imm);
4573 		tmp_reg->type = SCALAR_VALUE;
4574 		save_register_state(env, state, spi, tmp_reg, size);
4575 	} else if (reg && is_spillable_regtype(reg->type)) {
4576 		/* register containing pointer is being spilled into stack */
4577 		if (size != BPF_REG_SIZE) {
4578 			verbose_linfo(env, insn_idx, "; ");
4579 			verbose(env, "invalid size of register spill\n");
4580 			return -EACCES;
4581 		}
4582 		if (state != cur && reg->type == PTR_TO_STACK) {
4583 			verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
4584 			return -EINVAL;
4585 		}
4586 		save_register_state(env, state, spi, reg, size);
4587 	} else {
4588 		u8 type = STACK_MISC;
4589 
4590 		/* regular write of data into stack destroys any spilled ptr */
4591 		state->stack[spi].spilled_ptr.type = NOT_INIT;
4592 		/* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */
4593 		if (is_stack_slot_special(&state->stack[spi]))
4594 			for (i = 0; i < BPF_REG_SIZE; i++)
4595 				scrub_spilled_slot(&state->stack[spi].slot_type[i]);
4596 
4597 		/* only mark the slot as written if all 8 bytes were written
4598 		 * otherwise read propagation may incorrectly stop too soon
4599 		 * when stack slots are partially written.
4600 		 * This heuristic means that read propagation will be
4601 		 * conservative, since it will add reg_live_read marks
4602 		 * to stack slots all the way to first state when programs
4603 		 * writes+reads less than 8 bytes
4604 		 */
4605 		if (size == BPF_REG_SIZE)
4606 			state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
4607 
4608 		/* when we zero initialize stack slots mark them as such */
4609 		if ((reg && register_is_null(reg)) ||
4610 		    (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) {
4611 			/* STACK_ZERO case happened because register spill
4612 			 * wasn't properly aligned at the stack slot boundary,
4613 			 * so it's not a register spill anymore; force
4614 			 * originating register to be precise to make
4615 			 * STACK_ZERO correct for subsequent states
4616 			 */
4617 			err = mark_chain_precision(env, value_regno);
4618 			if (err)
4619 				return err;
4620 			type = STACK_ZERO;
4621 		}
4622 
4623 		/* Mark slots affected by this stack write. */
4624 		for (i = 0; i < size; i++)
4625 			state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type;
4626 		insn_flags = 0; /* not a register spill */
4627 	}
4628 
4629 	if (insn_flags)
4630 		return push_jmp_history(env, env->cur_state, insn_flags);
4631 	return 0;
4632 }
4633 
4634 /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is
4635  * known to contain a variable offset.
4636  * This function checks whether the write is permitted and conservatively
4637  * tracks the effects of the write, considering that each stack slot in the
4638  * dynamic range is potentially written to.
4639  *
4640  * 'off' includes 'regno->off'.
4641  * 'value_regno' can be -1, meaning that an unknown value is being written to
4642  * the stack.
4643  *
4644  * Spilled pointers in range are not marked as written because we don't know
4645  * what's going to be actually written. This means that read propagation for
4646  * future reads cannot be terminated by this write.
4647  *
4648  * For privileged programs, uninitialized stack slots are considered
4649  * initialized by this write (even though we don't know exactly what offsets
4650  * are going to be written to). The idea is that we don't want the verifier to
4651  * reject future reads that access slots written to through variable offsets.
4652  */
4653 static int check_stack_write_var_off(struct bpf_verifier_env *env,
4654 				     /* func where register points to */
4655 				     struct bpf_func_state *state,
4656 				     int ptr_regno, int off, int size,
4657 				     int value_regno, int insn_idx)
4658 {
4659 	struct bpf_func_state *cur; /* state of the current function */
4660 	int min_off, max_off;
4661 	int i, err;
4662 	struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;
4663 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
4664 	bool writing_zero = false;
4665 	/* set if the fact that we're writing a zero is used to let any
4666 	 * stack slots remain STACK_ZERO
4667 	 */
4668 	bool zero_used = false;
4669 
4670 	cur = env->cur_state->frame[env->cur_state->curframe];
4671 	ptr_reg = &cur->regs[ptr_regno];
4672 	min_off = ptr_reg->smin_value + off;
4673 	max_off = ptr_reg->smax_value + off + size;
4674 	if (value_regno >= 0)
4675 		value_reg = &cur->regs[value_regno];
4676 	if ((value_reg && register_is_null(value_reg)) ||
4677 	    (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0))
4678 		writing_zero = true;
4679 
4680 	for (i = min_off; i < max_off; i++) {
4681 		int spi;
4682 
4683 		spi = __get_spi(i);
4684 		err = destroy_if_dynptr_stack_slot(env, state, spi);
4685 		if (err)
4686 			return err;
4687 	}
4688 
4689 	/* Variable offset writes destroy any spilled pointers in range. */
4690 	for (i = min_off; i < max_off; i++) {
4691 		u8 new_type, *stype;
4692 		int slot, spi;
4693 
4694 		slot = -i - 1;
4695 		spi = slot / BPF_REG_SIZE;
4696 		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
4697 		mark_stack_slot_scratched(env, spi);
4698 
4699 		if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) {
4700 			/* Reject the write if range we may write to has not
4701 			 * been initialized beforehand. If we didn't reject
4702 			 * here, the ptr status would be erased below (even
4703 			 * though not all slots are actually overwritten),
4704 			 * possibly opening the door to leaks.
4705 			 *
4706 			 * We do however catch STACK_INVALID case below, and
4707 			 * only allow reading possibly uninitialized memory
4708 			 * later for CAP_PERFMON, as the write may not happen to
4709 			 * that slot.
4710 			 */
4711 			verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
4712 				insn_idx, i);
4713 			return -EINVAL;
4714 		}
4715 
4716 		/* If writing_zero and the spi slot contains a spill of value 0,
4717 		 * maintain the spill type.
4718 		 */
4719 		if (writing_zero && *stype == STACK_SPILL &&
4720 		    is_spilled_scalar_reg(&state->stack[spi])) {
4721 			struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr;
4722 
4723 			if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) {
4724 				zero_used = true;
4725 				continue;
4726 			}
4727 		}
4728 
4729 		/* Erase all other spilled pointers. */
4730 		state->stack[spi].spilled_ptr.type = NOT_INIT;
4731 
4732 		/* Update the slot type. */
4733 		new_type = STACK_MISC;
4734 		if (writing_zero && *stype == STACK_ZERO) {
4735 			new_type = STACK_ZERO;
4736 			zero_used = true;
4737 		}
4738 		/* If the slot is STACK_INVALID, we check whether it's OK to
4739 		 * pretend that it will be initialized by this write. The slot
4740 		 * might not actually be written to, and so if we mark it as
4741 		 * initialized future reads might leak uninitialized memory.
4742 		 * For privileged programs, we will accept such reads to slots
4743 		 * that may or may not be written because, if we're reject
4744 		 * them, the error would be too confusing.
4745 		 */
4746 		if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
4747 			verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
4748 					insn_idx, i);
4749 			return -EINVAL;
4750 		}
4751 		*stype = new_type;
4752 	}
4753 	if (zero_used) {
4754 		/* backtracking doesn't work for STACK_ZERO yet. */
4755 		err = mark_chain_precision(env, value_regno);
4756 		if (err)
4757 			return err;
4758 	}
4759 	return 0;
4760 }
4761 
4762 /* When register 'dst_regno' is assigned some values from stack[min_off,
4763  * max_off), we set the register's type according to the types of the
4764  * respective stack slots. If all the stack values are known to be zeros, then
4765  * so is the destination reg. Otherwise, the register is considered to be
4766  * SCALAR. This function does not deal with register filling; the caller must
4767  * ensure that all spilled registers in the stack range have been marked as
4768  * read.
4769  */
4770 static void mark_reg_stack_read(struct bpf_verifier_env *env,
4771 				/* func where src register points to */
4772 				struct bpf_func_state *ptr_state,
4773 				int min_off, int max_off, int dst_regno)
4774 {
4775 	struct bpf_verifier_state *vstate = env->cur_state;
4776 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
4777 	int i, slot, spi;
4778 	u8 *stype;
4779 	int zeros = 0;
4780 
4781 	for (i = min_off; i < max_off; i++) {
4782 		slot = -i - 1;
4783 		spi = slot / BPF_REG_SIZE;
4784 		mark_stack_slot_scratched(env, spi);
4785 		stype = ptr_state->stack[spi].slot_type;
4786 		if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
4787 			break;
4788 		zeros++;
4789 	}
4790 	if (zeros == max_off - min_off) {
4791 		/* Any access_size read into register is zero extended,
4792 		 * so the whole register == const_zero.
4793 		 */
4794 		__mark_reg_const_zero(env, &state->regs[dst_regno]);
4795 	} else {
4796 		/* have read misc data from the stack */
4797 		mark_reg_unknown(env, state->regs, dst_regno);
4798 	}
4799 	state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
4800 }
4801 
4802 /* Read the stack at 'off' and put the results into the register indicated by
4803  * 'dst_regno'. It handles reg filling if the addressed stack slot is a
4804  * spilled reg.
4805  *
4806  * 'dst_regno' can be -1, meaning that the read value is not going to a
4807  * register.
4808  *
4809  * The access is assumed to be within the current stack bounds.
4810  */
4811 static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
4812 				      /* func where src register points to */
4813 				      struct bpf_func_state *reg_state,
4814 				      int off, int size, int dst_regno)
4815 {
4816 	struct bpf_verifier_state *vstate = env->cur_state;
4817 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
4818 	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
4819 	struct bpf_reg_state *reg;
4820 	u8 *stype, type;
4821 	int insn_flags = insn_stack_access_flags(reg_state->frameno, spi);
4822 
4823 	stype = reg_state->stack[spi].slot_type;
4824 	reg = &reg_state->stack[spi].spilled_ptr;
4825 
4826 	mark_stack_slot_scratched(env, spi);
4827 
4828 	if (is_spilled_reg(&reg_state->stack[spi])) {
4829 		u8 spill_size = 1;
4830 
4831 		for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--)
4832 			spill_size++;
4833 
4834 		if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) {
4835 			if (reg->type != SCALAR_VALUE) {
4836 				verbose_linfo(env, env->insn_idx, "; ");
4837 				verbose(env, "invalid size of register fill\n");
4838 				return -EACCES;
4839 			}
4840 
4841 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
4842 			if (dst_regno < 0)
4843 				return 0;
4844 
4845 			if (size <= spill_size &&
4846 			    bpf_stack_narrow_access_ok(off, size, spill_size)) {
4847 				/* The earlier check_reg_arg() has decided the
4848 				 * subreg_def for this insn.  Save it first.
4849 				 */
4850 				s32 subreg_def = state->regs[dst_regno].subreg_def;
4851 
4852 				copy_register_state(&state->regs[dst_regno], reg);
4853 				state->regs[dst_regno].subreg_def = subreg_def;
4854 
4855 				/* Break the relation on a narrowing fill.
4856 				 * coerce_reg_to_size will adjust the boundaries.
4857 				 */
4858 				if (get_reg_width(reg) > size * BITS_PER_BYTE)
4859 					state->regs[dst_regno].id = 0;
4860 			} else {
4861 				int spill_cnt = 0, zero_cnt = 0;
4862 
4863 				for (i = 0; i < size; i++) {
4864 					type = stype[(slot - i) % BPF_REG_SIZE];
4865 					if (type == STACK_SPILL) {
4866 						spill_cnt++;
4867 						continue;
4868 					}
4869 					if (type == STACK_MISC)
4870 						continue;
4871 					if (type == STACK_ZERO) {
4872 						zero_cnt++;
4873 						continue;
4874 					}
4875 					if (type == STACK_INVALID && env->allow_uninit_stack)
4876 						continue;
4877 					verbose(env, "invalid read from stack off %d+%d size %d\n",
4878 						off, i, size);
4879 					return -EACCES;
4880 				}
4881 
4882 				if (spill_cnt == size &&
4883 				    tnum_is_const(reg->var_off) && reg->var_off.value == 0) {
4884 					__mark_reg_const_zero(env, &state->regs[dst_regno]);
4885 					/* this IS register fill, so keep insn_flags */
4886 				} else if (zero_cnt == size) {
4887 					/* similarly to mark_reg_stack_read(), preserve zeroes */
4888 					__mark_reg_const_zero(env, &state->regs[dst_regno]);
4889 					insn_flags = 0; /* not restoring original register state */
4890 				} else {
4891 					mark_reg_unknown(env, state->regs, dst_regno);
4892 					insn_flags = 0; /* not restoring original register state */
4893 				}
4894 			}
4895 			state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
4896 		} else if (dst_regno >= 0) {
4897 			/* restore register state from stack */
4898 			copy_register_state(&state->regs[dst_regno], reg);
4899 			/* mark reg as written since spilled pointer state likely
4900 			 * has its liveness marks cleared by is_state_visited()
4901 			 * which resets stack/reg liveness for state transitions
4902 			 */
4903 			state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
4904 		} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
4905 			/* If dst_regno==-1, the caller is asking us whether
4906 			 * it is acceptable to use this value as a SCALAR_VALUE
4907 			 * (e.g. for XADD).
4908 			 * We must not allow unprivileged callers to do that
4909 			 * with spilled pointers.
4910 			 */
4911 			verbose(env, "leaking pointer from stack off %d\n",
4912 				off);
4913 			return -EACCES;
4914 		}
4915 		mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
4916 	} else {
4917 		for (i = 0; i < size; i++) {
4918 			type = stype[(slot - i) % BPF_REG_SIZE];
4919 			if (type == STACK_MISC)
4920 				continue;
4921 			if (type == STACK_ZERO)
4922 				continue;
4923 			if (type == STACK_INVALID && env->allow_uninit_stack)
4924 				continue;
4925 			verbose(env, "invalid read from stack off %d+%d size %d\n",
4926 				off, i, size);
4927 			return -EACCES;
4928 		}
4929 		mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
4930 		if (dst_regno >= 0)
4931 			mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
4932 		insn_flags = 0; /* we are not restoring spilled register */
4933 	}
4934 	if (insn_flags)
4935 		return push_jmp_history(env, env->cur_state, insn_flags);
4936 	return 0;
4937 }
4938 
4939 enum bpf_access_src {
4940 	ACCESS_DIRECT = 1,  /* the access is performed by an instruction */
4941 	ACCESS_HELPER = 2,  /* the access is performed by a helper */
4942 };
4943 
4944 static int check_stack_range_initialized(struct bpf_verifier_env *env,
4945 					 int regno, int off, int access_size,
4946 					 bool zero_size_allowed,
4947 					 enum bpf_access_src type,
4948 					 struct bpf_call_arg_meta *meta);
4949 
4950 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
4951 {
4952 	return cur_regs(env) + regno;
4953 }
4954 
4955 /* Read the stack at 'ptr_regno + off' and put the result into the register
4956  * 'dst_regno'.
4957  * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'),
4958  * but not its variable offset.
4959  * 'size' is assumed to be <= reg size and the access is assumed to be aligned.
4960  *
4961  * As opposed to check_stack_read_fixed_off, this function doesn't deal with
4962  * filling registers (i.e. reads of spilled register cannot be detected when
4963  * the offset is not fixed). We conservatively mark 'dst_regno' as containing
4964  * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable
4965  * offset; for a fixed offset check_stack_read_fixed_off should be used
4966  * instead.
4967  */
4968 static int check_stack_read_var_off(struct bpf_verifier_env *env,
4969 				    int ptr_regno, int off, int size, int dst_regno)
4970 {
4971 	/* The state of the source register. */
4972 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
4973 	struct bpf_func_state *ptr_state = func(env, reg);
4974 	int err;
4975 	int min_off, max_off;
4976 
4977 	/* Note that we pass a NULL meta, so raw access will not be permitted.
4978 	 */
4979 	err = check_stack_range_initialized(env, ptr_regno, off, size,
4980 					    false, ACCESS_DIRECT, NULL);
4981 	if (err)
4982 		return err;
4983 
4984 	min_off = reg->smin_value + off;
4985 	max_off = reg->smax_value + off;
4986 	mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
4987 	return 0;
4988 }
4989 
4990 /* check_stack_read dispatches to check_stack_read_fixed_off or
4991  * check_stack_read_var_off.
4992  *
4993  * The caller must ensure that the offset falls within the allocated stack
4994  * bounds.
4995  *
4996  * 'dst_regno' is a register which will receive the value from the stack. It
4997  * can be -1, meaning that the read value is not going to a register.
4998  */
4999 static int check_stack_read(struct bpf_verifier_env *env,
5000 			    int ptr_regno, int off, int size,
5001 			    int dst_regno)
5002 {
5003 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
5004 	struct bpf_func_state *state = func(env, reg);
5005 	int err;
5006 	/* Some accesses are only permitted with a static offset. */
5007 	bool var_off = !tnum_is_const(reg->var_off);
5008 
5009 	/* The offset is required to be static when reads don't go to a
5010 	 * register, in order to not leak pointers (see
5011 	 * check_stack_read_fixed_off).
5012 	 */
5013 	if (dst_regno < 0 && var_off) {
5014 		char tn_buf[48];
5015 
5016 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5017 		verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
5018 			tn_buf, off, size);
5019 		return -EACCES;
5020 	}
5021 	/* Variable offset is prohibited for unprivileged mode for simplicity
5022 	 * since it requires corresponding support in Spectre masking for stack
5023 	 * ALU. See also retrieve_ptr_limit(). The check in
5024 	 * check_stack_access_for_ptr_arithmetic() called by
5025 	 * adjust_ptr_min_max_vals() prevents users from creating stack pointers
5026 	 * with variable offsets, therefore no check is required here. Further,
5027 	 * just checking it here would be insufficient as speculative stack
5028 	 * writes could still lead to unsafe speculative behaviour.
5029 	 */
5030 	if (!var_off) {
5031 		off += reg->var_off.value;
5032 		err = check_stack_read_fixed_off(env, state, off, size,
5033 						 dst_regno);
5034 	} else {
5035 		/* Variable offset stack reads need more conservative handling
5036 		 * than fixed offset ones. Note that dst_regno >= 0 on this
5037 		 * branch.
5038 		 */
5039 		err = check_stack_read_var_off(env, ptr_regno, off, size,
5040 					       dst_regno);
5041 	}
5042 	return err;
5043 }
5044 
5045 
5046 /* check_stack_write dispatches to check_stack_write_fixed_off or
5047  * check_stack_write_var_off.
5048  *
5049  * 'ptr_regno' is the register used as a pointer into the stack.
5050  * 'off' includes 'ptr_regno->off', but not its variable offset (if any).
5051  * 'value_regno' is the register whose value we're writing to the stack. It can
5052  * be -1, meaning that we're not writing from a register.
5053  *
5054  * The caller must ensure that the offset falls within the maximum stack size.
5055  */
5056 static int check_stack_write(struct bpf_verifier_env *env,
5057 			     int ptr_regno, int off, int size,
5058 			     int value_regno, int insn_idx)
5059 {
5060 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
5061 	struct bpf_func_state *state = func(env, reg);
5062 	int err;
5063 
5064 	if (tnum_is_const(reg->var_off)) {
5065 		off += reg->var_off.value;
5066 		err = check_stack_write_fixed_off(env, state, off, size,
5067 						  value_regno, insn_idx);
5068 	} else {
5069 		/* Variable offset stack reads need more conservative handling
5070 		 * than fixed offset ones.
5071 		 */
5072 		err = check_stack_write_var_off(env, state,
5073 						ptr_regno, off, size,
5074 						value_regno, insn_idx);
5075 	}
5076 	return err;
5077 }
5078 
5079 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
5080 				 int off, int size, enum bpf_access_type type)
5081 {
5082 	struct bpf_reg_state *regs = cur_regs(env);
5083 	struct bpf_map *map = regs[regno].map_ptr;
5084 	u32 cap = bpf_map_flags_to_cap(map);
5085 
5086 	if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) {
5087 		verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",
5088 			map->value_size, off, size);
5089 		return -EACCES;
5090 	}
5091 
5092 	if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) {
5093 		verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",
5094 			map->value_size, off, size);
5095 		return -EACCES;
5096 	}
5097 
5098 	return 0;
5099 }
5100 
5101 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */
5102 static int __check_mem_access(struct bpf_verifier_env *env, int regno,
5103 			      int off, int size, u32 mem_size,
5104 			      bool zero_size_allowed)
5105 {
5106 	bool size_ok = size > 0 || (size == 0 && zero_size_allowed);
5107 	struct bpf_reg_state *reg;
5108 
5109 	if (off >= 0 && size_ok && (u64)off + size <= mem_size)
5110 		return 0;
5111 
5112 	reg = &cur_regs(env)[regno];
5113 	switch (reg->type) {
5114 	case PTR_TO_MAP_KEY:
5115 		verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",
5116 			mem_size, off, size);
5117 		break;
5118 	case PTR_TO_MAP_VALUE:
5119 		verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
5120 			mem_size, off, size);
5121 		break;
5122 	case PTR_TO_PACKET:
5123 	case PTR_TO_PACKET_META:
5124 	case PTR_TO_PACKET_END:
5125 		verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
5126 			off, size, regno, reg->id, off, mem_size);
5127 		break;
5128 	case PTR_TO_MEM:
5129 	default:
5130 		verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",
5131 			mem_size, off, size);
5132 	}
5133 
5134 	return -EACCES;
5135 }
5136 
5137 /* check read/write into a memory region with possible variable offset */
5138 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,
5139 				   int off, int size, u32 mem_size,
5140 				   bool zero_size_allowed)
5141 {
5142 	struct bpf_verifier_state *vstate = env->cur_state;
5143 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
5144 	struct bpf_reg_state *reg = &state->regs[regno];
5145 	int err;
5146 
5147 	/* We may have adjusted the register pointing to memory region, so we
5148 	 * need to try adding each of min_value and max_value to off
5149 	 * to make sure our theoretical access will be safe.
5150 	 *
5151 	 * The minimum value is only important with signed
5152 	 * comparisons where we can't assume the floor of a
5153 	 * value is 0.  If we are using signed variables for our
5154 	 * index'es we need to make sure that whatever we use
5155 	 * will have a set floor within our range.
5156 	 */
5157 	if (reg->smin_value < 0 &&
5158 	    (reg->smin_value == S64_MIN ||
5159 	     (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
5160 	      reg->smin_value + off < 0)) {
5161 		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
5162 			regno);
5163 		return -EACCES;
5164 	}
5165 	err = __check_mem_access(env, regno, reg->smin_value + off, size,
5166 				 mem_size, zero_size_allowed);
5167 	if (err) {
5168 		verbose(env, "R%d min value is outside of the allowed memory range\n",
5169 			regno);
5170 		return err;
5171 	}
5172 
5173 	/* If we haven't set a max value then we need to bail since we can't be
5174 	 * sure we won't do bad things.
5175 	 * If reg->umax_value + off could overflow, treat that as unbounded too.
5176 	 */
5177 	if (reg->umax_value >= BPF_MAX_VAR_OFF) {
5178 		verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n",
5179 			regno);
5180 		return -EACCES;
5181 	}
5182 	err = __check_mem_access(env, regno, reg->umax_value + off, size,
5183 				 mem_size, zero_size_allowed);
5184 	if (err) {
5185 		verbose(env, "R%d max value is outside of the allowed memory range\n",
5186 			regno);
5187 		return err;
5188 	}
5189 
5190 	return 0;
5191 }
5192 
5193 static int __check_ptr_off_reg(struct bpf_verifier_env *env,
5194 			       const struct bpf_reg_state *reg, int regno,
5195 			       bool fixed_off_ok)
5196 {
5197 	/* Access to this pointer-typed register or passing it to a helper
5198 	 * is only allowed in its original, unmodified form.
5199 	 */
5200 
5201 	if (reg->off < 0) {
5202 		verbose(env, "negative offset %s ptr R%d off=%d disallowed\n",
5203 			reg_type_str(env, reg->type), regno, reg->off);
5204 		return -EACCES;
5205 	}
5206 
5207 	if (!fixed_off_ok && reg->off) {
5208 		verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n",
5209 			reg_type_str(env, reg->type), regno, reg->off);
5210 		return -EACCES;
5211 	}
5212 
5213 	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
5214 		char tn_buf[48];
5215 
5216 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5217 		verbose(env, "variable %s access var_off=%s disallowed\n",
5218 			reg_type_str(env, reg->type), tn_buf);
5219 		return -EACCES;
5220 	}
5221 
5222 	return 0;
5223 }
5224 
5225 static int check_ptr_off_reg(struct bpf_verifier_env *env,
5226 		             const struct bpf_reg_state *reg, int regno)
5227 {
5228 	return __check_ptr_off_reg(env, reg, regno, false);
5229 }
5230 
5231 static int map_kptr_match_type(struct bpf_verifier_env *env,
5232 			       struct btf_field *kptr_field,
5233 			       struct bpf_reg_state *reg, u32 regno)
5234 {
5235 	const char *targ_name = btf_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id);
5236 	int perm_flags;
5237 	const char *reg_name = "";
5238 
5239 	if (btf_is_kernel(reg->btf)) {
5240 		perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU;
5241 
5242 		/* Only unreferenced case accepts untrusted pointers */
5243 		if (kptr_field->type == BPF_KPTR_UNREF)
5244 			perm_flags |= PTR_UNTRUSTED;
5245 	} else {
5246 		perm_flags = PTR_MAYBE_NULL | MEM_ALLOC;
5247 		if (kptr_field->type == BPF_KPTR_PERCPU)
5248 			perm_flags |= MEM_PERCPU;
5249 	}
5250 
5251 	if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags))
5252 		goto bad_type;
5253 
5254 	/* We need to verify reg->type and reg->btf, before accessing reg->btf */
5255 	reg_name = btf_type_name(reg->btf, reg->btf_id);
5256 
5257 	/* For ref_ptr case, release function check should ensure we get one
5258 	 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the
5259 	 * normal store of unreferenced kptr, we must ensure var_off is zero.
5260 	 * Since ref_ptr cannot be accessed directly by BPF insns, checks for
5261 	 * reg->off and reg->ref_obj_id are not needed here.
5262 	 */
5263 	if (__check_ptr_off_reg(env, reg, regno, true))
5264 		return -EACCES;
5265 
5266 	/* A full type match is needed, as BTF can be vmlinux, module or prog BTF, and
5267 	 * we also need to take into account the reg->off.
5268 	 *
5269 	 * We want to support cases like:
5270 	 *
5271 	 * struct foo {
5272 	 *         struct bar br;
5273 	 *         struct baz bz;
5274 	 * };
5275 	 *
5276 	 * struct foo *v;
5277 	 * v = func();	      // PTR_TO_BTF_ID
5278 	 * val->foo = v;      // reg->off is zero, btf and btf_id match type
5279 	 * val->bar = &v->br; // reg->off is still zero, but we need to retry with
5280 	 *                    // first member type of struct after comparison fails
5281 	 * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked
5282 	 *                    // to match type
5283 	 *
5284 	 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off
5285 	 * is zero. We must also ensure that btf_struct_ids_match does not walk
5286 	 * the struct to match type against first member of struct, i.e. reject
5287 	 * second case from above. Hence, when type is BPF_KPTR_REF, we set
5288 	 * strict mode to true for type match.
5289 	 */
5290 	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
5291 				  kptr_field->kptr.btf, kptr_field->kptr.btf_id,
5292 				  kptr_field->type != BPF_KPTR_UNREF))
5293 		goto bad_type;
5294 	return 0;
5295 bad_type:
5296 	verbose(env, "invalid kptr access, R%d type=%s%s ", regno,
5297 		reg_type_str(env, reg->type), reg_name);
5298 	verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name);
5299 	if (kptr_field->type == BPF_KPTR_UNREF)
5300 		verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED),
5301 			targ_name);
5302 	else
5303 		verbose(env, "\n");
5304 	return -EINVAL;
5305 }
5306 
5307 static bool in_sleepable(struct bpf_verifier_env *env)
5308 {
5309 	return env->prog->sleepable ||
5310 	       (env->cur_state && env->cur_state->in_sleepable);
5311 }
5312 
5313 /* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock()
5314  * can dereference RCU protected pointers and result is PTR_TRUSTED.
5315  */
5316 static bool in_rcu_cs(struct bpf_verifier_env *env)
5317 {
5318 	return env->cur_state->active_rcu_lock ||
5319 	       env->cur_state->active_lock.ptr ||
5320 	       !in_sleepable(env);
5321 }
5322 
5323 /* Once GCC supports btf_type_tag the following mechanism will be replaced with tag check */
5324 BTF_SET_START(rcu_protected_types)
5325 BTF_ID(struct, prog_test_ref_kfunc)
5326 #ifdef CONFIG_CGROUPS
5327 BTF_ID(struct, cgroup)
5328 #endif
5329 #ifdef CONFIG_BPF_JIT
5330 BTF_ID(struct, bpf_cpumask)
5331 #endif
5332 BTF_ID(struct, task_struct)
5333 BTF_ID(struct, bpf_crypto_ctx)
5334 BTF_SET_END(rcu_protected_types)
5335 
5336 static bool rcu_protected_object(const struct btf *btf, u32 btf_id)
5337 {
5338 	if (!btf_is_kernel(btf))
5339 		return true;
5340 	return btf_id_set_contains(&rcu_protected_types, btf_id);
5341 }
5342 
5343 static struct btf_record *kptr_pointee_btf_record(struct btf_field *kptr_field)
5344 {
5345 	struct btf_struct_meta *meta;
5346 
5347 	if (btf_is_kernel(kptr_field->kptr.btf))
5348 		return NULL;
5349 
5350 	meta = btf_find_struct_meta(kptr_field->kptr.btf,
5351 				    kptr_field->kptr.btf_id);
5352 
5353 	return meta ? meta->record : NULL;
5354 }
5355 
5356 static bool rcu_safe_kptr(const struct btf_field *field)
5357 {
5358 	const struct btf_field_kptr *kptr = &field->kptr;
5359 
5360 	return field->type == BPF_KPTR_PERCPU ||
5361 	       (field->type == BPF_KPTR_REF && rcu_protected_object(kptr->btf, kptr->btf_id));
5362 }
5363 
5364 static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field)
5365 {
5366 	struct btf_record *rec;
5367 	u32 ret;
5368 
5369 	ret = PTR_MAYBE_NULL;
5370 	if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) {
5371 		ret |= MEM_RCU;
5372 		if (kptr_field->type == BPF_KPTR_PERCPU)
5373 			ret |= MEM_PERCPU;
5374 		else if (!btf_is_kernel(kptr_field->kptr.btf))
5375 			ret |= MEM_ALLOC;
5376 
5377 		rec = kptr_pointee_btf_record(kptr_field);
5378 		if (rec && btf_record_has_field(rec, BPF_GRAPH_NODE))
5379 			ret |= NON_OWN_REF;
5380 	} else {
5381 		ret |= PTR_UNTRUSTED;
5382 	}
5383 
5384 	return ret;
5385 }
5386 
5387 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,
5388 				 int value_regno, int insn_idx,
5389 				 struct btf_field *kptr_field)
5390 {
5391 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
5392 	int class = BPF_CLASS(insn->code);
5393 	struct bpf_reg_state *val_reg;
5394 
5395 	/* Things we already checked for in check_map_access and caller:
5396 	 *  - Reject cases where variable offset may touch kptr
5397 	 *  - size of access (must be BPF_DW)
5398 	 *  - tnum_is_const(reg->var_off)
5399 	 *  - kptr_field->offset == off + reg->var_off.value
5400 	 */
5401 	/* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */
5402 	if (BPF_MODE(insn->code) != BPF_MEM) {
5403 		verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n");
5404 		return -EACCES;
5405 	}
5406 
5407 	/* We only allow loading referenced kptr, since it will be marked as
5408 	 * untrusted, similar to unreferenced kptr.
5409 	 */
5410 	if (class != BPF_LDX &&
5411 	    (kptr_field->type == BPF_KPTR_REF || kptr_field->type == BPF_KPTR_PERCPU)) {
5412 		verbose(env, "store to referenced kptr disallowed\n");
5413 		return -EACCES;
5414 	}
5415 
5416 	if (class == BPF_LDX) {
5417 		val_reg = reg_state(env, value_regno);
5418 		/* We can simply mark the value_regno receiving the pointer
5419 		 * value from map as PTR_TO_BTF_ID, with the correct type.
5420 		 */
5421 		mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf,
5422 				kptr_field->kptr.btf_id, btf_ld_kptr_type(env, kptr_field));
5423 	} else if (class == BPF_STX) {
5424 		val_reg = reg_state(env, value_regno);
5425 		if (!register_is_null(val_reg) &&
5426 		    map_kptr_match_type(env, kptr_field, val_reg, value_regno))
5427 			return -EACCES;
5428 	} else if (class == BPF_ST) {
5429 		if (insn->imm) {
5430 			verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n",
5431 				kptr_field->offset);
5432 			return -EACCES;
5433 		}
5434 	} else {
5435 		verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n");
5436 		return -EACCES;
5437 	}
5438 	return 0;
5439 }
5440 
5441 /* check read/write into a map element with possible variable offset */
5442 static int check_map_access(struct bpf_verifier_env *env, u32 regno,
5443 			    int off, int size, bool zero_size_allowed,
5444 			    enum bpf_access_src src)
5445 {
5446 	struct bpf_verifier_state *vstate = env->cur_state;
5447 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
5448 	struct bpf_reg_state *reg = &state->regs[regno];
5449 	struct bpf_map *map = reg->map_ptr;
5450 	struct btf_record *rec;
5451 	int err, i;
5452 
5453 	err = check_mem_region_access(env, regno, off, size, map->value_size,
5454 				      zero_size_allowed);
5455 	if (err)
5456 		return err;
5457 
5458 	if (IS_ERR_OR_NULL(map->record))
5459 		return 0;
5460 	rec = map->record;
5461 	for (i = 0; i < rec->cnt; i++) {
5462 		struct btf_field *field = &rec->fields[i];
5463 		u32 p = field->offset;
5464 
5465 		/* If any part of a field  can be touched by load/store, reject
5466 		 * this program. To check that [x1, x2) overlaps with [y1, y2),
5467 		 * it is sufficient to check x1 < y2 && y1 < x2.
5468 		 */
5469 		if (reg->smin_value + off < p + field->size &&
5470 		    p < reg->umax_value + off + size) {
5471 			switch (field->type) {
5472 			case BPF_KPTR_UNREF:
5473 			case BPF_KPTR_REF:
5474 			case BPF_KPTR_PERCPU:
5475 				if (src != ACCESS_DIRECT) {
5476 					verbose(env, "kptr cannot be accessed indirectly by helper\n");
5477 					return -EACCES;
5478 				}
5479 				if (!tnum_is_const(reg->var_off)) {
5480 					verbose(env, "kptr access cannot have variable offset\n");
5481 					return -EACCES;
5482 				}
5483 				if (p != off + reg->var_off.value) {
5484 					verbose(env, "kptr access misaligned expected=%u off=%llu\n",
5485 						p, off + reg->var_off.value);
5486 					return -EACCES;
5487 				}
5488 				if (size != bpf_size_to_bytes(BPF_DW)) {
5489 					verbose(env, "kptr access size must be BPF_DW\n");
5490 					return -EACCES;
5491 				}
5492 				break;
5493 			default:
5494 				verbose(env, "%s cannot be accessed directly by load/store\n",
5495 					btf_field_type_name(field->type));
5496 				return -EACCES;
5497 			}
5498 		}
5499 	}
5500 	return 0;
5501 }
5502 
5503 #define MAX_PACKET_OFF 0xffff
5504 
5505 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
5506 				       const struct bpf_call_arg_meta *meta,
5507 				       enum bpf_access_type t)
5508 {
5509 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
5510 
5511 	switch (prog_type) {
5512 	/* Program types only with direct read access go here! */
5513 	case BPF_PROG_TYPE_LWT_IN:
5514 	case BPF_PROG_TYPE_LWT_OUT:
5515 	case BPF_PROG_TYPE_LWT_SEG6LOCAL:
5516 	case BPF_PROG_TYPE_SK_REUSEPORT:
5517 	case BPF_PROG_TYPE_FLOW_DISSECTOR:
5518 	case BPF_PROG_TYPE_CGROUP_SKB:
5519 		if (t == BPF_WRITE)
5520 			return false;
5521 		fallthrough;
5522 
5523 	/* Program types with direct read + write access go here! */
5524 	case BPF_PROG_TYPE_SCHED_CLS:
5525 	case BPF_PROG_TYPE_SCHED_ACT:
5526 	case BPF_PROG_TYPE_XDP:
5527 	case BPF_PROG_TYPE_LWT_XMIT:
5528 	case BPF_PROG_TYPE_SK_SKB:
5529 	case BPF_PROG_TYPE_SK_MSG:
5530 		if (meta)
5531 			return meta->pkt_access;
5532 
5533 		env->seen_direct_write = true;
5534 		return true;
5535 
5536 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
5537 		if (t == BPF_WRITE)
5538 			env->seen_direct_write = true;
5539 
5540 		return true;
5541 
5542 	default:
5543 		return false;
5544 	}
5545 }
5546 
5547 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
5548 			       int size, bool zero_size_allowed)
5549 {
5550 	struct bpf_reg_state *regs = cur_regs(env);
5551 	struct bpf_reg_state *reg = &regs[regno];
5552 	int err;
5553 
5554 	/* We may have added a variable offset to the packet pointer; but any
5555 	 * reg->range we have comes after that.  We are only checking the fixed
5556 	 * offset.
5557 	 */
5558 
5559 	/* We don't allow negative numbers, because we aren't tracking enough
5560 	 * detail to prove they're safe.
5561 	 */
5562 	if (reg->smin_value < 0) {
5563 		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
5564 			regno);
5565 		return -EACCES;
5566 	}
5567 
5568 	err = reg->range < 0 ? -EINVAL :
5569 	      __check_mem_access(env, regno, off, size, reg->range,
5570 				 zero_size_allowed);
5571 	if (err) {
5572 		verbose(env, "R%d offset is outside of the packet\n", regno);
5573 		return err;
5574 	}
5575 
5576 	/* __check_mem_access has made sure "off + size - 1" is within u16.
5577 	 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff,
5578 	 * otherwise find_good_pkt_pointers would have refused to set range info
5579 	 * that __check_mem_access would have rejected this pkt access.
5580 	 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32.
5581 	 */
5582 	env->prog->aux->max_pkt_offset =
5583 		max_t(u32, env->prog->aux->max_pkt_offset,
5584 		      off + reg->umax_value + size - 1);
5585 
5586 	return err;
5587 }
5588 
5589 /* check access to 'struct bpf_context' fields.  Supports fixed offsets only */
5590 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
5591 			    enum bpf_access_type t, enum bpf_reg_type *reg_type,
5592 			    struct btf **btf, u32 *btf_id)
5593 {
5594 	struct bpf_insn_access_aux info = {
5595 		.reg_type = *reg_type,
5596 		.log = &env->log,
5597 	};
5598 
5599 	if (env->ops->is_valid_access &&
5600 	    env->ops->is_valid_access(off, size, t, env->prog, &info)) {
5601 		/* A non zero info.ctx_field_size indicates that this field is a
5602 		 * candidate for later verifier transformation to load the whole
5603 		 * field and then apply a mask when accessed with a narrower
5604 		 * access than actual ctx access size. A zero info.ctx_field_size
5605 		 * will only allow for whole field access and rejects any other
5606 		 * type of narrower access.
5607 		 */
5608 		*reg_type = info.reg_type;
5609 
5610 		if (base_type(*reg_type) == PTR_TO_BTF_ID) {
5611 			*btf = info.btf;
5612 			*btf_id = info.btf_id;
5613 		} else {
5614 			env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
5615 		}
5616 		/* remember the offset of last byte accessed in ctx */
5617 		if (env->prog->aux->max_ctx_offset < off + size)
5618 			env->prog->aux->max_ctx_offset = off + size;
5619 		return 0;
5620 	}
5621 
5622 	verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
5623 	return -EACCES;
5624 }
5625 
5626 static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
5627 				  int size)
5628 {
5629 	if (size < 0 || off < 0 ||
5630 	    (u64)off + size > sizeof(struct bpf_flow_keys)) {
5631 		verbose(env, "invalid access to flow keys off=%d size=%d\n",
5632 			off, size);
5633 		return -EACCES;
5634 	}
5635 	return 0;
5636 }
5637 
5638 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
5639 			     u32 regno, int off, int size,
5640 			     enum bpf_access_type t)
5641 {
5642 	struct bpf_reg_state *regs = cur_regs(env);
5643 	struct bpf_reg_state *reg = &regs[regno];
5644 	struct bpf_insn_access_aux info = {};
5645 	bool valid;
5646 
5647 	if (reg->smin_value < 0) {
5648 		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
5649 			regno);
5650 		return -EACCES;
5651 	}
5652 
5653 	switch (reg->type) {
5654 	case PTR_TO_SOCK_COMMON:
5655 		valid = bpf_sock_common_is_valid_access(off, size, t, &info);
5656 		break;
5657 	case PTR_TO_SOCKET:
5658 		valid = bpf_sock_is_valid_access(off, size, t, &info);
5659 		break;
5660 	case PTR_TO_TCP_SOCK:
5661 		valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
5662 		break;
5663 	case PTR_TO_XDP_SOCK:
5664 		valid = bpf_xdp_sock_is_valid_access(off, size, t, &info);
5665 		break;
5666 	default:
5667 		valid = false;
5668 	}
5669 
5670 
5671 	if (valid) {
5672 		env->insn_aux_data[insn_idx].ctx_field_size =
5673 			info.ctx_field_size;
5674 		return 0;
5675 	}
5676 
5677 	verbose(env, "R%d invalid %s access off=%d size=%d\n",
5678 		regno, reg_type_str(env, reg->type), off, size);
5679 
5680 	return -EACCES;
5681 }
5682 
5683 static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
5684 {
5685 	return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
5686 }
5687 
5688 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
5689 {
5690 	const struct bpf_reg_state *reg = reg_state(env, regno);
5691 
5692 	return reg->type == PTR_TO_CTX;
5693 }
5694 
5695 static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
5696 {
5697 	const struct bpf_reg_state *reg = reg_state(env, regno);
5698 
5699 	return type_is_sk_pointer(reg->type);
5700 }
5701 
5702 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
5703 {
5704 	const struct bpf_reg_state *reg = reg_state(env, regno);
5705 
5706 	return type_is_pkt_pointer(reg->type);
5707 }
5708 
5709 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
5710 {
5711 	const struct bpf_reg_state *reg = reg_state(env, regno);
5712 
5713 	/* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */
5714 	return reg->type == PTR_TO_FLOW_KEYS;
5715 }
5716 
5717 static bool is_arena_reg(struct bpf_verifier_env *env, int regno)
5718 {
5719 	const struct bpf_reg_state *reg = reg_state(env, regno);
5720 
5721 	return reg->type == PTR_TO_ARENA;
5722 }
5723 
5724 static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
5725 #ifdef CONFIG_NET
5726 	[PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
5727 	[PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
5728 	[PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
5729 #endif
5730 	[CONST_PTR_TO_MAP] = btf_bpf_map_id,
5731 };
5732 
5733 static bool is_trusted_reg(const struct bpf_reg_state *reg)
5734 {
5735 	/* A referenced register is always trusted. */
5736 	if (reg->ref_obj_id)
5737 		return true;
5738 
5739 	/* Types listed in the reg2btf_ids are always trusted */
5740 	if (reg2btf_ids[base_type(reg->type)] &&
5741 	    !bpf_type_has_unsafe_modifiers(reg->type))
5742 		return true;
5743 
5744 	/* If a register is not referenced, it is trusted if it has the
5745 	 * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the
5746 	 * other type modifiers may be safe, but we elect to take an opt-in
5747 	 * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are
5748 	 * not.
5749 	 *
5750 	 * Eventually, we should make PTR_TRUSTED the single source of truth
5751 	 * for whether a register is trusted.
5752 	 */
5753 	return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS &&
5754 	       !bpf_type_has_unsafe_modifiers(reg->type);
5755 }
5756 
5757 static bool is_rcu_reg(const struct bpf_reg_state *reg)
5758 {
5759 	return reg->type & MEM_RCU;
5760 }
5761 
5762 static void clear_trusted_flags(enum bpf_type_flag *flag)
5763 {
5764 	*flag &= ~(BPF_REG_TRUSTED_MODIFIERS | MEM_RCU);
5765 }
5766 
5767 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
5768 				   const struct bpf_reg_state *reg,
5769 				   int off, int size, bool strict)
5770 {
5771 	struct tnum reg_off;
5772 	int ip_align;
5773 
5774 	/* Byte size accesses are always allowed. */
5775 	if (!strict || size == 1)
5776 		return 0;
5777 
5778 	/* For platforms that do not have a Kconfig enabling
5779 	 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of
5780 	 * NET_IP_ALIGN is universally set to '2'.  And on platforms
5781 	 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get
5782 	 * to this code only in strict mode where we want to emulate
5783 	 * the NET_IP_ALIGN==2 checking.  Therefore use an
5784 	 * unconditional IP align value of '2'.
5785 	 */
5786 	ip_align = 2;
5787 
5788 	reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
5789 	if (!tnum_is_aligned(reg_off, size)) {
5790 		char tn_buf[48];
5791 
5792 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5793 		verbose(env,
5794 			"misaligned packet access off %d+%s+%d+%d size %d\n",
5795 			ip_align, tn_buf, reg->off, off, size);
5796 		return -EACCES;
5797 	}
5798 
5799 	return 0;
5800 }
5801 
5802 static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
5803 				       const struct bpf_reg_state *reg,
5804 				       const char *pointer_desc,
5805 				       int off, int size, bool strict)
5806 {
5807 	struct tnum reg_off;
5808 
5809 	/* Byte size accesses are always allowed. */
5810 	if (!strict || size == 1)
5811 		return 0;
5812 
5813 	reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
5814 	if (!tnum_is_aligned(reg_off, size)) {
5815 		char tn_buf[48];
5816 
5817 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5818 		verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
5819 			pointer_desc, tn_buf, reg->off, off, size);
5820 		return -EACCES;
5821 	}
5822 
5823 	return 0;
5824 }
5825 
5826 static int check_ptr_alignment(struct bpf_verifier_env *env,
5827 			       const struct bpf_reg_state *reg, int off,
5828 			       int size, bool strict_alignment_once)
5829 {
5830 	bool strict = env->strict_alignment || strict_alignment_once;
5831 	const char *pointer_desc = "";
5832 
5833 	switch (reg->type) {
5834 	case PTR_TO_PACKET:
5835 	case PTR_TO_PACKET_META:
5836 		/* Special case, because of NET_IP_ALIGN. Given metadata sits
5837 		 * right in front, treat it the very same way.
5838 		 */
5839 		return check_pkt_ptr_alignment(env, reg, off, size, strict);
5840 	case PTR_TO_FLOW_KEYS:
5841 		pointer_desc = "flow keys ";
5842 		break;
5843 	case PTR_TO_MAP_KEY:
5844 		pointer_desc = "key ";
5845 		break;
5846 	case PTR_TO_MAP_VALUE:
5847 		pointer_desc = "value ";
5848 		break;
5849 	case PTR_TO_CTX:
5850 		pointer_desc = "context ";
5851 		break;
5852 	case PTR_TO_STACK:
5853 		pointer_desc = "stack ";
5854 		/* The stack spill tracking logic in check_stack_write_fixed_off()
5855 		 * and check_stack_read_fixed_off() relies on stack accesses being
5856 		 * aligned.
5857 		 */
5858 		strict = true;
5859 		break;
5860 	case PTR_TO_SOCKET:
5861 		pointer_desc = "sock ";
5862 		break;
5863 	case PTR_TO_SOCK_COMMON:
5864 		pointer_desc = "sock_common ";
5865 		break;
5866 	case PTR_TO_TCP_SOCK:
5867 		pointer_desc = "tcp_sock ";
5868 		break;
5869 	case PTR_TO_XDP_SOCK:
5870 		pointer_desc = "xdp_sock ";
5871 		break;
5872 	case PTR_TO_ARENA:
5873 		return 0;
5874 	default:
5875 		break;
5876 	}
5877 	return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
5878 					   strict);
5879 }
5880 
5881 static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
5882 {
5883 	if (env->prog->jit_requested)
5884 		return round_up(stack_depth, 16);
5885 
5886 	/* round up to 32-bytes, since this is granularity
5887 	 * of interpreter stack size
5888 	 */
5889 	return round_up(max_t(u32, stack_depth, 1), 32);
5890 }
5891 
5892 /* starting from main bpf function walk all instructions of the function
5893  * and recursively walk all callees that given function can call.
5894  * Ignore jump and exit insns.
5895  * Since recursion is prevented by check_cfg() this algorithm
5896  * only needs a local stack of MAX_CALL_FRAMES to remember callsites
5897  */
5898 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx)
5899 {
5900 	struct bpf_subprog_info *subprog = env->subprog_info;
5901 	struct bpf_insn *insn = env->prog->insnsi;
5902 	int depth = 0, frame = 0, i, subprog_end;
5903 	bool tail_call_reachable = false;
5904 	int ret_insn[MAX_CALL_FRAMES];
5905 	int ret_prog[MAX_CALL_FRAMES];
5906 	int j;
5907 
5908 	i = subprog[idx].start;
5909 process_func:
5910 	/* protect against potential stack overflow that might happen when
5911 	 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack
5912 	 * depth for such case down to 256 so that the worst case scenario
5913 	 * would result in 8k stack size (32 which is tailcall limit * 256 =
5914 	 * 8k).
5915 	 *
5916 	 * To get the idea what might happen, see an example:
5917 	 * func1 -> sub rsp, 128
5918 	 *  subfunc1 -> sub rsp, 256
5919 	 *  tailcall1 -> add rsp, 256
5920 	 *   func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320)
5921 	 *   subfunc2 -> sub rsp, 64
5922 	 *   subfunc22 -> sub rsp, 128
5923 	 *   tailcall2 -> add rsp, 128
5924 	 *    func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416)
5925 	 *
5926 	 * tailcall will unwind the current stack frame but it will not get rid
5927 	 * of caller's stack as shown on the example above.
5928 	 */
5929 	if (idx && subprog[idx].has_tail_call && depth >= 256) {
5930 		verbose(env,
5931 			"tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n",
5932 			depth);
5933 		return -EACCES;
5934 	}
5935 	depth += round_up_stack_depth(env, subprog[idx].stack_depth);
5936 	if (depth > MAX_BPF_STACK) {
5937 		verbose(env, "combined stack size of %d calls is %d. Too large\n",
5938 			frame + 1, depth);
5939 		return -EACCES;
5940 	}
5941 continue_func:
5942 	subprog_end = subprog[idx + 1].start;
5943 	for (; i < subprog_end; i++) {
5944 		int next_insn, sidx;
5945 
5946 		if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) {
5947 			bool err = false;
5948 
5949 			if (!is_bpf_throw_kfunc(insn + i))
5950 				continue;
5951 			if (subprog[idx].is_cb)
5952 				err = true;
5953 			for (int c = 0; c < frame && !err; c++) {
5954 				if (subprog[ret_prog[c]].is_cb) {
5955 					err = true;
5956 					break;
5957 				}
5958 			}
5959 			if (!err)
5960 				continue;
5961 			verbose(env,
5962 				"bpf_throw kfunc (insn %d) cannot be called from callback subprog %d\n",
5963 				i, idx);
5964 			return -EINVAL;
5965 		}
5966 
5967 		if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i))
5968 			continue;
5969 		/* remember insn and function to return to */
5970 		ret_insn[frame] = i + 1;
5971 		ret_prog[frame] = idx;
5972 
5973 		/* find the callee */
5974 		next_insn = i + insn[i].imm + 1;
5975 		sidx = find_subprog(env, next_insn);
5976 		if (sidx < 0) {
5977 			WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
5978 				  next_insn);
5979 			return -EFAULT;
5980 		}
5981 		if (subprog[sidx].is_async_cb) {
5982 			if (subprog[sidx].has_tail_call) {
5983 				verbose(env, "verifier bug. subprog has tail_call and async cb\n");
5984 				return -EFAULT;
5985 			}
5986 			/* async callbacks don't increase bpf prog stack size unless called directly */
5987 			if (!bpf_pseudo_call(insn + i))
5988 				continue;
5989 			if (subprog[sidx].is_exception_cb) {
5990 				verbose(env, "insn %d cannot call exception cb directly\n", i);
5991 				return -EINVAL;
5992 			}
5993 		}
5994 		i = next_insn;
5995 		idx = sidx;
5996 
5997 		if (subprog[idx].has_tail_call)
5998 			tail_call_reachable = true;
5999 
6000 		frame++;
6001 		if (frame >= MAX_CALL_FRAMES) {
6002 			verbose(env, "the call stack of %d frames is too deep !\n",
6003 				frame);
6004 			return -E2BIG;
6005 		}
6006 		goto process_func;
6007 	}
6008 	/* if tail call got detected across bpf2bpf calls then mark each of the
6009 	 * currently present subprog frames as tail call reachable subprogs;
6010 	 * this info will be utilized by JIT so that we will be preserving the
6011 	 * tail call counter throughout bpf2bpf calls combined with tailcalls
6012 	 */
6013 	if (tail_call_reachable)
6014 		for (j = 0; j < frame; j++) {
6015 			if (subprog[ret_prog[j]].is_exception_cb) {
6016 				verbose(env, "cannot tail call within exception cb\n");
6017 				return -EINVAL;
6018 			}
6019 			subprog[ret_prog[j]].tail_call_reachable = true;
6020 		}
6021 	if (subprog[0].tail_call_reachable)
6022 		env->prog->aux->tail_call_reachable = true;
6023 
6024 	/* end of for() loop means the last insn of the 'subprog'
6025 	 * was reached. Doesn't matter whether it was JA or EXIT
6026 	 */
6027 	if (frame == 0)
6028 		return 0;
6029 	depth -= round_up_stack_depth(env, subprog[idx].stack_depth);
6030 	frame--;
6031 	i = ret_insn[frame];
6032 	idx = ret_prog[frame];
6033 	goto continue_func;
6034 }
6035 
6036 static int check_max_stack_depth(struct bpf_verifier_env *env)
6037 {
6038 	struct bpf_subprog_info *si = env->subprog_info;
6039 	int ret;
6040 
6041 	for (int i = 0; i < env->subprog_cnt; i++) {
6042 		if (!i || si[i].is_async_cb) {
6043 			ret = check_max_stack_depth_subprog(env, i);
6044 			if (ret < 0)
6045 				return ret;
6046 		}
6047 		continue;
6048 	}
6049 	return 0;
6050 }
6051 
6052 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
6053 static int get_callee_stack_depth(struct bpf_verifier_env *env,
6054 				  const struct bpf_insn *insn, int idx)
6055 {
6056 	int start = idx + insn->imm + 1, subprog;
6057 
6058 	subprog = find_subprog(env, start);
6059 	if (subprog < 0) {
6060 		WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
6061 			  start);
6062 		return -EFAULT;
6063 	}
6064 	return env->subprog_info[subprog].stack_depth;
6065 }
6066 #endif
6067 
6068 static int __check_buffer_access(struct bpf_verifier_env *env,
6069 				 const char *buf_info,
6070 				 const struct bpf_reg_state *reg,
6071 				 int regno, int off, int size)
6072 {
6073 	if (off < 0) {
6074 		verbose(env,
6075 			"R%d invalid %s buffer access: off=%d, size=%d\n",
6076 			regno, buf_info, off, size);
6077 		return -EACCES;
6078 	}
6079 	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
6080 		char tn_buf[48];
6081 
6082 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
6083 		verbose(env,
6084 			"R%d invalid variable buffer offset: off=%d, var_off=%s\n",
6085 			regno, off, tn_buf);
6086 		return -EACCES;
6087 	}
6088 
6089 	return 0;
6090 }
6091 
6092 static int check_tp_buffer_access(struct bpf_verifier_env *env,
6093 				  const struct bpf_reg_state *reg,
6094 				  int regno, int off, int size)
6095 {
6096 	int err;
6097 
6098 	err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);
6099 	if (err)
6100 		return err;
6101 
6102 	if (off + size > env->prog->aux->max_tp_access)
6103 		env->prog->aux->max_tp_access = off + size;
6104 
6105 	return 0;
6106 }
6107 
6108 static int check_buffer_access(struct bpf_verifier_env *env,
6109 			       const struct bpf_reg_state *reg,
6110 			       int regno, int off, int size,
6111 			       bool zero_size_allowed,
6112 			       u32 *max_access)
6113 {
6114 	const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr";
6115 	int err;
6116 
6117 	err = __check_buffer_access(env, buf_info, reg, regno, off, size);
6118 	if (err)
6119 		return err;
6120 
6121 	if (off + size > *max_access)
6122 		*max_access = off + size;
6123 
6124 	return 0;
6125 }
6126 
6127 /* BPF architecture zero extends alu32 ops into 64-bit registesr */
6128 static void zext_32_to_64(struct bpf_reg_state *reg)
6129 {
6130 	reg->var_off = tnum_subreg(reg->var_off);
6131 	__reg_assign_32_into_64(reg);
6132 }
6133 
6134 /* truncate register to smaller size (in bytes)
6135  * must be called with size < BPF_REG_SIZE
6136  */
6137 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
6138 {
6139 	u64 mask;
6140 
6141 	/* clear high bits in bit representation */
6142 	reg->var_off = tnum_cast(reg->var_off, size);
6143 
6144 	/* fix arithmetic bounds */
6145 	mask = ((u64)1 << (size * 8)) - 1;
6146 	if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
6147 		reg->umin_value &= mask;
6148 		reg->umax_value &= mask;
6149 	} else {
6150 		reg->umin_value = 0;
6151 		reg->umax_value = mask;
6152 	}
6153 	reg->smin_value = reg->umin_value;
6154 	reg->smax_value = reg->umax_value;
6155 
6156 	/* If size is smaller than 32bit register the 32bit register
6157 	 * values are also truncated so we push 64-bit bounds into
6158 	 * 32-bit bounds. Above were truncated < 32-bits already.
6159 	 */
6160 	if (size < 4)
6161 		__mark_reg32_unbounded(reg);
6162 
6163 	reg_bounds_sync(reg);
6164 }
6165 
6166 static void set_sext64_default_val(struct bpf_reg_state *reg, int size)
6167 {
6168 	if (size == 1) {
6169 		reg->smin_value = reg->s32_min_value = S8_MIN;
6170 		reg->smax_value = reg->s32_max_value = S8_MAX;
6171 	} else if (size == 2) {
6172 		reg->smin_value = reg->s32_min_value = S16_MIN;
6173 		reg->smax_value = reg->s32_max_value = S16_MAX;
6174 	} else {
6175 		/* size == 4 */
6176 		reg->smin_value = reg->s32_min_value = S32_MIN;
6177 		reg->smax_value = reg->s32_max_value = S32_MAX;
6178 	}
6179 	reg->umin_value = reg->u32_min_value = 0;
6180 	reg->umax_value = U64_MAX;
6181 	reg->u32_max_value = U32_MAX;
6182 	reg->var_off = tnum_unknown;
6183 }
6184 
6185 static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size)
6186 {
6187 	s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval;
6188 	u64 top_smax_value, top_smin_value;
6189 	u64 num_bits = size * 8;
6190 
6191 	if (tnum_is_const(reg->var_off)) {
6192 		u64_cval = reg->var_off.value;
6193 		if (size == 1)
6194 			reg->var_off = tnum_const((s8)u64_cval);
6195 		else if (size == 2)
6196 			reg->var_off = tnum_const((s16)u64_cval);
6197 		else
6198 			/* size == 4 */
6199 			reg->var_off = tnum_const((s32)u64_cval);
6200 
6201 		u64_cval = reg->var_off.value;
6202 		reg->smax_value = reg->smin_value = u64_cval;
6203 		reg->umax_value = reg->umin_value = u64_cval;
6204 		reg->s32_max_value = reg->s32_min_value = u64_cval;
6205 		reg->u32_max_value = reg->u32_min_value = u64_cval;
6206 		return;
6207 	}
6208 
6209 	top_smax_value = ((u64)reg->smax_value >> num_bits) << num_bits;
6210 	top_smin_value = ((u64)reg->smin_value >> num_bits) << num_bits;
6211 
6212 	if (top_smax_value != top_smin_value)
6213 		goto out;
6214 
6215 	/* find the s64_min and s64_min after sign extension */
6216 	if (size == 1) {
6217 		init_s64_max = (s8)reg->smax_value;
6218 		init_s64_min = (s8)reg->smin_value;
6219 	} else if (size == 2) {
6220 		init_s64_max = (s16)reg->smax_value;
6221 		init_s64_min = (s16)reg->smin_value;
6222 	} else {
6223 		init_s64_max = (s32)reg->smax_value;
6224 		init_s64_min = (s32)reg->smin_value;
6225 	}
6226 
6227 	s64_max = max(init_s64_max, init_s64_min);
6228 	s64_min = min(init_s64_max, init_s64_min);
6229 
6230 	/* both of s64_max/s64_min positive or negative */
6231 	if ((s64_max >= 0) == (s64_min >= 0)) {
6232 		reg->smin_value = reg->s32_min_value = s64_min;
6233 		reg->smax_value = reg->s32_max_value = s64_max;
6234 		reg->umin_value = reg->u32_min_value = s64_min;
6235 		reg->umax_value = reg->u32_max_value = s64_max;
6236 		reg->var_off = tnum_range(s64_min, s64_max);
6237 		return;
6238 	}
6239 
6240 out:
6241 	set_sext64_default_val(reg, size);
6242 }
6243 
6244 static void set_sext32_default_val(struct bpf_reg_state *reg, int size)
6245 {
6246 	if (size == 1) {
6247 		reg->s32_min_value = S8_MIN;
6248 		reg->s32_max_value = S8_MAX;
6249 	} else {
6250 		/* size == 2 */
6251 		reg->s32_min_value = S16_MIN;
6252 		reg->s32_max_value = S16_MAX;
6253 	}
6254 	reg->u32_min_value = 0;
6255 	reg->u32_max_value = U32_MAX;
6256 	reg->var_off = tnum_subreg(tnum_unknown);
6257 }
6258 
6259 static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size)
6260 {
6261 	s32 init_s32_max, init_s32_min, s32_max, s32_min, u32_val;
6262 	u32 top_smax_value, top_smin_value;
6263 	u32 num_bits = size * 8;
6264 
6265 	if (tnum_is_const(reg->var_off)) {
6266 		u32_val = reg->var_off.value;
6267 		if (size == 1)
6268 			reg->var_off = tnum_const((s8)u32_val);
6269 		else
6270 			reg->var_off = tnum_const((s16)u32_val);
6271 
6272 		u32_val = reg->var_off.value;
6273 		reg->s32_min_value = reg->s32_max_value = u32_val;
6274 		reg->u32_min_value = reg->u32_max_value = u32_val;
6275 		return;
6276 	}
6277 
6278 	top_smax_value = ((u32)reg->s32_max_value >> num_bits) << num_bits;
6279 	top_smin_value = ((u32)reg->s32_min_value >> num_bits) << num_bits;
6280 
6281 	if (top_smax_value != top_smin_value)
6282 		goto out;
6283 
6284 	/* find the s32_min and s32_min after sign extension */
6285 	if (size == 1) {
6286 		init_s32_max = (s8)reg->s32_max_value;
6287 		init_s32_min = (s8)reg->s32_min_value;
6288 	} else {
6289 		/* size == 2 */
6290 		init_s32_max = (s16)reg->s32_max_value;
6291 		init_s32_min = (s16)reg->s32_min_value;
6292 	}
6293 	s32_max = max(init_s32_max, init_s32_min);
6294 	s32_min = min(init_s32_max, init_s32_min);
6295 
6296 	if ((s32_min >= 0) == (s32_max >= 0)) {
6297 		reg->s32_min_value = s32_min;
6298 		reg->s32_max_value = s32_max;
6299 		reg->u32_min_value = (u32)s32_min;
6300 		reg->u32_max_value = (u32)s32_max;
6301 		reg->var_off = tnum_subreg(tnum_range(s32_min, s32_max));
6302 		return;
6303 	}
6304 
6305 out:
6306 	set_sext32_default_val(reg, size);
6307 }
6308 
6309 static bool bpf_map_is_rdonly(const struct bpf_map *map)
6310 {
6311 	/* A map is considered read-only if the following condition are true:
6312 	 *
6313 	 * 1) BPF program side cannot change any of the map content. The
6314 	 *    BPF_F_RDONLY_PROG flag is throughout the lifetime of a map
6315 	 *    and was set at map creation time.
6316 	 * 2) The map value(s) have been initialized from user space by a
6317 	 *    loader and then "frozen", such that no new map update/delete
6318 	 *    operations from syscall side are possible for the rest of
6319 	 *    the map's lifetime from that point onwards.
6320 	 * 3) Any parallel/pending map update/delete operations from syscall
6321 	 *    side have been completed. Only after that point, it's safe to
6322 	 *    assume that map value(s) are immutable.
6323 	 */
6324 	return (map->map_flags & BPF_F_RDONLY_PROG) &&
6325 	       READ_ONCE(map->frozen) &&
6326 	       !bpf_map_write_active(map);
6327 }
6328 
6329 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
6330 			       bool is_ldsx)
6331 {
6332 	void *ptr;
6333 	u64 addr;
6334 	int err;
6335 
6336 	err = map->ops->map_direct_value_addr(map, &addr, off);
6337 	if (err)
6338 		return err;
6339 	ptr = (void *)(long)addr + off;
6340 
6341 	switch (size) {
6342 	case sizeof(u8):
6343 		*val = is_ldsx ? (s64)*(s8 *)ptr : (u64)*(u8 *)ptr;
6344 		break;
6345 	case sizeof(u16):
6346 		*val = is_ldsx ? (s64)*(s16 *)ptr : (u64)*(u16 *)ptr;
6347 		break;
6348 	case sizeof(u32):
6349 		*val = is_ldsx ? (s64)*(s32 *)ptr : (u64)*(u32 *)ptr;
6350 		break;
6351 	case sizeof(u64):
6352 		*val = *(u64 *)ptr;
6353 		break;
6354 	default:
6355 		return -EINVAL;
6356 	}
6357 	return 0;
6358 }
6359 
6360 #define BTF_TYPE_SAFE_RCU(__type)  __PASTE(__type, __safe_rcu)
6361 #define BTF_TYPE_SAFE_RCU_OR_NULL(__type)  __PASTE(__type, __safe_rcu_or_null)
6362 #define BTF_TYPE_SAFE_TRUSTED(__type)  __PASTE(__type, __safe_trusted)
6363 #define BTF_TYPE_SAFE_TRUSTED_OR_NULL(__type)  __PASTE(__type, __safe_trusted_or_null)
6364 
6365 /*
6366  * Allow list few fields as RCU trusted or full trusted.
6367  * This logic doesn't allow mix tagging and will be removed once GCC supports
6368  * btf_type_tag.
6369  */
6370 
6371 /* RCU trusted: these fields are trusted in RCU CS and never NULL */
6372 BTF_TYPE_SAFE_RCU(struct task_struct) {
6373 	const cpumask_t *cpus_ptr;
6374 	struct css_set __rcu *cgroups;
6375 	struct task_struct __rcu *real_parent;
6376 	struct task_struct *group_leader;
6377 };
6378 
6379 BTF_TYPE_SAFE_RCU(struct cgroup) {
6380 	/* cgrp->kn is always accessible as documented in kernel/cgroup/cgroup.c */
6381 	struct kernfs_node *kn;
6382 };
6383 
6384 BTF_TYPE_SAFE_RCU(struct css_set) {
6385 	struct cgroup *dfl_cgrp;
6386 };
6387 
6388 /* RCU trusted: these fields are trusted in RCU CS and can be NULL */
6389 BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct) {
6390 	struct file __rcu *exe_file;
6391 };
6392 
6393 /* skb->sk, req->sk are not RCU protected, but we mark them as such
6394  * because bpf prog accessible sockets are SOCK_RCU_FREE.
6395  */
6396 BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff) {
6397 	struct sock *sk;
6398 };
6399 
6400 BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock) {
6401 	struct sock *sk;
6402 };
6403 
6404 /* full trusted: these fields are trusted even outside of RCU CS and never NULL */
6405 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta) {
6406 	struct seq_file *seq;
6407 };
6408 
6409 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task) {
6410 	struct bpf_iter_meta *meta;
6411 	struct task_struct *task;
6412 };
6413 
6414 BTF_TYPE_SAFE_TRUSTED(struct linux_binprm) {
6415 	struct file *file;
6416 };
6417 
6418 BTF_TYPE_SAFE_TRUSTED(struct file) {
6419 	struct inode *f_inode;
6420 };
6421 
6422 BTF_TYPE_SAFE_TRUSTED(struct dentry) {
6423 	/* no negative dentry-s in places where bpf can see it */
6424 	struct inode *d_inode;
6425 };
6426 
6427 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) {
6428 	struct sock *sk;
6429 };
6430 
6431 static bool type_is_rcu(struct bpf_verifier_env *env,
6432 			struct bpf_reg_state *reg,
6433 			const char *field_name, u32 btf_id)
6434 {
6435 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct task_struct));
6436 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup));
6437 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct css_set));
6438 
6439 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu");
6440 }
6441 
6442 static bool type_is_rcu_or_null(struct bpf_verifier_env *env,
6443 				struct bpf_reg_state *reg,
6444 				const char *field_name, u32 btf_id)
6445 {
6446 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct));
6447 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff));
6448 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock));
6449 
6450 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null");
6451 }
6452 
6453 static bool type_is_trusted(struct bpf_verifier_env *env,
6454 			    struct bpf_reg_state *reg,
6455 			    const char *field_name, u32 btf_id)
6456 {
6457 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta));
6458 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task));
6459 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm));
6460 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file));
6461 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct dentry));
6462 
6463 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted");
6464 }
6465 
6466 static bool type_is_trusted_or_null(struct bpf_verifier_env *env,
6467 				    struct bpf_reg_state *reg,
6468 				    const char *field_name, u32 btf_id)
6469 {
6470 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket));
6471 
6472 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id,
6473 					  "__safe_trusted_or_null");
6474 }
6475 
6476 static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
6477 				   struct bpf_reg_state *regs,
6478 				   int regno, int off, int size,
6479 				   enum bpf_access_type atype,
6480 				   int value_regno)
6481 {
6482 	struct bpf_reg_state *reg = regs + regno;
6483 	const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
6484 	const char *tname = btf_name_by_offset(reg->btf, t->name_off);
6485 	const char *field_name = NULL;
6486 	enum bpf_type_flag flag = 0;
6487 	u32 btf_id = 0;
6488 	int ret;
6489 
6490 	if (!env->allow_ptr_leaks) {
6491 		verbose(env,
6492 			"'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
6493 			tname);
6494 		return -EPERM;
6495 	}
6496 	if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) {
6497 		verbose(env,
6498 			"Cannot access kernel 'struct %s' from non-GPL compatible program\n",
6499 			tname);
6500 		return -EINVAL;
6501 	}
6502 	if (off < 0) {
6503 		verbose(env,
6504 			"R%d is ptr_%s invalid negative access: off=%d\n",
6505 			regno, tname, off);
6506 		return -EACCES;
6507 	}
6508 	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
6509 		char tn_buf[48];
6510 
6511 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
6512 		verbose(env,
6513 			"R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n",
6514 			regno, tname, off, tn_buf);
6515 		return -EACCES;
6516 	}
6517 
6518 	if (reg->type & MEM_USER) {
6519 		verbose(env,
6520 			"R%d is ptr_%s access user memory: off=%d\n",
6521 			regno, tname, off);
6522 		return -EACCES;
6523 	}
6524 
6525 	if (reg->type & MEM_PERCPU) {
6526 		verbose(env,
6527 			"R%d is ptr_%s access percpu memory: off=%d\n",
6528 			regno, tname, off);
6529 		return -EACCES;
6530 	}
6531 
6532 	if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) {
6533 		if (!btf_is_kernel(reg->btf)) {
6534 			verbose(env, "verifier internal error: reg->btf must be kernel btf\n");
6535 			return -EFAULT;
6536 		}
6537 		ret = env->ops->btf_struct_access(&env->log, reg, off, size);
6538 	} else {
6539 		/* Writes are permitted with default btf_struct_access for
6540 		 * program allocated objects (which always have ref_obj_id > 0),
6541 		 * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC.
6542 		 */
6543 		if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) {
6544 			verbose(env, "only read is supported\n");
6545 			return -EACCES;
6546 		}
6547 
6548 		if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) &&
6549 		    !(reg->type & MEM_RCU) && !reg->ref_obj_id) {
6550 			verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n");
6551 			return -EFAULT;
6552 		}
6553 
6554 		ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name);
6555 	}
6556 
6557 	if (ret < 0)
6558 		return ret;
6559 
6560 	if (ret != PTR_TO_BTF_ID) {
6561 		/* just mark; */
6562 
6563 	} else if (type_flag(reg->type) & PTR_UNTRUSTED) {
6564 		/* If this is an untrusted pointer, all pointers formed by walking it
6565 		 * also inherit the untrusted flag.
6566 		 */
6567 		flag = PTR_UNTRUSTED;
6568 
6569 	} else if (is_trusted_reg(reg) || is_rcu_reg(reg)) {
6570 		/* By default any pointer obtained from walking a trusted pointer is no
6571 		 * longer trusted, unless the field being accessed has explicitly been
6572 		 * marked as inheriting its parent's state of trust (either full or RCU).
6573 		 * For example:
6574 		 * 'cgroups' pointer is untrusted if task->cgroups dereference
6575 		 * happened in a sleepable program outside of bpf_rcu_read_lock()
6576 		 * section. In a non-sleepable program it's trusted while in RCU CS (aka MEM_RCU).
6577 		 * Note bpf_rcu_read_unlock() converts MEM_RCU pointers to PTR_UNTRUSTED.
6578 		 *
6579 		 * A regular RCU-protected pointer with __rcu tag can also be deemed
6580 		 * trusted if we are in an RCU CS. Such pointer can be NULL.
6581 		 */
6582 		if (type_is_trusted(env, reg, field_name, btf_id)) {
6583 			flag |= PTR_TRUSTED;
6584 		} else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) {
6585 			flag |= PTR_TRUSTED | PTR_MAYBE_NULL;
6586 		} else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) {
6587 			if (type_is_rcu(env, reg, field_name, btf_id)) {
6588 				/* ignore __rcu tag and mark it MEM_RCU */
6589 				flag |= MEM_RCU;
6590 			} else if (flag & MEM_RCU ||
6591 				   type_is_rcu_or_null(env, reg, field_name, btf_id)) {
6592 				/* __rcu tagged pointers can be NULL */
6593 				flag |= MEM_RCU | PTR_MAYBE_NULL;
6594 
6595 				/* We always trust them */
6596 				if (type_is_rcu_or_null(env, reg, field_name, btf_id) &&
6597 				    flag & PTR_UNTRUSTED)
6598 					flag &= ~PTR_UNTRUSTED;
6599 			} else if (flag & (MEM_PERCPU | MEM_USER)) {
6600 				/* keep as-is */
6601 			} else {
6602 				/* walking unknown pointers yields old deprecated PTR_TO_BTF_ID */
6603 				clear_trusted_flags(&flag);
6604 			}
6605 		} else {
6606 			/*
6607 			 * If not in RCU CS or MEM_RCU pointer can be NULL then
6608 			 * aggressively mark as untrusted otherwise such
6609 			 * pointers will be plain PTR_TO_BTF_ID without flags
6610 			 * and will be allowed to be passed into helpers for
6611 			 * compat reasons.
6612 			 */
6613 			flag = PTR_UNTRUSTED;
6614 		}
6615 	} else {
6616 		/* Old compat. Deprecated */
6617 		clear_trusted_flags(&flag);
6618 	}
6619 
6620 	if (atype == BPF_READ && value_regno >= 0)
6621 		mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);
6622 
6623 	return 0;
6624 }
6625 
6626 static int check_ptr_to_map_access(struct bpf_verifier_env *env,
6627 				   struct bpf_reg_state *regs,
6628 				   int regno, int off, int size,
6629 				   enum bpf_access_type atype,
6630 				   int value_regno)
6631 {
6632 	struct bpf_reg_state *reg = regs + regno;
6633 	struct bpf_map *map = reg->map_ptr;
6634 	struct bpf_reg_state map_reg;
6635 	enum bpf_type_flag flag = 0;
6636 	const struct btf_type *t;
6637 	const char *tname;
6638 	u32 btf_id;
6639 	int ret;
6640 
6641 	if (!btf_vmlinux) {
6642 		verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");
6643 		return -ENOTSUPP;
6644 	}
6645 
6646 	if (!map->ops->map_btf_id || !*map->ops->map_btf_id) {
6647 		verbose(env, "map_ptr access not supported for map type %d\n",
6648 			map->map_type);
6649 		return -ENOTSUPP;
6650 	}
6651 
6652 	t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id);
6653 	tname = btf_name_by_offset(btf_vmlinux, t->name_off);
6654 
6655 	if (!env->allow_ptr_leaks) {
6656 		verbose(env,
6657 			"'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
6658 			tname);
6659 		return -EPERM;
6660 	}
6661 
6662 	if (off < 0) {
6663 		verbose(env, "R%d is %s invalid negative access: off=%d\n",
6664 			regno, tname, off);
6665 		return -EACCES;
6666 	}
6667 
6668 	if (atype != BPF_READ) {
6669 		verbose(env, "only read from %s is supported\n", tname);
6670 		return -EACCES;
6671 	}
6672 
6673 	/* Simulate access to a PTR_TO_BTF_ID */
6674 	memset(&map_reg, 0, sizeof(map_reg));
6675 	mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0);
6676 	ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL);
6677 	if (ret < 0)
6678 		return ret;
6679 
6680 	if (value_regno >= 0)
6681 		mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag);
6682 
6683 	return 0;
6684 }
6685 
6686 /* Check that the stack access at the given offset is within bounds. The
6687  * maximum valid offset is -1.
6688  *
6689  * The minimum valid offset is -MAX_BPF_STACK for writes, and
6690  * -state->allocated_stack for reads.
6691  */
6692 static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,
6693                                           s64 off,
6694                                           struct bpf_func_state *state,
6695                                           enum bpf_access_type t)
6696 {
6697 	int min_valid_off;
6698 
6699 	if (t == BPF_WRITE || env->allow_uninit_stack)
6700 		min_valid_off = -MAX_BPF_STACK;
6701 	else
6702 		min_valid_off = -state->allocated_stack;
6703 
6704 	if (off < min_valid_off || off > -1)
6705 		return -EACCES;
6706 	return 0;
6707 }
6708 
6709 /* Check that the stack access at 'regno + off' falls within the maximum stack
6710  * bounds.
6711  *
6712  * 'off' includes `regno->offset`, but not its dynamic part (if any).
6713  */
6714 static int check_stack_access_within_bounds(
6715 		struct bpf_verifier_env *env,
6716 		int regno, int off, int access_size,
6717 		enum bpf_access_src src, enum bpf_access_type type)
6718 {
6719 	struct bpf_reg_state *regs = cur_regs(env);
6720 	struct bpf_reg_state *reg = regs + regno;
6721 	struct bpf_func_state *state = func(env, reg);
6722 	s64 min_off, max_off;
6723 	int err;
6724 	char *err_extra;
6725 
6726 	if (src == ACCESS_HELPER)
6727 		/* We don't know if helpers are reading or writing (or both). */
6728 		err_extra = " indirect access to";
6729 	else if (type == BPF_READ)
6730 		err_extra = " read from";
6731 	else
6732 		err_extra = " write to";
6733 
6734 	if (tnum_is_const(reg->var_off)) {
6735 		min_off = (s64)reg->var_off.value + off;
6736 		max_off = min_off + access_size;
6737 	} else {
6738 		if (reg->smax_value >= BPF_MAX_VAR_OFF ||
6739 		    reg->smin_value <= -BPF_MAX_VAR_OFF) {
6740 			verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
6741 				err_extra, regno);
6742 			return -EACCES;
6743 		}
6744 		min_off = reg->smin_value + off;
6745 		max_off = reg->smax_value + off + access_size;
6746 	}
6747 
6748 	err = check_stack_slot_within_bounds(env, min_off, state, type);
6749 	if (!err && max_off > 0)
6750 		err = -EINVAL; /* out of stack access into non-negative offsets */
6751 	if (!err && access_size < 0)
6752 		/* access_size should not be negative (or overflow an int); others checks
6753 		 * along the way should have prevented such an access.
6754 		 */
6755 		err = -EFAULT; /* invalid negative access size; integer overflow? */
6756 
6757 	if (err) {
6758 		if (tnum_is_const(reg->var_off)) {
6759 			verbose(env, "invalid%s stack R%d off=%d size=%d\n",
6760 				err_extra, regno, off, access_size);
6761 		} else {
6762 			char tn_buf[48];
6763 
6764 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
6765 			verbose(env, "invalid variable-offset%s stack R%d var_off=%s off=%d size=%d\n",
6766 				err_extra, regno, tn_buf, off, access_size);
6767 		}
6768 		return err;
6769 	}
6770 
6771 	/* Note that there is no stack access with offset zero, so the needed stack
6772 	 * size is -min_off, not -min_off+1.
6773 	 */
6774 	return grow_stack_state(env, state, -min_off /* size */);
6775 }
6776 
6777 /* check whether memory at (regno + off) is accessible for t = (read | write)
6778  * if t==write, value_regno is a register which value is stored into memory
6779  * if t==read, value_regno is a register which will receive the value from memory
6780  * if t==write && value_regno==-1, some unknown value is stored into memory
6781  * if t==read && value_regno==-1, don't care what we read from memory
6782  */
6783 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
6784 			    int off, int bpf_size, enum bpf_access_type t,
6785 			    int value_regno, bool strict_alignment_once, bool is_ldsx)
6786 {
6787 	struct bpf_reg_state *regs = cur_regs(env);
6788 	struct bpf_reg_state *reg = regs + regno;
6789 	int size, err = 0;
6790 
6791 	size = bpf_size_to_bytes(bpf_size);
6792 	if (size < 0)
6793 		return size;
6794 
6795 	/* alignment checks will add in reg->off themselves */
6796 	err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
6797 	if (err)
6798 		return err;
6799 
6800 	/* for access checks, reg->off is just part of off */
6801 	off += reg->off;
6802 
6803 	if (reg->type == PTR_TO_MAP_KEY) {
6804 		if (t == BPF_WRITE) {
6805 			verbose(env, "write to change key R%d not allowed\n", regno);
6806 			return -EACCES;
6807 		}
6808 
6809 		err = check_mem_region_access(env, regno, off, size,
6810 					      reg->map_ptr->key_size, false);
6811 		if (err)
6812 			return err;
6813 		if (value_regno >= 0)
6814 			mark_reg_unknown(env, regs, value_regno);
6815 	} else if (reg->type == PTR_TO_MAP_VALUE) {
6816 		struct btf_field *kptr_field = NULL;
6817 
6818 		if (t == BPF_WRITE && value_regno >= 0 &&
6819 		    is_pointer_value(env, value_regno)) {
6820 			verbose(env, "R%d leaks addr into map\n", value_regno);
6821 			return -EACCES;
6822 		}
6823 		err = check_map_access_type(env, regno, off, size, t);
6824 		if (err)
6825 			return err;
6826 		err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT);
6827 		if (err)
6828 			return err;
6829 		if (tnum_is_const(reg->var_off))
6830 			kptr_field = btf_record_find(reg->map_ptr->record,
6831 						     off + reg->var_off.value, BPF_KPTR);
6832 		if (kptr_field) {
6833 			err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field);
6834 		} else if (t == BPF_READ && value_regno >= 0) {
6835 			struct bpf_map *map = reg->map_ptr;
6836 
6837 			/* if map is read-only, track its contents as scalars */
6838 			if (tnum_is_const(reg->var_off) &&
6839 			    bpf_map_is_rdonly(map) &&
6840 			    map->ops->map_direct_value_addr) {
6841 				int map_off = off + reg->var_off.value;
6842 				u64 val = 0;
6843 
6844 				err = bpf_map_direct_read(map, map_off, size,
6845 							  &val, is_ldsx);
6846 				if (err)
6847 					return err;
6848 
6849 				regs[value_regno].type = SCALAR_VALUE;
6850 				__mark_reg_known(&regs[value_regno], val);
6851 			} else {
6852 				mark_reg_unknown(env, regs, value_regno);
6853 			}
6854 		}
6855 	} else if (base_type(reg->type) == PTR_TO_MEM) {
6856 		bool rdonly_mem = type_is_rdonly_mem(reg->type);
6857 
6858 		if (type_may_be_null(reg->type)) {
6859 			verbose(env, "R%d invalid mem access '%s'\n", regno,
6860 				reg_type_str(env, reg->type));
6861 			return -EACCES;
6862 		}
6863 
6864 		if (t == BPF_WRITE && rdonly_mem) {
6865 			verbose(env, "R%d cannot write into %s\n",
6866 				regno, reg_type_str(env, reg->type));
6867 			return -EACCES;
6868 		}
6869 
6870 		if (t == BPF_WRITE && value_regno >= 0 &&
6871 		    is_pointer_value(env, value_regno)) {
6872 			verbose(env, "R%d leaks addr into mem\n", value_regno);
6873 			return -EACCES;
6874 		}
6875 
6876 		err = check_mem_region_access(env, regno, off, size,
6877 					      reg->mem_size, false);
6878 		if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem))
6879 			mark_reg_unknown(env, regs, value_regno);
6880 	} else if (reg->type == PTR_TO_CTX) {
6881 		enum bpf_reg_type reg_type = SCALAR_VALUE;
6882 		struct btf *btf = NULL;
6883 		u32 btf_id = 0;
6884 
6885 		if (t == BPF_WRITE && value_regno >= 0 &&
6886 		    is_pointer_value(env, value_regno)) {
6887 			verbose(env, "R%d leaks addr into ctx\n", value_regno);
6888 			return -EACCES;
6889 		}
6890 
6891 		err = check_ptr_off_reg(env, reg, regno);
6892 		if (err < 0)
6893 			return err;
6894 
6895 		err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf,
6896 				       &btf_id);
6897 		if (err)
6898 			verbose_linfo(env, insn_idx, "; ");
6899 		if (!err && t == BPF_READ && value_regno >= 0) {
6900 			/* ctx access returns either a scalar, or a
6901 			 * PTR_TO_PACKET[_META,_END]. In the latter
6902 			 * case, we know the offset is zero.
6903 			 */
6904 			if (reg_type == SCALAR_VALUE) {
6905 				mark_reg_unknown(env, regs, value_regno);
6906 			} else {
6907 				mark_reg_known_zero(env, regs,
6908 						    value_regno);
6909 				if (type_may_be_null(reg_type))
6910 					regs[value_regno].id = ++env->id_gen;
6911 				/* A load of ctx field could have different
6912 				 * actual load size with the one encoded in the
6913 				 * insn. When the dst is PTR, it is for sure not
6914 				 * a sub-register.
6915 				 */
6916 				regs[value_regno].subreg_def = DEF_NOT_SUBREG;
6917 				if (base_type(reg_type) == PTR_TO_BTF_ID) {
6918 					regs[value_regno].btf = btf;
6919 					regs[value_regno].btf_id = btf_id;
6920 				}
6921 			}
6922 			regs[value_regno].type = reg_type;
6923 		}
6924 
6925 	} else if (reg->type == PTR_TO_STACK) {
6926 		/* Basic bounds checks. */
6927 		err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);
6928 		if (err)
6929 			return err;
6930 
6931 		if (t == BPF_READ)
6932 			err = check_stack_read(env, regno, off, size,
6933 					       value_regno);
6934 		else
6935 			err = check_stack_write(env, regno, off, size,
6936 						value_regno, insn_idx);
6937 	} else if (reg_is_pkt_pointer(reg)) {
6938 		if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
6939 			verbose(env, "cannot write into packet\n");
6940 			return -EACCES;
6941 		}
6942 		if (t == BPF_WRITE && value_regno >= 0 &&
6943 		    is_pointer_value(env, value_regno)) {
6944 			verbose(env, "R%d leaks addr into packet\n",
6945 				value_regno);
6946 			return -EACCES;
6947 		}
6948 		err = check_packet_access(env, regno, off, size, false);
6949 		if (!err && t == BPF_READ && value_regno >= 0)
6950 			mark_reg_unknown(env, regs, value_regno);
6951 	} else if (reg->type == PTR_TO_FLOW_KEYS) {
6952 		if (t == BPF_WRITE && value_regno >= 0 &&
6953 		    is_pointer_value(env, value_regno)) {
6954 			verbose(env, "R%d leaks addr into flow keys\n",
6955 				value_regno);
6956 			return -EACCES;
6957 		}
6958 
6959 		err = check_flow_keys_access(env, off, size);
6960 		if (!err && t == BPF_READ && value_regno >= 0)
6961 			mark_reg_unknown(env, regs, value_regno);
6962 	} else if (type_is_sk_pointer(reg->type)) {
6963 		if (t == BPF_WRITE) {
6964 			verbose(env, "R%d cannot write into %s\n",
6965 				regno, reg_type_str(env, reg->type));
6966 			return -EACCES;
6967 		}
6968 		err = check_sock_access(env, insn_idx, regno, off, size, t);
6969 		if (!err && value_regno >= 0)
6970 			mark_reg_unknown(env, regs, value_regno);
6971 	} else if (reg->type == PTR_TO_TP_BUFFER) {
6972 		err = check_tp_buffer_access(env, reg, regno, off, size);
6973 		if (!err && t == BPF_READ && value_regno >= 0)
6974 			mark_reg_unknown(env, regs, value_regno);
6975 	} else if (base_type(reg->type) == PTR_TO_BTF_ID &&
6976 		   !type_may_be_null(reg->type)) {
6977 		err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
6978 					      value_regno);
6979 	} else if (reg->type == CONST_PTR_TO_MAP) {
6980 		err = check_ptr_to_map_access(env, regs, regno, off, size, t,
6981 					      value_regno);
6982 	} else if (base_type(reg->type) == PTR_TO_BUF) {
6983 		bool rdonly_mem = type_is_rdonly_mem(reg->type);
6984 		u32 *max_access;
6985 
6986 		if (rdonly_mem) {
6987 			if (t == BPF_WRITE) {
6988 				verbose(env, "R%d cannot write into %s\n",
6989 					regno, reg_type_str(env, reg->type));
6990 				return -EACCES;
6991 			}
6992 			max_access = &env->prog->aux->max_rdonly_access;
6993 		} else {
6994 			max_access = &env->prog->aux->max_rdwr_access;
6995 		}
6996 
6997 		err = check_buffer_access(env, reg, regno, off, size, false,
6998 					  max_access);
6999 
7000 		if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ))
7001 			mark_reg_unknown(env, regs, value_regno);
7002 	} else if (reg->type == PTR_TO_ARENA) {
7003 		if (t == BPF_READ && value_regno >= 0)
7004 			mark_reg_unknown(env, regs, value_regno);
7005 	} else {
7006 		verbose(env, "R%d invalid mem access '%s'\n", regno,
7007 			reg_type_str(env, reg->type));
7008 		return -EACCES;
7009 	}
7010 
7011 	if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
7012 	    regs[value_regno].type == SCALAR_VALUE) {
7013 		if (!is_ldsx)
7014 			/* b/h/w load zero-extends, mark upper bits as known 0 */
7015 			coerce_reg_to_size(&regs[value_regno], size);
7016 		else
7017 			coerce_reg_to_size_sx(&regs[value_regno], size);
7018 	}
7019 	return err;
7020 }
7021 
7022 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
7023 			     bool allow_trust_mismatch);
7024 
7025 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
7026 {
7027 	int load_reg;
7028 	int err;
7029 
7030 	switch (insn->imm) {
7031 	case BPF_ADD:
7032 	case BPF_ADD | BPF_FETCH:
7033 	case BPF_AND:
7034 	case BPF_AND | BPF_FETCH:
7035 	case BPF_OR:
7036 	case BPF_OR | BPF_FETCH:
7037 	case BPF_XOR:
7038 	case BPF_XOR | BPF_FETCH:
7039 	case BPF_XCHG:
7040 	case BPF_CMPXCHG:
7041 		break;
7042 	default:
7043 		verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm);
7044 		return -EINVAL;
7045 	}
7046 
7047 	if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) {
7048 		verbose(env, "invalid atomic operand size\n");
7049 		return -EINVAL;
7050 	}
7051 
7052 	/* check src1 operand */
7053 	err = check_reg_arg(env, insn->src_reg, SRC_OP);
7054 	if (err)
7055 		return err;
7056 
7057 	/* check src2 operand */
7058 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
7059 	if (err)
7060 		return err;
7061 
7062 	if (insn->imm == BPF_CMPXCHG) {
7063 		/* Check comparison of R0 with memory location */
7064 		const u32 aux_reg = BPF_REG_0;
7065 
7066 		err = check_reg_arg(env, aux_reg, SRC_OP);
7067 		if (err)
7068 			return err;
7069 
7070 		if (is_pointer_value(env, aux_reg)) {
7071 			verbose(env, "R%d leaks addr into mem\n", aux_reg);
7072 			return -EACCES;
7073 		}
7074 	}
7075 
7076 	if (is_pointer_value(env, insn->src_reg)) {
7077 		verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
7078 		return -EACCES;
7079 	}
7080 
7081 	if (is_ctx_reg(env, insn->dst_reg) ||
7082 	    is_pkt_reg(env, insn->dst_reg) ||
7083 	    is_flow_key_reg(env, insn->dst_reg) ||
7084 	    is_sk_reg(env, insn->dst_reg) ||
7085 	    (is_arena_reg(env, insn->dst_reg) && !bpf_jit_supports_insn(insn, true))) {
7086 		verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
7087 			insn->dst_reg,
7088 			reg_type_str(env, reg_state(env, insn->dst_reg)->type));
7089 		return -EACCES;
7090 	}
7091 
7092 	if (insn->imm & BPF_FETCH) {
7093 		if (insn->imm == BPF_CMPXCHG)
7094 			load_reg = BPF_REG_0;
7095 		else
7096 			load_reg = insn->src_reg;
7097 
7098 		/* check and record load of old value */
7099 		err = check_reg_arg(env, load_reg, DST_OP);
7100 		if (err)
7101 			return err;
7102 	} else {
7103 		/* This instruction accesses a memory location but doesn't
7104 		 * actually load it into a register.
7105 		 */
7106 		load_reg = -1;
7107 	}
7108 
7109 	/* Check whether we can read the memory, with second call for fetch
7110 	 * case to simulate the register fill.
7111 	 */
7112 	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
7113 			       BPF_SIZE(insn->code), BPF_READ, -1, true, false);
7114 	if (!err && load_reg >= 0)
7115 		err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
7116 				       BPF_SIZE(insn->code), BPF_READ, load_reg,
7117 				       true, false);
7118 	if (err)
7119 		return err;
7120 
7121 	if (is_arena_reg(env, insn->dst_reg)) {
7122 		err = save_aux_ptr_type(env, PTR_TO_ARENA, false);
7123 		if (err)
7124 			return err;
7125 	}
7126 	/* Check whether we can write into the same memory. */
7127 	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
7128 			       BPF_SIZE(insn->code), BPF_WRITE, -1, true, false);
7129 	if (err)
7130 		return err;
7131 	return 0;
7132 }
7133 
7134 /* When register 'regno' is used to read the stack (either directly or through
7135  * a helper function) make sure that it's within stack boundary and, depending
7136  * on the access type and privileges, that all elements of the stack are
7137  * initialized.
7138  *
7139  * 'off' includes 'regno->off', but not its dynamic part (if any).
7140  *
7141  * All registers that have been spilled on the stack in the slots within the
7142  * read offsets are marked as read.
7143  */
7144 static int check_stack_range_initialized(
7145 		struct bpf_verifier_env *env, int regno, int off,
7146 		int access_size, bool zero_size_allowed,
7147 		enum bpf_access_src type, struct bpf_call_arg_meta *meta)
7148 {
7149 	struct bpf_reg_state *reg = reg_state(env, regno);
7150 	struct bpf_func_state *state = func(env, reg);
7151 	int err, min_off, max_off, i, j, slot, spi;
7152 	char *err_extra = type == ACCESS_HELPER ? " indirect" : "";
7153 	enum bpf_access_type bounds_check_type;
7154 	/* Some accesses can write anything into the stack, others are
7155 	 * read-only.
7156 	 */
7157 	bool clobber = false;
7158 
7159 	if (access_size == 0 && !zero_size_allowed) {
7160 		verbose(env, "invalid zero-sized read\n");
7161 		return -EACCES;
7162 	}
7163 
7164 	if (type == ACCESS_HELPER) {
7165 		/* The bounds checks for writes are more permissive than for
7166 		 * reads. However, if raw_mode is not set, we'll do extra
7167 		 * checks below.
7168 		 */
7169 		bounds_check_type = BPF_WRITE;
7170 		clobber = true;
7171 	} else {
7172 		bounds_check_type = BPF_READ;
7173 	}
7174 	err = check_stack_access_within_bounds(env, regno, off, access_size,
7175 					       type, bounds_check_type);
7176 	if (err)
7177 		return err;
7178 
7179 
7180 	if (tnum_is_const(reg->var_off)) {
7181 		min_off = max_off = reg->var_off.value + off;
7182 	} else {
7183 		/* Variable offset is prohibited for unprivileged mode for
7184 		 * simplicity since it requires corresponding support in
7185 		 * Spectre masking for stack ALU.
7186 		 * See also retrieve_ptr_limit().
7187 		 */
7188 		if (!env->bypass_spec_v1) {
7189 			char tn_buf[48];
7190 
7191 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
7192 			verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n",
7193 				regno, err_extra, tn_buf);
7194 			return -EACCES;
7195 		}
7196 		/* Only initialized buffer on stack is allowed to be accessed
7197 		 * with variable offset. With uninitialized buffer it's hard to
7198 		 * guarantee that whole memory is marked as initialized on
7199 		 * helper return since specific bounds are unknown what may
7200 		 * cause uninitialized stack leaking.
7201 		 */
7202 		if (meta && meta->raw_mode)
7203 			meta = NULL;
7204 
7205 		min_off = reg->smin_value + off;
7206 		max_off = reg->smax_value + off;
7207 	}
7208 
7209 	if (meta && meta->raw_mode) {
7210 		/* Ensure we won't be overwriting dynptrs when simulating byte
7211 		 * by byte access in check_helper_call using meta.access_size.
7212 		 * This would be a problem if we have a helper in the future
7213 		 * which takes:
7214 		 *
7215 		 *	helper(uninit_mem, len, dynptr)
7216 		 *
7217 		 * Now, uninint_mem may overlap with dynptr pointer. Hence, it
7218 		 * may end up writing to dynptr itself when touching memory from
7219 		 * arg 1. This can be relaxed on a case by case basis for known
7220 		 * safe cases, but reject due to the possibilitiy of aliasing by
7221 		 * default.
7222 		 */
7223 		for (i = min_off; i < max_off + access_size; i++) {
7224 			int stack_off = -i - 1;
7225 
7226 			spi = __get_spi(i);
7227 			/* raw_mode may write past allocated_stack */
7228 			if (state->allocated_stack <= stack_off)
7229 				continue;
7230 			if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) {
7231 				verbose(env, "potential write to dynptr at off=%d disallowed\n", i);
7232 				return -EACCES;
7233 			}
7234 		}
7235 		meta->access_size = access_size;
7236 		meta->regno = regno;
7237 		return 0;
7238 	}
7239 
7240 	for (i = min_off; i < max_off + access_size; i++) {
7241 		u8 *stype;
7242 
7243 		slot = -i - 1;
7244 		spi = slot / BPF_REG_SIZE;
7245 		if (state->allocated_stack <= slot) {
7246 			verbose(env, "verifier bug: allocated_stack too small");
7247 			return -EFAULT;
7248 		}
7249 
7250 		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
7251 		if (*stype == STACK_MISC)
7252 			goto mark;
7253 		if ((*stype == STACK_ZERO) ||
7254 		    (*stype == STACK_INVALID && env->allow_uninit_stack)) {
7255 			if (clobber) {
7256 				/* helper can write anything into the stack */
7257 				*stype = STACK_MISC;
7258 			}
7259 			goto mark;
7260 		}
7261 
7262 		if (is_spilled_reg(&state->stack[spi]) &&
7263 		    (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
7264 		     env->allow_ptr_leaks)) {
7265 			if (clobber) {
7266 				__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
7267 				for (j = 0; j < BPF_REG_SIZE; j++)
7268 					scrub_spilled_slot(&state->stack[spi].slot_type[j]);
7269 			}
7270 			goto mark;
7271 		}
7272 
7273 		if (tnum_is_const(reg->var_off)) {
7274 			verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n",
7275 				err_extra, regno, min_off, i - min_off, access_size);
7276 		} else {
7277 			char tn_buf[48];
7278 
7279 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
7280 			verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n",
7281 				err_extra, regno, tn_buf, i - min_off, access_size);
7282 		}
7283 		return -EACCES;
7284 mark:
7285 		/* reading any byte out of 8-byte 'spill_slot' will cause
7286 		 * the whole slot to be marked as 'read'
7287 		 */
7288 		mark_reg_read(env, &state->stack[spi].spilled_ptr,
7289 			      state->stack[spi].spilled_ptr.parent,
7290 			      REG_LIVE_READ64);
7291 		/* We do not set REG_LIVE_WRITTEN for stack slot, as we can not
7292 		 * be sure that whether stack slot is written to or not. Hence,
7293 		 * we must still conservatively propagate reads upwards even if
7294 		 * helper may write to the entire memory range.
7295 		 */
7296 	}
7297 	return 0;
7298 }
7299 
7300 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
7301 				   int access_size, bool zero_size_allowed,
7302 				   struct bpf_call_arg_meta *meta)
7303 {
7304 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7305 	u32 *max_access;
7306 
7307 	switch (base_type(reg->type)) {
7308 	case PTR_TO_PACKET:
7309 	case PTR_TO_PACKET_META:
7310 		return check_packet_access(env, regno, reg->off, access_size,
7311 					   zero_size_allowed);
7312 	case PTR_TO_MAP_KEY:
7313 		if (meta && meta->raw_mode) {
7314 			verbose(env, "R%d cannot write into %s\n", regno,
7315 				reg_type_str(env, reg->type));
7316 			return -EACCES;
7317 		}
7318 		return check_mem_region_access(env, regno, reg->off, access_size,
7319 					       reg->map_ptr->key_size, false);
7320 	case PTR_TO_MAP_VALUE:
7321 		if (check_map_access_type(env, regno, reg->off, access_size,
7322 					  meta && meta->raw_mode ? BPF_WRITE :
7323 					  BPF_READ))
7324 			return -EACCES;
7325 		return check_map_access(env, regno, reg->off, access_size,
7326 					zero_size_allowed, ACCESS_HELPER);
7327 	case PTR_TO_MEM:
7328 		if (type_is_rdonly_mem(reg->type)) {
7329 			if (meta && meta->raw_mode) {
7330 				verbose(env, "R%d cannot write into %s\n", regno,
7331 					reg_type_str(env, reg->type));
7332 				return -EACCES;
7333 			}
7334 		}
7335 		return check_mem_region_access(env, regno, reg->off,
7336 					       access_size, reg->mem_size,
7337 					       zero_size_allowed);
7338 	case PTR_TO_BUF:
7339 		if (type_is_rdonly_mem(reg->type)) {
7340 			if (meta && meta->raw_mode) {
7341 				verbose(env, "R%d cannot write into %s\n", regno,
7342 					reg_type_str(env, reg->type));
7343 				return -EACCES;
7344 			}
7345 
7346 			max_access = &env->prog->aux->max_rdonly_access;
7347 		} else {
7348 			max_access = &env->prog->aux->max_rdwr_access;
7349 		}
7350 		return check_buffer_access(env, reg, regno, reg->off,
7351 					   access_size, zero_size_allowed,
7352 					   max_access);
7353 	case PTR_TO_STACK:
7354 		return check_stack_range_initialized(
7355 				env,
7356 				regno, reg->off, access_size,
7357 				zero_size_allowed, ACCESS_HELPER, meta);
7358 	case PTR_TO_BTF_ID:
7359 		return check_ptr_to_btf_access(env, regs, regno, reg->off,
7360 					       access_size, BPF_READ, -1);
7361 	case PTR_TO_CTX:
7362 		/* in case the function doesn't know how to access the context,
7363 		 * (because we are in a program of type SYSCALL for example), we
7364 		 * can not statically check its size.
7365 		 * Dynamically check it now.
7366 		 */
7367 		if (!env->ops->convert_ctx_access) {
7368 			enum bpf_access_type atype = meta && meta->raw_mode ? BPF_WRITE : BPF_READ;
7369 			int offset = access_size - 1;
7370 
7371 			/* Allow zero-byte read from PTR_TO_CTX */
7372 			if (access_size == 0)
7373 				return zero_size_allowed ? 0 : -EACCES;
7374 
7375 			return check_mem_access(env, env->insn_idx, regno, offset, BPF_B,
7376 						atype, -1, false, false);
7377 		}
7378 
7379 		fallthrough;
7380 	default: /* scalar_value or invalid ptr */
7381 		/* Allow zero-byte read from NULL, regardless of pointer type */
7382 		if (zero_size_allowed && access_size == 0 &&
7383 		    register_is_null(reg))
7384 			return 0;
7385 
7386 		verbose(env, "R%d type=%s ", regno,
7387 			reg_type_str(env, reg->type));
7388 		verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
7389 		return -EACCES;
7390 	}
7391 }
7392 
7393 /* verify arguments to helpers or kfuncs consisting of a pointer and an access
7394  * size.
7395  *
7396  * @regno is the register containing the access size. regno-1 is the register
7397  * containing the pointer.
7398  */
7399 static int check_mem_size_reg(struct bpf_verifier_env *env,
7400 			      struct bpf_reg_state *reg, u32 regno,
7401 			      bool zero_size_allowed,
7402 			      struct bpf_call_arg_meta *meta)
7403 {
7404 	int err;
7405 
7406 	/* This is used to refine r0 return value bounds for helpers
7407 	 * that enforce this value as an upper bound on return values.
7408 	 * See do_refine_retval_range() for helpers that can refine
7409 	 * the return value. C type of helper is u32 so we pull register
7410 	 * bound from umax_value however, if negative verifier errors
7411 	 * out. Only upper bounds can be learned because retval is an
7412 	 * int type and negative retvals are allowed.
7413 	 */
7414 	meta->msize_max_value = reg->umax_value;
7415 
7416 	/* The register is SCALAR_VALUE; the access check
7417 	 * happens using its boundaries.
7418 	 */
7419 	if (!tnum_is_const(reg->var_off))
7420 		/* For unprivileged variable accesses, disable raw
7421 		 * mode so that the program is required to
7422 		 * initialize all the memory that the helper could
7423 		 * just partially fill up.
7424 		 */
7425 		meta = NULL;
7426 
7427 	if (reg->smin_value < 0) {
7428 		verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
7429 			regno);
7430 		return -EACCES;
7431 	}
7432 
7433 	if (reg->umin_value == 0 && !zero_size_allowed) {
7434 		verbose(env, "R%d invalid zero-sized read: u64=[%lld,%lld]\n",
7435 			regno, reg->umin_value, reg->umax_value);
7436 		return -EACCES;
7437 	}
7438 
7439 	if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
7440 		verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
7441 			regno);
7442 		return -EACCES;
7443 	}
7444 	err = check_helper_mem_access(env, regno - 1,
7445 				      reg->umax_value,
7446 				      zero_size_allowed, meta);
7447 	if (!err)
7448 		err = mark_chain_precision(env, regno);
7449 	return err;
7450 }
7451 
7452 static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
7453 			 u32 regno, u32 mem_size)
7454 {
7455 	bool may_be_null = type_may_be_null(reg->type);
7456 	struct bpf_reg_state saved_reg;
7457 	struct bpf_call_arg_meta meta;
7458 	int err;
7459 
7460 	if (register_is_null(reg))
7461 		return 0;
7462 
7463 	memset(&meta, 0, sizeof(meta));
7464 	/* Assuming that the register contains a value check if the memory
7465 	 * access is safe. Temporarily save and restore the register's state as
7466 	 * the conversion shouldn't be visible to a caller.
7467 	 */
7468 	if (may_be_null) {
7469 		saved_reg = *reg;
7470 		mark_ptr_not_null_reg(reg);
7471 	}
7472 
7473 	err = check_helper_mem_access(env, regno, mem_size, true, &meta);
7474 	/* Check access for BPF_WRITE */
7475 	meta.raw_mode = true;
7476 	err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta);
7477 
7478 	if (may_be_null)
7479 		*reg = saved_reg;
7480 
7481 	return err;
7482 }
7483 
7484 static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
7485 				    u32 regno)
7486 {
7487 	struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1];
7488 	bool may_be_null = type_may_be_null(mem_reg->type);
7489 	struct bpf_reg_state saved_reg;
7490 	struct bpf_call_arg_meta meta;
7491 	int err;
7492 
7493 	WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5);
7494 
7495 	memset(&meta, 0, sizeof(meta));
7496 
7497 	if (may_be_null) {
7498 		saved_reg = *mem_reg;
7499 		mark_ptr_not_null_reg(mem_reg);
7500 	}
7501 
7502 	err = check_mem_size_reg(env, reg, regno, true, &meta);
7503 	/* Check access for BPF_WRITE */
7504 	meta.raw_mode = true;
7505 	err = err ?: check_mem_size_reg(env, reg, regno, true, &meta);
7506 
7507 	if (may_be_null)
7508 		*mem_reg = saved_reg;
7509 	return err;
7510 }
7511 
7512 /* Implementation details:
7513  * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL.
7514  * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL.
7515  * Two bpf_map_lookups (even with the same key) will have different reg->id.
7516  * Two separate bpf_obj_new will also have different reg->id.
7517  * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier
7518  * clears reg->id after value_or_null->value transition, since the verifier only
7519  * cares about the range of access to valid map value pointer and doesn't care
7520  * about actual address of the map element.
7521  * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps
7522  * reg->id > 0 after value_or_null->value transition. By doing so
7523  * two bpf_map_lookups will be considered two different pointers that
7524  * point to different bpf_spin_locks. Likewise for pointers to allocated objects
7525  * returned from bpf_obj_new.
7526  * The verifier allows taking only one bpf_spin_lock at a time to avoid
7527  * dead-locks.
7528  * Since only one bpf_spin_lock is allowed the checks are simpler than
7529  * reg_is_refcounted() logic. The verifier needs to remember only
7530  * one spin_lock instead of array of acquired_refs.
7531  * cur_state->active_lock remembers which map value element or allocated
7532  * object got locked and clears it after bpf_spin_unlock.
7533  */
7534 static int process_spin_lock(struct bpf_verifier_env *env, int regno,
7535 			     bool is_lock)
7536 {
7537 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7538 	struct bpf_verifier_state *cur = env->cur_state;
7539 	bool is_const = tnum_is_const(reg->var_off);
7540 	u64 val = reg->var_off.value;
7541 	struct bpf_map *map = NULL;
7542 	struct btf *btf = NULL;
7543 	struct btf_record *rec;
7544 
7545 	if (!is_const) {
7546 		verbose(env,
7547 			"R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n",
7548 			regno);
7549 		return -EINVAL;
7550 	}
7551 	if (reg->type == PTR_TO_MAP_VALUE) {
7552 		map = reg->map_ptr;
7553 		if (!map->btf) {
7554 			verbose(env,
7555 				"map '%s' has to have BTF in order to use bpf_spin_lock\n",
7556 				map->name);
7557 			return -EINVAL;
7558 		}
7559 	} else {
7560 		btf = reg->btf;
7561 	}
7562 
7563 	rec = reg_btf_record(reg);
7564 	if (!btf_record_has_field(rec, BPF_SPIN_LOCK)) {
7565 		verbose(env, "%s '%s' has no valid bpf_spin_lock\n", map ? "map" : "local",
7566 			map ? map->name : "kptr");
7567 		return -EINVAL;
7568 	}
7569 	if (rec->spin_lock_off != val + reg->off) {
7570 		verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock' that is at %d\n",
7571 			val + reg->off, rec->spin_lock_off);
7572 		return -EINVAL;
7573 	}
7574 	if (is_lock) {
7575 		if (cur->active_lock.ptr) {
7576 			verbose(env,
7577 				"Locking two bpf_spin_locks are not allowed\n");
7578 			return -EINVAL;
7579 		}
7580 		if (map)
7581 			cur->active_lock.ptr = map;
7582 		else
7583 			cur->active_lock.ptr = btf;
7584 		cur->active_lock.id = reg->id;
7585 	} else {
7586 		void *ptr;
7587 
7588 		if (map)
7589 			ptr = map;
7590 		else
7591 			ptr = btf;
7592 
7593 		if (!cur->active_lock.ptr) {
7594 			verbose(env, "bpf_spin_unlock without taking a lock\n");
7595 			return -EINVAL;
7596 		}
7597 		if (cur->active_lock.ptr != ptr ||
7598 		    cur->active_lock.id != reg->id) {
7599 			verbose(env, "bpf_spin_unlock of different lock\n");
7600 			return -EINVAL;
7601 		}
7602 
7603 		invalidate_non_owning_refs(env);
7604 
7605 		cur->active_lock.ptr = NULL;
7606 		cur->active_lock.id = 0;
7607 	}
7608 	return 0;
7609 }
7610 
7611 static int process_timer_func(struct bpf_verifier_env *env, int regno,
7612 			      struct bpf_call_arg_meta *meta)
7613 {
7614 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7615 	bool is_const = tnum_is_const(reg->var_off);
7616 	struct bpf_map *map = reg->map_ptr;
7617 	u64 val = reg->var_off.value;
7618 
7619 	if (!is_const) {
7620 		verbose(env,
7621 			"R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n",
7622 			regno);
7623 		return -EINVAL;
7624 	}
7625 	if (!map->btf) {
7626 		verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n",
7627 			map->name);
7628 		return -EINVAL;
7629 	}
7630 	if (!btf_record_has_field(map->record, BPF_TIMER)) {
7631 		verbose(env, "map '%s' has no valid bpf_timer\n", map->name);
7632 		return -EINVAL;
7633 	}
7634 	if (map->record->timer_off != val + reg->off) {
7635 		verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n",
7636 			val + reg->off, map->record->timer_off);
7637 		return -EINVAL;
7638 	}
7639 	if (meta->map_ptr) {
7640 		verbose(env, "verifier bug. Two map pointers in a timer helper\n");
7641 		return -EFAULT;
7642 	}
7643 	meta->map_uid = reg->map_uid;
7644 	meta->map_ptr = map;
7645 	return 0;
7646 }
7647 
7648 static int process_wq_func(struct bpf_verifier_env *env, int regno,
7649 			   struct bpf_kfunc_call_arg_meta *meta)
7650 {
7651 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7652 	struct bpf_map *map = reg->map_ptr;
7653 	u64 val = reg->var_off.value;
7654 
7655 	if (map->record->wq_off != val + reg->off) {
7656 		verbose(env, "off %lld doesn't point to 'struct bpf_wq' that is at %d\n",
7657 			val + reg->off, map->record->wq_off);
7658 		return -EINVAL;
7659 	}
7660 	meta->map.uid = reg->map_uid;
7661 	meta->map.ptr = map;
7662 	return 0;
7663 }
7664 
7665 static int process_kptr_func(struct bpf_verifier_env *env, int regno,
7666 			     struct bpf_call_arg_meta *meta)
7667 {
7668 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7669 	struct bpf_map *map_ptr = reg->map_ptr;
7670 	struct btf_field *kptr_field;
7671 	u32 kptr_off;
7672 
7673 	if (!tnum_is_const(reg->var_off)) {
7674 		verbose(env,
7675 			"R%d doesn't have constant offset. kptr has to be at the constant offset\n",
7676 			regno);
7677 		return -EINVAL;
7678 	}
7679 	if (!map_ptr->btf) {
7680 		verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n",
7681 			map_ptr->name);
7682 		return -EINVAL;
7683 	}
7684 	if (!btf_record_has_field(map_ptr->record, BPF_KPTR)) {
7685 		verbose(env, "map '%s' has no valid kptr\n", map_ptr->name);
7686 		return -EINVAL;
7687 	}
7688 
7689 	meta->map_ptr = map_ptr;
7690 	kptr_off = reg->off + reg->var_off.value;
7691 	kptr_field = btf_record_find(map_ptr->record, kptr_off, BPF_KPTR);
7692 	if (!kptr_field) {
7693 		verbose(env, "off=%d doesn't point to kptr\n", kptr_off);
7694 		return -EACCES;
7695 	}
7696 	if (kptr_field->type != BPF_KPTR_REF && kptr_field->type != BPF_KPTR_PERCPU) {
7697 		verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off);
7698 		return -EACCES;
7699 	}
7700 	meta->kptr_field = kptr_field;
7701 	return 0;
7702 }
7703 
7704 /* There are two register types representing a bpf_dynptr, one is PTR_TO_STACK
7705  * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR.
7706  *
7707  * In both cases we deal with the first 8 bytes, but need to mark the next 8
7708  * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of
7709  * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object.
7710  *
7711  * Mutability of bpf_dynptr is at two levels, one is at the level of struct
7712  * bpf_dynptr itself, i.e. whether the helper is receiving a pointer to struct
7713  * bpf_dynptr or pointer to const struct bpf_dynptr. In the former case, it can
7714  * mutate the view of the dynptr and also possibly destroy it. In the latter
7715  * case, it cannot mutate the bpf_dynptr itself but it can still mutate the
7716  * memory that dynptr points to.
7717  *
7718  * The verifier will keep track both levels of mutation (bpf_dynptr's in
7719  * reg->type and the memory's in reg->dynptr.type), but there is no support for
7720  * readonly dynptr view yet, hence only the first case is tracked and checked.
7721  *
7722  * This is consistent with how C applies the const modifier to a struct object,
7723  * where the pointer itself inside bpf_dynptr becomes const but not what it
7724  * points to.
7725  *
7726  * Helpers which do not mutate the bpf_dynptr set MEM_RDONLY in their argument
7727  * type, and declare it as 'const struct bpf_dynptr *' in their prototype.
7728  */
7729 static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,
7730 			       enum bpf_arg_type arg_type, int clone_ref_obj_id)
7731 {
7732 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7733 	int err;
7734 
7735 	if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) {
7736 		verbose(env,
7737 			"arg#%d expected pointer to stack or const struct bpf_dynptr\n",
7738 			regno);
7739 		return -EINVAL;
7740 	}
7741 
7742 	/* MEM_UNINIT and MEM_RDONLY are exclusive, when applied to an
7743 	 * ARG_PTR_TO_DYNPTR (or ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_*):
7744 	 */
7745 	if ((arg_type & (MEM_UNINIT | MEM_RDONLY)) == (MEM_UNINIT | MEM_RDONLY)) {
7746 		verbose(env, "verifier internal error: misconfigured dynptr helper type flags\n");
7747 		return -EFAULT;
7748 	}
7749 
7750 	/*  MEM_UNINIT - Points to memory that is an appropriate candidate for
7751 	 *		 constructing a mutable bpf_dynptr object.
7752 	 *
7753 	 *		 Currently, this is only possible with PTR_TO_STACK
7754 	 *		 pointing to a region of at least 16 bytes which doesn't
7755 	 *		 contain an existing bpf_dynptr.
7756 	 *
7757 	 *  MEM_RDONLY - Points to a initialized bpf_dynptr that will not be
7758 	 *		 mutated or destroyed. However, the memory it points to
7759 	 *		 may be mutated.
7760 	 *
7761 	 *  None       - Points to a initialized dynptr that can be mutated and
7762 	 *		 destroyed, including mutation of the memory it points
7763 	 *		 to.
7764 	 */
7765 	if (arg_type & MEM_UNINIT) {
7766 		int i;
7767 
7768 		if (!is_dynptr_reg_valid_uninit(env, reg)) {
7769 			verbose(env, "Dynptr has to be an uninitialized dynptr\n");
7770 			return -EINVAL;
7771 		}
7772 
7773 		/* we write BPF_DW bits (8 bytes) at a time */
7774 		for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) {
7775 			err = check_mem_access(env, insn_idx, regno,
7776 					       i, BPF_DW, BPF_WRITE, -1, false, false);
7777 			if (err)
7778 				return err;
7779 		}
7780 
7781 		err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id);
7782 	} else /* MEM_RDONLY and None case from above */ {
7783 		/* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */
7784 		if (reg->type == CONST_PTR_TO_DYNPTR && !(arg_type & MEM_RDONLY)) {
7785 			verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n");
7786 			return -EINVAL;
7787 		}
7788 
7789 		if (!is_dynptr_reg_valid_init(env, reg)) {
7790 			verbose(env,
7791 				"Expected an initialized dynptr as arg #%d\n",
7792 				regno);
7793 			return -EINVAL;
7794 		}
7795 
7796 		/* Fold modifiers (in this case, MEM_RDONLY) when checking expected type */
7797 		if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) {
7798 			verbose(env,
7799 				"Expected a dynptr of type %s as arg #%d\n",
7800 				dynptr_type_str(arg_to_dynptr_type(arg_type)), regno);
7801 			return -EINVAL;
7802 		}
7803 
7804 		err = mark_dynptr_read(env, reg);
7805 	}
7806 	return err;
7807 }
7808 
7809 static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi)
7810 {
7811 	struct bpf_func_state *state = func(env, reg);
7812 
7813 	return state->stack[spi].spilled_ptr.ref_obj_id;
7814 }
7815 
7816 static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta)
7817 {
7818 	return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY);
7819 }
7820 
7821 static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta)
7822 {
7823 	return meta->kfunc_flags & KF_ITER_NEW;
7824 }
7825 
7826 static bool is_iter_next_kfunc(struct bpf_kfunc_call_arg_meta *meta)
7827 {
7828 	return meta->kfunc_flags & KF_ITER_NEXT;
7829 }
7830 
7831 static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta)
7832 {
7833 	return meta->kfunc_flags & KF_ITER_DESTROY;
7834 }
7835 
7836 static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg)
7837 {
7838 	/* btf_check_iter_kfuncs() guarantees that first argument of any iter
7839 	 * kfunc is iter state pointer
7840 	 */
7841 	return arg == 0 && is_iter_kfunc(meta);
7842 }
7843 
7844 static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx,
7845 			    struct bpf_kfunc_call_arg_meta *meta)
7846 {
7847 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7848 	const struct btf_type *t;
7849 	const struct btf_param *arg;
7850 	int spi, err, i, nr_slots;
7851 	u32 btf_id;
7852 
7853 	/* btf_check_iter_kfuncs() ensures we don't need to validate anything here */
7854 	arg = &btf_params(meta->func_proto)[0];
7855 	t = btf_type_skip_modifiers(meta->btf, arg->type, NULL);	/* PTR */
7856 	t = btf_type_skip_modifiers(meta->btf, t->type, &btf_id);	/* STRUCT */
7857 	nr_slots = t->size / BPF_REG_SIZE;
7858 
7859 	if (is_iter_new_kfunc(meta)) {
7860 		/* bpf_iter_<type>_new() expects pointer to uninit iter state */
7861 		if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) {
7862 			verbose(env, "expected uninitialized iter_%s as arg #%d\n",
7863 				iter_type_str(meta->btf, btf_id), regno);
7864 			return -EINVAL;
7865 		}
7866 
7867 		for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) {
7868 			err = check_mem_access(env, insn_idx, regno,
7869 					       i, BPF_DW, BPF_WRITE, -1, false, false);
7870 			if (err)
7871 				return err;
7872 		}
7873 
7874 		err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots);
7875 		if (err)
7876 			return err;
7877 	} else {
7878 		/* iter_next() or iter_destroy() expect initialized iter state*/
7879 		err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots);
7880 		switch (err) {
7881 		case 0:
7882 			break;
7883 		case -EINVAL:
7884 			verbose(env, "expected an initialized iter_%s as arg #%d\n",
7885 				iter_type_str(meta->btf, btf_id), regno);
7886 			return err;
7887 		case -EPROTO:
7888 			verbose(env, "expected an RCU CS when using %s\n", meta->func_name);
7889 			return err;
7890 		default:
7891 			return err;
7892 		}
7893 
7894 		spi = iter_get_spi(env, reg, nr_slots);
7895 		if (spi < 0)
7896 			return spi;
7897 
7898 		err = mark_iter_read(env, reg, spi, nr_slots);
7899 		if (err)
7900 			return err;
7901 
7902 		/* remember meta->iter info for process_iter_next_call() */
7903 		meta->iter.spi = spi;
7904 		meta->iter.frameno = reg->frameno;
7905 		meta->ref_obj_id = iter_ref_obj_id(env, reg, spi);
7906 
7907 		if (is_iter_destroy_kfunc(meta)) {
7908 			err = unmark_stack_slots_iter(env, reg, nr_slots);
7909 			if (err)
7910 				return err;
7911 		}
7912 	}
7913 
7914 	return 0;
7915 }
7916 
7917 /* Look for a previous loop entry at insn_idx: nearest parent state
7918  * stopped at insn_idx with callsites matching those in cur->frame.
7919  */
7920 static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env,
7921 						  struct bpf_verifier_state *cur,
7922 						  int insn_idx)
7923 {
7924 	struct bpf_verifier_state_list *sl;
7925 	struct bpf_verifier_state *st;
7926 
7927 	/* Explored states are pushed in stack order, most recent states come first */
7928 	sl = *explored_state(env, insn_idx);
7929 	for (; sl; sl = sl->next) {
7930 		/* If st->branches != 0 state is a part of current DFS verification path,
7931 		 * hence cur & st for a loop.
7932 		 */
7933 		st = &sl->state;
7934 		if (st->insn_idx == insn_idx && st->branches && same_callsites(st, cur) &&
7935 		    st->dfs_depth < cur->dfs_depth)
7936 			return st;
7937 	}
7938 
7939 	return NULL;
7940 }
7941 
7942 static void reset_idmap_scratch(struct bpf_verifier_env *env);
7943 static bool regs_exact(const struct bpf_reg_state *rold,
7944 		       const struct bpf_reg_state *rcur,
7945 		       struct bpf_idmap *idmap);
7946 
7947 static void maybe_widen_reg(struct bpf_verifier_env *env,
7948 			    struct bpf_reg_state *rold, struct bpf_reg_state *rcur,
7949 			    struct bpf_idmap *idmap)
7950 {
7951 	if (rold->type != SCALAR_VALUE)
7952 		return;
7953 	if (rold->type != rcur->type)
7954 		return;
7955 	if (rold->precise || rcur->precise || regs_exact(rold, rcur, idmap))
7956 		return;
7957 	__mark_reg_unknown(env, rcur);
7958 }
7959 
7960 static int widen_imprecise_scalars(struct bpf_verifier_env *env,
7961 				   struct bpf_verifier_state *old,
7962 				   struct bpf_verifier_state *cur)
7963 {
7964 	struct bpf_func_state *fold, *fcur;
7965 	int i, fr;
7966 
7967 	reset_idmap_scratch(env);
7968 	for (fr = old->curframe; fr >= 0; fr--) {
7969 		fold = old->frame[fr];
7970 		fcur = cur->frame[fr];
7971 
7972 		for (i = 0; i < MAX_BPF_REG; i++)
7973 			maybe_widen_reg(env,
7974 					&fold->regs[i],
7975 					&fcur->regs[i],
7976 					&env->idmap_scratch);
7977 
7978 		for (i = 0; i < fold->allocated_stack / BPF_REG_SIZE; i++) {
7979 			if (!is_spilled_reg(&fold->stack[i]) ||
7980 			    !is_spilled_reg(&fcur->stack[i]))
7981 				continue;
7982 
7983 			maybe_widen_reg(env,
7984 					&fold->stack[i].spilled_ptr,
7985 					&fcur->stack[i].spilled_ptr,
7986 					&env->idmap_scratch);
7987 		}
7988 	}
7989 	return 0;
7990 }
7991 
7992 /* process_iter_next_call() is called when verifier gets to iterator's next
7993  * "method" (e.g., bpf_iter_num_next() for numbers iterator) call. We'll refer
7994  * to it as just "iter_next()" in comments below.
7995  *
7996  * BPF verifier relies on a crucial contract for any iter_next()
7997  * implementation: it should *eventually* return NULL, and once that happens
7998  * it should keep returning NULL. That is, once iterator exhausts elements to
7999  * iterate, it should never reset or spuriously return new elements.
8000  *
8001  * With the assumption of such contract, process_iter_next_call() simulates
8002  * a fork in the verifier state to validate loop logic correctness and safety
8003  * without having to simulate infinite amount of iterations.
8004  *
8005  * In current state, we first assume that iter_next() returned NULL and
8006  * iterator state is set to DRAINED (BPF_ITER_STATE_DRAINED). In such
8007  * conditions we should not form an infinite loop and should eventually reach
8008  * exit.
8009  *
8010  * Besides that, we also fork current state and enqueue it for later
8011  * verification. In a forked state we keep iterator state as ACTIVE
8012  * (BPF_ITER_STATE_ACTIVE) and assume non-NULL return from iter_next(). We
8013  * also bump iteration depth to prevent erroneous infinite loop detection
8014  * later on (see iter_active_depths_differ() comment for details). In this
8015  * state we assume that we'll eventually loop back to another iter_next()
8016  * calls (it could be in exactly same location or in some other instruction,
8017  * it doesn't matter, we don't make any unnecessary assumptions about this,
8018  * everything revolves around iterator state in a stack slot, not which
8019  * instruction is calling iter_next()). When that happens, we either will come
8020  * to iter_next() with equivalent state and can conclude that next iteration
8021  * will proceed in exactly the same way as we just verified, so it's safe to
8022  * assume that loop converges. If not, we'll go on another iteration
8023  * simulation with a different input state, until all possible starting states
8024  * are validated or we reach maximum number of instructions limit.
8025  *
8026  * This way, we will either exhaustively discover all possible input states
8027  * that iterator loop can start with and eventually will converge, or we'll
8028  * effectively regress into bounded loop simulation logic and either reach
8029  * maximum number of instructions if loop is not provably convergent, or there
8030  * is some statically known limit on number of iterations (e.g., if there is
8031  * an explicit `if n > 100 then break;` statement somewhere in the loop).
8032  *
8033  * Iteration convergence logic in is_state_visited() relies on exact
8034  * states comparison, which ignores read and precision marks.
8035  * This is necessary because read and precision marks are not finalized
8036  * while in the loop. Exact comparison might preclude convergence for
8037  * simple programs like below:
8038  *
8039  *     i = 0;
8040  *     while(iter_next(&it))
8041  *       i++;
8042  *
8043  * At each iteration step i++ would produce a new distinct state and
8044  * eventually instruction processing limit would be reached.
8045  *
8046  * To avoid such behavior speculatively forget (widen) range for
8047  * imprecise scalar registers, if those registers were not precise at the
8048  * end of the previous iteration and do not match exactly.
8049  *
8050  * This is a conservative heuristic that allows to verify wide range of programs,
8051  * however it precludes verification of programs that conjure an
8052  * imprecise value on the first loop iteration and use it as precise on a second.
8053  * For example, the following safe program would fail to verify:
8054  *
8055  *     struct bpf_num_iter it;
8056  *     int arr[10];
8057  *     int i = 0, a = 0;
8058  *     bpf_iter_num_new(&it, 0, 10);
8059  *     while (bpf_iter_num_next(&it)) {
8060  *       if (a == 0) {
8061  *         a = 1;
8062  *         i = 7; // Because i changed verifier would forget
8063  *                // it's range on second loop entry.
8064  *       } else {
8065  *         arr[i] = 42; // This would fail to verify.
8066  *       }
8067  *     }
8068  *     bpf_iter_num_destroy(&it);
8069  */
8070 static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx,
8071 				  struct bpf_kfunc_call_arg_meta *meta)
8072 {
8073 	struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;
8074 	struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr;
8075 	struct bpf_reg_state *cur_iter, *queued_iter;
8076 	int iter_frameno = meta->iter.frameno;
8077 	int iter_spi = meta->iter.spi;
8078 
8079 	BTF_TYPE_EMIT(struct bpf_iter);
8080 
8081 	cur_iter = &env->cur_state->frame[iter_frameno]->stack[iter_spi].spilled_ptr;
8082 
8083 	if (cur_iter->iter.state != BPF_ITER_STATE_ACTIVE &&
8084 	    cur_iter->iter.state != BPF_ITER_STATE_DRAINED) {
8085 		verbose(env, "verifier internal error: unexpected iterator state %d (%s)\n",
8086 			cur_iter->iter.state, iter_state_str(cur_iter->iter.state));
8087 		return -EFAULT;
8088 	}
8089 
8090 	if (cur_iter->iter.state == BPF_ITER_STATE_ACTIVE) {
8091 		/* Because iter_next() call is a checkpoint is_state_visitied()
8092 		 * should guarantee parent state with same call sites and insn_idx.
8093 		 */
8094 		if (!cur_st->parent || cur_st->parent->insn_idx != insn_idx ||
8095 		    !same_callsites(cur_st->parent, cur_st)) {
8096 			verbose(env, "bug: bad parent state for iter next call");
8097 			return -EFAULT;
8098 		}
8099 		/* Note cur_st->parent in the call below, it is necessary to skip
8100 		 * checkpoint created for cur_st by is_state_visited()
8101 		 * right at this instruction.
8102 		 */
8103 		prev_st = find_prev_entry(env, cur_st->parent, insn_idx);
8104 		/* branch out active iter state */
8105 		queued_st = push_stack(env, insn_idx + 1, insn_idx, false);
8106 		if (!queued_st)
8107 			return -ENOMEM;
8108 
8109 		queued_iter = &queued_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr;
8110 		queued_iter->iter.state = BPF_ITER_STATE_ACTIVE;
8111 		queued_iter->iter.depth++;
8112 		if (prev_st)
8113 			widen_imprecise_scalars(env, prev_st, queued_st);
8114 
8115 		queued_fr = queued_st->frame[queued_st->curframe];
8116 		mark_ptr_not_null_reg(&queued_fr->regs[BPF_REG_0]);
8117 	}
8118 
8119 	/* switch to DRAINED state, but keep the depth unchanged */
8120 	/* mark current iter state as drained and assume returned NULL */
8121 	cur_iter->iter.state = BPF_ITER_STATE_DRAINED;
8122 	__mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]);
8123 
8124 	return 0;
8125 }
8126 
8127 static bool arg_type_is_mem_size(enum bpf_arg_type type)
8128 {
8129 	return type == ARG_CONST_SIZE ||
8130 	       type == ARG_CONST_SIZE_OR_ZERO;
8131 }
8132 
8133 static bool arg_type_is_release(enum bpf_arg_type type)
8134 {
8135 	return type & OBJ_RELEASE;
8136 }
8137 
8138 static bool arg_type_is_dynptr(enum bpf_arg_type type)
8139 {
8140 	return base_type(type) == ARG_PTR_TO_DYNPTR;
8141 }
8142 
8143 static int int_ptr_type_to_size(enum bpf_arg_type type)
8144 {
8145 	if (type == ARG_PTR_TO_INT)
8146 		return sizeof(u32);
8147 	else if (type == ARG_PTR_TO_LONG)
8148 		return sizeof(u64);
8149 
8150 	return -EINVAL;
8151 }
8152 
8153 static int resolve_map_arg_type(struct bpf_verifier_env *env,
8154 				 const struct bpf_call_arg_meta *meta,
8155 				 enum bpf_arg_type *arg_type)
8156 {
8157 	if (!meta->map_ptr) {
8158 		/* kernel subsystem misconfigured verifier */
8159 		verbose(env, "invalid map_ptr to access map->type\n");
8160 		return -EACCES;
8161 	}
8162 
8163 	switch (meta->map_ptr->map_type) {
8164 	case BPF_MAP_TYPE_SOCKMAP:
8165 	case BPF_MAP_TYPE_SOCKHASH:
8166 		if (*arg_type == ARG_PTR_TO_MAP_VALUE) {
8167 			*arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON;
8168 		} else {
8169 			verbose(env, "invalid arg_type for sockmap/sockhash\n");
8170 			return -EINVAL;
8171 		}
8172 		break;
8173 	case BPF_MAP_TYPE_BLOOM_FILTER:
8174 		if (meta->func_id == BPF_FUNC_map_peek_elem)
8175 			*arg_type = ARG_PTR_TO_MAP_VALUE;
8176 		break;
8177 	default:
8178 		break;
8179 	}
8180 	return 0;
8181 }
8182 
8183 struct bpf_reg_types {
8184 	const enum bpf_reg_type types[10];
8185 	u32 *btf_id;
8186 };
8187 
8188 static const struct bpf_reg_types sock_types = {
8189 	.types = {
8190 		PTR_TO_SOCK_COMMON,
8191 		PTR_TO_SOCKET,
8192 		PTR_TO_TCP_SOCK,
8193 		PTR_TO_XDP_SOCK,
8194 	},
8195 };
8196 
8197 #ifdef CONFIG_NET
8198 static const struct bpf_reg_types btf_id_sock_common_types = {
8199 	.types = {
8200 		PTR_TO_SOCK_COMMON,
8201 		PTR_TO_SOCKET,
8202 		PTR_TO_TCP_SOCK,
8203 		PTR_TO_XDP_SOCK,
8204 		PTR_TO_BTF_ID,
8205 		PTR_TO_BTF_ID | PTR_TRUSTED,
8206 	},
8207 	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
8208 };
8209 #endif
8210 
8211 static const struct bpf_reg_types mem_types = {
8212 	.types = {
8213 		PTR_TO_STACK,
8214 		PTR_TO_PACKET,
8215 		PTR_TO_PACKET_META,
8216 		PTR_TO_MAP_KEY,
8217 		PTR_TO_MAP_VALUE,
8218 		PTR_TO_MEM,
8219 		PTR_TO_MEM | MEM_RINGBUF,
8220 		PTR_TO_BUF,
8221 		PTR_TO_BTF_ID | PTR_TRUSTED,
8222 	},
8223 };
8224 
8225 static const struct bpf_reg_types int_ptr_types = {
8226 	.types = {
8227 		PTR_TO_STACK,
8228 		PTR_TO_PACKET,
8229 		PTR_TO_PACKET_META,
8230 		PTR_TO_MAP_KEY,
8231 		PTR_TO_MAP_VALUE,
8232 	},
8233 };
8234 
8235 static const struct bpf_reg_types spin_lock_types = {
8236 	.types = {
8237 		PTR_TO_MAP_VALUE,
8238 		PTR_TO_BTF_ID | MEM_ALLOC,
8239 	}
8240 };
8241 
8242 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } };
8243 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } };
8244 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } };
8245 static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } };
8246 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
8247 static const struct bpf_reg_types btf_ptr_types = {
8248 	.types = {
8249 		PTR_TO_BTF_ID,
8250 		PTR_TO_BTF_ID | PTR_TRUSTED,
8251 		PTR_TO_BTF_ID | MEM_RCU,
8252 	},
8253 };
8254 static const struct bpf_reg_types percpu_btf_ptr_types = {
8255 	.types = {
8256 		PTR_TO_BTF_ID | MEM_PERCPU,
8257 		PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU,
8258 		PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED,
8259 	}
8260 };
8261 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } };
8262 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } };
8263 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } };
8264 static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } };
8265 static const struct bpf_reg_types kptr_types = { .types = { PTR_TO_MAP_VALUE } };
8266 static const struct bpf_reg_types dynptr_types = {
8267 	.types = {
8268 		PTR_TO_STACK,
8269 		CONST_PTR_TO_DYNPTR,
8270 	}
8271 };
8272 
8273 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
8274 	[ARG_PTR_TO_MAP_KEY]		= &mem_types,
8275 	[ARG_PTR_TO_MAP_VALUE]		= &mem_types,
8276 	[ARG_CONST_SIZE]		= &scalar_types,
8277 	[ARG_CONST_SIZE_OR_ZERO]	= &scalar_types,
8278 	[ARG_CONST_ALLOC_SIZE_OR_ZERO]	= &scalar_types,
8279 	[ARG_CONST_MAP_PTR]		= &const_map_ptr_types,
8280 	[ARG_PTR_TO_CTX]		= &context_types,
8281 	[ARG_PTR_TO_SOCK_COMMON]	= &sock_types,
8282 #ifdef CONFIG_NET
8283 	[ARG_PTR_TO_BTF_ID_SOCK_COMMON]	= &btf_id_sock_common_types,
8284 #endif
8285 	[ARG_PTR_TO_SOCKET]		= &fullsock_types,
8286 	[ARG_PTR_TO_BTF_ID]		= &btf_ptr_types,
8287 	[ARG_PTR_TO_SPIN_LOCK]		= &spin_lock_types,
8288 	[ARG_PTR_TO_MEM]		= &mem_types,
8289 	[ARG_PTR_TO_RINGBUF_MEM]	= &ringbuf_mem_types,
8290 	[ARG_PTR_TO_INT]		= &int_ptr_types,
8291 	[ARG_PTR_TO_LONG]		= &int_ptr_types,
8292 	[ARG_PTR_TO_PERCPU_BTF_ID]	= &percpu_btf_ptr_types,
8293 	[ARG_PTR_TO_FUNC]		= &func_ptr_types,
8294 	[ARG_PTR_TO_STACK]		= &stack_ptr_types,
8295 	[ARG_PTR_TO_CONST_STR]		= &const_str_ptr_types,
8296 	[ARG_PTR_TO_TIMER]		= &timer_types,
8297 	[ARG_PTR_TO_KPTR]		= &kptr_types,
8298 	[ARG_PTR_TO_DYNPTR]		= &dynptr_types,
8299 };
8300 
8301 static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
8302 			  enum bpf_arg_type arg_type,
8303 			  const u32 *arg_btf_id,
8304 			  struct bpf_call_arg_meta *meta)
8305 {
8306 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
8307 	enum bpf_reg_type expected, type = reg->type;
8308 	const struct bpf_reg_types *compatible;
8309 	int i, j;
8310 
8311 	compatible = compatible_reg_types[base_type(arg_type)];
8312 	if (!compatible) {
8313 		verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
8314 		return -EFAULT;
8315 	}
8316 
8317 	/* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY,
8318 	 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY
8319 	 *
8320 	 * Same for MAYBE_NULL:
8321 	 *
8322 	 * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL,
8323 	 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL
8324 	 *
8325 	 * ARG_PTR_TO_MEM is compatible with PTR_TO_MEM that is tagged with a dynptr type.
8326 	 *
8327 	 * Therefore we fold these flags depending on the arg_type before comparison.
8328 	 */
8329 	if (arg_type & MEM_RDONLY)
8330 		type &= ~MEM_RDONLY;
8331 	if (arg_type & PTR_MAYBE_NULL)
8332 		type &= ~PTR_MAYBE_NULL;
8333 	if (base_type(arg_type) == ARG_PTR_TO_MEM)
8334 		type &= ~DYNPTR_TYPE_FLAG_MASK;
8335 
8336 	if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type)) {
8337 		type &= ~MEM_ALLOC;
8338 		type &= ~MEM_PERCPU;
8339 	}
8340 
8341 	for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
8342 		expected = compatible->types[i];
8343 		if (expected == NOT_INIT)
8344 			break;
8345 
8346 		if (type == expected)
8347 			goto found;
8348 	}
8349 
8350 	verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));
8351 	for (j = 0; j + 1 < i; j++)
8352 		verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
8353 	verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
8354 	return -EACCES;
8355 
8356 found:
8357 	if (base_type(reg->type) != PTR_TO_BTF_ID)
8358 		return 0;
8359 
8360 	if (compatible == &mem_types) {
8361 		if (!(arg_type & MEM_RDONLY)) {
8362 			verbose(env,
8363 				"%s() may write into memory pointed by R%d type=%s\n",
8364 				func_id_name(meta->func_id),
8365 				regno, reg_type_str(env, reg->type));
8366 			return -EACCES;
8367 		}
8368 		return 0;
8369 	}
8370 
8371 	switch ((int)reg->type) {
8372 	case PTR_TO_BTF_ID:
8373 	case PTR_TO_BTF_ID | PTR_TRUSTED:
8374 	case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL:
8375 	case PTR_TO_BTF_ID | MEM_RCU:
8376 	case PTR_TO_BTF_ID | PTR_MAYBE_NULL:
8377 	case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU:
8378 	{
8379 		/* For bpf_sk_release, it needs to match against first member
8380 		 * 'struct sock_common', hence make an exception for it. This
8381 		 * allows bpf_sk_release to work for multiple socket types.
8382 		 */
8383 		bool strict_type_match = arg_type_is_release(arg_type) &&
8384 					 meta->func_id != BPF_FUNC_sk_release;
8385 
8386 		if (type_may_be_null(reg->type) &&
8387 		    (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) {
8388 			verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno);
8389 			return -EACCES;
8390 		}
8391 
8392 		if (!arg_btf_id) {
8393 			if (!compatible->btf_id) {
8394 				verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
8395 				return -EFAULT;
8396 			}
8397 			arg_btf_id = compatible->btf_id;
8398 		}
8399 
8400 		if (meta->func_id == BPF_FUNC_kptr_xchg) {
8401 			if (map_kptr_match_type(env, meta->kptr_field, reg, regno))
8402 				return -EACCES;
8403 		} else {
8404 			if (arg_btf_id == BPF_PTR_POISON) {
8405 				verbose(env, "verifier internal error:");
8406 				verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n",
8407 					regno);
8408 				return -EACCES;
8409 			}
8410 
8411 			if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
8412 						  btf_vmlinux, *arg_btf_id,
8413 						  strict_type_match)) {
8414 				verbose(env, "R%d is of type %s but %s is expected\n",
8415 					regno, btf_type_name(reg->btf, reg->btf_id),
8416 					btf_type_name(btf_vmlinux, *arg_btf_id));
8417 				return -EACCES;
8418 			}
8419 		}
8420 		break;
8421 	}
8422 	case PTR_TO_BTF_ID | MEM_ALLOC:
8423 	case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC:
8424 		if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock &&
8425 		    meta->func_id != BPF_FUNC_kptr_xchg) {
8426 			verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n");
8427 			return -EFAULT;
8428 		}
8429 		if (meta->func_id == BPF_FUNC_kptr_xchg) {
8430 			if (map_kptr_match_type(env, meta->kptr_field, reg, regno))
8431 				return -EACCES;
8432 		}
8433 		break;
8434 	case PTR_TO_BTF_ID | MEM_PERCPU:
8435 	case PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU:
8436 	case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED:
8437 		/* Handled by helper specific checks */
8438 		break;
8439 	default:
8440 		verbose(env, "verifier internal error: invalid PTR_TO_BTF_ID register for type match\n");
8441 		return -EFAULT;
8442 	}
8443 	return 0;
8444 }
8445 
8446 static struct btf_field *
8447 reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields)
8448 {
8449 	struct btf_field *field;
8450 	struct btf_record *rec;
8451 
8452 	rec = reg_btf_record(reg);
8453 	if (!rec)
8454 		return NULL;
8455 
8456 	field = btf_record_find(rec, off, fields);
8457 	if (!field)
8458 		return NULL;
8459 
8460 	return field;
8461 }
8462 
8463 static int check_func_arg_reg_off(struct bpf_verifier_env *env,
8464 				  const struct bpf_reg_state *reg, int regno,
8465 				  enum bpf_arg_type arg_type)
8466 {
8467 	u32 type = reg->type;
8468 
8469 	/* When referenced register is passed to release function, its fixed
8470 	 * offset must be 0.
8471 	 *
8472 	 * We will check arg_type_is_release reg has ref_obj_id when storing
8473 	 * meta->release_regno.
8474 	 */
8475 	if (arg_type_is_release(arg_type)) {
8476 		/* ARG_PTR_TO_DYNPTR with OBJ_RELEASE is a bit special, as it
8477 		 * may not directly point to the object being released, but to
8478 		 * dynptr pointing to such object, which might be at some offset
8479 		 * on the stack. In that case, we simply to fallback to the
8480 		 * default handling.
8481 		 */
8482 		if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK)
8483 			return 0;
8484 
8485 		/* Doing check_ptr_off_reg check for the offset will catch this
8486 		 * because fixed_off_ok is false, but checking here allows us
8487 		 * to give the user a better error message.
8488 		 */
8489 		if (reg->off) {
8490 			verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n",
8491 				regno);
8492 			return -EINVAL;
8493 		}
8494 		return __check_ptr_off_reg(env, reg, regno, false);
8495 	}
8496 
8497 	switch (type) {
8498 	/* Pointer types where both fixed and variable offset is explicitly allowed: */
8499 	case PTR_TO_STACK:
8500 	case PTR_TO_PACKET:
8501 	case PTR_TO_PACKET_META:
8502 	case PTR_TO_MAP_KEY:
8503 	case PTR_TO_MAP_VALUE:
8504 	case PTR_TO_MEM:
8505 	case PTR_TO_MEM | MEM_RDONLY:
8506 	case PTR_TO_MEM | MEM_RINGBUF:
8507 	case PTR_TO_BUF:
8508 	case PTR_TO_BUF | MEM_RDONLY:
8509 	case PTR_TO_ARENA:
8510 	case SCALAR_VALUE:
8511 		return 0;
8512 	/* All the rest must be rejected, except PTR_TO_BTF_ID which allows
8513 	 * fixed offset.
8514 	 */
8515 	case PTR_TO_BTF_ID:
8516 	case PTR_TO_BTF_ID | MEM_ALLOC:
8517 	case PTR_TO_BTF_ID | PTR_TRUSTED:
8518 	case PTR_TO_BTF_ID | MEM_RCU:
8519 	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF:
8520 	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU:
8521 		/* When referenced PTR_TO_BTF_ID is passed to release function,
8522 		 * its fixed offset must be 0. In the other cases, fixed offset
8523 		 * can be non-zero. This was already checked above. So pass
8524 		 * fixed_off_ok as true to allow fixed offset for all other
8525 		 * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we
8526 		 * still need to do checks instead of returning.
8527 		 */
8528 		return __check_ptr_off_reg(env, reg, regno, true);
8529 	default:
8530 		return __check_ptr_off_reg(env, reg, regno, false);
8531 	}
8532 }
8533 
8534 static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env,
8535 						const struct bpf_func_proto *fn,
8536 						struct bpf_reg_state *regs)
8537 {
8538 	struct bpf_reg_state *state = NULL;
8539 	int i;
8540 
8541 	for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++)
8542 		if (arg_type_is_dynptr(fn->arg_type[i])) {
8543 			if (state) {
8544 				verbose(env, "verifier internal error: multiple dynptr args\n");
8545 				return NULL;
8546 			}
8547 			state = &regs[BPF_REG_1 + i];
8548 		}
8549 
8550 	if (!state)
8551 		verbose(env, "verifier internal error: no dynptr arg found\n");
8552 
8553 	return state;
8554 }
8555 
8556 static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
8557 {
8558 	struct bpf_func_state *state = func(env, reg);
8559 	int spi;
8560 
8561 	if (reg->type == CONST_PTR_TO_DYNPTR)
8562 		return reg->id;
8563 	spi = dynptr_get_spi(env, reg);
8564 	if (spi < 0)
8565 		return spi;
8566 	return state->stack[spi].spilled_ptr.id;
8567 }
8568 
8569 static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
8570 {
8571 	struct bpf_func_state *state = func(env, reg);
8572 	int spi;
8573 
8574 	if (reg->type == CONST_PTR_TO_DYNPTR)
8575 		return reg->ref_obj_id;
8576 	spi = dynptr_get_spi(env, reg);
8577 	if (spi < 0)
8578 		return spi;
8579 	return state->stack[spi].spilled_ptr.ref_obj_id;
8580 }
8581 
8582 static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env,
8583 					    struct bpf_reg_state *reg)
8584 {
8585 	struct bpf_func_state *state = func(env, reg);
8586 	int spi;
8587 
8588 	if (reg->type == CONST_PTR_TO_DYNPTR)
8589 		return reg->dynptr.type;
8590 
8591 	spi = __get_spi(reg->off);
8592 	if (spi < 0) {
8593 		verbose(env, "verifier internal error: invalid spi when querying dynptr type\n");
8594 		return BPF_DYNPTR_TYPE_INVALID;
8595 	}
8596 
8597 	return state->stack[spi].spilled_ptr.dynptr.type;
8598 }
8599 
8600 static int check_reg_const_str(struct bpf_verifier_env *env,
8601 			       struct bpf_reg_state *reg, u32 regno)
8602 {
8603 	struct bpf_map *map = reg->map_ptr;
8604 	int err;
8605 	int map_off;
8606 	u64 map_addr;
8607 	char *str_ptr;
8608 
8609 	if (reg->type != PTR_TO_MAP_VALUE)
8610 		return -EINVAL;
8611 
8612 	if (!bpf_map_is_rdonly(map)) {
8613 		verbose(env, "R%d does not point to a readonly map'\n", regno);
8614 		return -EACCES;
8615 	}
8616 
8617 	if (!tnum_is_const(reg->var_off)) {
8618 		verbose(env, "R%d is not a constant address'\n", regno);
8619 		return -EACCES;
8620 	}
8621 
8622 	if (!map->ops->map_direct_value_addr) {
8623 		verbose(env, "no direct value access support for this map type\n");
8624 		return -EACCES;
8625 	}
8626 
8627 	err = check_map_access(env, regno, reg->off,
8628 			       map->value_size - reg->off, false,
8629 			       ACCESS_HELPER);
8630 	if (err)
8631 		return err;
8632 
8633 	map_off = reg->off + reg->var_off.value;
8634 	err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
8635 	if (err) {
8636 		verbose(env, "direct value access on string failed\n");
8637 		return err;
8638 	}
8639 
8640 	str_ptr = (char *)(long)(map_addr);
8641 	if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) {
8642 		verbose(env, "string is not zero-terminated\n");
8643 		return -EINVAL;
8644 	}
8645 	return 0;
8646 }
8647 
8648 static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
8649 			  struct bpf_call_arg_meta *meta,
8650 			  const struct bpf_func_proto *fn,
8651 			  int insn_idx)
8652 {
8653 	u32 regno = BPF_REG_1 + arg;
8654 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
8655 	enum bpf_arg_type arg_type = fn->arg_type[arg];
8656 	enum bpf_reg_type type = reg->type;
8657 	u32 *arg_btf_id = NULL;
8658 	int err = 0;
8659 
8660 	if (arg_type == ARG_DONTCARE)
8661 		return 0;
8662 
8663 	err = check_reg_arg(env, regno, SRC_OP);
8664 	if (err)
8665 		return err;
8666 
8667 	if (arg_type == ARG_ANYTHING) {
8668 		if (is_pointer_value(env, regno)) {
8669 			verbose(env, "R%d leaks addr into helper function\n",
8670 				regno);
8671 			return -EACCES;
8672 		}
8673 		return 0;
8674 	}
8675 
8676 	if (type_is_pkt_pointer(type) &&
8677 	    !may_access_direct_pkt_data(env, meta, BPF_READ)) {
8678 		verbose(env, "helper access to the packet is not allowed\n");
8679 		return -EACCES;
8680 	}
8681 
8682 	if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) {
8683 		err = resolve_map_arg_type(env, meta, &arg_type);
8684 		if (err)
8685 			return err;
8686 	}
8687 
8688 	if (register_is_null(reg) && type_may_be_null(arg_type))
8689 		/* A NULL register has a SCALAR_VALUE type, so skip
8690 		 * type checking.
8691 		 */
8692 		goto skip_type_check;
8693 
8694 	/* arg_btf_id and arg_size are in a union. */
8695 	if (base_type(arg_type) == ARG_PTR_TO_BTF_ID ||
8696 	    base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK)
8697 		arg_btf_id = fn->arg_btf_id[arg];
8698 
8699 	err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);
8700 	if (err)
8701 		return err;
8702 
8703 	err = check_func_arg_reg_off(env, reg, regno, arg_type);
8704 	if (err)
8705 		return err;
8706 
8707 skip_type_check:
8708 	if (arg_type_is_release(arg_type)) {
8709 		if (arg_type_is_dynptr(arg_type)) {
8710 			struct bpf_func_state *state = func(env, reg);
8711 			int spi;
8712 
8713 			/* Only dynptr created on stack can be released, thus
8714 			 * the get_spi and stack state checks for spilled_ptr
8715 			 * should only be done before process_dynptr_func for
8716 			 * PTR_TO_STACK.
8717 			 */
8718 			if (reg->type == PTR_TO_STACK) {
8719 				spi = dynptr_get_spi(env, reg);
8720 				if (spi < 0 || !state->stack[spi].spilled_ptr.ref_obj_id) {
8721 					verbose(env, "arg %d is an unacquired reference\n", regno);
8722 					return -EINVAL;
8723 				}
8724 			} else {
8725 				verbose(env, "cannot release unowned const bpf_dynptr\n");
8726 				return -EINVAL;
8727 			}
8728 		} else if (!reg->ref_obj_id && !register_is_null(reg)) {
8729 			verbose(env, "R%d must be referenced when passed to release function\n",
8730 				regno);
8731 			return -EINVAL;
8732 		}
8733 		if (meta->release_regno) {
8734 			verbose(env, "verifier internal error: more than one release argument\n");
8735 			return -EFAULT;
8736 		}
8737 		meta->release_regno = regno;
8738 	}
8739 
8740 	if (reg->ref_obj_id) {
8741 		if (meta->ref_obj_id) {
8742 			verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
8743 				regno, reg->ref_obj_id,
8744 				meta->ref_obj_id);
8745 			return -EFAULT;
8746 		}
8747 		meta->ref_obj_id = reg->ref_obj_id;
8748 	}
8749 
8750 	switch (base_type(arg_type)) {
8751 	case ARG_CONST_MAP_PTR:
8752 		/* bpf_map_xxx(map_ptr) call: remember that map_ptr */
8753 		if (meta->map_ptr) {
8754 			/* Use map_uid (which is unique id of inner map) to reject:
8755 			 * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
8756 			 * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
8757 			 * if (inner_map1 && inner_map2) {
8758 			 *     timer = bpf_map_lookup_elem(inner_map1);
8759 			 *     if (timer)
8760 			 *         // mismatch would have been allowed
8761 			 *         bpf_timer_init(timer, inner_map2);
8762 			 * }
8763 			 *
8764 			 * Comparing map_ptr is enough to distinguish normal and outer maps.
8765 			 */
8766 			if (meta->map_ptr != reg->map_ptr ||
8767 			    meta->map_uid != reg->map_uid) {
8768 				verbose(env,
8769 					"timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n",
8770 					meta->map_uid, reg->map_uid);
8771 				return -EINVAL;
8772 			}
8773 		}
8774 		meta->map_ptr = reg->map_ptr;
8775 		meta->map_uid = reg->map_uid;
8776 		break;
8777 	case ARG_PTR_TO_MAP_KEY:
8778 		/* bpf_map_xxx(..., map_ptr, ..., key) call:
8779 		 * check that [key, key + map->key_size) are within
8780 		 * stack limits and initialized
8781 		 */
8782 		if (!meta->map_ptr) {
8783 			/* in function declaration map_ptr must come before
8784 			 * map_key, so that it's verified and known before
8785 			 * we have to check map_key here. Otherwise it means
8786 			 * that kernel subsystem misconfigured verifier
8787 			 */
8788 			verbose(env, "invalid map_ptr to access map->key\n");
8789 			return -EACCES;
8790 		}
8791 		err = check_helper_mem_access(env, regno,
8792 					      meta->map_ptr->key_size, false,
8793 					      NULL);
8794 		break;
8795 	case ARG_PTR_TO_MAP_VALUE:
8796 		if (type_may_be_null(arg_type) && register_is_null(reg))
8797 			return 0;
8798 
8799 		/* bpf_map_xxx(..., map_ptr, ..., value) call:
8800 		 * check [value, value + map->value_size) validity
8801 		 */
8802 		if (!meta->map_ptr) {
8803 			/* kernel subsystem misconfigured verifier */
8804 			verbose(env, "invalid map_ptr to access map->value\n");
8805 			return -EACCES;
8806 		}
8807 		meta->raw_mode = arg_type & MEM_UNINIT;
8808 		err = check_helper_mem_access(env, regno,
8809 					      meta->map_ptr->value_size, false,
8810 					      meta);
8811 		break;
8812 	case ARG_PTR_TO_PERCPU_BTF_ID:
8813 		if (!reg->btf_id) {
8814 			verbose(env, "Helper has invalid btf_id in R%d\n", regno);
8815 			return -EACCES;
8816 		}
8817 		meta->ret_btf = reg->btf;
8818 		meta->ret_btf_id = reg->btf_id;
8819 		break;
8820 	case ARG_PTR_TO_SPIN_LOCK:
8821 		if (in_rbtree_lock_required_cb(env)) {
8822 			verbose(env, "can't spin_{lock,unlock} in rbtree cb\n");
8823 			return -EACCES;
8824 		}
8825 		if (meta->func_id == BPF_FUNC_spin_lock) {
8826 			err = process_spin_lock(env, regno, true);
8827 			if (err)
8828 				return err;
8829 		} else if (meta->func_id == BPF_FUNC_spin_unlock) {
8830 			err = process_spin_lock(env, regno, false);
8831 			if (err)
8832 				return err;
8833 		} else {
8834 			verbose(env, "verifier internal error\n");
8835 			return -EFAULT;
8836 		}
8837 		break;
8838 	case ARG_PTR_TO_TIMER:
8839 		err = process_timer_func(env, regno, meta);
8840 		if (err)
8841 			return err;
8842 		break;
8843 	case ARG_PTR_TO_FUNC:
8844 		meta->subprogno = reg->subprogno;
8845 		break;
8846 	case ARG_PTR_TO_MEM:
8847 		/* The access to this pointer is only checked when we hit the
8848 		 * next is_mem_size argument below.
8849 		 */
8850 		meta->raw_mode = arg_type & MEM_UNINIT;
8851 		if (arg_type & MEM_FIXED_SIZE) {
8852 			err = check_helper_mem_access(env, regno,
8853 						      fn->arg_size[arg], false,
8854 						      meta);
8855 		}
8856 		break;
8857 	case ARG_CONST_SIZE:
8858 		err = check_mem_size_reg(env, reg, regno, false, meta);
8859 		break;
8860 	case ARG_CONST_SIZE_OR_ZERO:
8861 		err = check_mem_size_reg(env, reg, regno, true, meta);
8862 		break;
8863 	case ARG_PTR_TO_DYNPTR:
8864 		err = process_dynptr_func(env, regno, insn_idx, arg_type, 0);
8865 		if (err)
8866 			return err;
8867 		break;
8868 	case ARG_CONST_ALLOC_SIZE_OR_ZERO:
8869 		if (!tnum_is_const(reg->var_off)) {
8870 			verbose(env, "R%d is not a known constant'\n",
8871 				regno);
8872 			return -EACCES;
8873 		}
8874 		meta->mem_size = reg->var_off.value;
8875 		err = mark_chain_precision(env, regno);
8876 		if (err)
8877 			return err;
8878 		break;
8879 	case ARG_PTR_TO_INT:
8880 	case ARG_PTR_TO_LONG:
8881 	{
8882 		int size = int_ptr_type_to_size(arg_type);
8883 
8884 		err = check_helper_mem_access(env, regno, size, false, meta);
8885 		if (err)
8886 			return err;
8887 		err = check_ptr_alignment(env, reg, 0, size, true);
8888 		break;
8889 	}
8890 	case ARG_PTR_TO_CONST_STR:
8891 	{
8892 		err = check_reg_const_str(env, reg, regno);
8893 		if (err)
8894 			return err;
8895 		break;
8896 	}
8897 	case ARG_PTR_TO_KPTR:
8898 		err = process_kptr_func(env, regno, meta);
8899 		if (err)
8900 			return err;
8901 		break;
8902 	}
8903 
8904 	return err;
8905 }
8906 
8907 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
8908 {
8909 	enum bpf_attach_type eatype = env->prog->expected_attach_type;
8910 	enum bpf_prog_type type = resolve_prog_type(env->prog);
8911 
8912 	if (func_id != BPF_FUNC_map_update_elem &&
8913 	    func_id != BPF_FUNC_map_delete_elem)
8914 		return false;
8915 
8916 	/* It's not possible to get access to a locked struct sock in these
8917 	 * contexts, so updating is safe.
8918 	 */
8919 	switch (type) {
8920 	case BPF_PROG_TYPE_TRACING:
8921 		if (eatype == BPF_TRACE_ITER)
8922 			return true;
8923 		break;
8924 	case BPF_PROG_TYPE_SOCK_OPS:
8925 		/* map_update allowed only via dedicated helpers with event type checks */
8926 		if (func_id == BPF_FUNC_map_delete_elem)
8927 			return true;
8928 		break;
8929 	case BPF_PROG_TYPE_SOCKET_FILTER:
8930 	case BPF_PROG_TYPE_SCHED_CLS:
8931 	case BPF_PROG_TYPE_SCHED_ACT:
8932 	case BPF_PROG_TYPE_XDP:
8933 	case BPF_PROG_TYPE_SK_REUSEPORT:
8934 	case BPF_PROG_TYPE_FLOW_DISSECTOR:
8935 	case BPF_PROG_TYPE_SK_LOOKUP:
8936 		return true;
8937 	default:
8938 		break;
8939 	}
8940 
8941 	verbose(env, "cannot update sockmap in this context\n");
8942 	return false;
8943 }
8944 
8945 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)
8946 {
8947 	return env->prog->jit_requested &&
8948 	       bpf_jit_supports_subprog_tailcalls();
8949 }
8950 
8951 static int check_map_func_compatibility(struct bpf_verifier_env *env,
8952 					struct bpf_map *map, int func_id)
8953 {
8954 	if (!map)
8955 		return 0;
8956 
8957 	/* We need a two way check, first is from map perspective ... */
8958 	switch (map->map_type) {
8959 	case BPF_MAP_TYPE_PROG_ARRAY:
8960 		if (func_id != BPF_FUNC_tail_call)
8961 			goto error;
8962 		break;
8963 	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
8964 		if (func_id != BPF_FUNC_perf_event_read &&
8965 		    func_id != BPF_FUNC_perf_event_output &&
8966 		    func_id != BPF_FUNC_skb_output &&
8967 		    func_id != BPF_FUNC_perf_event_read_value &&
8968 		    func_id != BPF_FUNC_xdp_output)
8969 			goto error;
8970 		break;
8971 	case BPF_MAP_TYPE_RINGBUF:
8972 		if (func_id != BPF_FUNC_ringbuf_output &&
8973 		    func_id != BPF_FUNC_ringbuf_reserve &&
8974 		    func_id != BPF_FUNC_ringbuf_query &&
8975 		    func_id != BPF_FUNC_ringbuf_reserve_dynptr &&
8976 		    func_id != BPF_FUNC_ringbuf_submit_dynptr &&
8977 		    func_id != BPF_FUNC_ringbuf_discard_dynptr)
8978 			goto error;
8979 		break;
8980 	case BPF_MAP_TYPE_USER_RINGBUF:
8981 		if (func_id != BPF_FUNC_user_ringbuf_drain)
8982 			goto error;
8983 		break;
8984 	case BPF_MAP_TYPE_STACK_TRACE:
8985 		if (func_id != BPF_FUNC_get_stackid)
8986 			goto error;
8987 		break;
8988 	case BPF_MAP_TYPE_CGROUP_ARRAY:
8989 		if (func_id != BPF_FUNC_skb_under_cgroup &&
8990 		    func_id != BPF_FUNC_current_task_under_cgroup)
8991 			goto error;
8992 		break;
8993 	case BPF_MAP_TYPE_CGROUP_STORAGE:
8994 	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
8995 		if (func_id != BPF_FUNC_get_local_storage)
8996 			goto error;
8997 		break;
8998 	case BPF_MAP_TYPE_DEVMAP:
8999 	case BPF_MAP_TYPE_DEVMAP_HASH:
9000 		if (func_id != BPF_FUNC_redirect_map &&
9001 		    func_id != BPF_FUNC_map_lookup_elem)
9002 			goto error;
9003 		break;
9004 	/* Restrict bpf side of cpumap and xskmap, open when use-cases
9005 	 * appear.
9006 	 */
9007 	case BPF_MAP_TYPE_CPUMAP:
9008 		if (func_id != BPF_FUNC_redirect_map)
9009 			goto error;
9010 		break;
9011 	case BPF_MAP_TYPE_XSKMAP:
9012 		if (func_id != BPF_FUNC_redirect_map &&
9013 		    func_id != BPF_FUNC_map_lookup_elem)
9014 			goto error;
9015 		break;
9016 	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
9017 	case BPF_MAP_TYPE_HASH_OF_MAPS:
9018 		if (func_id != BPF_FUNC_map_lookup_elem)
9019 			goto error;
9020 		break;
9021 	case BPF_MAP_TYPE_SOCKMAP:
9022 		if (func_id != BPF_FUNC_sk_redirect_map &&
9023 		    func_id != BPF_FUNC_sock_map_update &&
9024 		    func_id != BPF_FUNC_msg_redirect_map &&
9025 		    func_id != BPF_FUNC_sk_select_reuseport &&
9026 		    func_id != BPF_FUNC_map_lookup_elem &&
9027 		    !may_update_sockmap(env, func_id))
9028 			goto error;
9029 		break;
9030 	case BPF_MAP_TYPE_SOCKHASH:
9031 		if (func_id != BPF_FUNC_sk_redirect_hash &&
9032 		    func_id != BPF_FUNC_sock_hash_update &&
9033 		    func_id != BPF_FUNC_msg_redirect_hash &&
9034 		    func_id != BPF_FUNC_sk_select_reuseport &&
9035 		    func_id != BPF_FUNC_map_lookup_elem &&
9036 		    !may_update_sockmap(env, func_id))
9037 			goto error;
9038 		break;
9039 	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
9040 		if (func_id != BPF_FUNC_sk_select_reuseport)
9041 			goto error;
9042 		break;
9043 	case BPF_MAP_TYPE_QUEUE:
9044 	case BPF_MAP_TYPE_STACK:
9045 		if (func_id != BPF_FUNC_map_peek_elem &&
9046 		    func_id != BPF_FUNC_map_pop_elem &&
9047 		    func_id != BPF_FUNC_map_push_elem)
9048 			goto error;
9049 		break;
9050 	case BPF_MAP_TYPE_SK_STORAGE:
9051 		if (func_id != BPF_FUNC_sk_storage_get &&
9052 		    func_id != BPF_FUNC_sk_storage_delete &&
9053 		    func_id != BPF_FUNC_kptr_xchg)
9054 			goto error;
9055 		break;
9056 	case BPF_MAP_TYPE_INODE_STORAGE:
9057 		if (func_id != BPF_FUNC_inode_storage_get &&
9058 		    func_id != BPF_FUNC_inode_storage_delete &&
9059 		    func_id != BPF_FUNC_kptr_xchg)
9060 			goto error;
9061 		break;
9062 	case BPF_MAP_TYPE_TASK_STORAGE:
9063 		if (func_id != BPF_FUNC_task_storage_get &&
9064 		    func_id != BPF_FUNC_task_storage_delete &&
9065 		    func_id != BPF_FUNC_kptr_xchg)
9066 			goto error;
9067 		break;
9068 	case BPF_MAP_TYPE_CGRP_STORAGE:
9069 		if (func_id != BPF_FUNC_cgrp_storage_get &&
9070 		    func_id != BPF_FUNC_cgrp_storage_delete &&
9071 		    func_id != BPF_FUNC_kptr_xchg)
9072 			goto error;
9073 		break;
9074 	case BPF_MAP_TYPE_BLOOM_FILTER:
9075 		if (func_id != BPF_FUNC_map_peek_elem &&
9076 		    func_id != BPF_FUNC_map_push_elem)
9077 			goto error;
9078 		break;
9079 	default:
9080 		break;
9081 	}
9082 
9083 	/* ... and second from the function itself. */
9084 	switch (func_id) {
9085 	case BPF_FUNC_tail_call:
9086 		if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
9087 			goto error;
9088 		if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {
9089 			verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
9090 			return -EINVAL;
9091 		}
9092 		break;
9093 	case BPF_FUNC_perf_event_read:
9094 	case BPF_FUNC_perf_event_output:
9095 	case BPF_FUNC_perf_event_read_value:
9096 	case BPF_FUNC_skb_output:
9097 	case BPF_FUNC_xdp_output:
9098 		if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
9099 			goto error;
9100 		break;
9101 	case BPF_FUNC_ringbuf_output:
9102 	case BPF_FUNC_ringbuf_reserve:
9103 	case BPF_FUNC_ringbuf_query:
9104 	case BPF_FUNC_ringbuf_reserve_dynptr:
9105 	case BPF_FUNC_ringbuf_submit_dynptr:
9106 	case BPF_FUNC_ringbuf_discard_dynptr:
9107 		if (map->map_type != BPF_MAP_TYPE_RINGBUF)
9108 			goto error;
9109 		break;
9110 	case BPF_FUNC_user_ringbuf_drain:
9111 		if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF)
9112 			goto error;
9113 		break;
9114 	case BPF_FUNC_get_stackid:
9115 		if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
9116 			goto error;
9117 		break;
9118 	case BPF_FUNC_current_task_under_cgroup:
9119 	case BPF_FUNC_skb_under_cgroup:
9120 		if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
9121 			goto error;
9122 		break;
9123 	case BPF_FUNC_redirect_map:
9124 		if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
9125 		    map->map_type != BPF_MAP_TYPE_DEVMAP_HASH &&
9126 		    map->map_type != BPF_MAP_TYPE_CPUMAP &&
9127 		    map->map_type != BPF_MAP_TYPE_XSKMAP)
9128 			goto error;
9129 		break;
9130 	case BPF_FUNC_sk_redirect_map:
9131 	case BPF_FUNC_msg_redirect_map:
9132 	case BPF_FUNC_sock_map_update:
9133 		if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
9134 			goto error;
9135 		break;
9136 	case BPF_FUNC_sk_redirect_hash:
9137 	case BPF_FUNC_msg_redirect_hash:
9138 	case BPF_FUNC_sock_hash_update:
9139 		if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
9140 			goto error;
9141 		break;
9142 	case BPF_FUNC_get_local_storage:
9143 		if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
9144 		    map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
9145 			goto error;
9146 		break;
9147 	case BPF_FUNC_sk_select_reuseport:
9148 		if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY &&
9149 		    map->map_type != BPF_MAP_TYPE_SOCKMAP &&
9150 		    map->map_type != BPF_MAP_TYPE_SOCKHASH)
9151 			goto error;
9152 		break;
9153 	case BPF_FUNC_map_pop_elem:
9154 		if (map->map_type != BPF_MAP_TYPE_QUEUE &&
9155 		    map->map_type != BPF_MAP_TYPE_STACK)
9156 			goto error;
9157 		break;
9158 	case BPF_FUNC_map_peek_elem:
9159 	case BPF_FUNC_map_push_elem:
9160 		if (map->map_type != BPF_MAP_TYPE_QUEUE &&
9161 		    map->map_type != BPF_MAP_TYPE_STACK &&
9162 		    map->map_type != BPF_MAP_TYPE_BLOOM_FILTER)
9163 			goto error;
9164 		break;
9165 	case BPF_FUNC_map_lookup_percpu_elem:
9166 		if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
9167 		    map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
9168 		    map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH)
9169 			goto error;
9170 		break;
9171 	case BPF_FUNC_sk_storage_get:
9172 	case BPF_FUNC_sk_storage_delete:
9173 		if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
9174 			goto error;
9175 		break;
9176 	case BPF_FUNC_inode_storage_get:
9177 	case BPF_FUNC_inode_storage_delete:
9178 		if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
9179 			goto error;
9180 		break;
9181 	case BPF_FUNC_task_storage_get:
9182 	case BPF_FUNC_task_storage_delete:
9183 		if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
9184 			goto error;
9185 		break;
9186 	case BPF_FUNC_cgrp_storage_get:
9187 	case BPF_FUNC_cgrp_storage_delete:
9188 		if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE)
9189 			goto error;
9190 		break;
9191 	default:
9192 		break;
9193 	}
9194 
9195 	return 0;
9196 error:
9197 	verbose(env, "cannot pass map_type %d into func %s#%d\n",
9198 		map->map_type, func_id_name(func_id), func_id);
9199 	return -EINVAL;
9200 }
9201 
9202 static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
9203 {
9204 	int count = 0;
9205 
9206 	if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM)
9207 		count++;
9208 	if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM)
9209 		count++;
9210 	if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM)
9211 		count++;
9212 	if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM)
9213 		count++;
9214 	if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM)
9215 		count++;
9216 
9217 	/* We only support one arg being in raw mode at the moment,
9218 	 * which is sufficient for the helper functions we have
9219 	 * right now.
9220 	 */
9221 	return count <= 1;
9222 }
9223 
9224 static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg)
9225 {
9226 	bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE;
9227 	bool has_size = fn->arg_size[arg] != 0;
9228 	bool is_next_size = false;
9229 
9230 	if (arg + 1 < ARRAY_SIZE(fn->arg_type))
9231 		is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]);
9232 
9233 	if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM)
9234 		return is_next_size;
9235 
9236 	return has_size == is_next_size || is_next_size == is_fixed;
9237 }
9238 
9239 static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
9240 {
9241 	/* bpf_xxx(..., buf, len) call will access 'len'
9242 	 * bytes from memory 'buf'. Both arg types need
9243 	 * to be paired, so make sure there's no buggy
9244 	 * helper function specification.
9245 	 */
9246 	if (arg_type_is_mem_size(fn->arg1_type) ||
9247 	    check_args_pair_invalid(fn, 0) ||
9248 	    check_args_pair_invalid(fn, 1) ||
9249 	    check_args_pair_invalid(fn, 2) ||
9250 	    check_args_pair_invalid(fn, 3) ||
9251 	    check_args_pair_invalid(fn, 4))
9252 		return false;
9253 
9254 	return true;
9255 }
9256 
9257 static bool check_btf_id_ok(const struct bpf_func_proto *fn)
9258 {
9259 	int i;
9260 
9261 	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
9262 		if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID)
9263 			return !!fn->arg_btf_id[i];
9264 		if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK)
9265 			return fn->arg_btf_id[i] == BPF_PTR_POISON;
9266 		if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] &&
9267 		    /* arg_btf_id and arg_size are in a union. */
9268 		    (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM ||
9269 		     !(fn->arg_type[i] & MEM_FIXED_SIZE)))
9270 			return false;
9271 	}
9272 
9273 	return true;
9274 }
9275 
9276 static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
9277 {
9278 	return check_raw_mode_ok(fn) &&
9279 	       check_arg_pair_ok(fn) &&
9280 	       check_btf_id_ok(fn) ? 0 : -EINVAL;
9281 }
9282 
9283 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
9284  * are now invalid, so turn them into unknown SCALAR_VALUE.
9285  *
9286  * This also applies to dynptr slices belonging to skb and xdp dynptrs,
9287  * since these slices point to packet data.
9288  */
9289 static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
9290 {
9291 	struct bpf_func_state *state;
9292 	struct bpf_reg_state *reg;
9293 
9294 	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
9295 		if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))
9296 			mark_reg_invalid(env, reg);
9297 	}));
9298 }
9299 
9300 enum {
9301 	AT_PKT_END = -1,
9302 	BEYOND_PKT_END = -2,
9303 };
9304 
9305 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open)
9306 {
9307 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
9308 	struct bpf_reg_state *reg = &state->regs[regn];
9309 
9310 	if (reg->type != PTR_TO_PACKET)
9311 		/* PTR_TO_PACKET_META is not supported yet */
9312 		return;
9313 
9314 	/* The 'reg' is pkt > pkt_end or pkt >= pkt_end.
9315 	 * How far beyond pkt_end it goes is unknown.
9316 	 * if (!range_open) it's the case of pkt >= pkt_end
9317 	 * if (range_open) it's the case of pkt > pkt_end
9318 	 * hence this pointer is at least 1 byte bigger than pkt_end
9319 	 */
9320 	if (range_open)
9321 		reg->range = BEYOND_PKT_END;
9322 	else
9323 		reg->range = AT_PKT_END;
9324 }
9325 
9326 /* The pointer with the specified id has released its reference to kernel
9327  * resources. Identify all copies of the same pointer and clear the reference.
9328  */
9329 static int release_reference(struct bpf_verifier_env *env,
9330 			     int ref_obj_id)
9331 {
9332 	struct bpf_func_state *state;
9333 	struct bpf_reg_state *reg;
9334 	int err;
9335 
9336 	err = release_reference_state(cur_func(env), ref_obj_id);
9337 	if (err)
9338 		return err;
9339 
9340 	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
9341 		if (reg->ref_obj_id == ref_obj_id)
9342 			mark_reg_invalid(env, reg);
9343 	}));
9344 
9345 	return 0;
9346 }
9347 
9348 static void invalidate_non_owning_refs(struct bpf_verifier_env *env)
9349 {
9350 	struct bpf_func_state *unused;
9351 	struct bpf_reg_state *reg;
9352 
9353 	bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
9354 		if (type_is_non_owning_ref(reg->type))
9355 			mark_reg_invalid(env, reg);
9356 	}));
9357 }
9358 
9359 static void clear_caller_saved_regs(struct bpf_verifier_env *env,
9360 				    struct bpf_reg_state *regs)
9361 {
9362 	int i;
9363 
9364 	/* after the call registers r0 - r5 were scratched */
9365 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
9366 		mark_reg_not_init(env, regs, caller_saved[i]);
9367 		__check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK);
9368 	}
9369 }
9370 
9371 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
9372 				   struct bpf_func_state *caller,
9373 				   struct bpf_func_state *callee,
9374 				   int insn_idx);
9375 
9376 static int set_callee_state(struct bpf_verifier_env *env,
9377 			    struct bpf_func_state *caller,
9378 			    struct bpf_func_state *callee, int insn_idx);
9379 
9380 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite,
9381 			    set_callee_state_fn set_callee_state_cb,
9382 			    struct bpf_verifier_state *state)
9383 {
9384 	struct bpf_func_state *caller, *callee;
9385 	int err;
9386 
9387 	if (state->curframe + 1 >= MAX_CALL_FRAMES) {
9388 		verbose(env, "the call stack of %d frames is too deep\n",
9389 			state->curframe + 2);
9390 		return -E2BIG;
9391 	}
9392 
9393 	if (state->frame[state->curframe + 1]) {
9394 		verbose(env, "verifier bug. Frame %d already allocated\n",
9395 			state->curframe + 1);
9396 		return -EFAULT;
9397 	}
9398 
9399 	caller = state->frame[state->curframe];
9400 	callee = kzalloc(sizeof(*callee), GFP_KERNEL);
9401 	if (!callee)
9402 		return -ENOMEM;
9403 	state->frame[state->curframe + 1] = callee;
9404 
9405 	/* callee cannot access r0, r6 - r9 for reading and has to write
9406 	 * into its own stack before reading from it.
9407 	 * callee can read/write into caller's stack
9408 	 */
9409 	init_func_state(env, callee,
9410 			/* remember the callsite, it will be used by bpf_exit */
9411 			callsite,
9412 			state->curframe + 1 /* frameno within this callchain */,
9413 			subprog /* subprog number within this prog */);
9414 	/* Transfer references to the callee */
9415 	err = copy_reference_state(callee, caller);
9416 	err = err ?: set_callee_state_cb(env, caller, callee, callsite);
9417 	if (err)
9418 		goto err_out;
9419 
9420 	/* only increment it after check_reg_arg() finished */
9421 	state->curframe++;
9422 
9423 	return 0;
9424 
9425 err_out:
9426 	free_func_state(callee);
9427 	state->frame[state->curframe + 1] = NULL;
9428 	return err;
9429 }
9430 
9431 static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
9432 				    const struct btf *btf,
9433 				    struct bpf_reg_state *regs)
9434 {
9435 	struct bpf_subprog_info *sub = subprog_info(env, subprog);
9436 	struct bpf_verifier_log *log = &env->log;
9437 	u32 i;
9438 	int ret;
9439 
9440 	ret = btf_prepare_func_args(env, subprog);
9441 	if (ret)
9442 		return ret;
9443 
9444 	/* check that BTF function arguments match actual types that the
9445 	 * verifier sees.
9446 	 */
9447 	for (i = 0; i < sub->arg_cnt; i++) {
9448 		u32 regno = i + 1;
9449 		struct bpf_reg_state *reg = &regs[regno];
9450 		struct bpf_subprog_arg_info *arg = &sub->args[i];
9451 
9452 		if (arg->arg_type == ARG_ANYTHING) {
9453 			if (reg->type != SCALAR_VALUE) {
9454 				bpf_log(log, "R%d is not a scalar\n", regno);
9455 				return -EINVAL;
9456 			}
9457 		} else if (arg->arg_type == ARG_PTR_TO_CTX) {
9458 			ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);
9459 			if (ret < 0)
9460 				return ret;
9461 			/* If function expects ctx type in BTF check that caller
9462 			 * is passing PTR_TO_CTX.
9463 			 */
9464 			if (reg->type != PTR_TO_CTX) {
9465 				bpf_log(log, "arg#%d expects pointer to ctx\n", i);
9466 				return -EINVAL;
9467 			}
9468 		} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
9469 			ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);
9470 			if (ret < 0)
9471 				return ret;
9472 			if (check_mem_reg(env, reg, regno, arg->mem_size))
9473 				return -EINVAL;
9474 			if (!(arg->arg_type & PTR_MAYBE_NULL) && (reg->type & PTR_MAYBE_NULL)) {
9475 				bpf_log(log, "arg#%d is expected to be non-NULL\n", i);
9476 				return -EINVAL;
9477 			}
9478 		} else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) {
9479 			/*
9480 			 * Can pass any value and the kernel won't crash, but
9481 			 * only PTR_TO_ARENA or SCALAR make sense. Everything
9482 			 * else is a bug in the bpf program. Point it out to
9483 			 * the user at the verification time instead of
9484 			 * run-time debug nightmare.
9485 			 */
9486 			if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) {
9487 				bpf_log(log, "R%d is not a pointer to arena or scalar.\n", regno);
9488 				return -EINVAL;
9489 			}
9490 		} else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) {
9491 			ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_DYNPTR);
9492 			if (ret)
9493 				return ret;
9494 
9495 			ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0);
9496 			if (ret)
9497 				return ret;
9498 		} else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
9499 			struct bpf_call_arg_meta meta;
9500 			int err;
9501 
9502 			if (register_is_null(reg) && type_may_be_null(arg->arg_type))
9503 				continue;
9504 
9505 			memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */
9506 			err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta);
9507 			err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type);
9508 			if (err)
9509 				return err;
9510 		} else {
9511 			bpf_log(log, "verifier bug: unrecognized arg#%d type %d\n",
9512 				i, arg->arg_type);
9513 			return -EFAULT;
9514 		}
9515 	}
9516 
9517 	return 0;
9518 }
9519 
9520 /* Compare BTF of a function call with given bpf_reg_state.
9521  * Returns:
9522  * EFAULT - there is a verifier bug. Abort verification.
9523  * EINVAL - there is a type mismatch or BTF is not available.
9524  * 0 - BTF matches with what bpf_reg_state expects.
9525  * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
9526  */
9527 static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
9528 				  struct bpf_reg_state *regs)
9529 {
9530 	struct bpf_prog *prog = env->prog;
9531 	struct btf *btf = prog->aux->btf;
9532 	u32 btf_id;
9533 	int err;
9534 
9535 	if (!prog->aux->func_info)
9536 		return -EINVAL;
9537 
9538 	btf_id = prog->aux->func_info[subprog].type_id;
9539 	if (!btf_id)
9540 		return -EFAULT;
9541 
9542 	if (prog->aux->func_info_aux[subprog].unreliable)
9543 		return -EINVAL;
9544 
9545 	err = btf_check_func_arg_match(env, subprog, btf, regs);
9546 	/* Compiler optimizations can remove arguments from static functions
9547 	 * or mismatched type can be passed into a global function.
9548 	 * In such cases mark the function as unreliable from BTF point of view.
9549 	 */
9550 	if (err)
9551 		prog->aux->func_info_aux[subprog].unreliable = true;
9552 	return err;
9553 }
9554 
9555 static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
9556 			      int insn_idx, int subprog,
9557 			      set_callee_state_fn set_callee_state_cb)
9558 {
9559 	struct bpf_verifier_state *state = env->cur_state, *callback_state;
9560 	struct bpf_func_state *caller, *callee;
9561 	int err;
9562 
9563 	caller = state->frame[state->curframe];
9564 	err = btf_check_subprog_call(env, subprog, caller->regs);
9565 	if (err == -EFAULT)
9566 		return err;
9567 
9568 	/* set_callee_state is used for direct subprog calls, but we are
9569 	 * interested in validating only BPF helpers that can call subprogs as
9570 	 * callbacks
9571 	 */
9572 	env->subprog_info[subprog].is_cb = true;
9573 	if (bpf_pseudo_kfunc_call(insn) &&
9574 	    !is_callback_calling_kfunc(insn->imm)) {
9575 		verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n",
9576 			func_id_name(insn->imm), insn->imm);
9577 		return -EFAULT;
9578 	} else if (!bpf_pseudo_kfunc_call(insn) &&
9579 		   !is_callback_calling_function(insn->imm)) { /* helper */
9580 		verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n",
9581 			func_id_name(insn->imm), insn->imm);
9582 		return -EFAULT;
9583 	}
9584 
9585 	if (is_async_callback_calling_insn(insn)) {
9586 		struct bpf_verifier_state *async_cb;
9587 
9588 		/* there is no real recursion here. timer and workqueue callbacks are async */
9589 		env->subprog_info[subprog].is_async_cb = true;
9590 		async_cb = push_async_cb(env, env->subprog_info[subprog].start,
9591 					 insn_idx, subprog,
9592 					 is_bpf_wq_set_callback_impl_kfunc(insn->imm));
9593 		if (!async_cb)
9594 			return -EFAULT;
9595 		callee = async_cb->frame[0];
9596 		callee->async_entry_cnt = caller->async_entry_cnt + 1;
9597 
9598 		/* Convert bpf_timer_set_callback() args into timer callback args */
9599 		err = set_callee_state_cb(env, caller, callee, insn_idx);
9600 		if (err)
9601 			return err;
9602 
9603 		return 0;
9604 	}
9605 
9606 	/* for callback functions enqueue entry to callback and
9607 	 * proceed with next instruction within current frame.
9608 	 */
9609 	callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);
9610 	if (!callback_state)
9611 		return -ENOMEM;
9612 
9613 	err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,
9614 			       callback_state);
9615 	if (err)
9616 		return err;
9617 
9618 	callback_state->callback_unroll_depth++;
9619 	callback_state->frame[callback_state->curframe - 1]->callback_depth++;
9620 	caller->callback_depth = 0;
9621 	return 0;
9622 }
9623 
9624 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
9625 			   int *insn_idx)
9626 {
9627 	struct bpf_verifier_state *state = env->cur_state;
9628 	struct bpf_func_state *caller;
9629 	int err, subprog, target_insn;
9630 
9631 	target_insn = *insn_idx + insn->imm + 1;
9632 	subprog = find_subprog(env, target_insn);
9633 	if (subprog < 0) {
9634 		verbose(env, "verifier bug. No program starts at insn %d\n", target_insn);
9635 		return -EFAULT;
9636 	}
9637 
9638 	caller = state->frame[state->curframe];
9639 	err = btf_check_subprog_call(env, subprog, caller->regs);
9640 	if (err == -EFAULT)
9641 		return err;
9642 	if (subprog_is_global(env, subprog)) {
9643 		const char *sub_name = subprog_name(env, subprog);
9644 
9645 		/* Only global subprogs cannot be called with a lock held. */
9646 		if (env->cur_state->active_lock.ptr) {
9647 			verbose(env, "global function calls are not allowed while holding a lock,\n"
9648 				     "use static function instead\n");
9649 			return -EINVAL;
9650 		}
9651 
9652 		/* Only global subprogs cannot be called with preemption disabled. */
9653 		if (env->cur_state->active_preempt_lock) {
9654 			verbose(env, "global function calls are not allowed with preemption disabled,\n"
9655 				     "use static function instead\n");
9656 			return -EINVAL;
9657 		}
9658 
9659 		if (err) {
9660 			verbose(env, "Caller passes invalid args into func#%d ('%s')\n",
9661 				subprog, sub_name);
9662 			return err;
9663 		}
9664 
9665 		verbose(env, "Func#%d ('%s') is global and assumed valid.\n",
9666 			subprog, sub_name);
9667 		/* mark global subprog for verifying after main prog */
9668 		subprog_aux(env, subprog)->called = true;
9669 		clear_caller_saved_regs(env, caller->regs);
9670 
9671 		/* All global functions return a 64-bit SCALAR_VALUE */
9672 		mark_reg_unknown(env, caller->regs, BPF_REG_0);
9673 		caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
9674 
9675 		/* continue with next insn after call */
9676 		return 0;
9677 	}
9678 
9679 	/* for regular function entry setup new frame and continue
9680 	 * from that frame.
9681 	 */
9682 	err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);
9683 	if (err)
9684 		return err;
9685 
9686 	clear_caller_saved_regs(env, caller->regs);
9687 
9688 	/* and go analyze first insn of the callee */
9689 	*insn_idx = env->subprog_info[subprog].start - 1;
9690 
9691 	if (env->log.level & BPF_LOG_LEVEL) {
9692 		verbose(env, "caller:\n");
9693 		print_verifier_state(env, caller, true);
9694 		verbose(env, "callee:\n");
9695 		print_verifier_state(env, state->frame[state->curframe], true);
9696 	}
9697 
9698 	return 0;
9699 }
9700 
9701 int map_set_for_each_callback_args(struct bpf_verifier_env *env,
9702 				   struct bpf_func_state *caller,
9703 				   struct bpf_func_state *callee)
9704 {
9705 	/* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn,
9706 	 *      void *callback_ctx, u64 flags);
9707 	 * callback_fn(struct bpf_map *map, void *key, void *value,
9708 	 *      void *callback_ctx);
9709 	 */
9710 	callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
9711 
9712 	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
9713 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
9714 	callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr;
9715 
9716 	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
9717 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
9718 	callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;
9719 
9720 	/* pointer to stack or null */
9721 	callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
9722 
9723 	/* unused */
9724 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9725 	return 0;
9726 }
9727 
9728 static int set_callee_state(struct bpf_verifier_env *env,
9729 			    struct bpf_func_state *caller,
9730 			    struct bpf_func_state *callee, int insn_idx)
9731 {
9732 	int i;
9733 
9734 	/* copy r1 - r5 args that callee can access.  The copy includes parent
9735 	 * pointers, which connects us up to the liveness chain
9736 	 */
9737 	for (i = BPF_REG_1; i <= BPF_REG_5; i++)
9738 		callee->regs[i] = caller->regs[i];
9739 	return 0;
9740 }
9741 
9742 static int set_map_elem_callback_state(struct bpf_verifier_env *env,
9743 				       struct bpf_func_state *caller,
9744 				       struct bpf_func_state *callee,
9745 				       int insn_idx)
9746 {
9747 	struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];
9748 	struct bpf_map *map;
9749 	int err;
9750 
9751 	/* valid map_ptr and poison value does not matter */
9752 	map = insn_aux->map_ptr_state.map_ptr;
9753 	if (!map->ops->map_set_for_each_callback_args ||
9754 	    !map->ops->map_for_each_callback) {
9755 		verbose(env, "callback function not allowed for map\n");
9756 		return -ENOTSUPP;
9757 	}
9758 
9759 	err = map->ops->map_set_for_each_callback_args(env, caller, callee);
9760 	if (err)
9761 		return err;
9762 
9763 	callee->in_callback_fn = true;
9764 	callee->callback_ret_range = retval_range(0, 1);
9765 	return 0;
9766 }
9767 
9768 static int set_loop_callback_state(struct bpf_verifier_env *env,
9769 				   struct bpf_func_state *caller,
9770 				   struct bpf_func_state *callee,
9771 				   int insn_idx)
9772 {
9773 	/* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx,
9774 	 *	    u64 flags);
9775 	 * callback_fn(u32 index, void *callback_ctx);
9776 	 */
9777 	callee->regs[BPF_REG_1].type = SCALAR_VALUE;
9778 	callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
9779 
9780 	/* unused */
9781 	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
9782 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
9783 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9784 
9785 	callee->in_callback_fn = true;
9786 	callee->callback_ret_range = retval_range(0, 1);
9787 	return 0;
9788 }
9789 
9790 static int set_timer_callback_state(struct bpf_verifier_env *env,
9791 				    struct bpf_func_state *caller,
9792 				    struct bpf_func_state *callee,
9793 				    int insn_idx)
9794 {
9795 	struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr;
9796 
9797 	/* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn);
9798 	 * callback_fn(struct bpf_map *map, void *key, void *value);
9799 	 */
9800 	callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
9801 	__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
9802 	callee->regs[BPF_REG_1].map_ptr = map_ptr;
9803 
9804 	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
9805 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
9806 	callee->regs[BPF_REG_2].map_ptr = map_ptr;
9807 
9808 	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
9809 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
9810 	callee->regs[BPF_REG_3].map_ptr = map_ptr;
9811 
9812 	/* unused */
9813 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
9814 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9815 	callee->in_async_callback_fn = true;
9816 	callee->callback_ret_range = retval_range(0, 1);
9817 	return 0;
9818 }
9819 
9820 static int set_find_vma_callback_state(struct bpf_verifier_env *env,
9821 				       struct bpf_func_state *caller,
9822 				       struct bpf_func_state *callee,
9823 				       int insn_idx)
9824 {
9825 	/* bpf_find_vma(struct task_struct *task, u64 addr,
9826 	 *               void *callback_fn, void *callback_ctx, u64 flags)
9827 	 * (callback_fn)(struct task_struct *task,
9828 	 *               struct vm_area_struct *vma, void *callback_ctx);
9829 	 */
9830 	callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
9831 
9832 	callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID;
9833 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
9834 	callee->regs[BPF_REG_2].btf =  btf_vmlinux;
9835 	callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA];
9836 
9837 	/* pointer to stack or null */
9838 	callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4];
9839 
9840 	/* unused */
9841 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
9842 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9843 	callee->in_callback_fn = true;
9844 	callee->callback_ret_range = retval_range(0, 1);
9845 	return 0;
9846 }
9847 
9848 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env,
9849 					   struct bpf_func_state *caller,
9850 					   struct bpf_func_state *callee,
9851 					   int insn_idx)
9852 {
9853 	/* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void
9854 	 *			  callback_ctx, u64 flags);
9855 	 * callback_fn(const struct bpf_dynptr_t* dynptr, void *callback_ctx);
9856 	 */
9857 	__mark_reg_not_init(env, &callee->regs[BPF_REG_0]);
9858 	mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL);
9859 	callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
9860 
9861 	/* unused */
9862 	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
9863 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
9864 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9865 
9866 	callee->in_callback_fn = true;
9867 	callee->callback_ret_range = retval_range(0, 1);
9868 	return 0;
9869 }
9870 
9871 static int set_rbtree_add_callback_state(struct bpf_verifier_env *env,
9872 					 struct bpf_func_state *caller,
9873 					 struct bpf_func_state *callee,
9874 					 int insn_idx)
9875 {
9876 	/* void bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
9877 	 *                     bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b));
9878 	 *
9879 	 * 'struct bpf_rb_node *node' arg to bpf_rbtree_add_impl is the same PTR_TO_BTF_ID w/ offset
9880 	 * that 'less' callback args will be receiving. However, 'node' arg was release_reference'd
9881 	 * by this point, so look at 'root'
9882 	 */
9883 	struct btf_field *field;
9884 
9885 	field = reg_find_field_offset(&caller->regs[BPF_REG_1], caller->regs[BPF_REG_1].off,
9886 				      BPF_RB_ROOT);
9887 	if (!field || !field->graph_root.value_btf_id)
9888 		return -EFAULT;
9889 
9890 	mark_reg_graph_node(callee->regs, BPF_REG_1, &field->graph_root);
9891 	ref_set_non_owning(env, &callee->regs[BPF_REG_1]);
9892 	mark_reg_graph_node(callee->regs, BPF_REG_2, &field->graph_root);
9893 	ref_set_non_owning(env, &callee->regs[BPF_REG_2]);
9894 
9895 	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
9896 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
9897 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9898 	callee->in_callback_fn = true;
9899 	callee->callback_ret_range = retval_range(0, 1);
9900 	return 0;
9901 }
9902 
9903 static bool is_rbtree_lock_required_kfunc(u32 btf_id);
9904 
9905 /* Are we currently verifying the callback for a rbtree helper that must
9906  * be called with lock held? If so, no need to complain about unreleased
9907  * lock
9908  */
9909 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
9910 {
9911 	struct bpf_verifier_state *state = env->cur_state;
9912 	struct bpf_insn *insn = env->prog->insnsi;
9913 	struct bpf_func_state *callee;
9914 	int kfunc_btf_id;
9915 
9916 	if (!state->curframe)
9917 		return false;
9918 
9919 	callee = state->frame[state->curframe];
9920 
9921 	if (!callee->in_callback_fn)
9922 		return false;
9923 
9924 	kfunc_btf_id = insn[callee->callsite].imm;
9925 	return is_rbtree_lock_required_kfunc(kfunc_btf_id);
9926 }
9927 
9928 static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg)
9929 {
9930 	return range.minval <= reg->smin_value && reg->smax_value <= range.maxval;
9931 }
9932 
9933 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
9934 {
9935 	struct bpf_verifier_state *state = env->cur_state, *prev_st;
9936 	struct bpf_func_state *caller, *callee;
9937 	struct bpf_reg_state *r0;
9938 	bool in_callback_fn;
9939 	int err;
9940 
9941 	callee = state->frame[state->curframe];
9942 	r0 = &callee->regs[BPF_REG_0];
9943 	if (r0->type == PTR_TO_STACK) {
9944 		/* technically it's ok to return caller's stack pointer
9945 		 * (or caller's caller's pointer) back to the caller,
9946 		 * since these pointers are valid. Only current stack
9947 		 * pointer will be invalid as soon as function exits,
9948 		 * but let's be conservative
9949 		 */
9950 		verbose(env, "cannot return stack pointer to the caller\n");
9951 		return -EINVAL;
9952 	}
9953 
9954 	caller = state->frame[state->curframe - 1];
9955 	if (callee->in_callback_fn) {
9956 		if (r0->type != SCALAR_VALUE) {
9957 			verbose(env, "R0 not a scalar value\n");
9958 			return -EACCES;
9959 		}
9960 
9961 		/* we are going to rely on register's precise value */
9962 		err = mark_reg_read(env, r0, r0->parent, REG_LIVE_READ64);
9963 		err = err ?: mark_chain_precision(env, BPF_REG_0);
9964 		if (err)
9965 			return err;
9966 
9967 		/* enforce R0 return value range */
9968 		if (!retval_range_within(callee->callback_ret_range, r0)) {
9969 			verbose_invalid_scalar(env, r0, callee->callback_ret_range,
9970 					       "At callback return", "R0");
9971 			return -EINVAL;
9972 		}
9973 		if (!calls_callback(env, callee->callsite)) {
9974 			verbose(env, "BUG: in callback at %d, callsite %d !calls_callback\n",
9975 				*insn_idx, callee->callsite);
9976 			return -EFAULT;
9977 		}
9978 	} else {
9979 		/* return to the caller whatever r0 had in the callee */
9980 		caller->regs[BPF_REG_0] = *r0;
9981 	}
9982 
9983 	/* callback_fn frame should have released its own additions to parent's
9984 	 * reference state at this point, or check_reference_leak would
9985 	 * complain, hence it must be the same as the caller. There is no need
9986 	 * to copy it back.
9987 	 */
9988 	if (!callee->in_callback_fn) {
9989 		/* Transfer references to the caller */
9990 		err = copy_reference_state(caller, callee);
9991 		if (err)
9992 			return err;
9993 	}
9994 
9995 	/* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
9996 	 * there function call logic would reschedule callback visit. If iteration
9997 	 * converges is_state_visited() would prune that visit eventually.
9998 	 */
9999 	in_callback_fn = callee->in_callback_fn;
10000 	if (in_callback_fn)
10001 		*insn_idx = callee->callsite;
10002 	else
10003 		*insn_idx = callee->callsite + 1;
10004 
10005 	if (env->log.level & BPF_LOG_LEVEL) {
10006 		verbose(env, "returning from callee:\n");
10007 		print_verifier_state(env, callee, true);
10008 		verbose(env, "to caller at %d:\n", *insn_idx);
10009 		print_verifier_state(env, caller, true);
10010 	}
10011 	/* clear everything in the callee. In case of exceptional exits using
10012 	 * bpf_throw, this will be done by copy_verifier_state for extra frames. */
10013 	free_func_state(callee);
10014 	state->frame[state->curframe--] = NULL;
10015 
10016 	/* for callbacks widen imprecise scalars to make programs like below verify:
10017 	 *
10018 	 *   struct ctx { int i; }
10019 	 *   void cb(int idx, struct ctx *ctx) { ctx->i++; ... }
10020 	 *   ...
10021 	 *   struct ctx = { .i = 0; }
10022 	 *   bpf_loop(100, cb, &ctx, 0);
10023 	 *
10024 	 * This is similar to what is done in process_iter_next_call() for open
10025 	 * coded iterators.
10026 	 */
10027 	prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL;
10028 	if (prev_st) {
10029 		err = widen_imprecise_scalars(env, prev_st, state);
10030 		if (err)
10031 			return err;
10032 	}
10033 	return 0;
10034 }
10035 
10036 static int do_refine_retval_range(struct bpf_verifier_env *env,
10037 				  struct bpf_reg_state *regs, int ret_type,
10038 				  int func_id,
10039 				  struct bpf_call_arg_meta *meta)
10040 {
10041 	struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];
10042 
10043 	if (ret_type != RET_INTEGER)
10044 		return 0;
10045 
10046 	switch (func_id) {
10047 	case BPF_FUNC_get_stack:
10048 	case BPF_FUNC_get_task_stack:
10049 	case BPF_FUNC_probe_read_str:
10050 	case BPF_FUNC_probe_read_kernel_str:
10051 	case BPF_FUNC_probe_read_user_str:
10052 		ret_reg->smax_value = meta->msize_max_value;
10053 		ret_reg->s32_max_value = meta->msize_max_value;
10054 		ret_reg->smin_value = -MAX_ERRNO;
10055 		ret_reg->s32_min_value = -MAX_ERRNO;
10056 		reg_bounds_sync(ret_reg);
10057 		break;
10058 	case BPF_FUNC_get_smp_processor_id:
10059 		ret_reg->umax_value = nr_cpu_ids - 1;
10060 		ret_reg->u32_max_value = nr_cpu_ids - 1;
10061 		ret_reg->smax_value = nr_cpu_ids - 1;
10062 		ret_reg->s32_max_value = nr_cpu_ids - 1;
10063 		ret_reg->umin_value = 0;
10064 		ret_reg->u32_min_value = 0;
10065 		ret_reg->smin_value = 0;
10066 		ret_reg->s32_min_value = 0;
10067 		reg_bounds_sync(ret_reg);
10068 		break;
10069 	}
10070 
10071 	return reg_bounds_sanity_check(env, ret_reg, "retval");
10072 }
10073 
10074 static int
10075 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
10076 		int func_id, int insn_idx)
10077 {
10078 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
10079 	struct bpf_map *map = meta->map_ptr;
10080 
10081 	if (func_id != BPF_FUNC_tail_call &&
10082 	    func_id != BPF_FUNC_map_lookup_elem &&
10083 	    func_id != BPF_FUNC_map_update_elem &&
10084 	    func_id != BPF_FUNC_map_delete_elem &&
10085 	    func_id != BPF_FUNC_map_push_elem &&
10086 	    func_id != BPF_FUNC_map_pop_elem &&
10087 	    func_id != BPF_FUNC_map_peek_elem &&
10088 	    func_id != BPF_FUNC_for_each_map_elem &&
10089 	    func_id != BPF_FUNC_redirect_map &&
10090 	    func_id != BPF_FUNC_map_lookup_percpu_elem)
10091 		return 0;
10092 
10093 	if (map == NULL) {
10094 		verbose(env, "kernel subsystem misconfigured verifier\n");
10095 		return -EINVAL;
10096 	}
10097 
10098 	/* In case of read-only, some additional restrictions
10099 	 * need to be applied in order to prevent altering the
10100 	 * state of the map from program side.
10101 	 */
10102 	if ((map->map_flags & BPF_F_RDONLY_PROG) &&
10103 	    (func_id == BPF_FUNC_map_delete_elem ||
10104 	     func_id == BPF_FUNC_map_update_elem ||
10105 	     func_id == BPF_FUNC_map_push_elem ||
10106 	     func_id == BPF_FUNC_map_pop_elem)) {
10107 		verbose(env, "write into map forbidden\n");
10108 		return -EACCES;
10109 	}
10110 
10111 	if (!aux->map_ptr_state.map_ptr)
10112 		bpf_map_ptr_store(aux, meta->map_ptr,
10113 				  !meta->map_ptr->bypass_spec_v1, false);
10114 	else if (aux->map_ptr_state.map_ptr != meta->map_ptr)
10115 		bpf_map_ptr_store(aux, meta->map_ptr,
10116 				  !meta->map_ptr->bypass_spec_v1, true);
10117 	return 0;
10118 }
10119 
10120 static int
10121 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
10122 		int func_id, int insn_idx)
10123 {
10124 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
10125 	struct bpf_reg_state *regs = cur_regs(env), *reg;
10126 	struct bpf_map *map = meta->map_ptr;
10127 	u64 val, max;
10128 	int err;
10129 
10130 	if (func_id != BPF_FUNC_tail_call)
10131 		return 0;
10132 	if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) {
10133 		verbose(env, "kernel subsystem misconfigured verifier\n");
10134 		return -EINVAL;
10135 	}
10136 
10137 	reg = &regs[BPF_REG_3];
10138 	val = reg->var_off.value;
10139 	max = map->max_entries;
10140 
10141 	if (!(is_reg_const(reg, false) && val < max)) {
10142 		bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
10143 		return 0;
10144 	}
10145 
10146 	err = mark_chain_precision(env, BPF_REG_3);
10147 	if (err)
10148 		return err;
10149 	if (bpf_map_key_unseen(aux))
10150 		bpf_map_key_store(aux, val);
10151 	else if (!bpf_map_key_poisoned(aux) &&
10152 		  bpf_map_key_immediate(aux) != val)
10153 		bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
10154 	return 0;
10155 }
10156 
10157 static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit)
10158 {
10159 	struct bpf_func_state *state = cur_func(env);
10160 	bool refs_lingering = false;
10161 	int i;
10162 
10163 	if (!exception_exit && state->frameno && !state->in_callback_fn)
10164 		return 0;
10165 
10166 	for (i = 0; i < state->acquired_refs; i++) {
10167 		if (!exception_exit && state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
10168 			continue;
10169 		verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
10170 			state->refs[i].id, state->refs[i].insn_idx);
10171 		refs_lingering = true;
10172 	}
10173 	return refs_lingering ? -EINVAL : 0;
10174 }
10175 
10176 static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
10177 				   struct bpf_reg_state *regs)
10178 {
10179 	struct bpf_reg_state *fmt_reg = &regs[BPF_REG_3];
10180 	struct bpf_reg_state *data_len_reg = &regs[BPF_REG_5];
10181 	struct bpf_map *fmt_map = fmt_reg->map_ptr;
10182 	struct bpf_bprintf_data data = {};
10183 	int err, fmt_map_off, num_args;
10184 	u64 fmt_addr;
10185 	char *fmt;
10186 
10187 	/* data must be an array of u64 */
10188 	if (data_len_reg->var_off.value % 8)
10189 		return -EINVAL;
10190 	num_args = data_len_reg->var_off.value / 8;
10191 
10192 	/* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const
10193 	 * and map_direct_value_addr is set.
10194 	 */
10195 	fmt_map_off = fmt_reg->off + fmt_reg->var_off.value;
10196 	err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr,
10197 						  fmt_map_off);
10198 	if (err) {
10199 		verbose(env, "verifier bug\n");
10200 		return -EFAULT;
10201 	}
10202 	fmt = (char *)(long)fmt_addr + fmt_map_off;
10203 
10204 	/* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we
10205 	 * can focus on validating the format specifiers.
10206 	 */
10207 	err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, num_args, &data);
10208 	if (err < 0)
10209 		verbose(env, "Invalid format string\n");
10210 
10211 	return err;
10212 }
10213 
10214 static int check_get_func_ip(struct bpf_verifier_env *env)
10215 {
10216 	enum bpf_prog_type type = resolve_prog_type(env->prog);
10217 	int func_id = BPF_FUNC_get_func_ip;
10218 
10219 	if (type == BPF_PROG_TYPE_TRACING) {
10220 		if (!bpf_prog_has_trampoline(env->prog)) {
10221 			verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",
10222 				func_id_name(func_id), func_id);
10223 			return -ENOTSUPP;
10224 		}
10225 		return 0;
10226 	} else if (type == BPF_PROG_TYPE_KPROBE) {
10227 		return 0;
10228 	}
10229 
10230 	verbose(env, "func %s#%d not supported for program type %d\n",
10231 		func_id_name(func_id), func_id, type);
10232 	return -ENOTSUPP;
10233 }
10234 
10235 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
10236 {
10237 	return &env->insn_aux_data[env->insn_idx];
10238 }
10239 
10240 static bool loop_flag_is_zero(struct bpf_verifier_env *env)
10241 {
10242 	struct bpf_reg_state *regs = cur_regs(env);
10243 	struct bpf_reg_state *reg = &regs[BPF_REG_4];
10244 	bool reg_is_null = register_is_null(reg);
10245 
10246 	if (reg_is_null)
10247 		mark_chain_precision(env, BPF_REG_4);
10248 
10249 	return reg_is_null;
10250 }
10251 
10252 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)
10253 {
10254 	struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;
10255 
10256 	if (!state->initialized) {
10257 		state->initialized = 1;
10258 		state->fit_for_inline = loop_flag_is_zero(env);
10259 		state->callback_subprogno = subprogno;
10260 		return;
10261 	}
10262 
10263 	if (!state->fit_for_inline)
10264 		return;
10265 
10266 	state->fit_for_inline = (loop_flag_is_zero(env) &&
10267 				 state->callback_subprogno == subprogno);
10268 }
10269 
10270 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
10271 			     int *insn_idx_p)
10272 {
10273 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
10274 	bool returns_cpu_specific_alloc_ptr = false;
10275 	const struct bpf_func_proto *fn = NULL;
10276 	enum bpf_return_type ret_type;
10277 	enum bpf_type_flag ret_flag;
10278 	struct bpf_reg_state *regs;
10279 	struct bpf_call_arg_meta meta;
10280 	int insn_idx = *insn_idx_p;
10281 	bool changes_data;
10282 	int i, err, func_id;
10283 
10284 	/* find function prototype */
10285 	func_id = insn->imm;
10286 	if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) {
10287 		verbose(env, "invalid func %s#%d\n", func_id_name(func_id),
10288 			func_id);
10289 		return -EINVAL;
10290 	}
10291 
10292 	if (env->ops->get_func_proto)
10293 		fn = env->ops->get_func_proto(func_id, env->prog);
10294 	if (!fn) {
10295 		verbose(env, "program of this type cannot use helper %s#%d\n",
10296 			func_id_name(func_id), func_id);
10297 		return -EINVAL;
10298 	}
10299 
10300 	/* eBPF programs must be GPL compatible to use GPL-ed functions */
10301 	if (!env->prog->gpl_compatible && fn->gpl_only) {
10302 		verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
10303 		return -EINVAL;
10304 	}
10305 
10306 	if (fn->allowed && !fn->allowed(env->prog)) {
10307 		verbose(env, "helper call is not allowed in probe\n");
10308 		return -EINVAL;
10309 	}
10310 
10311 	if (!in_sleepable(env) && fn->might_sleep) {
10312 		verbose(env, "helper call might sleep in a non-sleepable prog\n");
10313 		return -EINVAL;
10314 	}
10315 
10316 	/* With LD_ABS/IND some JITs save/restore skb from r1. */
10317 	changes_data = bpf_helper_changes_pkt_data(fn->func);
10318 	if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
10319 		verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",
10320 			func_id_name(func_id), func_id);
10321 		return -EINVAL;
10322 	}
10323 
10324 	memset(&meta, 0, sizeof(meta));
10325 	meta.pkt_access = fn->pkt_access;
10326 
10327 	err = check_func_proto(fn, func_id);
10328 	if (err) {
10329 		verbose(env, "kernel subsystem misconfigured func %s#%d\n",
10330 			func_id_name(func_id), func_id);
10331 		return err;
10332 	}
10333 
10334 	if (env->cur_state->active_rcu_lock) {
10335 		if (fn->might_sleep) {
10336 			verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n",
10337 				func_id_name(func_id), func_id);
10338 			return -EINVAL;
10339 		}
10340 
10341 		if (in_sleepable(env) && is_storage_get_function(func_id))
10342 			env->insn_aux_data[insn_idx].storage_get_func_atomic = true;
10343 	}
10344 
10345 	if (env->cur_state->active_preempt_lock) {
10346 		if (fn->might_sleep) {
10347 			verbose(env, "sleepable helper %s#%d in non-preemptible region\n",
10348 				func_id_name(func_id), func_id);
10349 			return -EINVAL;
10350 		}
10351 
10352 		if (in_sleepable(env) && is_storage_get_function(func_id))
10353 			env->insn_aux_data[insn_idx].storage_get_func_atomic = true;
10354 	}
10355 
10356 	meta.func_id = func_id;
10357 	/* check args */
10358 	for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
10359 		err = check_func_arg(env, i, &meta, fn, insn_idx);
10360 		if (err)
10361 			return err;
10362 	}
10363 
10364 	err = record_func_map(env, &meta, func_id, insn_idx);
10365 	if (err)
10366 		return err;
10367 
10368 	err = record_func_key(env, &meta, func_id, insn_idx);
10369 	if (err)
10370 		return err;
10371 
10372 	/* Mark slots with STACK_MISC in case of raw mode, stack offset
10373 	 * is inferred from register state.
10374 	 */
10375 	for (i = 0; i < meta.access_size; i++) {
10376 		err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
10377 				       BPF_WRITE, -1, false, false);
10378 		if (err)
10379 			return err;
10380 	}
10381 
10382 	regs = cur_regs(env);
10383 
10384 	if (meta.release_regno) {
10385 		err = -EINVAL;
10386 		/* This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot
10387 		 * be released by any dynptr helper. Hence, unmark_stack_slots_dynptr
10388 		 * is safe to do directly.
10389 		 */
10390 		if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) {
10391 			if (regs[meta.release_regno].type == CONST_PTR_TO_DYNPTR) {
10392 				verbose(env, "verifier internal error: CONST_PTR_TO_DYNPTR cannot be released\n");
10393 				return -EFAULT;
10394 			}
10395 			err = unmark_stack_slots_dynptr(env, &regs[meta.release_regno]);
10396 		} else if (func_id == BPF_FUNC_kptr_xchg && meta.ref_obj_id) {
10397 			u32 ref_obj_id = meta.ref_obj_id;
10398 			bool in_rcu = in_rcu_cs(env);
10399 			struct bpf_func_state *state;
10400 			struct bpf_reg_state *reg;
10401 
10402 			err = release_reference_state(cur_func(env), ref_obj_id);
10403 			if (!err) {
10404 				bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
10405 					if (reg->ref_obj_id == ref_obj_id) {
10406 						if (in_rcu && (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) {
10407 							reg->ref_obj_id = 0;
10408 							reg->type &= ~MEM_ALLOC;
10409 							reg->type |= MEM_RCU;
10410 						} else {
10411 							mark_reg_invalid(env, reg);
10412 						}
10413 					}
10414 				}));
10415 			}
10416 		} else if (meta.ref_obj_id) {
10417 			err = release_reference(env, meta.ref_obj_id);
10418 		} else if (register_is_null(&regs[meta.release_regno])) {
10419 			/* meta.ref_obj_id can only be 0 if register that is meant to be
10420 			 * released is NULL, which must be > R0.
10421 			 */
10422 			err = 0;
10423 		}
10424 		if (err) {
10425 			verbose(env, "func %s#%d reference has not been acquired before\n",
10426 				func_id_name(func_id), func_id);
10427 			return err;
10428 		}
10429 	}
10430 
10431 	switch (func_id) {
10432 	case BPF_FUNC_tail_call:
10433 		err = check_reference_leak(env, false);
10434 		if (err) {
10435 			verbose(env, "tail_call would lead to reference leak\n");
10436 			return err;
10437 		}
10438 		break;
10439 	case BPF_FUNC_get_local_storage:
10440 		/* check that flags argument in get_local_storage(map, flags) is 0,
10441 		 * this is required because get_local_storage() can't return an error.
10442 		 */
10443 		if (!register_is_null(&regs[BPF_REG_2])) {
10444 			verbose(env, "get_local_storage() doesn't support non-zero flags\n");
10445 			return -EINVAL;
10446 		}
10447 		break;
10448 	case BPF_FUNC_for_each_map_elem:
10449 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
10450 					 set_map_elem_callback_state);
10451 		break;
10452 	case BPF_FUNC_timer_set_callback:
10453 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
10454 					 set_timer_callback_state);
10455 		break;
10456 	case BPF_FUNC_find_vma:
10457 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
10458 					 set_find_vma_callback_state);
10459 		break;
10460 	case BPF_FUNC_snprintf:
10461 		err = check_bpf_snprintf_call(env, regs);
10462 		break;
10463 	case BPF_FUNC_loop:
10464 		update_loop_inline_state(env, meta.subprogno);
10465 		/* Verifier relies on R1 value to determine if bpf_loop() iteration
10466 		 * is finished, thus mark it precise.
10467 		 */
10468 		err = mark_chain_precision(env, BPF_REG_1);
10469 		if (err)
10470 			return err;
10471 		if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) {
10472 			err = push_callback_call(env, insn, insn_idx, meta.subprogno,
10473 						 set_loop_callback_state);
10474 		} else {
10475 			cur_func(env)->callback_depth = 0;
10476 			if (env->log.level & BPF_LOG_LEVEL2)
10477 				verbose(env, "frame%d bpf_loop iteration limit reached\n",
10478 					env->cur_state->curframe);
10479 		}
10480 		break;
10481 	case BPF_FUNC_dynptr_from_mem:
10482 		if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) {
10483 			verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n",
10484 				reg_type_str(env, regs[BPF_REG_1].type));
10485 			return -EACCES;
10486 		}
10487 		break;
10488 	case BPF_FUNC_set_retval:
10489 		if (prog_type == BPF_PROG_TYPE_LSM &&
10490 		    env->prog->expected_attach_type == BPF_LSM_CGROUP) {
10491 			if (!env->prog->aux->attach_func_proto->type) {
10492 				/* Make sure programs that attach to void
10493 				 * hooks don't try to modify return value.
10494 				 */
10495 				verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
10496 				return -EINVAL;
10497 			}
10498 		}
10499 		break;
10500 	case BPF_FUNC_dynptr_data:
10501 	{
10502 		struct bpf_reg_state *reg;
10503 		int id, ref_obj_id;
10504 
10505 		reg = get_dynptr_arg_reg(env, fn, regs);
10506 		if (!reg)
10507 			return -EFAULT;
10508 
10509 
10510 		if (meta.dynptr_id) {
10511 			verbose(env, "verifier internal error: meta.dynptr_id already set\n");
10512 			return -EFAULT;
10513 		}
10514 		if (meta.ref_obj_id) {
10515 			verbose(env, "verifier internal error: meta.ref_obj_id already set\n");
10516 			return -EFAULT;
10517 		}
10518 
10519 		id = dynptr_id(env, reg);
10520 		if (id < 0) {
10521 			verbose(env, "verifier internal error: failed to obtain dynptr id\n");
10522 			return id;
10523 		}
10524 
10525 		ref_obj_id = dynptr_ref_obj_id(env, reg);
10526 		if (ref_obj_id < 0) {
10527 			verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n");
10528 			return ref_obj_id;
10529 		}
10530 
10531 		meta.dynptr_id = id;
10532 		meta.ref_obj_id = ref_obj_id;
10533 
10534 		break;
10535 	}
10536 	case BPF_FUNC_dynptr_write:
10537 	{
10538 		enum bpf_dynptr_type dynptr_type;
10539 		struct bpf_reg_state *reg;
10540 
10541 		reg = get_dynptr_arg_reg(env, fn, regs);
10542 		if (!reg)
10543 			return -EFAULT;
10544 
10545 		dynptr_type = dynptr_get_type(env, reg);
10546 		if (dynptr_type == BPF_DYNPTR_TYPE_INVALID)
10547 			return -EFAULT;
10548 
10549 		if (dynptr_type == BPF_DYNPTR_TYPE_SKB)
10550 			/* this will trigger clear_all_pkt_pointers(), which will
10551 			 * invalidate all dynptr slices associated with the skb
10552 			 */
10553 			changes_data = true;
10554 
10555 		break;
10556 	}
10557 	case BPF_FUNC_per_cpu_ptr:
10558 	case BPF_FUNC_this_cpu_ptr:
10559 	{
10560 		struct bpf_reg_state *reg = &regs[BPF_REG_1];
10561 		const struct btf_type *type;
10562 
10563 		if (reg->type & MEM_RCU) {
10564 			type = btf_type_by_id(reg->btf, reg->btf_id);
10565 			if (!type || !btf_type_is_struct(type)) {
10566 				verbose(env, "Helper has invalid btf/btf_id in R1\n");
10567 				return -EFAULT;
10568 			}
10569 			returns_cpu_specific_alloc_ptr = true;
10570 			env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true;
10571 		}
10572 		break;
10573 	}
10574 	case BPF_FUNC_user_ringbuf_drain:
10575 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
10576 					 set_user_ringbuf_callback_state);
10577 		break;
10578 	}
10579 
10580 	if (err)
10581 		return err;
10582 
10583 	/* reset caller saved regs */
10584 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
10585 		mark_reg_not_init(env, regs, caller_saved[i]);
10586 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
10587 	}
10588 
10589 	/* helper call returns 64-bit value. */
10590 	regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
10591 
10592 	/* update return register (already marked as written above) */
10593 	ret_type = fn->ret_type;
10594 	ret_flag = type_flag(ret_type);
10595 
10596 	switch (base_type(ret_type)) {
10597 	case RET_INTEGER:
10598 		/* sets type to SCALAR_VALUE */
10599 		mark_reg_unknown(env, regs, BPF_REG_0);
10600 		break;
10601 	case RET_VOID:
10602 		regs[BPF_REG_0].type = NOT_INIT;
10603 		break;
10604 	case RET_PTR_TO_MAP_VALUE:
10605 		/* There is no offset yet applied, variable or fixed */
10606 		mark_reg_known_zero(env, regs, BPF_REG_0);
10607 		/* remember map_ptr, so that check_map_access()
10608 		 * can check 'value_size' boundary of memory access
10609 		 * to map element returned from bpf_map_lookup_elem()
10610 		 */
10611 		if (meta.map_ptr == NULL) {
10612 			verbose(env,
10613 				"kernel subsystem misconfigured verifier\n");
10614 			return -EINVAL;
10615 		}
10616 		regs[BPF_REG_0].map_ptr = meta.map_ptr;
10617 		regs[BPF_REG_0].map_uid = meta.map_uid;
10618 		regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag;
10619 		if (!type_may_be_null(ret_type) &&
10620 		    btf_record_has_field(meta.map_ptr->record, BPF_SPIN_LOCK)) {
10621 			regs[BPF_REG_0].id = ++env->id_gen;
10622 		}
10623 		break;
10624 	case RET_PTR_TO_SOCKET:
10625 		mark_reg_known_zero(env, regs, BPF_REG_0);
10626 		regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag;
10627 		break;
10628 	case RET_PTR_TO_SOCK_COMMON:
10629 		mark_reg_known_zero(env, regs, BPF_REG_0);
10630 		regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag;
10631 		break;
10632 	case RET_PTR_TO_TCP_SOCK:
10633 		mark_reg_known_zero(env, regs, BPF_REG_0);
10634 		regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag;
10635 		break;
10636 	case RET_PTR_TO_MEM:
10637 		mark_reg_known_zero(env, regs, BPF_REG_0);
10638 		regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
10639 		regs[BPF_REG_0].mem_size = meta.mem_size;
10640 		break;
10641 	case RET_PTR_TO_MEM_OR_BTF_ID:
10642 	{
10643 		const struct btf_type *t;
10644 
10645 		mark_reg_known_zero(env, regs, BPF_REG_0);
10646 		t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL);
10647 		if (!btf_type_is_struct(t)) {
10648 			u32 tsize;
10649 			const struct btf_type *ret;
10650 			const char *tname;
10651 
10652 			/* resolve the type size of ksym. */
10653 			ret = btf_resolve_size(meta.ret_btf, t, &tsize);
10654 			if (IS_ERR(ret)) {
10655 				tname = btf_name_by_offset(meta.ret_btf, t->name_off);
10656 				verbose(env, "unable to resolve the size of type '%s': %ld\n",
10657 					tname, PTR_ERR(ret));
10658 				return -EINVAL;
10659 			}
10660 			regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
10661 			regs[BPF_REG_0].mem_size = tsize;
10662 		} else {
10663 			if (returns_cpu_specific_alloc_ptr) {
10664 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU;
10665 			} else {
10666 				/* MEM_RDONLY may be carried from ret_flag, but it
10667 				 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise
10668 				 * it will confuse the check of PTR_TO_BTF_ID in
10669 				 * check_mem_access().
10670 				 */
10671 				ret_flag &= ~MEM_RDONLY;
10672 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
10673 			}
10674 
10675 			regs[BPF_REG_0].btf = meta.ret_btf;
10676 			regs[BPF_REG_0].btf_id = meta.ret_btf_id;
10677 		}
10678 		break;
10679 	}
10680 	case RET_PTR_TO_BTF_ID:
10681 	{
10682 		struct btf *ret_btf;
10683 		int ret_btf_id;
10684 
10685 		mark_reg_known_zero(env, regs, BPF_REG_0);
10686 		regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
10687 		if (func_id == BPF_FUNC_kptr_xchg) {
10688 			ret_btf = meta.kptr_field->kptr.btf;
10689 			ret_btf_id = meta.kptr_field->kptr.btf_id;
10690 			if (!btf_is_kernel(ret_btf)) {
10691 				regs[BPF_REG_0].type |= MEM_ALLOC;
10692 				if (meta.kptr_field->type == BPF_KPTR_PERCPU)
10693 					regs[BPF_REG_0].type |= MEM_PERCPU;
10694 			}
10695 		} else {
10696 			if (fn->ret_btf_id == BPF_PTR_POISON) {
10697 				verbose(env, "verifier internal error:");
10698 				verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n",
10699 					func_id_name(func_id));
10700 				return -EINVAL;
10701 			}
10702 			ret_btf = btf_vmlinux;
10703 			ret_btf_id = *fn->ret_btf_id;
10704 		}
10705 		if (ret_btf_id == 0) {
10706 			verbose(env, "invalid return type %u of func %s#%d\n",
10707 				base_type(ret_type), func_id_name(func_id),
10708 				func_id);
10709 			return -EINVAL;
10710 		}
10711 		regs[BPF_REG_0].btf = ret_btf;
10712 		regs[BPF_REG_0].btf_id = ret_btf_id;
10713 		break;
10714 	}
10715 	default:
10716 		verbose(env, "unknown return type %u of func %s#%d\n",
10717 			base_type(ret_type), func_id_name(func_id), func_id);
10718 		return -EINVAL;
10719 	}
10720 
10721 	if (type_may_be_null(regs[BPF_REG_0].type))
10722 		regs[BPF_REG_0].id = ++env->id_gen;
10723 
10724 	if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) {
10725 		verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n",
10726 			func_id_name(func_id), func_id);
10727 		return -EFAULT;
10728 	}
10729 
10730 	if (is_dynptr_ref_function(func_id))
10731 		regs[BPF_REG_0].dynptr_id = meta.dynptr_id;
10732 
10733 	if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) {
10734 		/* For release_reference() */
10735 		regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
10736 	} else if (is_acquire_function(func_id, meta.map_ptr)) {
10737 		int id = acquire_reference_state(env, insn_idx);
10738 
10739 		if (id < 0)
10740 			return id;
10741 		/* For mark_ptr_or_null_reg() */
10742 		regs[BPF_REG_0].id = id;
10743 		/* For release_reference() */
10744 		regs[BPF_REG_0].ref_obj_id = id;
10745 	}
10746 
10747 	err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta);
10748 	if (err)
10749 		return err;
10750 
10751 	err = check_map_func_compatibility(env, meta.map_ptr, func_id);
10752 	if (err)
10753 		return err;
10754 
10755 	if ((func_id == BPF_FUNC_get_stack ||
10756 	     func_id == BPF_FUNC_get_task_stack) &&
10757 	    !env->prog->has_callchain_buf) {
10758 		const char *err_str;
10759 
10760 #ifdef CONFIG_PERF_EVENTS
10761 		err = get_callchain_buffers(sysctl_perf_event_max_stack);
10762 		err_str = "cannot get callchain buffer for func %s#%d\n";
10763 #else
10764 		err = -ENOTSUPP;
10765 		err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
10766 #endif
10767 		if (err) {
10768 			verbose(env, err_str, func_id_name(func_id), func_id);
10769 			return err;
10770 		}
10771 
10772 		env->prog->has_callchain_buf = true;
10773 	}
10774 
10775 	if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack)
10776 		env->prog->call_get_stack = true;
10777 
10778 	if (func_id == BPF_FUNC_get_func_ip) {
10779 		if (check_get_func_ip(env))
10780 			return -ENOTSUPP;
10781 		env->prog->call_get_func_ip = true;
10782 	}
10783 
10784 	if (changes_data)
10785 		clear_all_pkt_pointers(env);
10786 	return 0;
10787 }
10788 
10789 /* mark_btf_func_reg_size() is used when the reg size is determined by
10790  * the BTF func_proto's return value size and argument.
10791  */
10792 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
10793 				   size_t reg_size)
10794 {
10795 	struct bpf_reg_state *reg = &cur_regs(env)[regno];
10796 
10797 	if (regno == BPF_REG_0) {
10798 		/* Function return value */
10799 		reg->live |= REG_LIVE_WRITTEN;
10800 		reg->subreg_def = reg_size == sizeof(u64) ?
10801 			DEF_NOT_SUBREG : env->insn_idx + 1;
10802 	} else {
10803 		/* Function argument */
10804 		if (reg_size == sizeof(u64)) {
10805 			mark_insn_zext(env, reg);
10806 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
10807 		} else {
10808 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
10809 		}
10810 	}
10811 }
10812 
10813 static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta)
10814 {
10815 	return meta->kfunc_flags & KF_ACQUIRE;
10816 }
10817 
10818 static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta)
10819 {
10820 	return meta->kfunc_flags & KF_RELEASE;
10821 }
10822 
10823 static bool is_kfunc_trusted_args(struct bpf_kfunc_call_arg_meta *meta)
10824 {
10825 	return (meta->kfunc_flags & KF_TRUSTED_ARGS) || is_kfunc_release(meta);
10826 }
10827 
10828 static bool is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta)
10829 {
10830 	return meta->kfunc_flags & KF_SLEEPABLE;
10831 }
10832 
10833 static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta)
10834 {
10835 	return meta->kfunc_flags & KF_DESTRUCTIVE;
10836 }
10837 
10838 static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta)
10839 {
10840 	return meta->kfunc_flags & KF_RCU;
10841 }
10842 
10843 static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta)
10844 {
10845 	return meta->kfunc_flags & KF_RCU_PROTECTED;
10846 }
10847 
10848 static bool is_kfunc_arg_mem_size(const struct btf *btf,
10849 				  const struct btf_param *arg,
10850 				  const struct bpf_reg_state *reg)
10851 {
10852 	const struct btf_type *t;
10853 
10854 	t = btf_type_skip_modifiers(btf, arg->type, NULL);
10855 	if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE)
10856 		return false;
10857 
10858 	return btf_param_match_suffix(btf, arg, "__sz");
10859 }
10860 
10861 static bool is_kfunc_arg_const_mem_size(const struct btf *btf,
10862 					const struct btf_param *arg,
10863 					const struct bpf_reg_state *reg)
10864 {
10865 	const struct btf_type *t;
10866 
10867 	t = btf_type_skip_modifiers(btf, arg->type, NULL);
10868 	if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE)
10869 		return false;
10870 
10871 	return btf_param_match_suffix(btf, arg, "__szk");
10872 }
10873 
10874 static bool is_kfunc_arg_optional(const struct btf *btf, const struct btf_param *arg)
10875 {
10876 	return btf_param_match_suffix(btf, arg, "__opt");
10877 }
10878 
10879 static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg)
10880 {
10881 	return btf_param_match_suffix(btf, arg, "__k");
10882 }
10883 
10884 static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg)
10885 {
10886 	return btf_param_match_suffix(btf, arg, "__ign");
10887 }
10888 
10889 static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg)
10890 {
10891 	return btf_param_match_suffix(btf, arg, "__map");
10892 }
10893 
10894 static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg)
10895 {
10896 	return btf_param_match_suffix(btf, arg, "__alloc");
10897 }
10898 
10899 static bool is_kfunc_arg_uninit(const struct btf *btf, const struct btf_param *arg)
10900 {
10901 	return btf_param_match_suffix(btf, arg, "__uninit");
10902 }
10903 
10904 static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf_param *arg)
10905 {
10906 	return btf_param_match_suffix(btf, arg, "__refcounted_kptr");
10907 }
10908 
10909 static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param *arg)
10910 {
10911 	return btf_param_match_suffix(btf, arg, "__nullable");
10912 }
10913 
10914 static bool is_kfunc_arg_const_str(const struct btf *btf, const struct btf_param *arg)
10915 {
10916 	return btf_param_match_suffix(btf, arg, "__str");
10917 }
10918 
10919 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf,
10920 					  const struct btf_param *arg,
10921 					  const char *name)
10922 {
10923 	int len, target_len = strlen(name);
10924 	const char *param_name;
10925 
10926 	param_name = btf_name_by_offset(btf, arg->name_off);
10927 	if (str_is_empty(param_name))
10928 		return false;
10929 	len = strlen(param_name);
10930 	if (len != target_len)
10931 		return false;
10932 	if (strcmp(param_name, name))
10933 		return false;
10934 
10935 	return true;
10936 }
10937 
10938 enum {
10939 	KF_ARG_DYNPTR_ID,
10940 	KF_ARG_LIST_HEAD_ID,
10941 	KF_ARG_LIST_NODE_ID,
10942 	KF_ARG_RB_ROOT_ID,
10943 	KF_ARG_RB_NODE_ID,
10944 	KF_ARG_WORKQUEUE_ID,
10945 };
10946 
10947 BTF_ID_LIST(kf_arg_btf_ids)
10948 BTF_ID(struct, bpf_dynptr)
10949 BTF_ID(struct, bpf_list_head)
10950 BTF_ID(struct, bpf_list_node)
10951 BTF_ID(struct, bpf_rb_root)
10952 BTF_ID(struct, bpf_rb_node)
10953 BTF_ID(struct, bpf_wq)
10954 
10955 static bool __is_kfunc_ptr_arg_type(const struct btf *btf,
10956 				    const struct btf_param *arg, int type)
10957 {
10958 	const struct btf_type *t;
10959 	u32 res_id;
10960 
10961 	t = btf_type_skip_modifiers(btf, arg->type, NULL);
10962 	if (!t)
10963 		return false;
10964 	if (!btf_type_is_ptr(t))
10965 		return false;
10966 	t = btf_type_skip_modifiers(btf, t->type, &res_id);
10967 	if (!t)
10968 		return false;
10969 	return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]);
10970 }
10971 
10972 static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg)
10973 {
10974 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID);
10975 }
10976 
10977 static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg)
10978 {
10979 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID);
10980 }
10981 
10982 static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg)
10983 {
10984 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID);
10985 }
10986 
10987 static bool is_kfunc_arg_rbtree_root(const struct btf *btf, const struct btf_param *arg)
10988 {
10989 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_ROOT_ID);
10990 }
10991 
10992 static bool is_kfunc_arg_rbtree_node(const struct btf *btf, const struct btf_param *arg)
10993 {
10994 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_NODE_ID);
10995 }
10996 
10997 static bool is_kfunc_arg_wq(const struct btf *btf, const struct btf_param *arg)
10998 {
10999 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_WORKQUEUE_ID);
11000 }
11001 
11002 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf,
11003 				  const struct btf_param *arg)
11004 {
11005 	const struct btf_type *t;
11006 
11007 	t = btf_type_resolve_func_ptr(btf, arg->type, NULL);
11008 	if (!t)
11009 		return false;
11010 
11011 	return true;
11012 }
11013 
11014 /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */
11015 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,
11016 					const struct btf *btf,
11017 					const struct btf_type *t, int rec)
11018 {
11019 	const struct btf_type *member_type;
11020 	const struct btf_member *member;
11021 	u32 i;
11022 
11023 	if (!btf_type_is_struct(t))
11024 		return false;
11025 
11026 	for_each_member(i, t, member) {
11027 		const struct btf_array *array;
11028 
11029 		member_type = btf_type_skip_modifiers(btf, member->type, NULL);
11030 		if (btf_type_is_struct(member_type)) {
11031 			if (rec >= 3) {
11032 				verbose(env, "max struct nesting depth exceeded\n");
11033 				return false;
11034 			}
11035 			if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1))
11036 				return false;
11037 			continue;
11038 		}
11039 		if (btf_type_is_array(member_type)) {
11040 			array = btf_array(member_type);
11041 			if (!array->nelems)
11042 				return false;
11043 			member_type = btf_type_skip_modifiers(btf, array->type, NULL);
11044 			if (!btf_type_is_scalar(member_type))
11045 				return false;
11046 			continue;
11047 		}
11048 		if (!btf_type_is_scalar(member_type))
11049 			return false;
11050 	}
11051 	return true;
11052 }
11053 
11054 enum kfunc_ptr_arg_type {
11055 	KF_ARG_PTR_TO_CTX,
11056 	KF_ARG_PTR_TO_ALLOC_BTF_ID,    /* Allocated object */
11057 	KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */
11058 	KF_ARG_PTR_TO_DYNPTR,
11059 	KF_ARG_PTR_TO_ITER,
11060 	KF_ARG_PTR_TO_LIST_HEAD,
11061 	KF_ARG_PTR_TO_LIST_NODE,
11062 	KF_ARG_PTR_TO_BTF_ID,	       /* Also covers reg2btf_ids conversions */
11063 	KF_ARG_PTR_TO_MEM,
11064 	KF_ARG_PTR_TO_MEM_SIZE,	       /* Size derived from next argument, skip it */
11065 	KF_ARG_PTR_TO_CALLBACK,
11066 	KF_ARG_PTR_TO_RB_ROOT,
11067 	KF_ARG_PTR_TO_RB_NODE,
11068 	KF_ARG_PTR_TO_NULL,
11069 	KF_ARG_PTR_TO_CONST_STR,
11070 	KF_ARG_PTR_TO_MAP,
11071 	KF_ARG_PTR_TO_WORKQUEUE,
11072 };
11073 
11074 enum special_kfunc_type {
11075 	KF_bpf_obj_new_impl,
11076 	KF_bpf_obj_drop_impl,
11077 	KF_bpf_refcount_acquire_impl,
11078 	KF_bpf_list_push_front_impl,
11079 	KF_bpf_list_push_back_impl,
11080 	KF_bpf_list_pop_front,
11081 	KF_bpf_list_pop_back,
11082 	KF_bpf_cast_to_kern_ctx,
11083 	KF_bpf_rdonly_cast,
11084 	KF_bpf_rcu_read_lock,
11085 	KF_bpf_rcu_read_unlock,
11086 	KF_bpf_rbtree_remove,
11087 	KF_bpf_rbtree_add_impl,
11088 	KF_bpf_rbtree_first,
11089 	KF_bpf_dynptr_from_skb,
11090 	KF_bpf_dynptr_from_xdp,
11091 	KF_bpf_dynptr_slice,
11092 	KF_bpf_dynptr_slice_rdwr,
11093 	KF_bpf_dynptr_clone,
11094 	KF_bpf_percpu_obj_new_impl,
11095 	KF_bpf_percpu_obj_drop_impl,
11096 	KF_bpf_throw,
11097 	KF_bpf_wq_set_callback_impl,
11098 	KF_bpf_preempt_disable,
11099 	KF_bpf_preempt_enable,
11100 	KF_bpf_iter_css_task_new,
11101 	KF_bpf_session_cookie,
11102 };
11103 
11104 BTF_SET_START(special_kfunc_set)
11105 BTF_ID(func, bpf_obj_new_impl)
11106 BTF_ID(func, bpf_obj_drop_impl)
11107 BTF_ID(func, bpf_refcount_acquire_impl)
11108 BTF_ID(func, bpf_list_push_front_impl)
11109 BTF_ID(func, bpf_list_push_back_impl)
11110 BTF_ID(func, bpf_list_pop_front)
11111 BTF_ID(func, bpf_list_pop_back)
11112 BTF_ID(func, bpf_cast_to_kern_ctx)
11113 BTF_ID(func, bpf_rdonly_cast)
11114 BTF_ID(func, bpf_rbtree_remove)
11115 BTF_ID(func, bpf_rbtree_add_impl)
11116 BTF_ID(func, bpf_rbtree_first)
11117 BTF_ID(func, bpf_dynptr_from_skb)
11118 BTF_ID(func, bpf_dynptr_from_xdp)
11119 BTF_ID(func, bpf_dynptr_slice)
11120 BTF_ID(func, bpf_dynptr_slice_rdwr)
11121 BTF_ID(func, bpf_dynptr_clone)
11122 BTF_ID(func, bpf_percpu_obj_new_impl)
11123 BTF_ID(func, bpf_percpu_obj_drop_impl)
11124 BTF_ID(func, bpf_throw)
11125 BTF_ID(func, bpf_wq_set_callback_impl)
11126 #ifdef CONFIG_CGROUPS
11127 BTF_ID(func, bpf_iter_css_task_new)
11128 #endif
11129 BTF_SET_END(special_kfunc_set)
11130 
11131 BTF_ID_LIST(special_kfunc_list)
11132 BTF_ID(func, bpf_obj_new_impl)
11133 BTF_ID(func, bpf_obj_drop_impl)
11134 BTF_ID(func, bpf_refcount_acquire_impl)
11135 BTF_ID(func, bpf_list_push_front_impl)
11136 BTF_ID(func, bpf_list_push_back_impl)
11137 BTF_ID(func, bpf_list_pop_front)
11138 BTF_ID(func, bpf_list_pop_back)
11139 BTF_ID(func, bpf_cast_to_kern_ctx)
11140 BTF_ID(func, bpf_rdonly_cast)
11141 BTF_ID(func, bpf_rcu_read_lock)
11142 BTF_ID(func, bpf_rcu_read_unlock)
11143 BTF_ID(func, bpf_rbtree_remove)
11144 BTF_ID(func, bpf_rbtree_add_impl)
11145 BTF_ID(func, bpf_rbtree_first)
11146 BTF_ID(func, bpf_dynptr_from_skb)
11147 BTF_ID(func, bpf_dynptr_from_xdp)
11148 BTF_ID(func, bpf_dynptr_slice)
11149 BTF_ID(func, bpf_dynptr_slice_rdwr)
11150 BTF_ID(func, bpf_dynptr_clone)
11151 BTF_ID(func, bpf_percpu_obj_new_impl)
11152 BTF_ID(func, bpf_percpu_obj_drop_impl)
11153 BTF_ID(func, bpf_throw)
11154 BTF_ID(func, bpf_wq_set_callback_impl)
11155 BTF_ID(func, bpf_preempt_disable)
11156 BTF_ID(func, bpf_preempt_enable)
11157 #ifdef CONFIG_CGROUPS
11158 BTF_ID(func, bpf_iter_css_task_new)
11159 #else
11160 BTF_ID_UNUSED
11161 #endif
11162 #ifdef CONFIG_BPF_EVENTS
11163 BTF_ID(func, bpf_session_cookie)
11164 #else
11165 BTF_ID_UNUSED
11166 #endif
11167 
11168 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
11169 {
11170 	if (meta->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] &&
11171 	    meta->arg_owning_ref) {
11172 		return false;
11173 	}
11174 
11175 	return meta->kfunc_flags & KF_RET_NULL;
11176 }
11177 
11178 static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta)
11179 {
11180 	return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock];
11181 }
11182 
11183 static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta)
11184 {
11185 	return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock];
11186 }
11187 
11188 static bool is_kfunc_bpf_preempt_disable(struct bpf_kfunc_call_arg_meta *meta)
11189 {
11190 	return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable];
11191 }
11192 
11193 static bool is_kfunc_bpf_preempt_enable(struct bpf_kfunc_call_arg_meta *meta)
11194 {
11195 	return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable];
11196 }
11197 
11198 static enum kfunc_ptr_arg_type
11199 get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,
11200 		       struct bpf_kfunc_call_arg_meta *meta,
11201 		       const struct btf_type *t, const struct btf_type *ref_t,
11202 		       const char *ref_tname, const struct btf_param *args,
11203 		       int argno, int nargs)
11204 {
11205 	u32 regno = argno + 1;
11206 	struct bpf_reg_state *regs = cur_regs(env);
11207 	struct bpf_reg_state *reg = &regs[regno];
11208 	bool arg_mem_size = false;
11209 
11210 	if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx])
11211 		return KF_ARG_PTR_TO_CTX;
11212 
11213 	/* In this function, we verify the kfunc's BTF as per the argument type,
11214 	 * leaving the rest of the verification with respect to the register
11215 	 * type to our caller. When a set of conditions hold in the BTF type of
11216 	 * arguments, we resolve it to a known kfunc_ptr_arg_type.
11217 	 */
11218 	if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno))
11219 		return KF_ARG_PTR_TO_CTX;
11220 
11221 	if (is_kfunc_arg_nullable(meta->btf, &args[argno]) && register_is_null(reg))
11222 		return KF_ARG_PTR_TO_NULL;
11223 
11224 	if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno]))
11225 		return KF_ARG_PTR_TO_ALLOC_BTF_ID;
11226 
11227 	if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[argno]))
11228 		return KF_ARG_PTR_TO_REFCOUNTED_KPTR;
11229 
11230 	if (is_kfunc_arg_dynptr(meta->btf, &args[argno]))
11231 		return KF_ARG_PTR_TO_DYNPTR;
11232 
11233 	if (is_kfunc_arg_iter(meta, argno))
11234 		return KF_ARG_PTR_TO_ITER;
11235 
11236 	if (is_kfunc_arg_list_head(meta->btf, &args[argno]))
11237 		return KF_ARG_PTR_TO_LIST_HEAD;
11238 
11239 	if (is_kfunc_arg_list_node(meta->btf, &args[argno]))
11240 		return KF_ARG_PTR_TO_LIST_NODE;
11241 
11242 	if (is_kfunc_arg_rbtree_root(meta->btf, &args[argno]))
11243 		return KF_ARG_PTR_TO_RB_ROOT;
11244 
11245 	if (is_kfunc_arg_rbtree_node(meta->btf, &args[argno]))
11246 		return KF_ARG_PTR_TO_RB_NODE;
11247 
11248 	if (is_kfunc_arg_const_str(meta->btf, &args[argno]))
11249 		return KF_ARG_PTR_TO_CONST_STR;
11250 
11251 	if (is_kfunc_arg_map(meta->btf, &args[argno]))
11252 		return KF_ARG_PTR_TO_MAP;
11253 
11254 	if (is_kfunc_arg_wq(meta->btf, &args[argno]))
11255 		return KF_ARG_PTR_TO_WORKQUEUE;
11256 
11257 	if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) {
11258 		if (!btf_type_is_struct(ref_t)) {
11259 			verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n",
11260 				meta->func_name, argno, btf_type_str(ref_t), ref_tname);
11261 			return -EINVAL;
11262 		}
11263 		return KF_ARG_PTR_TO_BTF_ID;
11264 	}
11265 
11266 	if (is_kfunc_arg_callback(env, meta->btf, &args[argno]))
11267 		return KF_ARG_PTR_TO_CALLBACK;
11268 
11269 	if (argno + 1 < nargs &&
11270 	    (is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], &regs[regno + 1]) ||
11271 	     is_kfunc_arg_const_mem_size(meta->btf, &args[argno + 1], &regs[regno + 1])))
11272 		arg_mem_size = true;
11273 
11274 	/* This is the catch all argument type of register types supported by
11275 	 * check_helper_mem_access. However, we only allow when argument type is
11276 	 * pointer to scalar, or struct composed (recursively) of scalars. When
11277 	 * arg_mem_size is true, the pointer can be void *.
11278 	 */
11279 	if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) &&
11280 	    (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) {
11281 		verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n",
11282 			argno, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : "");
11283 		return -EINVAL;
11284 	}
11285 	return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM;
11286 }
11287 
11288 static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
11289 					struct bpf_reg_state *reg,
11290 					const struct btf_type *ref_t,
11291 					const char *ref_tname, u32 ref_id,
11292 					struct bpf_kfunc_call_arg_meta *meta,
11293 					int argno)
11294 {
11295 	const struct btf_type *reg_ref_t;
11296 	bool strict_type_match = false;
11297 	const struct btf *reg_btf;
11298 	const char *reg_ref_tname;
11299 	bool taking_projection;
11300 	bool struct_same;
11301 	u32 reg_ref_id;
11302 
11303 	if (base_type(reg->type) == PTR_TO_BTF_ID) {
11304 		reg_btf = reg->btf;
11305 		reg_ref_id = reg->btf_id;
11306 	} else {
11307 		reg_btf = btf_vmlinux;
11308 		reg_ref_id = *reg2btf_ids[base_type(reg->type)];
11309 	}
11310 
11311 	/* Enforce strict type matching for calls to kfuncs that are acquiring
11312 	 * or releasing a reference, or are no-cast aliases. We do _not_
11313 	 * enforce strict matching for plain KF_TRUSTED_ARGS kfuncs by default,
11314 	 * as we want to enable BPF programs to pass types that are bitwise
11315 	 * equivalent without forcing them to explicitly cast with something
11316 	 * like bpf_cast_to_kern_ctx().
11317 	 *
11318 	 * For example, say we had a type like the following:
11319 	 *
11320 	 * struct bpf_cpumask {
11321 	 *	cpumask_t cpumask;
11322 	 *	refcount_t usage;
11323 	 * };
11324 	 *
11325 	 * Note that as specified in <linux/cpumask.h>, cpumask_t is typedef'ed
11326 	 * to a struct cpumask, so it would be safe to pass a struct
11327 	 * bpf_cpumask * to a kfunc expecting a struct cpumask *.
11328 	 *
11329 	 * The philosophy here is similar to how we allow scalars of different
11330 	 * types to be passed to kfuncs as long as the size is the same. The
11331 	 * only difference here is that we're simply allowing
11332 	 * btf_struct_ids_match() to walk the struct at the 0th offset, and
11333 	 * resolve types.
11334 	 */
11335 	if (is_kfunc_acquire(meta) ||
11336 	    (is_kfunc_release(meta) && reg->ref_obj_id) ||
11337 	    btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id))
11338 		strict_type_match = true;
11339 
11340 	WARN_ON_ONCE(is_kfunc_release(meta) &&
11341 		     (reg->off || !tnum_is_const(reg->var_off) ||
11342 		      reg->var_off.value));
11343 
11344 	reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, &reg_ref_id);
11345 	reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off);
11346 	struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type_match);
11347 	/* If kfunc is accepting a projection type (ie. __sk_buff), it cannot
11348 	 * actually use it -- it must cast to the underlying type. So we allow
11349 	 * caller to pass in the underlying type.
11350 	 */
11351 	taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname);
11352 	if (!taking_projection && !struct_same) {
11353 		verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
11354 			meta->func_name, argno, btf_type_str(ref_t), ref_tname, argno + 1,
11355 			btf_type_str(reg_ref_t), reg_ref_tname);
11356 		return -EINVAL;
11357 	}
11358 	return 0;
11359 }
11360 
11361 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
11362 {
11363 	struct bpf_verifier_state *state = env->cur_state;
11364 	struct btf_record *rec = reg_btf_record(reg);
11365 
11366 	if (!state->active_lock.ptr) {
11367 		verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n");
11368 		return -EFAULT;
11369 	}
11370 
11371 	if (type_flag(reg->type) & NON_OWN_REF) {
11372 		verbose(env, "verifier internal error: NON_OWN_REF already set\n");
11373 		return -EFAULT;
11374 	}
11375 
11376 	reg->type |= NON_OWN_REF;
11377 	if (rec->refcount_off >= 0)
11378 		reg->type |= MEM_RCU;
11379 
11380 	return 0;
11381 }
11382 
11383 static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id)
11384 {
11385 	struct bpf_func_state *state, *unused;
11386 	struct bpf_reg_state *reg;
11387 	int i;
11388 
11389 	state = cur_func(env);
11390 
11391 	if (!ref_obj_id) {
11392 		verbose(env, "verifier internal error: ref_obj_id is zero for "
11393 			     "owning -> non-owning conversion\n");
11394 		return -EFAULT;
11395 	}
11396 
11397 	for (i = 0; i < state->acquired_refs; i++) {
11398 		if (state->refs[i].id != ref_obj_id)
11399 			continue;
11400 
11401 		/* Clear ref_obj_id here so release_reference doesn't clobber
11402 		 * the whole reg
11403 		 */
11404 		bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
11405 			if (reg->ref_obj_id == ref_obj_id) {
11406 				reg->ref_obj_id = 0;
11407 				ref_set_non_owning(env, reg);
11408 			}
11409 		}));
11410 		return 0;
11411 	}
11412 
11413 	verbose(env, "verifier internal error: ref state missing for ref_obj_id\n");
11414 	return -EFAULT;
11415 }
11416 
11417 /* Implementation details:
11418  *
11419  * Each register points to some region of memory, which we define as an
11420  * allocation. Each allocation may embed a bpf_spin_lock which protects any
11421  * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same
11422  * allocation. The lock and the data it protects are colocated in the same
11423  * memory region.
11424  *
11425  * Hence, everytime a register holds a pointer value pointing to such
11426  * allocation, the verifier preserves a unique reg->id for it.
11427  *
11428  * The verifier remembers the lock 'ptr' and the lock 'id' whenever
11429  * bpf_spin_lock is called.
11430  *
11431  * To enable this, lock state in the verifier captures two values:
11432  *	active_lock.ptr = Register's type specific pointer
11433  *	active_lock.id  = A unique ID for each register pointer value
11434  *
11435  * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two
11436  * supported register types.
11437  *
11438  * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of
11439  * allocated objects is the reg->btf pointer.
11440  *
11441  * The active_lock.id is non-unique for maps supporting direct_value_addr, as we
11442  * can establish the provenance of the map value statically for each distinct
11443  * lookup into such maps. They always contain a single map value hence unique
11444  * IDs for each pseudo load pessimizes the algorithm and rejects valid programs.
11445  *
11446  * So, in case of global variables, they use array maps with max_entries = 1,
11447  * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point
11448  * into the same map value as max_entries is 1, as described above).
11449  *
11450  * In case of inner map lookups, the inner map pointer has same map_ptr as the
11451  * outer map pointer (in verifier context), but each lookup into an inner map
11452  * assigns a fresh reg->id to the lookup, so while lookups into distinct inner
11453  * maps from the same outer map share the same map_ptr as active_lock.ptr, they
11454  * will get different reg->id assigned to each lookup, hence different
11455  * active_lock.id.
11456  *
11457  * In case of allocated objects, active_lock.ptr is the reg->btf, and the
11458  * reg->id is a unique ID preserved after the NULL pointer check on the pointer
11459  * returned from bpf_obj_new. Each allocation receives a new reg->id.
11460  */
11461 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
11462 {
11463 	void *ptr;
11464 	u32 id;
11465 
11466 	switch ((int)reg->type) {
11467 	case PTR_TO_MAP_VALUE:
11468 		ptr = reg->map_ptr;
11469 		break;
11470 	case PTR_TO_BTF_ID | MEM_ALLOC:
11471 		ptr = reg->btf;
11472 		break;
11473 	default:
11474 		verbose(env, "verifier internal error: unknown reg type for lock check\n");
11475 		return -EFAULT;
11476 	}
11477 	id = reg->id;
11478 
11479 	if (!env->cur_state->active_lock.ptr)
11480 		return -EINVAL;
11481 	if (env->cur_state->active_lock.ptr != ptr ||
11482 	    env->cur_state->active_lock.id != id) {
11483 		verbose(env, "held lock and object are not in the same allocation\n");
11484 		return -EINVAL;
11485 	}
11486 	return 0;
11487 }
11488 
11489 static bool is_bpf_list_api_kfunc(u32 btf_id)
11490 {
11491 	return btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
11492 	       btf_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
11493 	       btf_id == special_kfunc_list[KF_bpf_list_pop_front] ||
11494 	       btf_id == special_kfunc_list[KF_bpf_list_pop_back];
11495 }
11496 
11497 static bool is_bpf_rbtree_api_kfunc(u32 btf_id)
11498 {
11499 	return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] ||
11500 	       btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
11501 	       btf_id == special_kfunc_list[KF_bpf_rbtree_first];
11502 }
11503 
11504 static bool is_bpf_graph_api_kfunc(u32 btf_id)
11505 {
11506 	return is_bpf_list_api_kfunc(btf_id) || is_bpf_rbtree_api_kfunc(btf_id) ||
11507 	       btf_id == special_kfunc_list[KF_bpf_refcount_acquire_impl];
11508 }
11509 
11510 static bool is_sync_callback_calling_kfunc(u32 btf_id)
11511 {
11512 	return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl];
11513 }
11514 
11515 static bool is_async_callback_calling_kfunc(u32 btf_id)
11516 {
11517 	return btf_id == special_kfunc_list[KF_bpf_wq_set_callback_impl];
11518 }
11519 
11520 static bool is_bpf_throw_kfunc(struct bpf_insn *insn)
11521 {
11522 	return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
11523 	       insn->imm == special_kfunc_list[KF_bpf_throw];
11524 }
11525 
11526 static bool is_bpf_wq_set_callback_impl_kfunc(u32 btf_id)
11527 {
11528 	return btf_id == special_kfunc_list[KF_bpf_wq_set_callback_impl];
11529 }
11530 
11531 static bool is_callback_calling_kfunc(u32 btf_id)
11532 {
11533 	return is_sync_callback_calling_kfunc(btf_id) ||
11534 	       is_async_callback_calling_kfunc(btf_id);
11535 }
11536 
11537 static bool is_rbtree_lock_required_kfunc(u32 btf_id)
11538 {
11539 	return is_bpf_rbtree_api_kfunc(btf_id);
11540 }
11541 
11542 static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env,
11543 					  enum btf_field_type head_field_type,
11544 					  u32 kfunc_btf_id)
11545 {
11546 	bool ret;
11547 
11548 	switch (head_field_type) {
11549 	case BPF_LIST_HEAD:
11550 		ret = is_bpf_list_api_kfunc(kfunc_btf_id);
11551 		break;
11552 	case BPF_RB_ROOT:
11553 		ret = is_bpf_rbtree_api_kfunc(kfunc_btf_id);
11554 		break;
11555 	default:
11556 		verbose(env, "verifier internal error: unexpected graph root argument type %s\n",
11557 			btf_field_type_name(head_field_type));
11558 		return false;
11559 	}
11560 
11561 	if (!ret)
11562 		verbose(env, "verifier internal error: %s head arg for unknown kfunc\n",
11563 			btf_field_type_name(head_field_type));
11564 	return ret;
11565 }
11566 
11567 static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env,
11568 					  enum btf_field_type node_field_type,
11569 					  u32 kfunc_btf_id)
11570 {
11571 	bool ret;
11572 
11573 	switch (node_field_type) {
11574 	case BPF_LIST_NODE:
11575 		ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
11576 		       kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_back_impl]);
11577 		break;
11578 	case BPF_RB_NODE:
11579 		ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
11580 		       kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]);
11581 		break;
11582 	default:
11583 		verbose(env, "verifier internal error: unexpected graph node argument type %s\n",
11584 			btf_field_type_name(node_field_type));
11585 		return false;
11586 	}
11587 
11588 	if (!ret)
11589 		verbose(env, "verifier internal error: %s node arg for unknown kfunc\n",
11590 			btf_field_type_name(node_field_type));
11591 	return ret;
11592 }
11593 
11594 static int
11595 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,
11596 				   struct bpf_reg_state *reg, u32 regno,
11597 				   struct bpf_kfunc_call_arg_meta *meta,
11598 				   enum btf_field_type head_field_type,
11599 				   struct btf_field **head_field)
11600 {
11601 	const char *head_type_name;
11602 	struct btf_field *field;
11603 	struct btf_record *rec;
11604 	u32 head_off;
11605 
11606 	if (meta->btf != btf_vmlinux) {
11607 		verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n");
11608 		return -EFAULT;
11609 	}
11610 
11611 	if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id))
11612 		return -EFAULT;
11613 
11614 	head_type_name = btf_field_type_name(head_field_type);
11615 	if (!tnum_is_const(reg->var_off)) {
11616 		verbose(env,
11617 			"R%d doesn't have constant offset. %s has to be at the constant offset\n",
11618 			regno, head_type_name);
11619 		return -EINVAL;
11620 	}
11621 
11622 	rec = reg_btf_record(reg);
11623 	head_off = reg->off + reg->var_off.value;
11624 	field = btf_record_find(rec, head_off, head_field_type);
11625 	if (!field) {
11626 		verbose(env, "%s not found at offset=%u\n", head_type_name, head_off);
11627 		return -EINVAL;
11628 	}
11629 
11630 	/* All functions require bpf_list_head to be protected using a bpf_spin_lock */
11631 	if (check_reg_allocation_locked(env, reg)) {
11632 		verbose(env, "bpf_spin_lock at off=%d must be held for %s\n",
11633 			rec->spin_lock_off, head_type_name);
11634 		return -EINVAL;
11635 	}
11636 
11637 	if (*head_field) {
11638 		verbose(env, "verifier internal error: repeating %s arg\n", head_type_name);
11639 		return -EFAULT;
11640 	}
11641 	*head_field = field;
11642 	return 0;
11643 }
11644 
11645 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,
11646 					   struct bpf_reg_state *reg, u32 regno,
11647 					   struct bpf_kfunc_call_arg_meta *meta)
11648 {
11649 	return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD,
11650 							  &meta->arg_list_head.field);
11651 }
11652 
11653 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,
11654 					     struct bpf_reg_state *reg, u32 regno,
11655 					     struct bpf_kfunc_call_arg_meta *meta)
11656 {
11657 	return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT,
11658 							  &meta->arg_rbtree_root.field);
11659 }
11660 
11661 static int
11662 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
11663 				   struct bpf_reg_state *reg, u32 regno,
11664 				   struct bpf_kfunc_call_arg_meta *meta,
11665 				   enum btf_field_type head_field_type,
11666 				   enum btf_field_type node_field_type,
11667 				   struct btf_field **node_field)
11668 {
11669 	const char *node_type_name;
11670 	const struct btf_type *et, *t;
11671 	struct btf_field *field;
11672 	u32 node_off;
11673 
11674 	if (meta->btf != btf_vmlinux) {
11675 		verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n");
11676 		return -EFAULT;
11677 	}
11678 
11679 	if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id))
11680 		return -EFAULT;
11681 
11682 	node_type_name = btf_field_type_name(node_field_type);
11683 	if (!tnum_is_const(reg->var_off)) {
11684 		verbose(env,
11685 			"R%d doesn't have constant offset. %s has to be at the constant offset\n",
11686 			regno, node_type_name);
11687 		return -EINVAL;
11688 	}
11689 
11690 	node_off = reg->off + reg->var_off.value;
11691 	field = reg_find_field_offset(reg, node_off, node_field_type);
11692 	if (!field) {
11693 		verbose(env, "%s not found at offset=%u\n", node_type_name, node_off);
11694 		return -EINVAL;
11695 	}
11696 
11697 	field = *node_field;
11698 
11699 	et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id);
11700 	t = btf_type_by_id(reg->btf, reg->btf_id);
11701 	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf,
11702 				  field->graph_root.value_btf_id, true)) {
11703 		verbose(env, "operation on %s expects arg#1 %s at offset=%d "
11704 			"in struct %s, but arg is at offset=%d in struct %s\n",
11705 			btf_field_type_name(head_field_type),
11706 			btf_field_type_name(node_field_type),
11707 			field->graph_root.node_offset,
11708 			btf_name_by_offset(field->graph_root.btf, et->name_off),
11709 			node_off, btf_name_by_offset(reg->btf, t->name_off));
11710 		return -EINVAL;
11711 	}
11712 	meta->arg_btf = reg->btf;
11713 	meta->arg_btf_id = reg->btf_id;
11714 
11715 	if (node_off != field->graph_root.node_offset) {
11716 		verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n",
11717 			node_off, btf_field_type_name(node_field_type),
11718 			field->graph_root.node_offset,
11719 			btf_name_by_offset(field->graph_root.btf, et->name_off));
11720 		return -EINVAL;
11721 	}
11722 
11723 	return 0;
11724 }
11725 
11726 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
11727 					   struct bpf_reg_state *reg, u32 regno,
11728 					   struct bpf_kfunc_call_arg_meta *meta)
11729 {
11730 	return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,
11731 						  BPF_LIST_HEAD, BPF_LIST_NODE,
11732 						  &meta->arg_list_head.field);
11733 }
11734 
11735 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,
11736 					     struct bpf_reg_state *reg, u32 regno,
11737 					     struct bpf_kfunc_call_arg_meta *meta)
11738 {
11739 	return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,
11740 						  BPF_RB_ROOT, BPF_RB_NODE,
11741 						  &meta->arg_rbtree_root.field);
11742 }
11743 
11744 /*
11745  * css_task iter allowlist is needed to avoid dead locking on css_set_lock.
11746  * LSM hooks and iters (both sleepable and non-sleepable) are safe.
11747  * Any sleepable progs are also safe since bpf_check_attach_target() enforce
11748  * them can only be attached to some specific hook points.
11749  */
11750 static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env)
11751 {
11752 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
11753 
11754 	switch (prog_type) {
11755 	case BPF_PROG_TYPE_LSM:
11756 		return true;
11757 	case BPF_PROG_TYPE_TRACING:
11758 		if (env->prog->expected_attach_type == BPF_TRACE_ITER)
11759 			return true;
11760 		fallthrough;
11761 	default:
11762 		return in_sleepable(env);
11763 	}
11764 }
11765 
11766 static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,
11767 			    int insn_idx)
11768 {
11769 	const char *func_name = meta->func_name, *ref_tname;
11770 	const struct btf *btf = meta->btf;
11771 	const struct btf_param *args;
11772 	struct btf_record *rec;
11773 	u32 i, nargs;
11774 	int ret;
11775 
11776 	args = (const struct btf_param *)(meta->func_proto + 1);
11777 	nargs = btf_type_vlen(meta->func_proto);
11778 	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
11779 		verbose(env, "Function %s has %d > %d args\n", func_name, nargs,
11780 			MAX_BPF_FUNC_REG_ARGS);
11781 		return -EINVAL;
11782 	}
11783 
11784 	/* Check that BTF function arguments match actual types that the
11785 	 * verifier sees.
11786 	 */
11787 	for (i = 0; i < nargs; i++) {
11788 		struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[i + 1];
11789 		const struct btf_type *t, *ref_t, *resolve_ret;
11790 		enum bpf_arg_type arg_type = ARG_DONTCARE;
11791 		u32 regno = i + 1, ref_id, type_size;
11792 		bool is_ret_buf_sz = false;
11793 		int kf_arg_type;
11794 
11795 		t = btf_type_skip_modifiers(btf, args[i].type, NULL);
11796 
11797 		if (is_kfunc_arg_ignore(btf, &args[i]))
11798 			continue;
11799 
11800 		if (btf_type_is_scalar(t)) {
11801 			if (reg->type != SCALAR_VALUE) {
11802 				verbose(env, "R%d is not a scalar\n", regno);
11803 				return -EINVAL;
11804 			}
11805 
11806 			if (is_kfunc_arg_constant(meta->btf, &args[i])) {
11807 				if (meta->arg_constant.found) {
11808 					verbose(env, "verifier internal error: only one constant argument permitted\n");
11809 					return -EFAULT;
11810 				}
11811 				if (!tnum_is_const(reg->var_off)) {
11812 					verbose(env, "R%d must be a known constant\n", regno);
11813 					return -EINVAL;
11814 				}
11815 				ret = mark_chain_precision(env, regno);
11816 				if (ret < 0)
11817 					return ret;
11818 				meta->arg_constant.found = true;
11819 				meta->arg_constant.value = reg->var_off.value;
11820 			} else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) {
11821 				meta->r0_rdonly = true;
11822 				is_ret_buf_sz = true;
11823 			} else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) {
11824 				is_ret_buf_sz = true;
11825 			}
11826 
11827 			if (is_ret_buf_sz) {
11828 				if (meta->r0_size) {
11829 					verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc");
11830 					return -EINVAL;
11831 				}
11832 
11833 				if (!tnum_is_const(reg->var_off)) {
11834 					verbose(env, "R%d is not a const\n", regno);
11835 					return -EINVAL;
11836 				}
11837 
11838 				meta->r0_size = reg->var_off.value;
11839 				ret = mark_chain_precision(env, regno);
11840 				if (ret)
11841 					return ret;
11842 			}
11843 			continue;
11844 		}
11845 
11846 		if (!btf_type_is_ptr(t)) {
11847 			verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t));
11848 			return -EINVAL;
11849 		}
11850 
11851 		if ((is_kfunc_trusted_args(meta) || is_kfunc_rcu(meta)) &&
11852 		    (register_is_null(reg) || type_may_be_null(reg->type)) &&
11853 			!is_kfunc_arg_nullable(meta->btf, &args[i])) {
11854 			verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i);
11855 			return -EACCES;
11856 		}
11857 
11858 		if (reg->ref_obj_id) {
11859 			if (is_kfunc_release(meta) && meta->ref_obj_id) {
11860 				verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
11861 					regno, reg->ref_obj_id,
11862 					meta->ref_obj_id);
11863 				return -EFAULT;
11864 			}
11865 			meta->ref_obj_id = reg->ref_obj_id;
11866 			if (is_kfunc_release(meta))
11867 				meta->release_regno = regno;
11868 		}
11869 
11870 		ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
11871 		ref_tname = btf_name_by_offset(btf, ref_t->name_off);
11872 
11873 		kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs);
11874 		if (kf_arg_type < 0)
11875 			return kf_arg_type;
11876 
11877 		switch (kf_arg_type) {
11878 		case KF_ARG_PTR_TO_NULL:
11879 			continue;
11880 		case KF_ARG_PTR_TO_MAP:
11881 			if (!reg->map_ptr) {
11882 				verbose(env, "pointer in R%d isn't map pointer\n", regno);
11883 				return -EINVAL;
11884 			}
11885 			if (meta->map.ptr && reg->map_ptr->record->wq_off >= 0) {
11886 				/* Use map_uid (which is unique id of inner map) to reject:
11887 				 * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
11888 				 * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
11889 				 * if (inner_map1 && inner_map2) {
11890 				 *     wq = bpf_map_lookup_elem(inner_map1);
11891 				 *     if (wq)
11892 				 *         // mismatch would have been allowed
11893 				 *         bpf_wq_init(wq, inner_map2);
11894 				 * }
11895 				 *
11896 				 * Comparing map_ptr is enough to distinguish normal and outer maps.
11897 				 */
11898 				if (meta->map.ptr != reg->map_ptr ||
11899 				    meta->map.uid != reg->map_uid) {
11900 					verbose(env,
11901 						"workqueue pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n",
11902 						meta->map.uid, reg->map_uid);
11903 					return -EINVAL;
11904 				}
11905 			}
11906 			meta->map.ptr = reg->map_ptr;
11907 			meta->map.uid = reg->map_uid;
11908 			fallthrough;
11909 		case KF_ARG_PTR_TO_ALLOC_BTF_ID:
11910 		case KF_ARG_PTR_TO_BTF_ID:
11911 			if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta))
11912 				break;
11913 
11914 			if (!is_trusted_reg(reg)) {
11915 				if (!is_kfunc_rcu(meta)) {
11916 					verbose(env, "R%d must be referenced or trusted\n", regno);
11917 					return -EINVAL;
11918 				}
11919 				if (!is_rcu_reg(reg)) {
11920 					verbose(env, "R%d must be a rcu pointer\n", regno);
11921 					return -EINVAL;
11922 				}
11923 			}
11924 			fallthrough;
11925 		case KF_ARG_PTR_TO_CTX:
11926 		case KF_ARG_PTR_TO_DYNPTR:
11927 		case KF_ARG_PTR_TO_ITER:
11928 		case KF_ARG_PTR_TO_LIST_HEAD:
11929 		case KF_ARG_PTR_TO_LIST_NODE:
11930 		case KF_ARG_PTR_TO_RB_ROOT:
11931 		case KF_ARG_PTR_TO_RB_NODE:
11932 		case KF_ARG_PTR_TO_MEM:
11933 		case KF_ARG_PTR_TO_MEM_SIZE:
11934 		case KF_ARG_PTR_TO_CALLBACK:
11935 		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
11936 		case KF_ARG_PTR_TO_CONST_STR:
11937 		case KF_ARG_PTR_TO_WORKQUEUE:
11938 			break;
11939 		default:
11940 			WARN_ON_ONCE(1);
11941 			return -EFAULT;
11942 		}
11943 
11944 		if (is_kfunc_release(meta) && reg->ref_obj_id)
11945 			arg_type |= OBJ_RELEASE;
11946 		ret = check_func_arg_reg_off(env, reg, regno, arg_type);
11947 		if (ret < 0)
11948 			return ret;
11949 
11950 		switch (kf_arg_type) {
11951 		case KF_ARG_PTR_TO_CTX:
11952 			if (reg->type != PTR_TO_CTX) {
11953 				verbose(env, "arg#%d expected pointer to ctx, but got %s\n", i, btf_type_str(t));
11954 				return -EINVAL;
11955 			}
11956 
11957 			if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
11958 				ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog));
11959 				if (ret < 0)
11960 					return -EINVAL;
11961 				meta->ret_btf_id  = ret;
11962 			}
11963 			break;
11964 		case KF_ARG_PTR_TO_ALLOC_BTF_ID:
11965 			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) {
11966 				if (meta->func_id != special_kfunc_list[KF_bpf_obj_drop_impl]) {
11967 					verbose(env, "arg#%d expected for bpf_obj_drop_impl()\n", i);
11968 					return -EINVAL;
11969 				}
11970 			} else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) {
11971 				if (meta->func_id != special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) {
11972 					verbose(env, "arg#%d expected for bpf_percpu_obj_drop_impl()\n", i);
11973 					return -EINVAL;
11974 				}
11975 			} else {
11976 				verbose(env, "arg#%d expected pointer to allocated object\n", i);
11977 				return -EINVAL;
11978 			}
11979 			if (!reg->ref_obj_id) {
11980 				verbose(env, "allocated object must be referenced\n");
11981 				return -EINVAL;
11982 			}
11983 			if (meta->btf == btf_vmlinux) {
11984 				meta->arg_btf = reg->btf;
11985 				meta->arg_btf_id = reg->btf_id;
11986 			}
11987 			break;
11988 		case KF_ARG_PTR_TO_DYNPTR:
11989 		{
11990 			enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
11991 			int clone_ref_obj_id = 0;
11992 
11993 			if (reg->type == CONST_PTR_TO_DYNPTR)
11994 				dynptr_arg_type |= MEM_RDONLY;
11995 
11996 			if (is_kfunc_arg_uninit(btf, &args[i]))
11997 				dynptr_arg_type |= MEM_UNINIT;
11998 
11999 			if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
12000 				dynptr_arg_type |= DYNPTR_TYPE_SKB;
12001 			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) {
12002 				dynptr_arg_type |= DYNPTR_TYPE_XDP;
12003 			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] &&
12004 				   (dynptr_arg_type & MEM_UNINIT)) {
12005 				enum bpf_dynptr_type parent_type = meta->initialized_dynptr.type;
12006 
12007 				if (parent_type == BPF_DYNPTR_TYPE_INVALID) {
12008 					verbose(env, "verifier internal error: no dynptr type for parent of clone\n");
12009 					return -EFAULT;
12010 				}
12011 
12012 				dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type);
12013 				clone_ref_obj_id = meta->initialized_dynptr.ref_obj_id;
12014 				if (dynptr_type_refcounted(parent_type) && !clone_ref_obj_id) {
12015 					verbose(env, "verifier internal error: missing ref obj id for parent of clone\n");
12016 					return -EFAULT;
12017 				}
12018 			}
12019 
12020 			ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id);
12021 			if (ret < 0)
12022 				return ret;
12023 
12024 			if (!(dynptr_arg_type & MEM_UNINIT)) {
12025 				int id = dynptr_id(env, reg);
12026 
12027 				if (id < 0) {
12028 					verbose(env, "verifier internal error: failed to obtain dynptr id\n");
12029 					return id;
12030 				}
12031 				meta->initialized_dynptr.id = id;
12032 				meta->initialized_dynptr.type = dynptr_get_type(env, reg);
12033 				meta->initialized_dynptr.ref_obj_id = dynptr_ref_obj_id(env, reg);
12034 			}
12035 
12036 			break;
12037 		}
12038 		case KF_ARG_PTR_TO_ITER:
12039 			if (meta->func_id == special_kfunc_list[KF_bpf_iter_css_task_new]) {
12040 				if (!check_css_task_iter_allowlist(env)) {
12041 					verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n");
12042 					return -EINVAL;
12043 				}
12044 			}
12045 			ret = process_iter_arg(env, regno, insn_idx, meta);
12046 			if (ret < 0)
12047 				return ret;
12048 			break;
12049 		case KF_ARG_PTR_TO_LIST_HEAD:
12050 			if (reg->type != PTR_TO_MAP_VALUE &&
12051 			    reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
12052 				verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);
12053 				return -EINVAL;
12054 			}
12055 			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
12056 				verbose(env, "allocated object must be referenced\n");
12057 				return -EINVAL;
12058 			}
12059 			ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta);
12060 			if (ret < 0)
12061 				return ret;
12062 			break;
12063 		case KF_ARG_PTR_TO_RB_ROOT:
12064 			if (reg->type != PTR_TO_MAP_VALUE &&
12065 			    reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
12066 				verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);
12067 				return -EINVAL;
12068 			}
12069 			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
12070 				verbose(env, "allocated object must be referenced\n");
12071 				return -EINVAL;
12072 			}
12073 			ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta);
12074 			if (ret < 0)
12075 				return ret;
12076 			break;
12077 		case KF_ARG_PTR_TO_LIST_NODE:
12078 			if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
12079 				verbose(env, "arg#%d expected pointer to allocated object\n", i);
12080 				return -EINVAL;
12081 			}
12082 			if (!reg->ref_obj_id) {
12083 				verbose(env, "allocated object must be referenced\n");
12084 				return -EINVAL;
12085 			}
12086 			ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta);
12087 			if (ret < 0)
12088 				return ret;
12089 			break;
12090 		case KF_ARG_PTR_TO_RB_NODE:
12091 			if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_remove]) {
12092 				if (!type_is_non_owning_ref(reg->type) || reg->ref_obj_id) {
12093 					verbose(env, "rbtree_remove node input must be non-owning ref\n");
12094 					return -EINVAL;
12095 				}
12096 				if (in_rbtree_lock_required_cb(env)) {
12097 					verbose(env, "rbtree_remove not allowed in rbtree cb\n");
12098 					return -EINVAL;
12099 				}
12100 			} else {
12101 				if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
12102 					verbose(env, "arg#%d expected pointer to allocated object\n", i);
12103 					return -EINVAL;
12104 				}
12105 				if (!reg->ref_obj_id) {
12106 					verbose(env, "allocated object must be referenced\n");
12107 					return -EINVAL;
12108 				}
12109 			}
12110 
12111 			ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta);
12112 			if (ret < 0)
12113 				return ret;
12114 			break;
12115 		case KF_ARG_PTR_TO_MAP:
12116 			/* If argument has '__map' suffix expect 'struct bpf_map *' */
12117 			ref_id = *reg2btf_ids[CONST_PTR_TO_MAP];
12118 			ref_t = btf_type_by_id(btf_vmlinux, ref_id);
12119 			ref_tname = btf_name_by_offset(btf, ref_t->name_off);
12120 			fallthrough;
12121 		case KF_ARG_PTR_TO_BTF_ID:
12122 			/* Only base_type is checked, further checks are done here */
12123 			if ((base_type(reg->type) != PTR_TO_BTF_ID ||
12124 			     (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) &&
12125 			    !reg2btf_ids[base_type(reg->type)]) {
12126 				verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type));
12127 				verbose(env, "expected %s or socket\n",
12128 					reg_type_str(env, base_type(reg->type) |
12129 							  (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS)));
12130 				return -EINVAL;
12131 			}
12132 			ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i);
12133 			if (ret < 0)
12134 				return ret;
12135 			break;
12136 		case KF_ARG_PTR_TO_MEM:
12137 			resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
12138 			if (IS_ERR(resolve_ret)) {
12139 				verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
12140 					i, btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret));
12141 				return -EINVAL;
12142 			}
12143 			ret = check_mem_reg(env, reg, regno, type_size);
12144 			if (ret < 0)
12145 				return ret;
12146 			break;
12147 		case KF_ARG_PTR_TO_MEM_SIZE:
12148 		{
12149 			struct bpf_reg_state *buff_reg = &regs[regno];
12150 			const struct btf_param *buff_arg = &args[i];
12151 			struct bpf_reg_state *size_reg = &regs[regno + 1];
12152 			const struct btf_param *size_arg = &args[i + 1];
12153 
12154 			if (!register_is_null(buff_reg) || !is_kfunc_arg_optional(meta->btf, buff_arg)) {
12155 				ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1);
12156 				if (ret < 0) {
12157 					verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1);
12158 					return ret;
12159 				}
12160 			}
12161 
12162 			if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) {
12163 				if (meta->arg_constant.found) {
12164 					verbose(env, "verifier internal error: only one constant argument permitted\n");
12165 					return -EFAULT;
12166 				}
12167 				if (!tnum_is_const(size_reg->var_off)) {
12168 					verbose(env, "R%d must be a known constant\n", regno + 1);
12169 					return -EINVAL;
12170 				}
12171 				meta->arg_constant.found = true;
12172 				meta->arg_constant.value = size_reg->var_off.value;
12173 			}
12174 
12175 			/* Skip next '__sz' or '__szk' argument */
12176 			i++;
12177 			break;
12178 		}
12179 		case KF_ARG_PTR_TO_CALLBACK:
12180 			if (reg->type != PTR_TO_FUNC) {
12181 				verbose(env, "arg%d expected pointer to func\n", i);
12182 				return -EINVAL;
12183 			}
12184 			meta->subprogno = reg->subprogno;
12185 			break;
12186 		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
12187 			if (!type_is_ptr_alloc_obj(reg->type)) {
12188 				verbose(env, "arg#%d is neither owning or non-owning ref\n", i);
12189 				return -EINVAL;
12190 			}
12191 			if (!type_is_non_owning_ref(reg->type))
12192 				meta->arg_owning_ref = true;
12193 
12194 			rec = reg_btf_record(reg);
12195 			if (!rec) {
12196 				verbose(env, "verifier internal error: Couldn't find btf_record\n");
12197 				return -EFAULT;
12198 			}
12199 
12200 			if (rec->refcount_off < 0) {
12201 				verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i);
12202 				return -EINVAL;
12203 			}
12204 
12205 			meta->arg_btf = reg->btf;
12206 			meta->arg_btf_id = reg->btf_id;
12207 			break;
12208 		case KF_ARG_PTR_TO_CONST_STR:
12209 			if (reg->type != PTR_TO_MAP_VALUE) {
12210 				verbose(env, "arg#%d doesn't point to a const string\n", i);
12211 				return -EINVAL;
12212 			}
12213 			ret = check_reg_const_str(env, reg, regno);
12214 			if (ret)
12215 				return ret;
12216 			break;
12217 		case KF_ARG_PTR_TO_WORKQUEUE:
12218 			if (reg->type != PTR_TO_MAP_VALUE) {
12219 				verbose(env, "arg#%d doesn't point to a map value\n", i);
12220 				return -EINVAL;
12221 			}
12222 			ret = process_wq_func(env, regno, meta);
12223 			if (ret < 0)
12224 				return ret;
12225 			break;
12226 		}
12227 	}
12228 
12229 	if (is_kfunc_release(meta) && !meta->release_regno) {
12230 		verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n",
12231 			func_name);
12232 		return -EINVAL;
12233 	}
12234 
12235 	return 0;
12236 }
12237 
12238 static int fetch_kfunc_meta(struct bpf_verifier_env *env,
12239 			    struct bpf_insn *insn,
12240 			    struct bpf_kfunc_call_arg_meta *meta,
12241 			    const char **kfunc_name)
12242 {
12243 	const struct btf_type *func, *func_proto;
12244 	u32 func_id, *kfunc_flags;
12245 	const char *func_name;
12246 	struct btf *desc_btf;
12247 
12248 	if (kfunc_name)
12249 		*kfunc_name = NULL;
12250 
12251 	if (!insn->imm)
12252 		return -EINVAL;
12253 
12254 	desc_btf = find_kfunc_desc_btf(env, insn->off);
12255 	if (IS_ERR(desc_btf))
12256 		return PTR_ERR(desc_btf);
12257 
12258 	func_id = insn->imm;
12259 	func = btf_type_by_id(desc_btf, func_id);
12260 	func_name = btf_name_by_offset(desc_btf, func->name_off);
12261 	if (kfunc_name)
12262 		*kfunc_name = func_name;
12263 	func_proto = btf_type_by_id(desc_btf, func->type);
12264 
12265 	kfunc_flags = btf_kfunc_id_set_contains(desc_btf, func_id, env->prog);
12266 	if (!kfunc_flags) {
12267 		return -EACCES;
12268 	}
12269 
12270 	memset(meta, 0, sizeof(*meta));
12271 	meta->btf = desc_btf;
12272 	meta->func_id = func_id;
12273 	meta->kfunc_flags = *kfunc_flags;
12274 	meta->func_proto = func_proto;
12275 	meta->func_name = func_name;
12276 
12277 	return 0;
12278 }
12279 
12280 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
12281 
12282 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
12283 			    int *insn_idx_p)
12284 {
12285 	bool sleepable, rcu_lock, rcu_unlock, preempt_disable, preempt_enable;
12286 	u32 i, nargs, ptr_type_id, release_ref_obj_id;
12287 	struct bpf_reg_state *regs = cur_regs(env);
12288 	const char *func_name, *ptr_type_name;
12289 	const struct btf_type *t, *ptr_type;
12290 	struct bpf_kfunc_call_arg_meta meta;
12291 	struct bpf_insn_aux_data *insn_aux;
12292 	int err, insn_idx = *insn_idx_p;
12293 	const struct btf_param *args;
12294 	const struct btf_type *ret_t;
12295 	struct btf *desc_btf;
12296 
12297 	/* skip for now, but return error when we find this in fixup_kfunc_call */
12298 	if (!insn->imm)
12299 		return 0;
12300 
12301 	err = fetch_kfunc_meta(env, insn, &meta, &func_name);
12302 	if (err == -EACCES && func_name)
12303 		verbose(env, "calling kernel function %s is not allowed\n", func_name);
12304 	if (err)
12305 		return err;
12306 	desc_btf = meta.btf;
12307 	insn_aux = &env->insn_aux_data[insn_idx];
12308 
12309 	insn_aux->is_iter_next = is_iter_next_kfunc(&meta);
12310 
12311 	if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) {
12312 		verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n");
12313 		return -EACCES;
12314 	}
12315 
12316 	sleepable = is_kfunc_sleepable(&meta);
12317 	if (sleepable && !in_sleepable(env)) {
12318 		verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name);
12319 		return -EACCES;
12320 	}
12321 
12322 	/* Check the arguments */
12323 	err = check_kfunc_args(env, &meta, insn_idx);
12324 	if (err < 0)
12325 		return err;
12326 
12327 	if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
12328 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
12329 					 set_rbtree_add_callback_state);
12330 		if (err) {
12331 			verbose(env, "kfunc %s#%d failed callback verification\n",
12332 				func_name, meta.func_id);
12333 			return err;
12334 		}
12335 	}
12336 
12337 	if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) {
12338 		meta.r0_size = sizeof(u64);
12339 		meta.r0_rdonly = false;
12340 	}
12341 
12342 	if (is_bpf_wq_set_callback_impl_kfunc(meta.func_id)) {
12343 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
12344 					 set_timer_callback_state);
12345 		if (err) {
12346 			verbose(env, "kfunc %s#%d failed callback verification\n",
12347 				func_name, meta.func_id);
12348 			return err;
12349 		}
12350 	}
12351 
12352 	rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta);
12353 	rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta);
12354 
12355 	preempt_disable = is_kfunc_bpf_preempt_disable(&meta);
12356 	preempt_enable = is_kfunc_bpf_preempt_enable(&meta);
12357 
12358 	if (env->cur_state->active_rcu_lock) {
12359 		struct bpf_func_state *state;
12360 		struct bpf_reg_state *reg;
12361 		u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);
12362 
12363 		if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {
12364 			verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");
12365 			return -EACCES;
12366 		}
12367 
12368 		if (rcu_lock) {
12369 			verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
12370 			return -EINVAL;
12371 		} else if (rcu_unlock) {
12372 			bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
12373 				if (reg->type & MEM_RCU) {
12374 					reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
12375 					reg->type |= PTR_UNTRUSTED;
12376 				}
12377 			}));
12378 			env->cur_state->active_rcu_lock = false;
12379 		} else if (sleepable) {
12380 			verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
12381 			return -EACCES;
12382 		}
12383 	} else if (rcu_lock) {
12384 		env->cur_state->active_rcu_lock = true;
12385 	} else if (rcu_unlock) {
12386 		verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
12387 		return -EINVAL;
12388 	}
12389 
12390 	if (env->cur_state->active_preempt_lock) {
12391 		if (preempt_disable) {
12392 			env->cur_state->active_preempt_lock++;
12393 		} else if (preempt_enable) {
12394 			env->cur_state->active_preempt_lock--;
12395 		} else if (sleepable) {
12396 			verbose(env, "kernel func %s is sleepable within non-preemptible region\n", func_name);
12397 			return -EACCES;
12398 		}
12399 	} else if (preempt_disable) {
12400 		env->cur_state->active_preempt_lock++;
12401 	} else if (preempt_enable) {
12402 		verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name);
12403 		return -EINVAL;
12404 	}
12405 
12406 	/* In case of release function, we get register number of refcounted
12407 	 * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now.
12408 	 */
12409 	if (meta.release_regno) {
12410 		err = release_reference(env, regs[meta.release_regno].ref_obj_id);
12411 		if (err) {
12412 			verbose(env, "kfunc %s#%d reference has not been acquired before\n",
12413 				func_name, meta.func_id);
12414 			return err;
12415 		}
12416 	}
12417 
12418 	if (meta.func_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
12419 	    meta.func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
12420 	    meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
12421 		release_ref_obj_id = regs[BPF_REG_2].ref_obj_id;
12422 		insn_aux->insert_off = regs[BPF_REG_2].off;
12423 		insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
12424 		err = ref_convert_owning_non_owning(env, release_ref_obj_id);
12425 		if (err) {
12426 			verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n",
12427 				func_name, meta.func_id);
12428 			return err;
12429 		}
12430 
12431 		err = release_reference(env, release_ref_obj_id);
12432 		if (err) {
12433 			verbose(env, "kfunc %s#%d reference has not been acquired before\n",
12434 				func_name, meta.func_id);
12435 			return err;
12436 		}
12437 	}
12438 
12439 	if (meta.func_id == special_kfunc_list[KF_bpf_throw]) {
12440 		if (!bpf_jit_supports_exceptions()) {
12441 			verbose(env, "JIT does not support calling kfunc %s#%d\n",
12442 				func_name, meta.func_id);
12443 			return -ENOTSUPP;
12444 		}
12445 		env->seen_exception = true;
12446 
12447 		/* In the case of the default callback, the cookie value passed
12448 		 * to bpf_throw becomes the return value of the program.
12449 		 */
12450 		if (!env->exception_callback_subprog) {
12451 			err = check_return_code(env, BPF_REG_1, "R1");
12452 			if (err < 0)
12453 				return err;
12454 		}
12455 	}
12456 
12457 	for (i = 0; i < CALLER_SAVED_REGS; i++)
12458 		mark_reg_not_init(env, regs, caller_saved[i]);
12459 
12460 	/* Check return type */
12461 	t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL);
12462 
12463 	if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) {
12464 		/* Only exception is bpf_obj_new_impl */
12465 		if (meta.btf != btf_vmlinux ||
12466 		    (meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl] &&
12467 		     meta.func_id != special_kfunc_list[KF_bpf_percpu_obj_new_impl] &&
12468 		     meta.func_id != special_kfunc_list[KF_bpf_refcount_acquire_impl])) {
12469 			verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n");
12470 			return -EINVAL;
12471 		}
12472 	}
12473 
12474 	if (btf_type_is_scalar(t)) {
12475 		mark_reg_unknown(env, regs, BPF_REG_0);
12476 		mark_btf_func_reg_size(env, BPF_REG_0, t->size);
12477 	} else if (btf_type_is_ptr(t)) {
12478 		ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id);
12479 
12480 		if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) {
12481 			if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl] ||
12482 			    meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
12483 				struct btf_struct_meta *struct_meta;
12484 				struct btf *ret_btf;
12485 				u32 ret_btf_id;
12486 
12487 				if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl] && !bpf_global_ma_set)
12488 					return -ENOMEM;
12489 
12490 				if (((u64)(u32)meta.arg_constant.value) != meta.arg_constant.value) {
12491 					verbose(env, "local type ID argument must be in range [0, U32_MAX]\n");
12492 					return -EINVAL;
12493 				}
12494 
12495 				ret_btf = env->prog->aux->btf;
12496 				ret_btf_id = meta.arg_constant.value;
12497 
12498 				/* This may be NULL due to user not supplying a BTF */
12499 				if (!ret_btf) {
12500 					verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n");
12501 					return -EINVAL;
12502 				}
12503 
12504 				ret_t = btf_type_by_id(ret_btf, ret_btf_id);
12505 				if (!ret_t || !__btf_type_is_struct(ret_t)) {
12506 					verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n");
12507 					return -EINVAL;
12508 				}
12509 
12510 				if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
12511 					if (ret_t->size > BPF_GLOBAL_PERCPU_MA_MAX_SIZE) {
12512 						verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n",
12513 							ret_t->size, BPF_GLOBAL_PERCPU_MA_MAX_SIZE);
12514 						return -EINVAL;
12515 					}
12516 
12517 					if (!bpf_global_percpu_ma_set) {
12518 						mutex_lock(&bpf_percpu_ma_lock);
12519 						if (!bpf_global_percpu_ma_set) {
12520 							/* Charge memory allocated with bpf_global_percpu_ma to
12521 							 * root memcg. The obj_cgroup for root memcg is NULL.
12522 							 */
12523 							err = bpf_mem_alloc_percpu_init(&bpf_global_percpu_ma, NULL);
12524 							if (!err)
12525 								bpf_global_percpu_ma_set = true;
12526 						}
12527 						mutex_unlock(&bpf_percpu_ma_lock);
12528 						if (err)
12529 							return err;
12530 					}
12531 
12532 					mutex_lock(&bpf_percpu_ma_lock);
12533 					err = bpf_mem_alloc_percpu_unit_init(&bpf_global_percpu_ma, ret_t->size);
12534 					mutex_unlock(&bpf_percpu_ma_lock);
12535 					if (err)
12536 						return err;
12537 				}
12538 
12539 				struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id);
12540 				if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
12541 					if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) {
12542 						verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n");
12543 						return -EINVAL;
12544 					}
12545 
12546 					if (struct_meta) {
12547 						verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n");
12548 						return -EINVAL;
12549 					}
12550 				}
12551 
12552 				mark_reg_known_zero(env, regs, BPF_REG_0);
12553 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
12554 				regs[BPF_REG_0].btf = ret_btf;
12555 				regs[BPF_REG_0].btf_id = ret_btf_id;
12556 				if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl])
12557 					regs[BPF_REG_0].type |= MEM_PERCPU;
12558 
12559 				insn_aux->obj_new_size = ret_t->size;
12560 				insn_aux->kptr_struct_meta = struct_meta;
12561 			} else if (meta.func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {
12562 				mark_reg_known_zero(env, regs, BPF_REG_0);
12563 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
12564 				regs[BPF_REG_0].btf = meta.arg_btf;
12565 				regs[BPF_REG_0].btf_id = meta.arg_btf_id;
12566 
12567 				insn_aux->kptr_struct_meta =
12568 					btf_find_struct_meta(meta.arg_btf,
12569 							     meta.arg_btf_id);
12570 			} else if (meta.func_id == special_kfunc_list[KF_bpf_list_pop_front] ||
12571 				   meta.func_id == special_kfunc_list[KF_bpf_list_pop_back]) {
12572 				struct btf_field *field = meta.arg_list_head.field;
12573 
12574 				mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
12575 			} else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
12576 				   meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) {
12577 				struct btf_field *field = meta.arg_rbtree_root.field;
12578 
12579 				mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
12580 			} else if (meta.func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
12581 				mark_reg_known_zero(env, regs, BPF_REG_0);
12582 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED;
12583 				regs[BPF_REG_0].btf = desc_btf;
12584 				regs[BPF_REG_0].btf_id = meta.ret_btf_id;
12585 			} else if (meta.func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
12586 				ret_t = btf_type_by_id(desc_btf, meta.arg_constant.value);
12587 				if (!ret_t || !btf_type_is_struct(ret_t)) {
12588 					verbose(env,
12589 						"kfunc bpf_rdonly_cast type ID argument must be of a struct\n");
12590 					return -EINVAL;
12591 				}
12592 
12593 				mark_reg_known_zero(env, regs, BPF_REG_0);
12594 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED;
12595 				regs[BPF_REG_0].btf = desc_btf;
12596 				regs[BPF_REG_0].btf_id = meta.arg_constant.value;
12597 			} else if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice] ||
12598 				   meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) {
12599 				enum bpf_type_flag type_flag = get_dynptr_type_flag(meta.initialized_dynptr.type);
12600 
12601 				mark_reg_known_zero(env, regs, BPF_REG_0);
12602 
12603 				if (!meta.arg_constant.found) {
12604 					verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n");
12605 					return -EFAULT;
12606 				}
12607 
12608 				regs[BPF_REG_0].mem_size = meta.arg_constant.value;
12609 
12610 				/* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */
12611 				regs[BPF_REG_0].type = PTR_TO_MEM | type_flag;
12612 
12613 				if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice]) {
12614 					regs[BPF_REG_0].type |= MEM_RDONLY;
12615 				} else {
12616 					/* this will set env->seen_direct_write to true */
12617 					if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) {
12618 						verbose(env, "the prog does not allow writes to packet data\n");
12619 						return -EINVAL;
12620 					}
12621 				}
12622 
12623 				if (!meta.initialized_dynptr.id) {
12624 					verbose(env, "verifier internal error: no dynptr id\n");
12625 					return -EFAULT;
12626 				}
12627 				regs[BPF_REG_0].dynptr_id = meta.initialized_dynptr.id;
12628 
12629 				/* we don't need to set BPF_REG_0's ref obj id
12630 				 * because packet slices are not refcounted (see
12631 				 * dynptr_type_refcounted)
12632 				 */
12633 			} else {
12634 				verbose(env, "kernel function %s unhandled dynamic return type\n",
12635 					meta.func_name);
12636 				return -EFAULT;
12637 			}
12638 		} else if (btf_type_is_void(ptr_type)) {
12639 			/* kfunc returning 'void *' is equivalent to returning scalar */
12640 			mark_reg_unknown(env, regs, BPF_REG_0);
12641 		} else if (!__btf_type_is_struct(ptr_type)) {
12642 			if (!meta.r0_size) {
12643 				__u32 sz;
12644 
12645 				if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) {
12646 					meta.r0_size = sz;
12647 					meta.r0_rdonly = true;
12648 				}
12649 			}
12650 			if (!meta.r0_size) {
12651 				ptr_type_name = btf_name_by_offset(desc_btf,
12652 								   ptr_type->name_off);
12653 				verbose(env,
12654 					"kernel function %s returns pointer type %s %s is not supported\n",
12655 					func_name,
12656 					btf_type_str(ptr_type),
12657 					ptr_type_name);
12658 				return -EINVAL;
12659 			}
12660 
12661 			mark_reg_known_zero(env, regs, BPF_REG_0);
12662 			regs[BPF_REG_0].type = PTR_TO_MEM;
12663 			regs[BPF_REG_0].mem_size = meta.r0_size;
12664 
12665 			if (meta.r0_rdonly)
12666 				regs[BPF_REG_0].type |= MEM_RDONLY;
12667 
12668 			/* Ensures we don't access the memory after a release_reference() */
12669 			if (meta.ref_obj_id)
12670 				regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
12671 		} else {
12672 			mark_reg_known_zero(env, regs, BPF_REG_0);
12673 			regs[BPF_REG_0].btf = desc_btf;
12674 			regs[BPF_REG_0].type = PTR_TO_BTF_ID;
12675 			regs[BPF_REG_0].btf_id = ptr_type_id;
12676 		}
12677 
12678 		if (is_kfunc_ret_null(&meta)) {
12679 			regs[BPF_REG_0].type |= PTR_MAYBE_NULL;
12680 			/* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */
12681 			regs[BPF_REG_0].id = ++env->id_gen;
12682 		}
12683 		mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
12684 		if (is_kfunc_acquire(&meta)) {
12685 			int id = acquire_reference_state(env, insn_idx);
12686 
12687 			if (id < 0)
12688 				return id;
12689 			if (is_kfunc_ret_null(&meta))
12690 				regs[BPF_REG_0].id = id;
12691 			regs[BPF_REG_0].ref_obj_id = id;
12692 		} else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) {
12693 			ref_set_non_owning(env, &regs[BPF_REG_0]);
12694 		}
12695 
12696 		if (reg_may_point_to_spin_lock(&regs[BPF_REG_0]) && !regs[BPF_REG_0].id)
12697 			regs[BPF_REG_0].id = ++env->id_gen;
12698 	} else if (btf_type_is_void(t)) {
12699 		if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) {
12700 			if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl] ||
12701 			    meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) {
12702 				insn_aux->kptr_struct_meta =
12703 					btf_find_struct_meta(meta.arg_btf,
12704 							     meta.arg_btf_id);
12705 			}
12706 		}
12707 	}
12708 
12709 	nargs = btf_type_vlen(meta.func_proto);
12710 	args = (const struct btf_param *)(meta.func_proto + 1);
12711 	for (i = 0; i < nargs; i++) {
12712 		u32 regno = i + 1;
12713 
12714 		t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL);
12715 		if (btf_type_is_ptr(t))
12716 			mark_btf_func_reg_size(env, regno, sizeof(void *));
12717 		else
12718 			/* scalar. ensured by btf_check_kfunc_arg_match() */
12719 			mark_btf_func_reg_size(env, regno, t->size);
12720 	}
12721 
12722 	if (is_iter_next_kfunc(&meta)) {
12723 		err = process_iter_next_call(env, insn_idx, &meta);
12724 		if (err)
12725 			return err;
12726 	}
12727 
12728 	return 0;
12729 }
12730 
12731 static bool check_reg_sane_offset(struct bpf_verifier_env *env,
12732 				  const struct bpf_reg_state *reg,
12733 				  enum bpf_reg_type type)
12734 {
12735 	bool known = tnum_is_const(reg->var_off);
12736 	s64 val = reg->var_off.value;
12737 	s64 smin = reg->smin_value;
12738 
12739 	if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
12740 		verbose(env, "math between %s pointer and %lld is not allowed\n",
12741 			reg_type_str(env, type), val);
12742 		return false;
12743 	}
12744 
12745 	if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
12746 		verbose(env, "%s pointer offset %d is not allowed\n",
12747 			reg_type_str(env, type), reg->off);
12748 		return false;
12749 	}
12750 
12751 	if (smin == S64_MIN) {
12752 		verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
12753 			reg_type_str(env, type));
12754 		return false;
12755 	}
12756 
12757 	if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
12758 		verbose(env, "value %lld makes %s pointer be out of bounds\n",
12759 			smin, reg_type_str(env, type));
12760 		return false;
12761 	}
12762 
12763 	return true;
12764 }
12765 
12766 enum {
12767 	REASON_BOUNDS	= -1,
12768 	REASON_TYPE	= -2,
12769 	REASON_PATHS	= -3,
12770 	REASON_LIMIT	= -4,
12771 	REASON_STACK	= -5,
12772 };
12773 
12774 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
12775 			      u32 *alu_limit, bool mask_to_left)
12776 {
12777 	u32 max = 0, ptr_limit = 0;
12778 
12779 	switch (ptr_reg->type) {
12780 	case PTR_TO_STACK:
12781 		/* Offset 0 is out-of-bounds, but acceptable start for the
12782 		 * left direction, see BPF_REG_FP. Also, unknown scalar
12783 		 * offset where we would need to deal with min/max bounds is
12784 		 * currently prohibited for unprivileged.
12785 		 */
12786 		max = MAX_BPF_STACK + mask_to_left;
12787 		ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off);
12788 		break;
12789 	case PTR_TO_MAP_VALUE:
12790 		max = ptr_reg->map_ptr->value_size;
12791 		ptr_limit = (mask_to_left ?
12792 			     ptr_reg->smin_value :
12793 			     ptr_reg->umax_value) + ptr_reg->off;
12794 		break;
12795 	default:
12796 		return REASON_TYPE;
12797 	}
12798 
12799 	if (ptr_limit >= max)
12800 		return REASON_LIMIT;
12801 	*alu_limit = ptr_limit;
12802 	return 0;
12803 }
12804 
12805 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
12806 				    const struct bpf_insn *insn)
12807 {
12808 	return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K;
12809 }
12810 
12811 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
12812 				       u32 alu_state, u32 alu_limit)
12813 {
12814 	/* If we arrived here from different branches with different
12815 	 * state or limits to sanitize, then this won't work.
12816 	 */
12817 	if (aux->alu_state &&
12818 	    (aux->alu_state != alu_state ||
12819 	     aux->alu_limit != alu_limit))
12820 		return REASON_PATHS;
12821 
12822 	/* Corresponding fixup done in do_misc_fixups(). */
12823 	aux->alu_state = alu_state;
12824 	aux->alu_limit = alu_limit;
12825 	return 0;
12826 }
12827 
12828 static int sanitize_val_alu(struct bpf_verifier_env *env,
12829 			    struct bpf_insn *insn)
12830 {
12831 	struct bpf_insn_aux_data *aux = cur_aux(env);
12832 
12833 	if (can_skip_alu_sanitation(env, insn))
12834 		return 0;
12835 
12836 	return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
12837 }
12838 
12839 static bool sanitize_needed(u8 opcode)
12840 {
12841 	return opcode == BPF_ADD || opcode == BPF_SUB;
12842 }
12843 
12844 struct bpf_sanitize_info {
12845 	struct bpf_insn_aux_data aux;
12846 	bool mask_to_left;
12847 };
12848 
12849 static struct bpf_verifier_state *
12850 sanitize_speculative_path(struct bpf_verifier_env *env,
12851 			  const struct bpf_insn *insn,
12852 			  u32 next_idx, u32 curr_idx)
12853 {
12854 	struct bpf_verifier_state *branch;
12855 	struct bpf_reg_state *regs;
12856 
12857 	branch = push_stack(env, next_idx, curr_idx, true);
12858 	if (branch && insn) {
12859 		regs = branch->frame[branch->curframe]->regs;
12860 		if (BPF_SRC(insn->code) == BPF_K) {
12861 			mark_reg_unknown(env, regs, insn->dst_reg);
12862 		} else if (BPF_SRC(insn->code) == BPF_X) {
12863 			mark_reg_unknown(env, regs, insn->dst_reg);
12864 			mark_reg_unknown(env, regs, insn->src_reg);
12865 		}
12866 	}
12867 	return branch;
12868 }
12869 
12870 static int sanitize_ptr_alu(struct bpf_verifier_env *env,
12871 			    struct bpf_insn *insn,
12872 			    const struct bpf_reg_state *ptr_reg,
12873 			    const struct bpf_reg_state *off_reg,
12874 			    struct bpf_reg_state *dst_reg,
12875 			    struct bpf_sanitize_info *info,
12876 			    const bool commit_window)
12877 {
12878 	struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
12879 	struct bpf_verifier_state *vstate = env->cur_state;
12880 	bool off_is_imm = tnum_is_const(off_reg->var_off);
12881 	bool off_is_neg = off_reg->smin_value < 0;
12882 	bool ptr_is_dst_reg = ptr_reg == dst_reg;
12883 	u8 opcode = BPF_OP(insn->code);
12884 	u32 alu_state, alu_limit;
12885 	struct bpf_reg_state tmp;
12886 	bool ret;
12887 	int err;
12888 
12889 	if (can_skip_alu_sanitation(env, insn))
12890 		return 0;
12891 
12892 	/* We already marked aux for masking from non-speculative
12893 	 * paths, thus we got here in the first place. We only care
12894 	 * to explore bad access from here.
12895 	 */
12896 	if (vstate->speculative)
12897 		goto do_sim;
12898 
12899 	if (!commit_window) {
12900 		if (!tnum_is_const(off_reg->var_off) &&
12901 		    (off_reg->smin_value < 0) != (off_reg->smax_value < 0))
12902 			return REASON_BOUNDS;
12903 
12904 		info->mask_to_left = (opcode == BPF_ADD &&  off_is_neg) ||
12905 				     (opcode == BPF_SUB && !off_is_neg);
12906 	}
12907 
12908 	err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left);
12909 	if (err < 0)
12910 		return err;
12911 
12912 	if (commit_window) {
12913 		/* In commit phase we narrow the masking window based on
12914 		 * the observed pointer move after the simulated operation.
12915 		 */
12916 		alu_state = info->aux.alu_state;
12917 		alu_limit = abs(info->aux.alu_limit - alu_limit);
12918 	} else {
12919 		alu_state  = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
12920 		alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;
12921 		alu_state |= ptr_is_dst_reg ?
12922 			     BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
12923 
12924 		/* Limit pruning on unknown scalars to enable deep search for
12925 		 * potential masking differences from other program paths.
12926 		 */
12927 		if (!off_is_imm)
12928 			env->explore_alu_limits = true;
12929 	}
12930 
12931 	err = update_alu_sanitation_state(aux, alu_state, alu_limit);
12932 	if (err < 0)
12933 		return err;
12934 do_sim:
12935 	/* If we're in commit phase, we're done here given we already
12936 	 * pushed the truncated dst_reg into the speculative verification
12937 	 * stack.
12938 	 *
12939 	 * Also, when register is a known constant, we rewrite register-based
12940 	 * operation to immediate-based, and thus do not need masking (and as
12941 	 * a consequence, do not need to simulate the zero-truncation either).
12942 	 */
12943 	if (commit_window || off_is_imm)
12944 		return 0;
12945 
12946 	/* Simulate and find potential out-of-bounds access under
12947 	 * speculative execution from truncation as a result of
12948 	 * masking when off was not within expected range. If off
12949 	 * sits in dst, then we temporarily need to move ptr there
12950 	 * to simulate dst (== 0) +/-= ptr. Needed, for example,
12951 	 * for cases where we use K-based arithmetic in one direction
12952 	 * and truncated reg-based in the other in order to explore
12953 	 * bad access.
12954 	 */
12955 	if (!ptr_is_dst_reg) {
12956 		tmp = *dst_reg;
12957 		copy_register_state(dst_reg, ptr_reg);
12958 	}
12959 	ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1,
12960 					env->insn_idx);
12961 	if (!ptr_is_dst_reg && ret)
12962 		*dst_reg = tmp;
12963 	return !ret ? REASON_STACK : 0;
12964 }
12965 
12966 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
12967 {
12968 	struct bpf_verifier_state *vstate = env->cur_state;
12969 
12970 	/* If we simulate paths under speculation, we don't update the
12971 	 * insn as 'seen' such that when we verify unreachable paths in
12972 	 * the non-speculative domain, sanitize_dead_code() can still
12973 	 * rewrite/sanitize them.
12974 	 */
12975 	if (!vstate->speculative)
12976 		env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;
12977 }
12978 
12979 static int sanitize_err(struct bpf_verifier_env *env,
12980 			const struct bpf_insn *insn, int reason,
12981 			const struct bpf_reg_state *off_reg,
12982 			const struct bpf_reg_state *dst_reg)
12983 {
12984 	static const char *err = "pointer arithmetic with it prohibited for !root";
12985 	const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub";
12986 	u32 dst = insn->dst_reg, src = insn->src_reg;
12987 
12988 	switch (reason) {
12989 	case REASON_BOUNDS:
12990 		verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",
12991 			off_reg == dst_reg ? dst : src, err);
12992 		break;
12993 	case REASON_TYPE:
12994 		verbose(env, "R%d has pointer with unsupported alu operation, %s\n",
12995 			off_reg == dst_reg ? src : dst, err);
12996 		break;
12997 	case REASON_PATHS:
12998 		verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",
12999 			dst, op, err);
13000 		break;
13001 	case REASON_LIMIT:
13002 		verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",
13003 			dst, op, err);
13004 		break;
13005 	case REASON_STACK:
13006 		verbose(env, "R%d could not be pushed for speculative verification, %s\n",
13007 			dst, err);
13008 		break;
13009 	default:
13010 		verbose(env, "verifier internal error: unknown reason (%d)\n",
13011 			reason);
13012 		break;
13013 	}
13014 
13015 	return -EACCES;
13016 }
13017 
13018 /* check that stack access falls within stack limits and that 'reg' doesn't
13019  * have a variable offset.
13020  *
13021  * Variable offset is prohibited for unprivileged mode for simplicity since it
13022  * requires corresponding support in Spectre masking for stack ALU.  See also
13023  * retrieve_ptr_limit().
13024  *
13025  *
13026  * 'off' includes 'reg->off'.
13027  */
13028 static int check_stack_access_for_ptr_arithmetic(
13029 				struct bpf_verifier_env *env,
13030 				int regno,
13031 				const struct bpf_reg_state *reg,
13032 				int off)
13033 {
13034 	if (!tnum_is_const(reg->var_off)) {
13035 		char tn_buf[48];
13036 
13037 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
13038 		verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
13039 			regno, tn_buf, off);
13040 		return -EACCES;
13041 	}
13042 
13043 	if (off >= 0 || off < -MAX_BPF_STACK) {
13044 		verbose(env, "R%d stack pointer arithmetic goes out of range, "
13045 			"prohibited for !root; off=%d\n", regno, off);
13046 		return -EACCES;
13047 	}
13048 
13049 	return 0;
13050 }
13051 
13052 static int sanitize_check_bounds(struct bpf_verifier_env *env,
13053 				 const struct bpf_insn *insn,
13054 				 const struct bpf_reg_state *dst_reg)
13055 {
13056 	u32 dst = insn->dst_reg;
13057 
13058 	/* For unprivileged we require that resulting offset must be in bounds
13059 	 * in order to be able to sanitize access later on.
13060 	 */
13061 	if (env->bypass_spec_v1)
13062 		return 0;
13063 
13064 	switch (dst_reg->type) {
13065 	case PTR_TO_STACK:
13066 		if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,
13067 					dst_reg->off + dst_reg->var_off.value))
13068 			return -EACCES;
13069 		break;
13070 	case PTR_TO_MAP_VALUE:
13071 		if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) {
13072 			verbose(env, "R%d pointer arithmetic of map value goes out of range, "
13073 				"prohibited for !root\n", dst);
13074 			return -EACCES;
13075 		}
13076 		break;
13077 	default:
13078 		break;
13079 	}
13080 
13081 	return 0;
13082 }
13083 
13084 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
13085  * Caller should also handle BPF_MOV case separately.
13086  * If we return -EACCES, caller may want to try again treating pointer as a
13087  * scalar.  So we only emit a diagnostic if !env->allow_ptr_leaks.
13088  */
13089 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
13090 				   struct bpf_insn *insn,
13091 				   const struct bpf_reg_state *ptr_reg,
13092 				   const struct bpf_reg_state *off_reg)
13093 {
13094 	struct bpf_verifier_state *vstate = env->cur_state;
13095 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
13096 	struct bpf_reg_state *regs = state->regs, *dst_reg;
13097 	bool known = tnum_is_const(off_reg->var_off);
13098 	s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
13099 	    smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
13100 	u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
13101 	    umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
13102 	struct bpf_sanitize_info info = {};
13103 	u8 opcode = BPF_OP(insn->code);
13104 	u32 dst = insn->dst_reg;
13105 	int ret;
13106 
13107 	dst_reg = &regs[dst];
13108 
13109 	if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
13110 	    smin_val > smax_val || umin_val > umax_val) {
13111 		/* Taint dst register if offset had invalid bounds derived from
13112 		 * e.g. dead branches.
13113 		 */
13114 		__mark_reg_unknown(env, dst_reg);
13115 		return 0;
13116 	}
13117 
13118 	if (BPF_CLASS(insn->code) != BPF_ALU64) {
13119 		/* 32-bit ALU ops on pointers produce (meaningless) scalars */
13120 		if (opcode == BPF_SUB && env->allow_ptr_leaks) {
13121 			__mark_reg_unknown(env, dst_reg);
13122 			return 0;
13123 		}
13124 
13125 		verbose(env,
13126 			"R%d 32-bit pointer arithmetic prohibited\n",
13127 			dst);
13128 		return -EACCES;
13129 	}
13130 
13131 	if (ptr_reg->type & PTR_MAYBE_NULL) {
13132 		verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
13133 			dst, reg_type_str(env, ptr_reg->type));
13134 		return -EACCES;
13135 	}
13136 
13137 	switch (base_type(ptr_reg->type)) {
13138 	case PTR_TO_CTX:
13139 	case PTR_TO_MAP_VALUE:
13140 	case PTR_TO_MAP_KEY:
13141 	case PTR_TO_STACK:
13142 	case PTR_TO_PACKET_META:
13143 	case PTR_TO_PACKET:
13144 	case PTR_TO_TP_BUFFER:
13145 	case PTR_TO_BTF_ID:
13146 	case PTR_TO_MEM:
13147 	case PTR_TO_BUF:
13148 	case PTR_TO_FUNC:
13149 	case CONST_PTR_TO_DYNPTR:
13150 		break;
13151 	case PTR_TO_FLOW_KEYS:
13152 		if (known)
13153 			break;
13154 		fallthrough;
13155 	case CONST_PTR_TO_MAP:
13156 		/* smin_val represents the known value */
13157 		if (known && smin_val == 0 && opcode == BPF_ADD)
13158 			break;
13159 		fallthrough;
13160 	default:
13161 		verbose(env, "R%d pointer arithmetic on %s prohibited\n",
13162 			dst, reg_type_str(env, ptr_reg->type));
13163 		return -EACCES;
13164 	}
13165 
13166 	/* In case of 'scalar += pointer', dst_reg inherits pointer type and id.
13167 	 * The id may be overwritten later if we create a new variable offset.
13168 	 */
13169 	dst_reg->type = ptr_reg->type;
13170 	dst_reg->id = ptr_reg->id;
13171 
13172 	if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
13173 	    !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
13174 		return -EINVAL;
13175 
13176 	/* pointer types do not carry 32-bit bounds at the moment. */
13177 	__mark_reg32_unbounded(dst_reg);
13178 
13179 	if (sanitize_needed(opcode)) {
13180 		ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
13181 				       &info, false);
13182 		if (ret < 0)
13183 			return sanitize_err(env, insn, ret, off_reg, dst_reg);
13184 	}
13185 
13186 	switch (opcode) {
13187 	case BPF_ADD:
13188 		/* We can take a fixed offset as long as it doesn't overflow
13189 		 * the s32 'off' field
13190 		 */
13191 		if (known && (ptr_reg->off + smin_val ==
13192 			      (s64)(s32)(ptr_reg->off + smin_val))) {
13193 			/* pointer += K.  Accumulate it into fixed offset */
13194 			dst_reg->smin_value = smin_ptr;
13195 			dst_reg->smax_value = smax_ptr;
13196 			dst_reg->umin_value = umin_ptr;
13197 			dst_reg->umax_value = umax_ptr;
13198 			dst_reg->var_off = ptr_reg->var_off;
13199 			dst_reg->off = ptr_reg->off + smin_val;
13200 			dst_reg->raw = ptr_reg->raw;
13201 			break;
13202 		}
13203 		/* A new variable offset is created.  Note that off_reg->off
13204 		 * == 0, since it's a scalar.
13205 		 * dst_reg gets the pointer type and since some positive
13206 		 * integer value was added to the pointer, give it a new 'id'
13207 		 * if it's a PTR_TO_PACKET.
13208 		 * this creates a new 'base' pointer, off_reg (variable) gets
13209 		 * added into the variable offset, and we copy the fixed offset
13210 		 * from ptr_reg.
13211 		 */
13212 		if (check_add_overflow(smin_ptr, smin_val, &dst_reg->smin_value) ||
13213 		    check_add_overflow(smax_ptr, smax_val, &dst_reg->smax_value)) {
13214 			dst_reg->smin_value = S64_MIN;
13215 			dst_reg->smax_value = S64_MAX;
13216 		}
13217 		if (check_add_overflow(umin_ptr, umin_val, &dst_reg->umin_value) ||
13218 		    check_add_overflow(umax_ptr, umax_val, &dst_reg->umax_value)) {
13219 			dst_reg->umin_value = 0;
13220 			dst_reg->umax_value = U64_MAX;
13221 		}
13222 		dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
13223 		dst_reg->off = ptr_reg->off;
13224 		dst_reg->raw = ptr_reg->raw;
13225 		if (reg_is_pkt_pointer(ptr_reg)) {
13226 			dst_reg->id = ++env->id_gen;
13227 			/* something was added to pkt_ptr, set range to zero */
13228 			memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
13229 		}
13230 		break;
13231 	case BPF_SUB:
13232 		if (dst_reg == off_reg) {
13233 			/* scalar -= pointer.  Creates an unknown scalar */
13234 			verbose(env, "R%d tried to subtract pointer from scalar\n",
13235 				dst);
13236 			return -EACCES;
13237 		}
13238 		/* We don't allow subtraction from FP, because (according to
13239 		 * test_verifier.c test "invalid fp arithmetic", JITs might not
13240 		 * be able to deal with it.
13241 		 */
13242 		if (ptr_reg->type == PTR_TO_STACK) {
13243 			verbose(env, "R%d subtraction from stack pointer prohibited\n",
13244 				dst);
13245 			return -EACCES;
13246 		}
13247 		if (known && (ptr_reg->off - smin_val ==
13248 			      (s64)(s32)(ptr_reg->off - smin_val))) {
13249 			/* pointer -= K.  Subtract it from fixed offset */
13250 			dst_reg->smin_value = smin_ptr;
13251 			dst_reg->smax_value = smax_ptr;
13252 			dst_reg->umin_value = umin_ptr;
13253 			dst_reg->umax_value = umax_ptr;
13254 			dst_reg->var_off = ptr_reg->var_off;
13255 			dst_reg->id = ptr_reg->id;
13256 			dst_reg->off = ptr_reg->off - smin_val;
13257 			dst_reg->raw = ptr_reg->raw;
13258 			break;
13259 		}
13260 		/* A new variable offset is created.  If the subtrahend is known
13261 		 * nonnegative, then any reg->range we had before is still good.
13262 		 */
13263 		if (check_sub_overflow(smin_ptr, smax_val, &dst_reg->smin_value) ||
13264 		    check_sub_overflow(smax_ptr, smin_val, &dst_reg->smax_value)) {
13265 			/* Overflow possible, we know nothing */
13266 			dst_reg->smin_value = S64_MIN;
13267 			dst_reg->smax_value = S64_MAX;
13268 		}
13269 		if (umin_ptr < umax_val) {
13270 			/* Overflow possible, we know nothing */
13271 			dst_reg->umin_value = 0;
13272 			dst_reg->umax_value = U64_MAX;
13273 		} else {
13274 			/* Cannot overflow (as long as bounds are consistent) */
13275 			dst_reg->umin_value = umin_ptr - umax_val;
13276 			dst_reg->umax_value = umax_ptr - umin_val;
13277 		}
13278 		dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
13279 		dst_reg->off = ptr_reg->off;
13280 		dst_reg->raw = ptr_reg->raw;
13281 		if (reg_is_pkt_pointer(ptr_reg)) {
13282 			dst_reg->id = ++env->id_gen;
13283 			/* something was added to pkt_ptr, set range to zero */
13284 			if (smin_val < 0)
13285 				memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
13286 		}
13287 		break;
13288 	case BPF_AND:
13289 	case BPF_OR:
13290 	case BPF_XOR:
13291 		/* bitwise ops on pointers are troublesome, prohibit. */
13292 		verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
13293 			dst, bpf_alu_string[opcode >> 4]);
13294 		return -EACCES;
13295 	default:
13296 		/* other operators (e.g. MUL,LSH) produce non-pointer results */
13297 		verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
13298 			dst, bpf_alu_string[opcode >> 4]);
13299 		return -EACCES;
13300 	}
13301 
13302 	if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
13303 		return -EINVAL;
13304 	reg_bounds_sync(dst_reg);
13305 	if (sanitize_check_bounds(env, insn, dst_reg) < 0)
13306 		return -EACCES;
13307 	if (sanitize_needed(opcode)) {
13308 		ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,
13309 				       &info, true);
13310 		if (ret < 0)
13311 			return sanitize_err(env, insn, ret, off_reg, dst_reg);
13312 	}
13313 
13314 	return 0;
13315 }
13316 
13317 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,
13318 				 struct bpf_reg_state *src_reg)
13319 {
13320 	s32 *dst_smin = &dst_reg->s32_min_value;
13321 	s32 *dst_smax = &dst_reg->s32_max_value;
13322 	u32 *dst_umin = &dst_reg->u32_min_value;
13323 	u32 *dst_umax = &dst_reg->u32_max_value;
13324 
13325 	if (check_add_overflow(*dst_smin, src_reg->s32_min_value, dst_smin) ||
13326 	    check_add_overflow(*dst_smax, src_reg->s32_max_value, dst_smax)) {
13327 		*dst_smin = S32_MIN;
13328 		*dst_smax = S32_MAX;
13329 	}
13330 	if (check_add_overflow(*dst_umin, src_reg->u32_min_value, dst_umin) ||
13331 	    check_add_overflow(*dst_umax, src_reg->u32_max_value, dst_umax)) {
13332 		*dst_umin = 0;
13333 		*dst_umax = U32_MAX;
13334 	}
13335 }
13336 
13337 static void scalar_min_max_add(struct bpf_reg_state *dst_reg,
13338 			       struct bpf_reg_state *src_reg)
13339 {
13340 	s64 *dst_smin = &dst_reg->smin_value;
13341 	s64 *dst_smax = &dst_reg->smax_value;
13342 	u64 *dst_umin = &dst_reg->umin_value;
13343 	u64 *dst_umax = &dst_reg->umax_value;
13344 
13345 	if (check_add_overflow(*dst_smin, src_reg->smin_value, dst_smin) ||
13346 	    check_add_overflow(*dst_smax, src_reg->smax_value, dst_smax)) {
13347 		*dst_smin = S64_MIN;
13348 		*dst_smax = S64_MAX;
13349 	}
13350 	if (check_add_overflow(*dst_umin, src_reg->umin_value, dst_umin) ||
13351 	    check_add_overflow(*dst_umax, src_reg->umax_value, dst_umax)) {
13352 		*dst_umin = 0;
13353 		*dst_umax = U64_MAX;
13354 	}
13355 }
13356 
13357 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,
13358 				 struct bpf_reg_state *src_reg)
13359 {
13360 	s32 *dst_smin = &dst_reg->s32_min_value;
13361 	s32 *dst_smax = &dst_reg->s32_max_value;
13362 	u32 umin_val = src_reg->u32_min_value;
13363 	u32 umax_val = src_reg->u32_max_value;
13364 
13365 	if (check_sub_overflow(*dst_smin, src_reg->s32_max_value, dst_smin) ||
13366 	    check_sub_overflow(*dst_smax, src_reg->s32_min_value, dst_smax)) {
13367 		/* Overflow possible, we know nothing */
13368 		*dst_smin = S32_MIN;
13369 		*dst_smax = S32_MAX;
13370 	}
13371 	if (dst_reg->u32_min_value < umax_val) {
13372 		/* Overflow possible, we know nothing */
13373 		dst_reg->u32_min_value = 0;
13374 		dst_reg->u32_max_value = U32_MAX;
13375 	} else {
13376 		/* Cannot overflow (as long as bounds are consistent) */
13377 		dst_reg->u32_min_value -= umax_val;
13378 		dst_reg->u32_max_value -= umin_val;
13379 	}
13380 }
13381 
13382 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,
13383 			       struct bpf_reg_state *src_reg)
13384 {
13385 	s64 *dst_smin = &dst_reg->smin_value;
13386 	s64 *dst_smax = &dst_reg->smax_value;
13387 	u64 umin_val = src_reg->umin_value;
13388 	u64 umax_val = src_reg->umax_value;
13389 
13390 	if (check_sub_overflow(*dst_smin, src_reg->smax_value, dst_smin) ||
13391 	    check_sub_overflow(*dst_smax, src_reg->smin_value, dst_smax)) {
13392 		/* Overflow possible, we know nothing */
13393 		*dst_smin = S64_MIN;
13394 		*dst_smax = S64_MAX;
13395 	}
13396 	if (dst_reg->umin_value < umax_val) {
13397 		/* Overflow possible, we know nothing */
13398 		dst_reg->umin_value = 0;
13399 		dst_reg->umax_value = U64_MAX;
13400 	} else {
13401 		/* Cannot overflow (as long as bounds are consistent) */
13402 		dst_reg->umin_value -= umax_val;
13403 		dst_reg->umax_value -= umin_val;
13404 	}
13405 }
13406 
13407 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,
13408 				 struct bpf_reg_state *src_reg)
13409 {
13410 	s32 smin_val = src_reg->s32_min_value;
13411 	u32 umin_val = src_reg->u32_min_value;
13412 	u32 umax_val = src_reg->u32_max_value;
13413 
13414 	if (smin_val < 0 || dst_reg->s32_min_value < 0) {
13415 		/* Ain't nobody got time to multiply that sign */
13416 		__mark_reg32_unbounded(dst_reg);
13417 		return;
13418 	}
13419 	/* Both values are positive, so we can work with unsigned and
13420 	 * copy the result to signed (unless it exceeds S32_MAX).
13421 	 */
13422 	if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) {
13423 		/* Potential overflow, we know nothing */
13424 		__mark_reg32_unbounded(dst_reg);
13425 		return;
13426 	}
13427 	dst_reg->u32_min_value *= umin_val;
13428 	dst_reg->u32_max_value *= umax_val;
13429 	if (dst_reg->u32_max_value > S32_MAX) {
13430 		/* Overflow possible, we know nothing */
13431 		dst_reg->s32_min_value = S32_MIN;
13432 		dst_reg->s32_max_value = S32_MAX;
13433 	} else {
13434 		dst_reg->s32_min_value = dst_reg->u32_min_value;
13435 		dst_reg->s32_max_value = dst_reg->u32_max_value;
13436 	}
13437 }
13438 
13439 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,
13440 			       struct bpf_reg_state *src_reg)
13441 {
13442 	s64 smin_val = src_reg->smin_value;
13443 	u64 umin_val = src_reg->umin_value;
13444 	u64 umax_val = src_reg->umax_value;
13445 
13446 	if (smin_val < 0 || dst_reg->smin_value < 0) {
13447 		/* Ain't nobody got time to multiply that sign */
13448 		__mark_reg64_unbounded(dst_reg);
13449 		return;
13450 	}
13451 	/* Both values are positive, so we can work with unsigned and
13452 	 * copy the result to signed (unless it exceeds S64_MAX).
13453 	 */
13454 	if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) {
13455 		/* Potential overflow, we know nothing */
13456 		__mark_reg64_unbounded(dst_reg);
13457 		return;
13458 	}
13459 	dst_reg->umin_value *= umin_val;
13460 	dst_reg->umax_value *= umax_val;
13461 	if (dst_reg->umax_value > S64_MAX) {
13462 		/* Overflow possible, we know nothing */
13463 		dst_reg->smin_value = S64_MIN;
13464 		dst_reg->smax_value = S64_MAX;
13465 	} else {
13466 		dst_reg->smin_value = dst_reg->umin_value;
13467 		dst_reg->smax_value = dst_reg->umax_value;
13468 	}
13469 }
13470 
13471 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,
13472 				 struct bpf_reg_state *src_reg)
13473 {
13474 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
13475 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
13476 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
13477 	u32 umax_val = src_reg->u32_max_value;
13478 
13479 	if (src_known && dst_known) {
13480 		__mark_reg32_known(dst_reg, var32_off.value);
13481 		return;
13482 	}
13483 
13484 	/* We get our minimum from the var_off, since that's inherently
13485 	 * bitwise.  Our maximum is the minimum of the operands' maxima.
13486 	 */
13487 	dst_reg->u32_min_value = var32_off.value;
13488 	dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val);
13489 
13490 	/* Safe to set s32 bounds by casting u32 result into s32 when u32
13491 	 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
13492 	 */
13493 	if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
13494 		dst_reg->s32_min_value = dst_reg->u32_min_value;
13495 		dst_reg->s32_max_value = dst_reg->u32_max_value;
13496 	} else {
13497 		dst_reg->s32_min_value = S32_MIN;
13498 		dst_reg->s32_max_value = S32_MAX;
13499 	}
13500 }
13501 
13502 static void scalar_min_max_and(struct bpf_reg_state *dst_reg,
13503 			       struct bpf_reg_state *src_reg)
13504 {
13505 	bool src_known = tnum_is_const(src_reg->var_off);
13506 	bool dst_known = tnum_is_const(dst_reg->var_off);
13507 	u64 umax_val = src_reg->umax_value;
13508 
13509 	if (src_known && dst_known) {
13510 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
13511 		return;
13512 	}
13513 
13514 	/* We get our minimum from the var_off, since that's inherently
13515 	 * bitwise.  Our maximum is the minimum of the operands' maxima.
13516 	 */
13517 	dst_reg->umin_value = dst_reg->var_off.value;
13518 	dst_reg->umax_value = min(dst_reg->umax_value, umax_val);
13519 
13520 	/* Safe to set s64 bounds by casting u64 result into s64 when u64
13521 	 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
13522 	 */
13523 	if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
13524 		dst_reg->smin_value = dst_reg->umin_value;
13525 		dst_reg->smax_value = dst_reg->umax_value;
13526 	} else {
13527 		dst_reg->smin_value = S64_MIN;
13528 		dst_reg->smax_value = S64_MAX;
13529 	}
13530 	/* We may learn something more from the var_off */
13531 	__update_reg_bounds(dst_reg);
13532 }
13533 
13534 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,
13535 				struct bpf_reg_state *src_reg)
13536 {
13537 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
13538 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
13539 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
13540 	u32 umin_val = src_reg->u32_min_value;
13541 
13542 	if (src_known && dst_known) {
13543 		__mark_reg32_known(dst_reg, var32_off.value);
13544 		return;
13545 	}
13546 
13547 	/* We get our maximum from the var_off, and our minimum is the
13548 	 * maximum of the operands' minima
13549 	 */
13550 	dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val);
13551 	dst_reg->u32_max_value = var32_off.value | var32_off.mask;
13552 
13553 	/* Safe to set s32 bounds by casting u32 result into s32 when u32
13554 	 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
13555 	 */
13556 	if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
13557 		dst_reg->s32_min_value = dst_reg->u32_min_value;
13558 		dst_reg->s32_max_value = dst_reg->u32_max_value;
13559 	} else {
13560 		dst_reg->s32_min_value = S32_MIN;
13561 		dst_reg->s32_max_value = S32_MAX;
13562 	}
13563 }
13564 
13565 static void scalar_min_max_or(struct bpf_reg_state *dst_reg,
13566 			      struct bpf_reg_state *src_reg)
13567 {
13568 	bool src_known = tnum_is_const(src_reg->var_off);
13569 	bool dst_known = tnum_is_const(dst_reg->var_off);
13570 	u64 umin_val = src_reg->umin_value;
13571 
13572 	if (src_known && dst_known) {
13573 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
13574 		return;
13575 	}
13576 
13577 	/* We get our maximum from the var_off, and our minimum is the
13578 	 * maximum of the operands' minima
13579 	 */
13580 	dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
13581 	dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
13582 
13583 	/* Safe to set s64 bounds by casting u64 result into s64 when u64
13584 	 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
13585 	 */
13586 	if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
13587 		dst_reg->smin_value = dst_reg->umin_value;
13588 		dst_reg->smax_value = dst_reg->umax_value;
13589 	} else {
13590 		dst_reg->smin_value = S64_MIN;
13591 		dst_reg->smax_value = S64_MAX;
13592 	}
13593 	/* We may learn something more from the var_off */
13594 	__update_reg_bounds(dst_reg);
13595 }
13596 
13597 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,
13598 				 struct bpf_reg_state *src_reg)
13599 {
13600 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
13601 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
13602 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
13603 
13604 	if (src_known && dst_known) {
13605 		__mark_reg32_known(dst_reg, var32_off.value);
13606 		return;
13607 	}
13608 
13609 	/* We get both minimum and maximum from the var32_off. */
13610 	dst_reg->u32_min_value = var32_off.value;
13611 	dst_reg->u32_max_value = var32_off.value | var32_off.mask;
13612 
13613 	/* Safe to set s32 bounds by casting u32 result into s32 when u32
13614 	 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
13615 	 */
13616 	if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
13617 		dst_reg->s32_min_value = dst_reg->u32_min_value;
13618 		dst_reg->s32_max_value = dst_reg->u32_max_value;
13619 	} else {
13620 		dst_reg->s32_min_value = S32_MIN;
13621 		dst_reg->s32_max_value = S32_MAX;
13622 	}
13623 }
13624 
13625 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
13626 			       struct bpf_reg_state *src_reg)
13627 {
13628 	bool src_known = tnum_is_const(src_reg->var_off);
13629 	bool dst_known = tnum_is_const(dst_reg->var_off);
13630 
13631 	if (src_known && dst_known) {
13632 		/* dst_reg->var_off.value has been updated earlier */
13633 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
13634 		return;
13635 	}
13636 
13637 	/* We get both minimum and maximum from the var_off. */
13638 	dst_reg->umin_value = dst_reg->var_off.value;
13639 	dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
13640 
13641 	/* Safe to set s64 bounds by casting u64 result into s64 when u64
13642 	 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
13643 	 */
13644 	if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
13645 		dst_reg->smin_value = dst_reg->umin_value;
13646 		dst_reg->smax_value = dst_reg->umax_value;
13647 	} else {
13648 		dst_reg->smin_value = S64_MIN;
13649 		dst_reg->smax_value = S64_MAX;
13650 	}
13651 
13652 	__update_reg_bounds(dst_reg);
13653 }
13654 
13655 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
13656 				   u64 umin_val, u64 umax_val)
13657 {
13658 	/* We lose all sign bit information (except what we can pick
13659 	 * up from var_off)
13660 	 */
13661 	dst_reg->s32_min_value = S32_MIN;
13662 	dst_reg->s32_max_value = S32_MAX;
13663 	/* If we might shift our top bit out, then we know nothing */
13664 	if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) {
13665 		dst_reg->u32_min_value = 0;
13666 		dst_reg->u32_max_value = U32_MAX;
13667 	} else {
13668 		dst_reg->u32_min_value <<= umin_val;
13669 		dst_reg->u32_max_value <<= umax_val;
13670 	}
13671 }
13672 
13673 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
13674 				 struct bpf_reg_state *src_reg)
13675 {
13676 	u32 umax_val = src_reg->u32_max_value;
13677 	u32 umin_val = src_reg->u32_min_value;
13678 	/* u32 alu operation will zext upper bits */
13679 	struct tnum subreg = tnum_subreg(dst_reg->var_off);
13680 
13681 	__scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
13682 	dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val));
13683 	/* Not required but being careful mark reg64 bounds as unknown so
13684 	 * that we are forced to pick them up from tnum and zext later and
13685 	 * if some path skips this step we are still safe.
13686 	 */
13687 	__mark_reg64_unbounded(dst_reg);
13688 	__update_reg32_bounds(dst_reg);
13689 }
13690 
13691 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,
13692 				   u64 umin_val, u64 umax_val)
13693 {
13694 	/* Special case <<32 because it is a common compiler pattern to sign
13695 	 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are
13696 	 * positive we know this shift will also be positive so we can track
13697 	 * bounds correctly. Otherwise we lose all sign bit information except
13698 	 * what we can pick up from var_off. Perhaps we can generalize this
13699 	 * later to shifts of any length.
13700 	 */
13701 	if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0)
13702 		dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32;
13703 	else
13704 		dst_reg->smax_value = S64_MAX;
13705 
13706 	if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0)
13707 		dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32;
13708 	else
13709 		dst_reg->smin_value = S64_MIN;
13710 
13711 	/* If we might shift our top bit out, then we know nothing */
13712 	if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
13713 		dst_reg->umin_value = 0;
13714 		dst_reg->umax_value = U64_MAX;
13715 	} else {
13716 		dst_reg->umin_value <<= umin_val;
13717 		dst_reg->umax_value <<= umax_val;
13718 	}
13719 }
13720 
13721 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,
13722 			       struct bpf_reg_state *src_reg)
13723 {
13724 	u64 umax_val = src_reg->umax_value;
13725 	u64 umin_val = src_reg->umin_value;
13726 
13727 	/* scalar64 calc uses 32bit unshifted bounds so must be called first */
13728 	__scalar64_min_max_lsh(dst_reg, umin_val, umax_val);
13729 	__scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
13730 
13731 	dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
13732 	/* We may learn something more from the var_off */
13733 	__update_reg_bounds(dst_reg);
13734 }
13735 
13736 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,
13737 				 struct bpf_reg_state *src_reg)
13738 {
13739 	struct tnum subreg = tnum_subreg(dst_reg->var_off);
13740 	u32 umax_val = src_reg->u32_max_value;
13741 	u32 umin_val = src_reg->u32_min_value;
13742 
13743 	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
13744 	 * be negative, then either:
13745 	 * 1) src_reg might be zero, so the sign bit of the result is
13746 	 *    unknown, so we lose our signed bounds
13747 	 * 2) it's known negative, thus the unsigned bounds capture the
13748 	 *    signed bounds
13749 	 * 3) the signed bounds cross zero, so they tell us nothing
13750 	 *    about the result
13751 	 * If the value in dst_reg is known nonnegative, then again the
13752 	 * unsigned bounds capture the signed bounds.
13753 	 * Thus, in all cases it suffices to blow away our signed bounds
13754 	 * and rely on inferring new ones from the unsigned bounds and
13755 	 * var_off of the result.
13756 	 */
13757 	dst_reg->s32_min_value = S32_MIN;
13758 	dst_reg->s32_max_value = S32_MAX;
13759 
13760 	dst_reg->var_off = tnum_rshift(subreg, umin_val);
13761 	dst_reg->u32_min_value >>= umax_val;
13762 	dst_reg->u32_max_value >>= umin_val;
13763 
13764 	__mark_reg64_unbounded(dst_reg);
13765 	__update_reg32_bounds(dst_reg);
13766 }
13767 
13768 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,
13769 			       struct bpf_reg_state *src_reg)
13770 {
13771 	u64 umax_val = src_reg->umax_value;
13772 	u64 umin_val = src_reg->umin_value;
13773 
13774 	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
13775 	 * be negative, then either:
13776 	 * 1) src_reg might be zero, so the sign bit of the result is
13777 	 *    unknown, so we lose our signed bounds
13778 	 * 2) it's known negative, thus the unsigned bounds capture the
13779 	 *    signed bounds
13780 	 * 3) the signed bounds cross zero, so they tell us nothing
13781 	 *    about the result
13782 	 * If the value in dst_reg is known nonnegative, then again the
13783 	 * unsigned bounds capture the signed bounds.
13784 	 * Thus, in all cases it suffices to blow away our signed bounds
13785 	 * and rely on inferring new ones from the unsigned bounds and
13786 	 * var_off of the result.
13787 	 */
13788 	dst_reg->smin_value = S64_MIN;
13789 	dst_reg->smax_value = S64_MAX;
13790 	dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
13791 	dst_reg->umin_value >>= umax_val;
13792 	dst_reg->umax_value >>= umin_val;
13793 
13794 	/* Its not easy to operate on alu32 bounds here because it depends
13795 	 * on bits being shifted in. Take easy way out and mark unbounded
13796 	 * so we can recalculate later from tnum.
13797 	 */
13798 	__mark_reg32_unbounded(dst_reg);
13799 	__update_reg_bounds(dst_reg);
13800 }
13801 
13802 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,
13803 				  struct bpf_reg_state *src_reg)
13804 {
13805 	u64 umin_val = src_reg->u32_min_value;
13806 
13807 	/* Upon reaching here, src_known is true and
13808 	 * umax_val is equal to umin_val.
13809 	 */
13810 	dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val);
13811 	dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val);
13812 
13813 	dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32);
13814 
13815 	/* blow away the dst_reg umin_value/umax_value and rely on
13816 	 * dst_reg var_off to refine the result.
13817 	 */
13818 	dst_reg->u32_min_value = 0;
13819 	dst_reg->u32_max_value = U32_MAX;
13820 
13821 	__mark_reg64_unbounded(dst_reg);
13822 	__update_reg32_bounds(dst_reg);
13823 }
13824 
13825 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,
13826 				struct bpf_reg_state *src_reg)
13827 {
13828 	u64 umin_val = src_reg->umin_value;
13829 
13830 	/* Upon reaching here, src_known is true and umax_val is equal
13831 	 * to umin_val.
13832 	 */
13833 	dst_reg->smin_value >>= umin_val;
13834 	dst_reg->smax_value >>= umin_val;
13835 
13836 	dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64);
13837 
13838 	/* blow away the dst_reg umin_value/umax_value and rely on
13839 	 * dst_reg var_off to refine the result.
13840 	 */
13841 	dst_reg->umin_value = 0;
13842 	dst_reg->umax_value = U64_MAX;
13843 
13844 	/* Its not easy to operate on alu32 bounds here because it depends
13845 	 * on bits being shifted in from upper 32-bits. Take easy way out
13846 	 * and mark unbounded so we can recalculate later from tnum.
13847 	 */
13848 	__mark_reg32_unbounded(dst_reg);
13849 	__update_reg_bounds(dst_reg);
13850 }
13851 
13852 static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn,
13853 					     const struct bpf_reg_state *src_reg)
13854 {
13855 	bool src_is_const = false;
13856 	u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
13857 
13858 	if (insn_bitness == 32) {
13859 		if (tnum_subreg_is_const(src_reg->var_off)
13860 		    && src_reg->s32_min_value == src_reg->s32_max_value
13861 		    && src_reg->u32_min_value == src_reg->u32_max_value)
13862 			src_is_const = true;
13863 	} else {
13864 		if (tnum_is_const(src_reg->var_off)
13865 		    && src_reg->smin_value == src_reg->smax_value
13866 		    && src_reg->umin_value == src_reg->umax_value)
13867 			src_is_const = true;
13868 	}
13869 
13870 	switch (BPF_OP(insn->code)) {
13871 	case BPF_ADD:
13872 	case BPF_SUB:
13873 	case BPF_AND:
13874 	case BPF_XOR:
13875 	case BPF_OR:
13876 	case BPF_MUL:
13877 		return true;
13878 
13879 	/* Shift operators range is only computable if shift dimension operand
13880 	 * is a constant. Shifts greater than 31 or 63 are undefined. This
13881 	 * includes shifts by a negative number.
13882 	 */
13883 	case BPF_LSH:
13884 	case BPF_RSH:
13885 	case BPF_ARSH:
13886 		return (src_is_const && src_reg->umax_value < insn_bitness);
13887 	default:
13888 		return false;
13889 	}
13890 }
13891 
13892 /* WARNING: This function does calculations on 64-bit values, but the actual
13893  * execution may occur on 32-bit values. Therefore, things like bitshifts
13894  * need extra checks in the 32-bit case.
13895  */
13896 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
13897 				      struct bpf_insn *insn,
13898 				      struct bpf_reg_state *dst_reg,
13899 				      struct bpf_reg_state src_reg)
13900 {
13901 	u8 opcode = BPF_OP(insn->code);
13902 	bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
13903 	int ret;
13904 
13905 	if (!is_safe_to_compute_dst_reg_range(insn, &src_reg)) {
13906 		__mark_reg_unknown(env, dst_reg);
13907 		return 0;
13908 	}
13909 
13910 	if (sanitize_needed(opcode)) {
13911 		ret = sanitize_val_alu(env, insn);
13912 		if (ret < 0)
13913 			return sanitize_err(env, insn, ret, NULL, NULL);
13914 	}
13915 
13916 	/* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops.
13917 	 * There are two classes of instructions: The first class we track both
13918 	 * alu32 and alu64 sign/unsigned bounds independently this provides the
13919 	 * greatest amount of precision when alu operations are mixed with jmp32
13920 	 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD,
13921 	 * and BPF_OR. This is possible because these ops have fairly easy to
13922 	 * understand and calculate behavior in both 32-bit and 64-bit alu ops.
13923 	 * See alu32 verifier tests for examples. The second class of
13924 	 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy
13925 	 * with regards to tracking sign/unsigned bounds because the bits may
13926 	 * cross subreg boundaries in the alu64 case. When this happens we mark
13927 	 * the reg unbounded in the subreg bound space and use the resulting
13928 	 * tnum to calculate an approximation of the sign/unsigned bounds.
13929 	 */
13930 	switch (opcode) {
13931 	case BPF_ADD:
13932 		scalar32_min_max_add(dst_reg, &src_reg);
13933 		scalar_min_max_add(dst_reg, &src_reg);
13934 		dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
13935 		break;
13936 	case BPF_SUB:
13937 		scalar32_min_max_sub(dst_reg, &src_reg);
13938 		scalar_min_max_sub(dst_reg, &src_reg);
13939 		dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
13940 		break;
13941 	case BPF_MUL:
13942 		dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
13943 		scalar32_min_max_mul(dst_reg, &src_reg);
13944 		scalar_min_max_mul(dst_reg, &src_reg);
13945 		break;
13946 	case BPF_AND:
13947 		dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
13948 		scalar32_min_max_and(dst_reg, &src_reg);
13949 		scalar_min_max_and(dst_reg, &src_reg);
13950 		break;
13951 	case BPF_OR:
13952 		dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
13953 		scalar32_min_max_or(dst_reg, &src_reg);
13954 		scalar_min_max_or(dst_reg, &src_reg);
13955 		break;
13956 	case BPF_XOR:
13957 		dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off);
13958 		scalar32_min_max_xor(dst_reg, &src_reg);
13959 		scalar_min_max_xor(dst_reg, &src_reg);
13960 		break;
13961 	case BPF_LSH:
13962 		if (alu32)
13963 			scalar32_min_max_lsh(dst_reg, &src_reg);
13964 		else
13965 			scalar_min_max_lsh(dst_reg, &src_reg);
13966 		break;
13967 	case BPF_RSH:
13968 		if (alu32)
13969 			scalar32_min_max_rsh(dst_reg, &src_reg);
13970 		else
13971 			scalar_min_max_rsh(dst_reg, &src_reg);
13972 		break;
13973 	case BPF_ARSH:
13974 		if (alu32)
13975 			scalar32_min_max_arsh(dst_reg, &src_reg);
13976 		else
13977 			scalar_min_max_arsh(dst_reg, &src_reg);
13978 		break;
13979 	default:
13980 		break;
13981 	}
13982 
13983 	/* ALU32 ops are zero extended into 64bit register */
13984 	if (alu32)
13985 		zext_32_to_64(dst_reg);
13986 	reg_bounds_sync(dst_reg);
13987 	return 0;
13988 }
13989 
13990 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max
13991  * and var_off.
13992  */
13993 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
13994 				   struct bpf_insn *insn)
13995 {
13996 	struct bpf_verifier_state *vstate = env->cur_state;
13997 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
13998 	struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
13999 	struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
14000 	bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
14001 	u8 opcode = BPF_OP(insn->code);
14002 	int err;
14003 
14004 	dst_reg = &regs[insn->dst_reg];
14005 	src_reg = NULL;
14006 
14007 	if (dst_reg->type == PTR_TO_ARENA) {
14008 		struct bpf_insn_aux_data *aux = cur_aux(env);
14009 
14010 		if (BPF_CLASS(insn->code) == BPF_ALU64)
14011 			/*
14012 			 * 32-bit operations zero upper bits automatically.
14013 			 * 64-bit operations need to be converted to 32.
14014 			 */
14015 			aux->needs_zext = true;
14016 
14017 		/* Any arithmetic operations are allowed on arena pointers */
14018 		return 0;
14019 	}
14020 
14021 	if (dst_reg->type != SCALAR_VALUE)
14022 		ptr_reg = dst_reg;
14023 
14024 	if (BPF_SRC(insn->code) == BPF_X) {
14025 		src_reg = &regs[insn->src_reg];
14026 		if (src_reg->type != SCALAR_VALUE) {
14027 			if (dst_reg->type != SCALAR_VALUE) {
14028 				/* Combining two pointers by any ALU op yields
14029 				 * an arbitrary scalar. Disallow all math except
14030 				 * pointer subtraction
14031 				 */
14032 				if (opcode == BPF_SUB && env->allow_ptr_leaks) {
14033 					mark_reg_unknown(env, regs, insn->dst_reg);
14034 					return 0;
14035 				}
14036 				verbose(env, "R%d pointer %s pointer prohibited\n",
14037 					insn->dst_reg,
14038 					bpf_alu_string[opcode >> 4]);
14039 				return -EACCES;
14040 			} else {
14041 				/* scalar += pointer
14042 				 * This is legal, but we have to reverse our
14043 				 * src/dest handling in computing the range
14044 				 */
14045 				err = mark_chain_precision(env, insn->dst_reg);
14046 				if (err)
14047 					return err;
14048 				return adjust_ptr_min_max_vals(env, insn,
14049 							       src_reg, dst_reg);
14050 			}
14051 		} else if (ptr_reg) {
14052 			/* pointer += scalar */
14053 			err = mark_chain_precision(env, insn->src_reg);
14054 			if (err)
14055 				return err;
14056 			return adjust_ptr_min_max_vals(env, insn,
14057 						       dst_reg, src_reg);
14058 		} else if (dst_reg->precise) {
14059 			/* if dst_reg is precise, src_reg should be precise as well */
14060 			err = mark_chain_precision(env, insn->src_reg);
14061 			if (err)
14062 				return err;
14063 		}
14064 	} else {
14065 		/* Pretend the src is a reg with a known value, since we only
14066 		 * need to be able to read from this state.
14067 		 */
14068 		off_reg.type = SCALAR_VALUE;
14069 		__mark_reg_known(&off_reg, insn->imm);
14070 		src_reg = &off_reg;
14071 		if (ptr_reg) /* pointer += K */
14072 			return adjust_ptr_min_max_vals(env, insn,
14073 						       ptr_reg, src_reg);
14074 	}
14075 
14076 	/* Got here implies adding two SCALAR_VALUEs */
14077 	if (WARN_ON_ONCE(ptr_reg)) {
14078 		print_verifier_state(env, state, true);
14079 		verbose(env, "verifier internal error: unexpected ptr_reg\n");
14080 		return -EINVAL;
14081 	}
14082 	if (WARN_ON(!src_reg)) {
14083 		print_verifier_state(env, state, true);
14084 		verbose(env, "verifier internal error: no src_reg\n");
14085 		return -EINVAL;
14086 	}
14087 	err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
14088 	if (err)
14089 		return err;
14090 	/*
14091 	 * Compilers can generate the code
14092 	 * r1 = r2
14093 	 * r1 += 0x1
14094 	 * if r2 < 1000 goto ...
14095 	 * use r1 in memory access
14096 	 * So remember constant delta between r2 and r1 and update r1 after
14097 	 * 'if' condition.
14098 	 */
14099 	if (env->bpf_capable && BPF_OP(insn->code) == BPF_ADD &&
14100 	    dst_reg->id && is_reg_const(src_reg, alu32)) {
14101 		u64 val = reg_const_value(src_reg, alu32);
14102 
14103 		if ((dst_reg->id & BPF_ADD_CONST) ||
14104 		    /* prevent overflow in find_equal_scalars() later */
14105 		    val > (u32)S32_MAX) {
14106 			/*
14107 			 * If the register already went through rX += val
14108 			 * we cannot accumulate another val into rx->off.
14109 			 */
14110 			dst_reg->off = 0;
14111 			dst_reg->id = 0;
14112 		} else {
14113 			dst_reg->id |= BPF_ADD_CONST;
14114 			dst_reg->off = val;
14115 		}
14116 	} else {
14117 		/*
14118 		 * Make sure ID is cleared otherwise dst_reg min/max could be
14119 		 * incorrectly propagated into other registers by find_equal_scalars()
14120 		 */
14121 		dst_reg->id = 0;
14122 	}
14123 	return 0;
14124 }
14125 
14126 /* check validity of 32-bit and 64-bit arithmetic operations */
14127 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
14128 {
14129 	struct bpf_reg_state *regs = cur_regs(env);
14130 	u8 opcode = BPF_OP(insn->code);
14131 	int err;
14132 
14133 	if (opcode == BPF_END || opcode == BPF_NEG) {
14134 		if (opcode == BPF_NEG) {
14135 			if (BPF_SRC(insn->code) != BPF_K ||
14136 			    insn->src_reg != BPF_REG_0 ||
14137 			    insn->off != 0 || insn->imm != 0) {
14138 				verbose(env, "BPF_NEG uses reserved fields\n");
14139 				return -EINVAL;
14140 			}
14141 		} else {
14142 			if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
14143 			    (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
14144 			    (BPF_CLASS(insn->code) == BPF_ALU64 &&
14145 			     BPF_SRC(insn->code) != BPF_TO_LE)) {
14146 				verbose(env, "BPF_END uses reserved fields\n");
14147 				return -EINVAL;
14148 			}
14149 		}
14150 
14151 		/* check src operand */
14152 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
14153 		if (err)
14154 			return err;
14155 
14156 		if (is_pointer_value(env, insn->dst_reg)) {
14157 			verbose(env, "R%d pointer arithmetic prohibited\n",
14158 				insn->dst_reg);
14159 			return -EACCES;
14160 		}
14161 
14162 		/* check dest operand */
14163 		err = check_reg_arg(env, insn->dst_reg, DST_OP);
14164 		if (err)
14165 			return err;
14166 
14167 	} else if (opcode == BPF_MOV) {
14168 
14169 		if (BPF_SRC(insn->code) == BPF_X) {
14170 			if (BPF_CLASS(insn->code) == BPF_ALU) {
14171 				if ((insn->off != 0 && insn->off != 8 && insn->off != 16) ||
14172 				    insn->imm) {
14173 					verbose(env, "BPF_MOV uses reserved fields\n");
14174 					return -EINVAL;
14175 				}
14176 			} else if (insn->off == BPF_ADDR_SPACE_CAST) {
14177 				if (insn->imm != 1 && insn->imm != 1u << 16) {
14178 					verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n");
14179 					return -EINVAL;
14180 				}
14181 				if (!env->prog->aux->arena) {
14182 					verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n");
14183 					return -EINVAL;
14184 				}
14185 			} else {
14186 				if ((insn->off != 0 && insn->off != 8 && insn->off != 16 &&
14187 				     insn->off != 32) || insn->imm) {
14188 					verbose(env, "BPF_MOV uses reserved fields\n");
14189 					return -EINVAL;
14190 				}
14191 			}
14192 
14193 			/* check src operand */
14194 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
14195 			if (err)
14196 				return err;
14197 		} else {
14198 			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
14199 				verbose(env, "BPF_MOV uses reserved fields\n");
14200 				return -EINVAL;
14201 			}
14202 		}
14203 
14204 		/* check dest operand, mark as required later */
14205 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
14206 		if (err)
14207 			return err;
14208 
14209 		if (BPF_SRC(insn->code) == BPF_X) {
14210 			struct bpf_reg_state *src_reg = regs + insn->src_reg;
14211 			struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
14212 
14213 			if (BPF_CLASS(insn->code) == BPF_ALU64) {
14214 				if (insn->imm) {
14215 					/* off == BPF_ADDR_SPACE_CAST */
14216 					mark_reg_unknown(env, regs, insn->dst_reg);
14217 					if (insn->imm == 1) { /* cast from as(1) to as(0) */
14218 						dst_reg->type = PTR_TO_ARENA;
14219 						/* PTR_TO_ARENA is 32-bit */
14220 						dst_reg->subreg_def = env->insn_idx + 1;
14221 					}
14222 				} else if (insn->off == 0) {
14223 					/* case: R1 = R2
14224 					 * copy register state to dest reg
14225 					 */
14226 					assign_scalar_id_before_mov(env, src_reg);
14227 					copy_register_state(dst_reg, src_reg);
14228 					dst_reg->live |= REG_LIVE_WRITTEN;
14229 					dst_reg->subreg_def = DEF_NOT_SUBREG;
14230 				} else {
14231 					/* case: R1 = (s8, s16 s32)R2 */
14232 					if (is_pointer_value(env, insn->src_reg)) {
14233 						verbose(env,
14234 							"R%d sign-extension part of pointer\n",
14235 							insn->src_reg);
14236 						return -EACCES;
14237 					} else if (src_reg->type == SCALAR_VALUE) {
14238 						bool no_sext;
14239 
14240 						no_sext = src_reg->umax_value < (1ULL << (insn->off - 1));
14241 						if (no_sext)
14242 							assign_scalar_id_before_mov(env, src_reg);
14243 						copy_register_state(dst_reg, src_reg);
14244 						if (!no_sext)
14245 							dst_reg->id = 0;
14246 						coerce_reg_to_size_sx(dst_reg, insn->off >> 3);
14247 						dst_reg->live |= REG_LIVE_WRITTEN;
14248 						dst_reg->subreg_def = DEF_NOT_SUBREG;
14249 					} else {
14250 						mark_reg_unknown(env, regs, insn->dst_reg);
14251 					}
14252 				}
14253 			} else {
14254 				/* R1 = (u32) R2 */
14255 				if (is_pointer_value(env, insn->src_reg)) {
14256 					verbose(env,
14257 						"R%d partial copy of pointer\n",
14258 						insn->src_reg);
14259 					return -EACCES;
14260 				} else if (src_reg->type == SCALAR_VALUE) {
14261 					if (insn->off == 0) {
14262 						bool is_src_reg_u32 = get_reg_width(src_reg) <= 32;
14263 
14264 						if (is_src_reg_u32)
14265 							assign_scalar_id_before_mov(env, src_reg);
14266 						copy_register_state(dst_reg, src_reg);
14267 						/* Make sure ID is cleared if src_reg is not in u32
14268 						 * range otherwise dst_reg min/max could be incorrectly
14269 						 * propagated into src_reg by find_equal_scalars()
14270 						 */
14271 						if (!is_src_reg_u32)
14272 							dst_reg->id = 0;
14273 						dst_reg->live |= REG_LIVE_WRITTEN;
14274 						dst_reg->subreg_def = env->insn_idx + 1;
14275 					} else {
14276 						/* case: W1 = (s8, s16)W2 */
14277 						bool no_sext = src_reg->umax_value < (1ULL << (insn->off - 1));
14278 
14279 						if (no_sext)
14280 							assign_scalar_id_before_mov(env, src_reg);
14281 						copy_register_state(dst_reg, src_reg);
14282 						if (!no_sext)
14283 							dst_reg->id = 0;
14284 						dst_reg->live |= REG_LIVE_WRITTEN;
14285 						dst_reg->subreg_def = env->insn_idx + 1;
14286 						coerce_subreg_to_size_sx(dst_reg, insn->off >> 3);
14287 					}
14288 				} else {
14289 					mark_reg_unknown(env, regs,
14290 							 insn->dst_reg);
14291 				}
14292 				zext_32_to_64(dst_reg);
14293 				reg_bounds_sync(dst_reg);
14294 			}
14295 		} else {
14296 			/* case: R = imm
14297 			 * remember the value we stored into this reg
14298 			 */
14299 			/* clear any state __mark_reg_known doesn't set */
14300 			mark_reg_unknown(env, regs, insn->dst_reg);
14301 			regs[insn->dst_reg].type = SCALAR_VALUE;
14302 			if (BPF_CLASS(insn->code) == BPF_ALU64) {
14303 				__mark_reg_known(regs + insn->dst_reg,
14304 						 insn->imm);
14305 			} else {
14306 				__mark_reg_known(regs + insn->dst_reg,
14307 						 (u32)insn->imm);
14308 			}
14309 		}
14310 
14311 	} else if (opcode > BPF_END) {
14312 		verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
14313 		return -EINVAL;
14314 
14315 	} else {	/* all other ALU ops: and, sub, xor, add, ... */
14316 
14317 		if (BPF_SRC(insn->code) == BPF_X) {
14318 			if (insn->imm != 0 || insn->off > 1 ||
14319 			    (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
14320 				verbose(env, "BPF_ALU uses reserved fields\n");
14321 				return -EINVAL;
14322 			}
14323 			/* check src1 operand */
14324 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
14325 			if (err)
14326 				return err;
14327 		} else {
14328 			if (insn->src_reg != BPF_REG_0 || insn->off > 1 ||
14329 			    (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
14330 				verbose(env, "BPF_ALU uses reserved fields\n");
14331 				return -EINVAL;
14332 			}
14333 		}
14334 
14335 		/* check src2 operand */
14336 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
14337 		if (err)
14338 			return err;
14339 
14340 		if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
14341 		    BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
14342 			verbose(env, "div by zero\n");
14343 			return -EINVAL;
14344 		}
14345 
14346 		if ((opcode == BPF_LSH || opcode == BPF_RSH ||
14347 		     opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
14348 			int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
14349 
14350 			if (insn->imm < 0 || insn->imm >= size) {
14351 				verbose(env, "invalid shift %d\n", insn->imm);
14352 				return -EINVAL;
14353 			}
14354 		}
14355 
14356 		/* check dest operand */
14357 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
14358 		err = err ?: adjust_reg_min_max_vals(env, insn);
14359 		if (err)
14360 			return err;
14361 	}
14362 
14363 	return reg_bounds_sanity_check(env, &regs[insn->dst_reg], "alu");
14364 }
14365 
14366 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
14367 				   struct bpf_reg_state *dst_reg,
14368 				   enum bpf_reg_type type,
14369 				   bool range_right_open)
14370 {
14371 	struct bpf_func_state *state;
14372 	struct bpf_reg_state *reg;
14373 	int new_range;
14374 
14375 	if (dst_reg->off < 0 ||
14376 	    (dst_reg->off == 0 && range_right_open))
14377 		/* This doesn't give us any range */
14378 		return;
14379 
14380 	if (dst_reg->umax_value > MAX_PACKET_OFF ||
14381 	    dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
14382 		/* Risk of overflow.  For instance, ptr + (1<<63) may be less
14383 		 * than pkt_end, but that's because it's also less than pkt.
14384 		 */
14385 		return;
14386 
14387 	new_range = dst_reg->off;
14388 	if (range_right_open)
14389 		new_range++;
14390 
14391 	/* Examples for register markings:
14392 	 *
14393 	 * pkt_data in dst register:
14394 	 *
14395 	 *   r2 = r3;
14396 	 *   r2 += 8;
14397 	 *   if (r2 > pkt_end) goto <handle exception>
14398 	 *   <access okay>
14399 	 *
14400 	 *   r2 = r3;
14401 	 *   r2 += 8;
14402 	 *   if (r2 < pkt_end) goto <access okay>
14403 	 *   <handle exception>
14404 	 *
14405 	 *   Where:
14406 	 *     r2 == dst_reg, pkt_end == src_reg
14407 	 *     r2=pkt(id=n,off=8,r=0)
14408 	 *     r3=pkt(id=n,off=0,r=0)
14409 	 *
14410 	 * pkt_data in src register:
14411 	 *
14412 	 *   r2 = r3;
14413 	 *   r2 += 8;
14414 	 *   if (pkt_end >= r2) goto <access okay>
14415 	 *   <handle exception>
14416 	 *
14417 	 *   r2 = r3;
14418 	 *   r2 += 8;
14419 	 *   if (pkt_end <= r2) goto <handle exception>
14420 	 *   <access okay>
14421 	 *
14422 	 *   Where:
14423 	 *     pkt_end == dst_reg, r2 == src_reg
14424 	 *     r2=pkt(id=n,off=8,r=0)
14425 	 *     r3=pkt(id=n,off=0,r=0)
14426 	 *
14427 	 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8)
14428 	 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8)
14429 	 * and [r3, r3 + 8-1) respectively is safe to access depending on
14430 	 * the check.
14431 	 */
14432 
14433 	/* If our ids match, then we must have the same max_value.  And we
14434 	 * don't care about the other reg's fixed offset, since if it's too big
14435 	 * the range won't allow anything.
14436 	 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16.
14437 	 */
14438 	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
14439 		if (reg->type == type && reg->id == dst_reg->id)
14440 			/* keep the maximum range already checked */
14441 			reg->range = max(reg->range, new_range);
14442 	}));
14443 }
14444 
14445 /*
14446  * <reg1> <op> <reg2>, currently assuming reg2 is a constant
14447  */
14448 static int is_scalar_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
14449 				  u8 opcode, bool is_jmp32)
14450 {
14451 	struct tnum t1 = is_jmp32 ? tnum_subreg(reg1->var_off) : reg1->var_off;
14452 	struct tnum t2 = is_jmp32 ? tnum_subreg(reg2->var_off) : reg2->var_off;
14453 	u64 umin1 = is_jmp32 ? (u64)reg1->u32_min_value : reg1->umin_value;
14454 	u64 umax1 = is_jmp32 ? (u64)reg1->u32_max_value : reg1->umax_value;
14455 	s64 smin1 = is_jmp32 ? (s64)reg1->s32_min_value : reg1->smin_value;
14456 	s64 smax1 = is_jmp32 ? (s64)reg1->s32_max_value : reg1->smax_value;
14457 	u64 umin2 = is_jmp32 ? (u64)reg2->u32_min_value : reg2->umin_value;
14458 	u64 umax2 = is_jmp32 ? (u64)reg2->u32_max_value : reg2->umax_value;
14459 	s64 smin2 = is_jmp32 ? (s64)reg2->s32_min_value : reg2->smin_value;
14460 	s64 smax2 = is_jmp32 ? (s64)reg2->s32_max_value : reg2->smax_value;
14461 
14462 	switch (opcode) {
14463 	case BPF_JEQ:
14464 		/* constants, umin/umax and smin/smax checks would be
14465 		 * redundant in this case because they all should match
14466 		 */
14467 		if (tnum_is_const(t1) && tnum_is_const(t2))
14468 			return t1.value == t2.value;
14469 		/* non-overlapping ranges */
14470 		if (umin1 > umax2 || umax1 < umin2)
14471 			return 0;
14472 		if (smin1 > smax2 || smax1 < smin2)
14473 			return 0;
14474 		if (!is_jmp32) {
14475 			/* if 64-bit ranges are inconclusive, see if we can
14476 			 * utilize 32-bit subrange knowledge to eliminate
14477 			 * branches that can't be taken a priori
14478 			 */
14479 			if (reg1->u32_min_value > reg2->u32_max_value ||
14480 			    reg1->u32_max_value < reg2->u32_min_value)
14481 				return 0;
14482 			if (reg1->s32_min_value > reg2->s32_max_value ||
14483 			    reg1->s32_max_value < reg2->s32_min_value)
14484 				return 0;
14485 		}
14486 		break;
14487 	case BPF_JNE:
14488 		/* constants, umin/umax and smin/smax checks would be
14489 		 * redundant in this case because they all should match
14490 		 */
14491 		if (tnum_is_const(t1) && tnum_is_const(t2))
14492 			return t1.value != t2.value;
14493 		/* non-overlapping ranges */
14494 		if (umin1 > umax2 || umax1 < umin2)
14495 			return 1;
14496 		if (smin1 > smax2 || smax1 < smin2)
14497 			return 1;
14498 		if (!is_jmp32) {
14499 			/* if 64-bit ranges are inconclusive, see if we can
14500 			 * utilize 32-bit subrange knowledge to eliminate
14501 			 * branches that can't be taken a priori
14502 			 */
14503 			if (reg1->u32_min_value > reg2->u32_max_value ||
14504 			    reg1->u32_max_value < reg2->u32_min_value)
14505 				return 1;
14506 			if (reg1->s32_min_value > reg2->s32_max_value ||
14507 			    reg1->s32_max_value < reg2->s32_min_value)
14508 				return 1;
14509 		}
14510 		break;
14511 	case BPF_JSET:
14512 		if (!is_reg_const(reg2, is_jmp32)) {
14513 			swap(reg1, reg2);
14514 			swap(t1, t2);
14515 		}
14516 		if (!is_reg_const(reg2, is_jmp32))
14517 			return -1;
14518 		if ((~t1.mask & t1.value) & t2.value)
14519 			return 1;
14520 		if (!((t1.mask | t1.value) & t2.value))
14521 			return 0;
14522 		break;
14523 	case BPF_JGT:
14524 		if (umin1 > umax2)
14525 			return 1;
14526 		else if (umax1 <= umin2)
14527 			return 0;
14528 		break;
14529 	case BPF_JSGT:
14530 		if (smin1 > smax2)
14531 			return 1;
14532 		else if (smax1 <= smin2)
14533 			return 0;
14534 		break;
14535 	case BPF_JLT:
14536 		if (umax1 < umin2)
14537 			return 1;
14538 		else if (umin1 >= umax2)
14539 			return 0;
14540 		break;
14541 	case BPF_JSLT:
14542 		if (smax1 < smin2)
14543 			return 1;
14544 		else if (smin1 >= smax2)
14545 			return 0;
14546 		break;
14547 	case BPF_JGE:
14548 		if (umin1 >= umax2)
14549 			return 1;
14550 		else if (umax1 < umin2)
14551 			return 0;
14552 		break;
14553 	case BPF_JSGE:
14554 		if (smin1 >= smax2)
14555 			return 1;
14556 		else if (smax1 < smin2)
14557 			return 0;
14558 		break;
14559 	case BPF_JLE:
14560 		if (umax1 <= umin2)
14561 			return 1;
14562 		else if (umin1 > umax2)
14563 			return 0;
14564 		break;
14565 	case BPF_JSLE:
14566 		if (smax1 <= smin2)
14567 			return 1;
14568 		else if (smin1 > smax2)
14569 			return 0;
14570 		break;
14571 	}
14572 
14573 	return -1;
14574 }
14575 
14576 static int flip_opcode(u32 opcode)
14577 {
14578 	/* How can we transform "a <op> b" into "b <op> a"? */
14579 	static const u8 opcode_flip[16] = {
14580 		/* these stay the same */
14581 		[BPF_JEQ  >> 4] = BPF_JEQ,
14582 		[BPF_JNE  >> 4] = BPF_JNE,
14583 		[BPF_JSET >> 4] = BPF_JSET,
14584 		/* these swap "lesser" and "greater" (L and G in the opcodes) */
14585 		[BPF_JGE  >> 4] = BPF_JLE,
14586 		[BPF_JGT  >> 4] = BPF_JLT,
14587 		[BPF_JLE  >> 4] = BPF_JGE,
14588 		[BPF_JLT  >> 4] = BPF_JGT,
14589 		[BPF_JSGE >> 4] = BPF_JSLE,
14590 		[BPF_JSGT >> 4] = BPF_JSLT,
14591 		[BPF_JSLE >> 4] = BPF_JSGE,
14592 		[BPF_JSLT >> 4] = BPF_JSGT
14593 	};
14594 	return opcode_flip[opcode >> 4];
14595 }
14596 
14597 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,
14598 				   struct bpf_reg_state *src_reg,
14599 				   u8 opcode)
14600 {
14601 	struct bpf_reg_state *pkt;
14602 
14603 	if (src_reg->type == PTR_TO_PACKET_END) {
14604 		pkt = dst_reg;
14605 	} else if (dst_reg->type == PTR_TO_PACKET_END) {
14606 		pkt = src_reg;
14607 		opcode = flip_opcode(opcode);
14608 	} else {
14609 		return -1;
14610 	}
14611 
14612 	if (pkt->range >= 0)
14613 		return -1;
14614 
14615 	switch (opcode) {
14616 	case BPF_JLE:
14617 		/* pkt <= pkt_end */
14618 		fallthrough;
14619 	case BPF_JGT:
14620 		/* pkt > pkt_end */
14621 		if (pkt->range == BEYOND_PKT_END)
14622 			/* pkt has at last one extra byte beyond pkt_end */
14623 			return opcode == BPF_JGT;
14624 		break;
14625 	case BPF_JLT:
14626 		/* pkt < pkt_end */
14627 		fallthrough;
14628 	case BPF_JGE:
14629 		/* pkt >= pkt_end */
14630 		if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END)
14631 			return opcode == BPF_JGE;
14632 		break;
14633 	}
14634 	return -1;
14635 }
14636 
14637 /* compute branch direction of the expression "if (<reg1> opcode <reg2>) goto target;"
14638  * and return:
14639  *  1 - branch will be taken and "goto target" will be executed
14640  *  0 - branch will not be taken and fall-through to next insn
14641  * -1 - unknown. Example: "if (reg1 < 5)" is unknown when register value
14642  *      range [0,10]
14643  */
14644 static int is_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
14645 			   u8 opcode, bool is_jmp32)
14646 {
14647 	if (reg_is_pkt_pointer_any(reg1) && reg_is_pkt_pointer_any(reg2) && !is_jmp32)
14648 		return is_pkt_ptr_branch_taken(reg1, reg2, opcode);
14649 
14650 	if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) {
14651 		u64 val;
14652 
14653 		/* arrange that reg2 is a scalar, and reg1 is a pointer */
14654 		if (!is_reg_const(reg2, is_jmp32)) {
14655 			opcode = flip_opcode(opcode);
14656 			swap(reg1, reg2);
14657 		}
14658 		/* and ensure that reg2 is a constant */
14659 		if (!is_reg_const(reg2, is_jmp32))
14660 			return -1;
14661 
14662 		if (!reg_not_null(reg1))
14663 			return -1;
14664 
14665 		/* If pointer is valid tests against zero will fail so we can
14666 		 * use this to direct branch taken.
14667 		 */
14668 		val = reg_const_value(reg2, is_jmp32);
14669 		if (val != 0)
14670 			return -1;
14671 
14672 		switch (opcode) {
14673 		case BPF_JEQ:
14674 			return 0;
14675 		case BPF_JNE:
14676 			return 1;
14677 		default:
14678 			return -1;
14679 		}
14680 	}
14681 
14682 	/* now deal with two scalars, but not necessarily constants */
14683 	return is_scalar_branch_taken(reg1, reg2, opcode, is_jmp32);
14684 }
14685 
14686 /* Opcode that corresponds to a *false* branch condition.
14687  * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2
14688  */
14689 static u8 rev_opcode(u8 opcode)
14690 {
14691 	switch (opcode) {
14692 	case BPF_JEQ:		return BPF_JNE;
14693 	case BPF_JNE:		return BPF_JEQ;
14694 	/* JSET doesn't have it's reverse opcode in BPF, so add
14695 	 * BPF_X flag to denote the reverse of that operation
14696 	 */
14697 	case BPF_JSET:		return BPF_JSET | BPF_X;
14698 	case BPF_JSET | BPF_X:	return BPF_JSET;
14699 	case BPF_JGE:		return BPF_JLT;
14700 	case BPF_JGT:		return BPF_JLE;
14701 	case BPF_JLE:		return BPF_JGT;
14702 	case BPF_JLT:		return BPF_JGE;
14703 	case BPF_JSGE:		return BPF_JSLT;
14704 	case BPF_JSGT:		return BPF_JSLE;
14705 	case BPF_JSLE:		return BPF_JSGT;
14706 	case BPF_JSLT:		return BPF_JSGE;
14707 	default:		return 0;
14708 	}
14709 }
14710 
14711 /* Refine range knowledge for <reg1> <op> <reg>2 conditional operation. */
14712 static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
14713 				u8 opcode, bool is_jmp32)
14714 {
14715 	struct tnum t;
14716 	u64 val;
14717 
14718 	/* In case of GE/GT/SGE/JST, reuse LE/LT/SLE/SLT logic from below */
14719 	switch (opcode) {
14720 	case BPF_JGE:
14721 	case BPF_JGT:
14722 	case BPF_JSGE:
14723 	case BPF_JSGT:
14724 		opcode = flip_opcode(opcode);
14725 		swap(reg1, reg2);
14726 		break;
14727 	default:
14728 		break;
14729 	}
14730 
14731 	switch (opcode) {
14732 	case BPF_JEQ:
14733 		if (is_jmp32) {
14734 			reg1->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value);
14735 			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value);
14736 			reg1->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value);
14737 			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value);
14738 			reg2->u32_min_value = reg1->u32_min_value;
14739 			reg2->u32_max_value = reg1->u32_max_value;
14740 			reg2->s32_min_value = reg1->s32_min_value;
14741 			reg2->s32_max_value = reg1->s32_max_value;
14742 
14743 			t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off));
14744 			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
14745 			reg2->var_off = tnum_with_subreg(reg2->var_off, t);
14746 		} else {
14747 			reg1->umin_value = max(reg1->umin_value, reg2->umin_value);
14748 			reg1->umax_value = min(reg1->umax_value, reg2->umax_value);
14749 			reg1->smin_value = max(reg1->smin_value, reg2->smin_value);
14750 			reg1->smax_value = min(reg1->smax_value, reg2->smax_value);
14751 			reg2->umin_value = reg1->umin_value;
14752 			reg2->umax_value = reg1->umax_value;
14753 			reg2->smin_value = reg1->smin_value;
14754 			reg2->smax_value = reg1->smax_value;
14755 
14756 			reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off);
14757 			reg2->var_off = reg1->var_off;
14758 		}
14759 		break;
14760 	case BPF_JNE:
14761 		if (!is_reg_const(reg2, is_jmp32))
14762 			swap(reg1, reg2);
14763 		if (!is_reg_const(reg2, is_jmp32))
14764 			break;
14765 
14766 		/* try to recompute the bound of reg1 if reg2 is a const and
14767 		 * is exactly the edge of reg1.
14768 		 */
14769 		val = reg_const_value(reg2, is_jmp32);
14770 		if (is_jmp32) {
14771 			/* u32_min_value is not equal to 0xffffffff at this point,
14772 			 * because otherwise u32_max_value is 0xffffffff as well,
14773 			 * in such a case both reg1 and reg2 would be constants,
14774 			 * jump would be predicted and reg_set_min_max() won't
14775 			 * be called.
14776 			 *
14777 			 * Same reasoning works for all {u,s}{min,max}{32,64} cases
14778 			 * below.
14779 			 */
14780 			if (reg1->u32_min_value == (u32)val)
14781 				reg1->u32_min_value++;
14782 			if (reg1->u32_max_value == (u32)val)
14783 				reg1->u32_max_value--;
14784 			if (reg1->s32_min_value == (s32)val)
14785 				reg1->s32_min_value++;
14786 			if (reg1->s32_max_value == (s32)val)
14787 				reg1->s32_max_value--;
14788 		} else {
14789 			if (reg1->umin_value == (u64)val)
14790 				reg1->umin_value++;
14791 			if (reg1->umax_value == (u64)val)
14792 				reg1->umax_value--;
14793 			if (reg1->smin_value == (s64)val)
14794 				reg1->smin_value++;
14795 			if (reg1->smax_value == (s64)val)
14796 				reg1->smax_value--;
14797 		}
14798 		break;
14799 	case BPF_JSET:
14800 		if (!is_reg_const(reg2, is_jmp32))
14801 			swap(reg1, reg2);
14802 		if (!is_reg_const(reg2, is_jmp32))
14803 			break;
14804 		val = reg_const_value(reg2, is_jmp32);
14805 		/* BPF_JSET (i.e., TRUE branch, *not* BPF_JSET | BPF_X)
14806 		 * requires single bit to learn something useful. E.g., if we
14807 		 * know that `r1 & 0x3` is true, then which bits (0, 1, or both)
14808 		 * are actually set? We can learn something definite only if
14809 		 * it's a single-bit value to begin with.
14810 		 *
14811 		 * BPF_JSET | BPF_X (i.e., negation of BPF_JSET) doesn't have
14812 		 * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor
14813 		 * bit 1 is set, which we can readily use in adjustments.
14814 		 */
14815 		if (!is_power_of_2(val))
14816 			break;
14817 		if (is_jmp32) {
14818 			t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val));
14819 			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
14820 		} else {
14821 			reg1->var_off = tnum_or(reg1->var_off, tnum_const(val));
14822 		}
14823 		break;
14824 	case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see rev_opcode() */
14825 		if (!is_reg_const(reg2, is_jmp32))
14826 			swap(reg1, reg2);
14827 		if (!is_reg_const(reg2, is_jmp32))
14828 			break;
14829 		val = reg_const_value(reg2, is_jmp32);
14830 		if (is_jmp32) {
14831 			t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val));
14832 			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
14833 		} else {
14834 			reg1->var_off = tnum_and(reg1->var_off, tnum_const(~val));
14835 		}
14836 		break;
14837 	case BPF_JLE:
14838 		if (is_jmp32) {
14839 			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value);
14840 			reg2->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value);
14841 		} else {
14842 			reg1->umax_value = min(reg1->umax_value, reg2->umax_value);
14843 			reg2->umin_value = max(reg1->umin_value, reg2->umin_value);
14844 		}
14845 		break;
14846 	case BPF_JLT:
14847 		if (is_jmp32) {
14848 			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value - 1);
14849 			reg2->u32_min_value = max(reg1->u32_min_value + 1, reg2->u32_min_value);
14850 		} else {
14851 			reg1->umax_value = min(reg1->umax_value, reg2->umax_value - 1);
14852 			reg2->umin_value = max(reg1->umin_value + 1, reg2->umin_value);
14853 		}
14854 		break;
14855 	case BPF_JSLE:
14856 		if (is_jmp32) {
14857 			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value);
14858 			reg2->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value);
14859 		} else {
14860 			reg1->smax_value = min(reg1->smax_value, reg2->smax_value);
14861 			reg2->smin_value = max(reg1->smin_value, reg2->smin_value);
14862 		}
14863 		break;
14864 	case BPF_JSLT:
14865 		if (is_jmp32) {
14866 			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value - 1);
14867 			reg2->s32_min_value = max(reg1->s32_min_value + 1, reg2->s32_min_value);
14868 		} else {
14869 			reg1->smax_value = min(reg1->smax_value, reg2->smax_value - 1);
14870 			reg2->smin_value = max(reg1->smin_value + 1, reg2->smin_value);
14871 		}
14872 		break;
14873 	default:
14874 		return;
14875 	}
14876 }
14877 
14878 /* Adjusts the register min/max values in the case that the dst_reg and
14879  * src_reg are both SCALAR_VALUE registers (or we are simply doing a BPF_K
14880  * check, in which case we have a fake SCALAR_VALUE representing insn->imm).
14881  * Technically we can do similar adjustments for pointers to the same object,
14882  * but we don't support that right now.
14883  */
14884 static int reg_set_min_max(struct bpf_verifier_env *env,
14885 			   struct bpf_reg_state *true_reg1,
14886 			   struct bpf_reg_state *true_reg2,
14887 			   struct bpf_reg_state *false_reg1,
14888 			   struct bpf_reg_state *false_reg2,
14889 			   u8 opcode, bool is_jmp32)
14890 {
14891 	int err;
14892 
14893 	/* If either register is a pointer, we can't learn anything about its
14894 	 * variable offset from the compare (unless they were a pointer into
14895 	 * the same object, but we don't bother with that).
14896 	 */
14897 	if (false_reg1->type != SCALAR_VALUE || false_reg2->type != SCALAR_VALUE)
14898 		return 0;
14899 
14900 	/* fallthrough (FALSE) branch */
14901 	regs_refine_cond_op(false_reg1, false_reg2, rev_opcode(opcode), is_jmp32);
14902 	reg_bounds_sync(false_reg1);
14903 	reg_bounds_sync(false_reg2);
14904 
14905 	/* jump (TRUE) branch */
14906 	regs_refine_cond_op(true_reg1, true_reg2, opcode, is_jmp32);
14907 	reg_bounds_sync(true_reg1);
14908 	reg_bounds_sync(true_reg2);
14909 
14910 	err = reg_bounds_sanity_check(env, true_reg1, "true_reg1");
14911 	err = err ?: reg_bounds_sanity_check(env, true_reg2, "true_reg2");
14912 	err = err ?: reg_bounds_sanity_check(env, false_reg1, "false_reg1");
14913 	err = err ?: reg_bounds_sanity_check(env, false_reg2, "false_reg2");
14914 	return err;
14915 }
14916 
14917 static void mark_ptr_or_null_reg(struct bpf_func_state *state,
14918 				 struct bpf_reg_state *reg, u32 id,
14919 				 bool is_null)
14920 {
14921 	if (type_may_be_null(reg->type) && reg->id == id &&
14922 	    (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) {
14923 		/* Old offset (both fixed and variable parts) should have been
14924 		 * known-zero, because we don't allow pointer arithmetic on
14925 		 * pointers that might be NULL. If we see this happening, don't
14926 		 * convert the register.
14927 		 *
14928 		 * But in some cases, some helpers that return local kptrs
14929 		 * advance offset for the returned pointer. In those cases, it
14930 		 * is fine to expect to see reg->off.
14931 		 */
14932 		if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || !tnum_equals_const(reg->var_off, 0)))
14933 			return;
14934 		if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) &&
14935 		    WARN_ON_ONCE(reg->off))
14936 			return;
14937 
14938 		if (is_null) {
14939 			reg->type = SCALAR_VALUE;
14940 			/* We don't need id and ref_obj_id from this point
14941 			 * onwards anymore, thus we should better reset it,
14942 			 * so that state pruning has chances to take effect.
14943 			 */
14944 			reg->id = 0;
14945 			reg->ref_obj_id = 0;
14946 
14947 			return;
14948 		}
14949 
14950 		mark_ptr_not_null_reg(reg);
14951 
14952 		if (!reg_may_point_to_spin_lock(reg)) {
14953 			/* For not-NULL ptr, reg->ref_obj_id will be reset
14954 			 * in release_reference().
14955 			 *
14956 			 * reg->id is still used by spin_lock ptr. Other
14957 			 * than spin_lock ptr type, reg->id can be reset.
14958 			 */
14959 			reg->id = 0;
14960 		}
14961 	}
14962 }
14963 
14964 /* The logic is similar to find_good_pkt_pointers(), both could eventually
14965  * be folded together at some point.
14966  */
14967 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
14968 				  bool is_null)
14969 {
14970 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
14971 	struct bpf_reg_state *regs = state->regs, *reg;
14972 	u32 ref_obj_id = regs[regno].ref_obj_id;
14973 	u32 id = regs[regno].id;
14974 
14975 	if (ref_obj_id && ref_obj_id == id && is_null)
14976 		/* regs[regno] is in the " == NULL" branch.
14977 		 * No one could have freed the reference state before
14978 		 * doing the NULL check.
14979 		 */
14980 		WARN_ON_ONCE(release_reference_state(state, id));
14981 
14982 	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
14983 		mark_ptr_or_null_reg(state, reg, id, is_null);
14984 	}));
14985 }
14986 
14987 static bool try_match_pkt_pointers(const struct bpf_insn *insn,
14988 				   struct bpf_reg_state *dst_reg,
14989 				   struct bpf_reg_state *src_reg,
14990 				   struct bpf_verifier_state *this_branch,
14991 				   struct bpf_verifier_state *other_branch)
14992 {
14993 	if (BPF_SRC(insn->code) != BPF_X)
14994 		return false;
14995 
14996 	/* Pointers are always 64-bit. */
14997 	if (BPF_CLASS(insn->code) == BPF_JMP32)
14998 		return false;
14999 
15000 	switch (BPF_OP(insn->code)) {
15001 	case BPF_JGT:
15002 		if ((dst_reg->type == PTR_TO_PACKET &&
15003 		     src_reg->type == PTR_TO_PACKET_END) ||
15004 		    (dst_reg->type == PTR_TO_PACKET_META &&
15005 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
15006 			/* pkt_data' > pkt_end, pkt_meta' > pkt_data */
15007 			find_good_pkt_pointers(this_branch, dst_reg,
15008 					       dst_reg->type, false);
15009 			mark_pkt_end(other_branch, insn->dst_reg, true);
15010 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
15011 			    src_reg->type == PTR_TO_PACKET) ||
15012 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
15013 			    src_reg->type == PTR_TO_PACKET_META)) {
15014 			/* pkt_end > pkt_data', pkt_data > pkt_meta' */
15015 			find_good_pkt_pointers(other_branch, src_reg,
15016 					       src_reg->type, true);
15017 			mark_pkt_end(this_branch, insn->src_reg, false);
15018 		} else {
15019 			return false;
15020 		}
15021 		break;
15022 	case BPF_JLT:
15023 		if ((dst_reg->type == PTR_TO_PACKET &&
15024 		     src_reg->type == PTR_TO_PACKET_END) ||
15025 		    (dst_reg->type == PTR_TO_PACKET_META &&
15026 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
15027 			/* pkt_data' < pkt_end, pkt_meta' < pkt_data */
15028 			find_good_pkt_pointers(other_branch, dst_reg,
15029 					       dst_reg->type, true);
15030 			mark_pkt_end(this_branch, insn->dst_reg, false);
15031 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
15032 			    src_reg->type == PTR_TO_PACKET) ||
15033 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
15034 			    src_reg->type == PTR_TO_PACKET_META)) {
15035 			/* pkt_end < pkt_data', pkt_data > pkt_meta' */
15036 			find_good_pkt_pointers(this_branch, src_reg,
15037 					       src_reg->type, false);
15038 			mark_pkt_end(other_branch, insn->src_reg, true);
15039 		} else {
15040 			return false;
15041 		}
15042 		break;
15043 	case BPF_JGE:
15044 		if ((dst_reg->type == PTR_TO_PACKET &&
15045 		     src_reg->type == PTR_TO_PACKET_END) ||
15046 		    (dst_reg->type == PTR_TO_PACKET_META &&
15047 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
15048 			/* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */
15049 			find_good_pkt_pointers(this_branch, dst_reg,
15050 					       dst_reg->type, true);
15051 			mark_pkt_end(other_branch, insn->dst_reg, false);
15052 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
15053 			    src_reg->type == PTR_TO_PACKET) ||
15054 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
15055 			    src_reg->type == PTR_TO_PACKET_META)) {
15056 			/* pkt_end >= pkt_data', pkt_data >= pkt_meta' */
15057 			find_good_pkt_pointers(other_branch, src_reg,
15058 					       src_reg->type, false);
15059 			mark_pkt_end(this_branch, insn->src_reg, true);
15060 		} else {
15061 			return false;
15062 		}
15063 		break;
15064 	case BPF_JLE:
15065 		if ((dst_reg->type == PTR_TO_PACKET &&
15066 		     src_reg->type == PTR_TO_PACKET_END) ||
15067 		    (dst_reg->type == PTR_TO_PACKET_META &&
15068 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
15069 			/* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */
15070 			find_good_pkt_pointers(other_branch, dst_reg,
15071 					       dst_reg->type, false);
15072 			mark_pkt_end(this_branch, insn->dst_reg, true);
15073 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
15074 			    src_reg->type == PTR_TO_PACKET) ||
15075 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
15076 			    src_reg->type == PTR_TO_PACKET_META)) {
15077 			/* pkt_end <= pkt_data', pkt_data <= pkt_meta' */
15078 			find_good_pkt_pointers(this_branch, src_reg,
15079 					       src_reg->type, true);
15080 			mark_pkt_end(other_branch, insn->src_reg, false);
15081 		} else {
15082 			return false;
15083 		}
15084 		break;
15085 	default:
15086 		return false;
15087 	}
15088 
15089 	return true;
15090 }
15091 
15092 static void find_equal_scalars(struct bpf_verifier_state *vstate,
15093 			       struct bpf_reg_state *known_reg)
15094 {
15095 	struct bpf_reg_state fake_reg;
15096 	struct bpf_func_state *state;
15097 	struct bpf_reg_state *reg;
15098 
15099 	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
15100 		if (reg->type != SCALAR_VALUE || reg == known_reg)
15101 			continue;
15102 		if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST))
15103 			continue;
15104 		if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) ||
15105 		    reg->off == known_reg->off) {
15106 			copy_register_state(reg, known_reg);
15107 		} else {
15108 			s32 saved_off = reg->off;
15109 
15110 			fake_reg.type = SCALAR_VALUE;
15111 			__mark_reg_known(&fake_reg, (s32)reg->off - (s32)known_reg->off);
15112 
15113 			/* reg = known_reg; reg += delta */
15114 			copy_register_state(reg, known_reg);
15115 			/*
15116 			 * Must preserve off, id and add_const flag,
15117 			 * otherwise another find_equal_scalars() will be incorrect.
15118 			 */
15119 			reg->off = saved_off;
15120 
15121 			scalar32_min_max_add(reg, &fake_reg);
15122 			scalar_min_max_add(reg, &fake_reg);
15123 			reg->var_off = tnum_add(reg->var_off, fake_reg.var_off);
15124 		}
15125 	}));
15126 }
15127 
15128 static int check_cond_jmp_op(struct bpf_verifier_env *env,
15129 			     struct bpf_insn *insn, int *insn_idx)
15130 {
15131 	struct bpf_verifier_state *this_branch = env->cur_state;
15132 	struct bpf_verifier_state *other_branch;
15133 	struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
15134 	struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;
15135 	struct bpf_reg_state *eq_branch_regs;
15136 	u8 opcode = BPF_OP(insn->code);
15137 	bool is_jmp32;
15138 	int pred = -1;
15139 	int err;
15140 
15141 	/* Only conditional jumps are expected to reach here. */
15142 	if (opcode == BPF_JA || opcode > BPF_JCOND) {
15143 		verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
15144 		return -EINVAL;
15145 	}
15146 
15147 	if (opcode == BPF_JCOND) {
15148 		struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;
15149 		int idx = *insn_idx;
15150 
15151 		if (insn->code != (BPF_JMP | BPF_JCOND) ||
15152 		    insn->src_reg != BPF_MAY_GOTO ||
15153 		    insn->dst_reg || insn->imm || insn->off == 0) {
15154 			verbose(env, "invalid may_goto off %d imm %d\n",
15155 				insn->off, insn->imm);
15156 			return -EINVAL;
15157 		}
15158 		prev_st = find_prev_entry(env, cur_st->parent, idx);
15159 
15160 		/* branch out 'fallthrough' insn as a new state to explore */
15161 		queued_st = push_stack(env, idx + 1, idx, false);
15162 		if (!queued_st)
15163 			return -ENOMEM;
15164 
15165 		queued_st->may_goto_depth++;
15166 		if (prev_st)
15167 			widen_imprecise_scalars(env, prev_st, queued_st);
15168 		*insn_idx += insn->off;
15169 		return 0;
15170 	}
15171 
15172 	/* check src2 operand */
15173 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
15174 	if (err)
15175 		return err;
15176 
15177 	dst_reg = &regs[insn->dst_reg];
15178 	if (BPF_SRC(insn->code) == BPF_X) {
15179 		if (insn->imm != 0) {
15180 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
15181 			return -EINVAL;
15182 		}
15183 
15184 		/* check src1 operand */
15185 		err = check_reg_arg(env, insn->src_reg, SRC_OP);
15186 		if (err)
15187 			return err;
15188 
15189 		src_reg = &regs[insn->src_reg];
15190 		if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) &&
15191 		    is_pointer_value(env, insn->src_reg)) {
15192 			verbose(env, "R%d pointer comparison prohibited\n",
15193 				insn->src_reg);
15194 			return -EACCES;
15195 		}
15196 	} else {
15197 		if (insn->src_reg != BPF_REG_0) {
15198 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
15199 			return -EINVAL;
15200 		}
15201 		src_reg = &env->fake_reg[0];
15202 		memset(src_reg, 0, sizeof(*src_reg));
15203 		src_reg->type = SCALAR_VALUE;
15204 		__mark_reg_known(src_reg, insn->imm);
15205 	}
15206 
15207 	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
15208 	pred = is_branch_taken(dst_reg, src_reg, opcode, is_jmp32);
15209 	if (pred >= 0) {
15210 		/* If we get here with a dst_reg pointer type it is because
15211 		 * above is_branch_taken() special cased the 0 comparison.
15212 		 */
15213 		if (!__is_pointer_value(false, dst_reg))
15214 			err = mark_chain_precision(env, insn->dst_reg);
15215 		if (BPF_SRC(insn->code) == BPF_X && !err &&
15216 		    !__is_pointer_value(false, src_reg))
15217 			err = mark_chain_precision(env, insn->src_reg);
15218 		if (err)
15219 			return err;
15220 	}
15221 
15222 	if (pred == 1) {
15223 		/* Only follow the goto, ignore fall-through. If needed, push
15224 		 * the fall-through branch for simulation under speculative
15225 		 * execution.
15226 		 */
15227 		if (!env->bypass_spec_v1 &&
15228 		    !sanitize_speculative_path(env, insn, *insn_idx + 1,
15229 					       *insn_idx))
15230 			return -EFAULT;
15231 		if (env->log.level & BPF_LOG_LEVEL)
15232 			print_insn_state(env, this_branch->frame[this_branch->curframe]);
15233 		*insn_idx += insn->off;
15234 		return 0;
15235 	} else if (pred == 0) {
15236 		/* Only follow the fall-through branch, since that's where the
15237 		 * program will go. If needed, push the goto branch for
15238 		 * simulation under speculative execution.
15239 		 */
15240 		if (!env->bypass_spec_v1 &&
15241 		    !sanitize_speculative_path(env, insn,
15242 					       *insn_idx + insn->off + 1,
15243 					       *insn_idx))
15244 			return -EFAULT;
15245 		if (env->log.level & BPF_LOG_LEVEL)
15246 			print_insn_state(env, this_branch->frame[this_branch->curframe]);
15247 		return 0;
15248 	}
15249 
15250 	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
15251 				  false);
15252 	if (!other_branch)
15253 		return -EFAULT;
15254 	other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
15255 
15256 	if (BPF_SRC(insn->code) == BPF_X) {
15257 		err = reg_set_min_max(env,
15258 				      &other_branch_regs[insn->dst_reg],
15259 				      &other_branch_regs[insn->src_reg],
15260 				      dst_reg, src_reg, opcode, is_jmp32);
15261 	} else /* BPF_SRC(insn->code) == BPF_K */ {
15262 		/* reg_set_min_max() can mangle the fake_reg. Make a copy
15263 		 * so that these are two different memory locations. The
15264 		 * src_reg is not used beyond here in context of K.
15265 		 */
15266 		memcpy(&env->fake_reg[1], &env->fake_reg[0],
15267 		       sizeof(env->fake_reg[0]));
15268 		err = reg_set_min_max(env,
15269 				      &other_branch_regs[insn->dst_reg],
15270 				      &env->fake_reg[0],
15271 				      dst_reg, &env->fake_reg[1],
15272 				      opcode, is_jmp32);
15273 	}
15274 	if (err)
15275 		return err;
15276 
15277 	if (BPF_SRC(insn->code) == BPF_X &&
15278 	    src_reg->type == SCALAR_VALUE && src_reg->id &&
15279 	    !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) {
15280 		find_equal_scalars(this_branch, src_reg);
15281 		find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);
15282 	}
15283 	if (dst_reg->type == SCALAR_VALUE && dst_reg->id &&
15284 	    !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) {
15285 		find_equal_scalars(this_branch, dst_reg);
15286 		find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]);
15287 	}
15288 
15289 	/* if one pointer register is compared to another pointer
15290 	 * register check if PTR_MAYBE_NULL could be lifted.
15291 	 * E.g. register A - maybe null
15292 	 *      register B - not null
15293 	 * for JNE A, B, ... - A is not null in the false branch;
15294 	 * for JEQ A, B, ... - A is not null in the true branch.
15295 	 *
15296 	 * Since PTR_TO_BTF_ID points to a kernel struct that does
15297 	 * not need to be null checked by the BPF program, i.e.,
15298 	 * could be null even without PTR_MAYBE_NULL marking, so
15299 	 * only propagate nullness when neither reg is that type.
15300 	 */
15301 	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X &&
15302 	    __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) &&
15303 	    type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) &&
15304 	    base_type(src_reg->type) != PTR_TO_BTF_ID &&
15305 	    base_type(dst_reg->type) != PTR_TO_BTF_ID) {
15306 		eq_branch_regs = NULL;
15307 		switch (opcode) {
15308 		case BPF_JEQ:
15309 			eq_branch_regs = other_branch_regs;
15310 			break;
15311 		case BPF_JNE:
15312 			eq_branch_regs = regs;
15313 			break;
15314 		default:
15315 			/* do nothing */
15316 			break;
15317 		}
15318 		if (eq_branch_regs) {
15319 			if (type_may_be_null(src_reg->type))
15320 				mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]);
15321 			else
15322 				mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]);
15323 		}
15324 	}
15325 
15326 	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().
15327 	 * NOTE: these optimizations below are related with pointer comparison
15328 	 *       which will never be JMP32.
15329 	 */
15330 	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
15331 	    insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
15332 	    type_may_be_null(dst_reg->type)) {
15333 		/* Mark all identical registers in each branch as either
15334 		 * safe or unknown depending R == 0 or R != 0 conditional.
15335 		 */
15336 		mark_ptr_or_null_regs(this_branch, insn->dst_reg,
15337 				      opcode == BPF_JNE);
15338 		mark_ptr_or_null_regs(other_branch, insn->dst_reg,
15339 				      opcode == BPF_JEQ);
15340 	} else if (!try_match_pkt_pointers(insn, dst_reg, &regs[insn->src_reg],
15341 					   this_branch, other_branch) &&
15342 		   is_pointer_value(env, insn->dst_reg)) {
15343 		verbose(env, "R%d pointer comparison prohibited\n",
15344 			insn->dst_reg);
15345 		return -EACCES;
15346 	}
15347 	if (env->log.level & BPF_LOG_LEVEL)
15348 		print_insn_state(env, this_branch->frame[this_branch->curframe]);
15349 	return 0;
15350 }
15351 
15352 /* verify BPF_LD_IMM64 instruction */
15353 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
15354 {
15355 	struct bpf_insn_aux_data *aux = cur_aux(env);
15356 	struct bpf_reg_state *regs = cur_regs(env);
15357 	struct bpf_reg_state *dst_reg;
15358 	struct bpf_map *map;
15359 	int err;
15360 
15361 	if (BPF_SIZE(insn->code) != BPF_DW) {
15362 		verbose(env, "invalid BPF_LD_IMM insn\n");
15363 		return -EINVAL;
15364 	}
15365 	if (insn->off != 0) {
15366 		verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
15367 		return -EINVAL;
15368 	}
15369 
15370 	err = check_reg_arg(env, insn->dst_reg, DST_OP);
15371 	if (err)
15372 		return err;
15373 
15374 	dst_reg = &regs[insn->dst_reg];
15375 	if (insn->src_reg == 0) {
15376 		u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
15377 
15378 		dst_reg->type = SCALAR_VALUE;
15379 		__mark_reg_known(&regs[insn->dst_reg], imm);
15380 		return 0;
15381 	}
15382 
15383 	/* All special src_reg cases are listed below. From this point onwards
15384 	 * we either succeed and assign a corresponding dst_reg->type after
15385 	 * zeroing the offset, or fail and reject the program.
15386 	 */
15387 	mark_reg_known_zero(env, regs, insn->dst_reg);
15388 
15389 	if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
15390 		dst_reg->type = aux->btf_var.reg_type;
15391 		switch (base_type(dst_reg->type)) {
15392 		case PTR_TO_MEM:
15393 			dst_reg->mem_size = aux->btf_var.mem_size;
15394 			break;
15395 		case PTR_TO_BTF_ID:
15396 			dst_reg->btf = aux->btf_var.btf;
15397 			dst_reg->btf_id = aux->btf_var.btf_id;
15398 			break;
15399 		default:
15400 			verbose(env, "bpf verifier is misconfigured\n");
15401 			return -EFAULT;
15402 		}
15403 		return 0;
15404 	}
15405 
15406 	if (insn->src_reg == BPF_PSEUDO_FUNC) {
15407 		struct bpf_prog_aux *aux = env->prog->aux;
15408 		u32 subprogno = find_subprog(env,
15409 					     env->insn_idx + insn->imm + 1);
15410 
15411 		if (!aux->func_info) {
15412 			verbose(env, "missing btf func_info\n");
15413 			return -EINVAL;
15414 		}
15415 		if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) {
15416 			verbose(env, "callback function not static\n");
15417 			return -EINVAL;
15418 		}
15419 
15420 		dst_reg->type = PTR_TO_FUNC;
15421 		dst_reg->subprogno = subprogno;
15422 		return 0;
15423 	}
15424 
15425 	map = env->used_maps[aux->map_index];
15426 	dst_reg->map_ptr = map;
15427 
15428 	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
15429 	    insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
15430 		if (map->map_type == BPF_MAP_TYPE_ARENA) {
15431 			__mark_reg_unknown(env, dst_reg);
15432 			return 0;
15433 		}
15434 		dst_reg->type = PTR_TO_MAP_VALUE;
15435 		dst_reg->off = aux->map_off;
15436 		WARN_ON_ONCE(map->max_entries != 1);
15437 		/* We want reg->id to be same (0) as map_value is not distinct */
15438 	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD ||
15439 		   insn->src_reg == BPF_PSEUDO_MAP_IDX) {
15440 		dst_reg->type = CONST_PTR_TO_MAP;
15441 	} else {
15442 		verbose(env, "bpf verifier is misconfigured\n");
15443 		return -EINVAL;
15444 	}
15445 
15446 	return 0;
15447 }
15448 
15449 static bool may_access_skb(enum bpf_prog_type type)
15450 {
15451 	switch (type) {
15452 	case BPF_PROG_TYPE_SOCKET_FILTER:
15453 	case BPF_PROG_TYPE_SCHED_CLS:
15454 	case BPF_PROG_TYPE_SCHED_ACT:
15455 		return true;
15456 	default:
15457 		return false;
15458 	}
15459 }
15460 
15461 /* verify safety of LD_ABS|LD_IND instructions:
15462  * - they can only appear in the programs where ctx == skb
15463  * - since they are wrappers of function calls, they scratch R1-R5 registers,
15464  *   preserve R6-R9, and store return value into R0
15465  *
15466  * Implicit input:
15467  *   ctx == skb == R6 == CTX
15468  *
15469  * Explicit input:
15470  *   SRC == any register
15471  *   IMM == 32-bit immediate
15472  *
15473  * Output:
15474  *   R0 - 8/16/32-bit skb data converted to cpu endianness
15475  */
15476 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
15477 {
15478 	struct bpf_reg_state *regs = cur_regs(env);
15479 	static const int ctx_reg = BPF_REG_6;
15480 	u8 mode = BPF_MODE(insn->code);
15481 	int i, err;
15482 
15483 	if (!may_access_skb(resolve_prog_type(env->prog))) {
15484 		verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
15485 		return -EINVAL;
15486 	}
15487 
15488 	if (!env->ops->gen_ld_abs) {
15489 		verbose(env, "bpf verifier is misconfigured\n");
15490 		return -EINVAL;
15491 	}
15492 
15493 	if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
15494 	    BPF_SIZE(insn->code) == BPF_DW ||
15495 	    (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
15496 		verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
15497 		return -EINVAL;
15498 	}
15499 
15500 	/* check whether implicit source operand (register R6) is readable */
15501 	err = check_reg_arg(env, ctx_reg, SRC_OP);
15502 	if (err)
15503 		return err;
15504 
15505 	/* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as
15506 	 * gen_ld_abs() may terminate the program at runtime, leading to
15507 	 * reference leak.
15508 	 */
15509 	err = check_reference_leak(env, false);
15510 	if (err) {
15511 		verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
15512 		return err;
15513 	}
15514 
15515 	if (env->cur_state->active_lock.ptr) {
15516 		verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
15517 		return -EINVAL;
15518 	}
15519 
15520 	if (env->cur_state->active_rcu_lock) {
15521 		verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_rcu_read_lock-ed region\n");
15522 		return -EINVAL;
15523 	}
15524 
15525 	if (env->cur_state->active_preempt_lock) {
15526 		verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_preempt_disable-ed region\n");
15527 		return -EINVAL;
15528 	}
15529 
15530 	if (regs[ctx_reg].type != PTR_TO_CTX) {
15531 		verbose(env,
15532 			"at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
15533 		return -EINVAL;
15534 	}
15535 
15536 	if (mode == BPF_IND) {
15537 		/* check explicit source operand */
15538 		err = check_reg_arg(env, insn->src_reg, SRC_OP);
15539 		if (err)
15540 			return err;
15541 	}
15542 
15543 	err = check_ptr_off_reg(env, &regs[ctx_reg], ctx_reg);
15544 	if (err < 0)
15545 		return err;
15546 
15547 	/* reset caller saved regs to unreadable */
15548 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
15549 		mark_reg_not_init(env, regs, caller_saved[i]);
15550 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
15551 	}
15552 
15553 	/* mark destination R0 register as readable, since it contains
15554 	 * the value fetched from the packet.
15555 	 * Already marked as written above.
15556 	 */
15557 	mark_reg_unknown(env, regs, BPF_REG_0);
15558 	/* ld_abs load up to 32-bit skb data. */
15559 	regs[BPF_REG_0].subreg_def = env->insn_idx + 1;
15560 	return 0;
15561 }
15562 
15563 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name)
15564 {
15565 	const char *exit_ctx = "At program exit";
15566 	struct tnum enforce_attach_type_range = tnum_unknown;
15567 	const struct bpf_prog *prog = env->prog;
15568 	struct bpf_reg_state *reg;
15569 	struct bpf_retval_range range = retval_range(0, 1);
15570 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
15571 	int err;
15572 	struct bpf_func_state *frame = env->cur_state->frame[0];
15573 	const bool is_subprog = frame->subprogno;
15574 
15575 	/* LSM and struct_ops func-ptr's return type could be "void" */
15576 	if (!is_subprog || frame->in_exception_callback_fn) {
15577 		switch (prog_type) {
15578 		case BPF_PROG_TYPE_LSM:
15579 			if (prog->expected_attach_type == BPF_LSM_CGROUP)
15580 				/* See below, can be 0 or 0-1 depending on hook. */
15581 				break;
15582 			fallthrough;
15583 		case BPF_PROG_TYPE_STRUCT_OPS:
15584 			if (!prog->aux->attach_func_proto->type)
15585 				return 0;
15586 			break;
15587 		default:
15588 			break;
15589 		}
15590 	}
15591 
15592 	/* eBPF calling convention is such that R0 is used
15593 	 * to return the value from eBPF program.
15594 	 * Make sure that it's readable at this time
15595 	 * of bpf_exit, which means that program wrote
15596 	 * something into it earlier
15597 	 */
15598 	err = check_reg_arg(env, regno, SRC_OP);
15599 	if (err)
15600 		return err;
15601 
15602 	if (is_pointer_value(env, regno)) {
15603 		verbose(env, "R%d leaks addr as return value\n", regno);
15604 		return -EACCES;
15605 	}
15606 
15607 	reg = cur_regs(env) + regno;
15608 
15609 	if (frame->in_async_callback_fn) {
15610 		/* enforce return zero from async callbacks like timer */
15611 		exit_ctx = "At async callback return";
15612 		range = retval_range(0, 0);
15613 		goto enforce_retval;
15614 	}
15615 
15616 	if (is_subprog && !frame->in_exception_callback_fn) {
15617 		if (reg->type != SCALAR_VALUE) {
15618 			verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n",
15619 				regno, reg_type_str(env, reg->type));
15620 			return -EINVAL;
15621 		}
15622 		return 0;
15623 	}
15624 
15625 	switch (prog_type) {
15626 	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
15627 		if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||
15628 		    env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||
15629 		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_RECVMSG ||
15630 		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||
15631 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||
15632 		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETPEERNAME ||
15633 		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
15634 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME ||
15635 		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETSOCKNAME)
15636 			range = retval_range(1, 1);
15637 		if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||
15638 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
15639 			range = retval_range(0, 3);
15640 		break;
15641 	case BPF_PROG_TYPE_CGROUP_SKB:
15642 		if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
15643 			range = retval_range(0, 3);
15644 			enforce_attach_type_range = tnum_range(2, 3);
15645 		}
15646 		break;
15647 	case BPF_PROG_TYPE_CGROUP_SOCK:
15648 	case BPF_PROG_TYPE_SOCK_OPS:
15649 	case BPF_PROG_TYPE_CGROUP_DEVICE:
15650 	case BPF_PROG_TYPE_CGROUP_SYSCTL:
15651 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
15652 		break;
15653 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
15654 		if (!env->prog->aux->attach_btf_id)
15655 			return 0;
15656 		range = retval_range(0, 0);
15657 		break;
15658 	case BPF_PROG_TYPE_TRACING:
15659 		switch (env->prog->expected_attach_type) {
15660 		case BPF_TRACE_FENTRY:
15661 		case BPF_TRACE_FEXIT:
15662 			range = retval_range(0, 0);
15663 			break;
15664 		case BPF_TRACE_RAW_TP:
15665 		case BPF_MODIFY_RETURN:
15666 			return 0;
15667 		case BPF_TRACE_ITER:
15668 			break;
15669 		default:
15670 			return -ENOTSUPP;
15671 		}
15672 		break;
15673 	case BPF_PROG_TYPE_SK_LOOKUP:
15674 		range = retval_range(SK_DROP, SK_PASS);
15675 		break;
15676 
15677 	case BPF_PROG_TYPE_LSM:
15678 		if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {
15679 			/* Regular BPF_PROG_TYPE_LSM programs can return
15680 			 * any value.
15681 			 */
15682 			return 0;
15683 		}
15684 		if (!env->prog->aux->attach_func_proto->type) {
15685 			/* Make sure programs that attach to void
15686 			 * hooks don't try to modify return value.
15687 			 */
15688 			range = retval_range(1, 1);
15689 		}
15690 		break;
15691 
15692 	case BPF_PROG_TYPE_NETFILTER:
15693 		range = retval_range(NF_DROP, NF_ACCEPT);
15694 		break;
15695 	case BPF_PROG_TYPE_EXT:
15696 		/* freplace program can return anything as its return value
15697 		 * depends on the to-be-replaced kernel func or bpf program.
15698 		 */
15699 	default:
15700 		return 0;
15701 	}
15702 
15703 enforce_retval:
15704 	if (reg->type != SCALAR_VALUE) {
15705 		verbose(env, "%s the register R%d is not a known value (%s)\n",
15706 			exit_ctx, regno, reg_type_str(env, reg->type));
15707 		return -EINVAL;
15708 	}
15709 
15710 	err = mark_chain_precision(env, regno);
15711 	if (err)
15712 		return err;
15713 
15714 	if (!retval_range_within(range, reg)) {
15715 		verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name);
15716 		if (!is_subprog &&
15717 		    prog->expected_attach_type == BPF_LSM_CGROUP &&
15718 		    prog_type == BPF_PROG_TYPE_LSM &&
15719 		    !prog->aux->attach_func_proto->type)
15720 			verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
15721 		return -EINVAL;
15722 	}
15723 
15724 	if (!tnum_is_unknown(enforce_attach_type_range) &&
15725 	    tnum_in(enforce_attach_type_range, reg->var_off))
15726 		env->prog->enforce_expected_attach_type = 1;
15727 	return 0;
15728 }
15729 
15730 /* non-recursive DFS pseudo code
15731  * 1  procedure DFS-iterative(G,v):
15732  * 2      label v as discovered
15733  * 3      let S be a stack
15734  * 4      S.push(v)
15735  * 5      while S is not empty
15736  * 6            t <- S.peek()
15737  * 7            if t is what we're looking for:
15738  * 8                return t
15739  * 9            for all edges e in G.adjacentEdges(t) do
15740  * 10               if edge e is already labelled
15741  * 11                   continue with the next edge
15742  * 12               w <- G.adjacentVertex(t,e)
15743  * 13               if vertex w is not discovered and not explored
15744  * 14                   label e as tree-edge
15745  * 15                   label w as discovered
15746  * 16                   S.push(w)
15747  * 17                   continue at 5
15748  * 18               else if vertex w is discovered
15749  * 19                   label e as back-edge
15750  * 20               else
15751  * 21                   // vertex w is explored
15752  * 22                   label e as forward- or cross-edge
15753  * 23           label t as explored
15754  * 24           S.pop()
15755  *
15756  * convention:
15757  * 0x10 - discovered
15758  * 0x11 - discovered and fall-through edge labelled
15759  * 0x12 - discovered and fall-through and branch edges labelled
15760  * 0x20 - explored
15761  */
15762 
15763 enum {
15764 	DISCOVERED = 0x10,
15765 	EXPLORED = 0x20,
15766 	FALLTHROUGH = 1,
15767 	BRANCH = 2,
15768 };
15769 
15770 static void mark_prune_point(struct bpf_verifier_env *env, int idx)
15771 {
15772 	env->insn_aux_data[idx].prune_point = true;
15773 }
15774 
15775 static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx)
15776 {
15777 	return env->insn_aux_data[insn_idx].prune_point;
15778 }
15779 
15780 static void mark_force_checkpoint(struct bpf_verifier_env *env, int idx)
15781 {
15782 	env->insn_aux_data[idx].force_checkpoint = true;
15783 }
15784 
15785 static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)
15786 {
15787 	return env->insn_aux_data[insn_idx].force_checkpoint;
15788 }
15789 
15790 static void mark_calls_callback(struct bpf_verifier_env *env, int idx)
15791 {
15792 	env->insn_aux_data[idx].calls_callback = true;
15793 }
15794 
15795 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx)
15796 {
15797 	return env->insn_aux_data[insn_idx].calls_callback;
15798 }
15799 
15800 enum {
15801 	DONE_EXPLORING = 0,
15802 	KEEP_EXPLORING = 1,
15803 };
15804 
15805 /* t, w, e - match pseudo-code above:
15806  * t - index of current instruction
15807  * w - next instruction
15808  * e - edge
15809  */
15810 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
15811 {
15812 	int *insn_stack = env->cfg.insn_stack;
15813 	int *insn_state = env->cfg.insn_state;
15814 
15815 	if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
15816 		return DONE_EXPLORING;
15817 
15818 	if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
15819 		return DONE_EXPLORING;
15820 
15821 	if (w < 0 || w >= env->prog->len) {
15822 		verbose_linfo(env, t, "%d: ", t);
15823 		verbose(env, "jump out of range from insn %d to %d\n", t, w);
15824 		return -EINVAL;
15825 	}
15826 
15827 	if (e == BRANCH) {
15828 		/* mark branch target for state pruning */
15829 		mark_prune_point(env, w);
15830 		mark_jmp_point(env, w);
15831 	}
15832 
15833 	if (insn_state[w] == 0) {
15834 		/* tree-edge */
15835 		insn_state[t] = DISCOVERED | e;
15836 		insn_state[w] = DISCOVERED;
15837 		if (env->cfg.cur_stack >= env->prog->len)
15838 			return -E2BIG;
15839 		insn_stack[env->cfg.cur_stack++] = w;
15840 		return KEEP_EXPLORING;
15841 	} else if ((insn_state[w] & 0xF0) == DISCOVERED) {
15842 		if (env->bpf_capable)
15843 			return DONE_EXPLORING;
15844 		verbose_linfo(env, t, "%d: ", t);
15845 		verbose_linfo(env, w, "%d: ", w);
15846 		verbose(env, "back-edge from insn %d to %d\n", t, w);
15847 		return -EINVAL;
15848 	} else if (insn_state[w] == EXPLORED) {
15849 		/* forward- or cross-edge */
15850 		insn_state[t] = DISCOVERED | e;
15851 	} else {
15852 		verbose(env, "insn state internal bug\n");
15853 		return -EFAULT;
15854 	}
15855 	return DONE_EXPLORING;
15856 }
15857 
15858 static int visit_func_call_insn(int t, struct bpf_insn *insns,
15859 				struct bpf_verifier_env *env,
15860 				bool visit_callee)
15861 {
15862 	int ret, insn_sz;
15863 
15864 	insn_sz = bpf_is_ldimm64(&insns[t]) ? 2 : 1;
15865 	ret = push_insn(t, t + insn_sz, FALLTHROUGH, env);
15866 	if (ret)
15867 		return ret;
15868 
15869 	mark_prune_point(env, t + insn_sz);
15870 	/* when we exit from subprog, we need to record non-linear history */
15871 	mark_jmp_point(env, t + insn_sz);
15872 
15873 	if (visit_callee) {
15874 		mark_prune_point(env, t);
15875 		ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env);
15876 	}
15877 	return ret;
15878 }
15879 
15880 /* Visits the instruction at index t and returns one of the following:
15881  *  < 0 - an error occurred
15882  *  DONE_EXPLORING - the instruction was fully explored
15883  *  KEEP_EXPLORING - there is still work to be done before it is fully explored
15884  */
15885 static int visit_insn(int t, struct bpf_verifier_env *env)
15886 {
15887 	struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t];
15888 	int ret, off, insn_sz;
15889 
15890 	if (bpf_pseudo_func(insn))
15891 		return visit_func_call_insn(t, insns, env, true);
15892 
15893 	/* All non-branch instructions have a single fall-through edge. */
15894 	if (BPF_CLASS(insn->code) != BPF_JMP &&
15895 	    BPF_CLASS(insn->code) != BPF_JMP32) {
15896 		insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
15897 		return push_insn(t, t + insn_sz, FALLTHROUGH, env);
15898 	}
15899 
15900 	switch (BPF_OP(insn->code)) {
15901 	case BPF_EXIT:
15902 		return DONE_EXPLORING;
15903 
15904 	case BPF_CALL:
15905 		if (is_async_callback_calling_insn(insn))
15906 			/* Mark this call insn as a prune point to trigger
15907 			 * is_state_visited() check before call itself is
15908 			 * processed by __check_func_call(). Otherwise new
15909 			 * async state will be pushed for further exploration.
15910 			 */
15911 			mark_prune_point(env, t);
15912 		/* For functions that invoke callbacks it is not known how many times
15913 		 * callback would be called. Verifier models callback calling functions
15914 		 * by repeatedly visiting callback bodies and returning to origin call
15915 		 * instruction.
15916 		 * In order to stop such iteration verifier needs to identify when a
15917 		 * state identical some state from a previous iteration is reached.
15918 		 * Check below forces creation of checkpoint before callback calling
15919 		 * instruction to allow search for such identical states.
15920 		 */
15921 		if (is_sync_callback_calling_insn(insn)) {
15922 			mark_calls_callback(env, t);
15923 			mark_force_checkpoint(env, t);
15924 			mark_prune_point(env, t);
15925 			mark_jmp_point(env, t);
15926 		}
15927 		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
15928 			struct bpf_kfunc_call_arg_meta meta;
15929 
15930 			ret = fetch_kfunc_meta(env, insn, &meta, NULL);
15931 			if (ret == 0 && is_iter_next_kfunc(&meta)) {
15932 				mark_prune_point(env, t);
15933 				/* Checking and saving state checkpoints at iter_next() call
15934 				 * is crucial for fast convergence of open-coded iterator loop
15935 				 * logic, so we need to force it. If we don't do that,
15936 				 * is_state_visited() might skip saving a checkpoint, causing
15937 				 * unnecessarily long sequence of not checkpointed
15938 				 * instructions and jumps, leading to exhaustion of jump
15939 				 * history buffer, and potentially other undesired outcomes.
15940 				 * It is expected that with correct open-coded iterators
15941 				 * convergence will happen quickly, so we don't run a risk of
15942 				 * exhausting memory.
15943 				 */
15944 				mark_force_checkpoint(env, t);
15945 			}
15946 		}
15947 		return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);
15948 
15949 	case BPF_JA:
15950 		if (BPF_SRC(insn->code) != BPF_K)
15951 			return -EINVAL;
15952 
15953 		if (BPF_CLASS(insn->code) == BPF_JMP)
15954 			off = insn->off;
15955 		else
15956 			off = insn->imm;
15957 
15958 		/* unconditional jump with single edge */
15959 		ret = push_insn(t, t + off + 1, FALLTHROUGH, env);
15960 		if (ret)
15961 			return ret;
15962 
15963 		mark_prune_point(env, t + off + 1);
15964 		mark_jmp_point(env, t + off + 1);
15965 
15966 		return ret;
15967 
15968 	default:
15969 		/* conditional jump with two edges */
15970 		mark_prune_point(env, t);
15971 		if (is_may_goto_insn(insn))
15972 			mark_force_checkpoint(env, t);
15973 
15974 		ret = push_insn(t, t + 1, FALLTHROUGH, env);
15975 		if (ret)
15976 			return ret;
15977 
15978 		return push_insn(t, t + insn->off + 1, BRANCH, env);
15979 	}
15980 }
15981 
15982 /* non-recursive depth-first-search to detect loops in BPF program
15983  * loop == back-edge in directed graph
15984  */
15985 static int check_cfg(struct bpf_verifier_env *env)
15986 {
15987 	int insn_cnt = env->prog->len;
15988 	int *insn_stack, *insn_state;
15989 	int ex_insn_beg, i, ret = 0;
15990 	bool ex_done = false;
15991 
15992 	insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
15993 	if (!insn_state)
15994 		return -ENOMEM;
15995 
15996 	insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
15997 	if (!insn_stack) {
15998 		kvfree(insn_state);
15999 		return -ENOMEM;
16000 	}
16001 
16002 	insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
16003 	insn_stack[0] = 0; /* 0 is the first instruction */
16004 	env->cfg.cur_stack = 1;
16005 
16006 walk_cfg:
16007 	while (env->cfg.cur_stack > 0) {
16008 		int t = insn_stack[env->cfg.cur_stack - 1];
16009 
16010 		ret = visit_insn(t, env);
16011 		switch (ret) {
16012 		case DONE_EXPLORING:
16013 			insn_state[t] = EXPLORED;
16014 			env->cfg.cur_stack--;
16015 			break;
16016 		case KEEP_EXPLORING:
16017 			break;
16018 		default:
16019 			if (ret > 0) {
16020 				verbose(env, "visit_insn internal bug\n");
16021 				ret = -EFAULT;
16022 			}
16023 			goto err_free;
16024 		}
16025 	}
16026 
16027 	if (env->cfg.cur_stack < 0) {
16028 		verbose(env, "pop stack internal bug\n");
16029 		ret = -EFAULT;
16030 		goto err_free;
16031 	}
16032 
16033 	if (env->exception_callback_subprog && !ex_done) {
16034 		ex_insn_beg = env->subprog_info[env->exception_callback_subprog].start;
16035 
16036 		insn_state[ex_insn_beg] = DISCOVERED;
16037 		insn_stack[0] = ex_insn_beg;
16038 		env->cfg.cur_stack = 1;
16039 		ex_done = true;
16040 		goto walk_cfg;
16041 	}
16042 
16043 	for (i = 0; i < insn_cnt; i++) {
16044 		struct bpf_insn *insn = &env->prog->insnsi[i];
16045 
16046 		if (insn_state[i] != EXPLORED) {
16047 			verbose(env, "unreachable insn %d\n", i);
16048 			ret = -EINVAL;
16049 			goto err_free;
16050 		}
16051 		if (bpf_is_ldimm64(insn)) {
16052 			if (insn_state[i + 1] != 0) {
16053 				verbose(env, "jump into the middle of ldimm64 insn %d\n", i);
16054 				ret = -EINVAL;
16055 				goto err_free;
16056 			}
16057 			i++; /* skip second half of ldimm64 */
16058 		}
16059 	}
16060 	ret = 0; /* cfg looks good */
16061 
16062 err_free:
16063 	kvfree(insn_state);
16064 	kvfree(insn_stack);
16065 	env->cfg.insn_state = env->cfg.insn_stack = NULL;
16066 	return ret;
16067 }
16068 
16069 static int check_abnormal_return(struct bpf_verifier_env *env)
16070 {
16071 	int i;
16072 
16073 	for (i = 1; i < env->subprog_cnt; i++) {
16074 		if (env->subprog_info[i].has_ld_abs) {
16075 			verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");
16076 			return -EINVAL;
16077 		}
16078 		if (env->subprog_info[i].has_tail_call) {
16079 			verbose(env, "tail_call is not allowed in subprogs without BTF\n");
16080 			return -EINVAL;
16081 		}
16082 	}
16083 	return 0;
16084 }
16085 
16086 /* The minimum supported BTF func info size */
16087 #define MIN_BPF_FUNCINFO_SIZE	8
16088 #define MAX_FUNCINFO_REC_SIZE	252
16089 
16090 static int check_btf_func_early(struct bpf_verifier_env *env,
16091 				const union bpf_attr *attr,
16092 				bpfptr_t uattr)
16093 {
16094 	u32 krec_size = sizeof(struct bpf_func_info);
16095 	const struct btf_type *type, *func_proto;
16096 	u32 i, nfuncs, urec_size, min_size;
16097 	struct bpf_func_info *krecord;
16098 	struct bpf_prog *prog;
16099 	const struct btf *btf;
16100 	u32 prev_offset = 0;
16101 	bpfptr_t urecord;
16102 	int ret = -ENOMEM;
16103 
16104 	nfuncs = attr->func_info_cnt;
16105 	if (!nfuncs) {
16106 		if (check_abnormal_return(env))
16107 			return -EINVAL;
16108 		return 0;
16109 	}
16110 
16111 	urec_size = attr->func_info_rec_size;
16112 	if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
16113 	    urec_size > MAX_FUNCINFO_REC_SIZE ||
16114 	    urec_size % sizeof(u32)) {
16115 		verbose(env, "invalid func info rec size %u\n", urec_size);
16116 		return -EINVAL;
16117 	}
16118 
16119 	prog = env->prog;
16120 	btf = prog->aux->btf;
16121 
16122 	urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
16123 	min_size = min_t(u32, krec_size, urec_size);
16124 
16125 	krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
16126 	if (!krecord)
16127 		return -ENOMEM;
16128 
16129 	for (i = 0; i < nfuncs; i++) {
16130 		ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
16131 		if (ret) {
16132 			if (ret == -E2BIG) {
16133 				verbose(env, "nonzero tailing record in func info");
16134 				/* set the size kernel expects so loader can zero
16135 				 * out the rest of the record.
16136 				 */
16137 				if (copy_to_bpfptr_offset(uattr,
16138 							  offsetof(union bpf_attr, func_info_rec_size),
16139 							  &min_size, sizeof(min_size)))
16140 					ret = -EFAULT;
16141 			}
16142 			goto err_free;
16143 		}
16144 
16145 		if (copy_from_bpfptr(&krecord[i], urecord, min_size)) {
16146 			ret = -EFAULT;
16147 			goto err_free;
16148 		}
16149 
16150 		/* check insn_off */
16151 		ret = -EINVAL;
16152 		if (i == 0) {
16153 			if (krecord[i].insn_off) {
16154 				verbose(env,
16155 					"nonzero insn_off %u for the first func info record",
16156 					krecord[i].insn_off);
16157 				goto err_free;
16158 			}
16159 		} else if (krecord[i].insn_off <= prev_offset) {
16160 			verbose(env,
16161 				"same or smaller insn offset (%u) than previous func info record (%u)",
16162 				krecord[i].insn_off, prev_offset);
16163 			goto err_free;
16164 		}
16165 
16166 		/* check type_id */
16167 		type = btf_type_by_id(btf, krecord[i].type_id);
16168 		if (!type || !btf_type_is_func(type)) {
16169 			verbose(env, "invalid type id %d in func info",
16170 				krecord[i].type_id);
16171 			goto err_free;
16172 		}
16173 
16174 		func_proto = btf_type_by_id(btf, type->type);
16175 		if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto)))
16176 			/* btf_func_check() already verified it during BTF load */
16177 			goto err_free;
16178 
16179 		prev_offset = krecord[i].insn_off;
16180 		bpfptr_add(&urecord, urec_size);
16181 	}
16182 
16183 	prog->aux->func_info = krecord;
16184 	prog->aux->func_info_cnt = nfuncs;
16185 	return 0;
16186 
16187 err_free:
16188 	kvfree(krecord);
16189 	return ret;
16190 }
16191 
16192 static int check_btf_func(struct bpf_verifier_env *env,
16193 			  const union bpf_attr *attr,
16194 			  bpfptr_t uattr)
16195 {
16196 	const struct btf_type *type, *func_proto, *ret_type;
16197 	u32 i, nfuncs, urec_size;
16198 	struct bpf_func_info *krecord;
16199 	struct bpf_func_info_aux *info_aux = NULL;
16200 	struct bpf_prog *prog;
16201 	const struct btf *btf;
16202 	bpfptr_t urecord;
16203 	bool scalar_return;
16204 	int ret = -ENOMEM;
16205 
16206 	nfuncs = attr->func_info_cnt;
16207 	if (!nfuncs) {
16208 		if (check_abnormal_return(env))
16209 			return -EINVAL;
16210 		return 0;
16211 	}
16212 	if (nfuncs != env->subprog_cnt) {
16213 		verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
16214 		return -EINVAL;
16215 	}
16216 
16217 	urec_size = attr->func_info_rec_size;
16218 
16219 	prog = env->prog;
16220 	btf = prog->aux->btf;
16221 
16222 	urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
16223 
16224 	krecord = prog->aux->func_info;
16225 	info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN);
16226 	if (!info_aux)
16227 		return -ENOMEM;
16228 
16229 	for (i = 0; i < nfuncs; i++) {
16230 		/* check insn_off */
16231 		ret = -EINVAL;
16232 
16233 		if (env->subprog_info[i].start != krecord[i].insn_off) {
16234 			verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
16235 			goto err_free;
16236 		}
16237 
16238 		/* Already checked type_id */
16239 		type = btf_type_by_id(btf, krecord[i].type_id);
16240 		info_aux[i].linkage = BTF_INFO_VLEN(type->info);
16241 		/* Already checked func_proto */
16242 		func_proto = btf_type_by_id(btf, type->type);
16243 
16244 		ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
16245 		scalar_return =
16246 			btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type);
16247 		if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {
16248 			verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");
16249 			goto err_free;
16250 		}
16251 		if (i && !scalar_return && env->subprog_info[i].has_tail_call) {
16252 			verbose(env, "tail_call is only allowed in functions that return 'int'.\n");
16253 			goto err_free;
16254 		}
16255 
16256 		bpfptr_add(&urecord, urec_size);
16257 	}
16258 
16259 	prog->aux->func_info_aux = info_aux;
16260 	return 0;
16261 
16262 err_free:
16263 	kfree(info_aux);
16264 	return ret;
16265 }
16266 
16267 static void adjust_btf_func(struct bpf_verifier_env *env)
16268 {
16269 	struct bpf_prog_aux *aux = env->prog->aux;
16270 	int i;
16271 
16272 	if (!aux->func_info)
16273 		return;
16274 
16275 	/* func_info is not available for hidden subprogs */
16276 	for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++)
16277 		aux->func_info[i].insn_off = env->subprog_info[i].start;
16278 }
16279 
16280 #define MIN_BPF_LINEINFO_SIZE	offsetofend(struct bpf_line_info, line_col)
16281 #define MAX_LINEINFO_REC_SIZE	MAX_FUNCINFO_REC_SIZE
16282 
16283 static int check_btf_line(struct bpf_verifier_env *env,
16284 			  const union bpf_attr *attr,
16285 			  bpfptr_t uattr)
16286 {
16287 	u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
16288 	struct bpf_subprog_info *sub;
16289 	struct bpf_line_info *linfo;
16290 	struct bpf_prog *prog;
16291 	const struct btf *btf;
16292 	bpfptr_t ulinfo;
16293 	int err;
16294 
16295 	nr_linfo = attr->line_info_cnt;
16296 	if (!nr_linfo)
16297 		return 0;
16298 	if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info))
16299 		return -EINVAL;
16300 
16301 	rec_size = attr->line_info_rec_size;
16302 	if (rec_size < MIN_BPF_LINEINFO_SIZE ||
16303 	    rec_size > MAX_LINEINFO_REC_SIZE ||
16304 	    rec_size & (sizeof(u32) - 1))
16305 		return -EINVAL;
16306 
16307 	/* Need to zero it in case the userspace may
16308 	 * pass in a smaller bpf_line_info object.
16309 	 */
16310 	linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
16311 			 GFP_KERNEL | __GFP_NOWARN);
16312 	if (!linfo)
16313 		return -ENOMEM;
16314 
16315 	prog = env->prog;
16316 	btf = prog->aux->btf;
16317 
16318 	s = 0;
16319 	sub = env->subprog_info;
16320 	ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel);
16321 	expected_size = sizeof(struct bpf_line_info);
16322 	ncopy = min_t(u32, expected_size, rec_size);
16323 	for (i = 0; i < nr_linfo; i++) {
16324 		err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
16325 		if (err) {
16326 			if (err == -E2BIG) {
16327 				verbose(env, "nonzero tailing record in line_info");
16328 				if (copy_to_bpfptr_offset(uattr,
16329 							  offsetof(union bpf_attr, line_info_rec_size),
16330 							  &expected_size, sizeof(expected_size)))
16331 					err = -EFAULT;
16332 			}
16333 			goto err_free;
16334 		}
16335 
16336 		if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) {
16337 			err = -EFAULT;
16338 			goto err_free;
16339 		}
16340 
16341 		/*
16342 		 * Check insn_off to ensure
16343 		 * 1) strictly increasing AND
16344 		 * 2) bounded by prog->len
16345 		 *
16346 		 * The linfo[0].insn_off == 0 check logically falls into
16347 		 * the later "missing bpf_line_info for func..." case
16348 		 * because the first linfo[0].insn_off must be the
16349 		 * first sub also and the first sub must have
16350 		 * subprog_info[0].start == 0.
16351 		 */
16352 		if ((i && linfo[i].insn_off <= prev_offset) ||
16353 		    linfo[i].insn_off >= prog->len) {
16354 			verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
16355 				i, linfo[i].insn_off, prev_offset,
16356 				prog->len);
16357 			err = -EINVAL;
16358 			goto err_free;
16359 		}
16360 
16361 		if (!prog->insnsi[linfo[i].insn_off].code) {
16362 			verbose(env,
16363 				"Invalid insn code at line_info[%u].insn_off\n",
16364 				i);
16365 			err = -EINVAL;
16366 			goto err_free;
16367 		}
16368 
16369 		if (!btf_name_by_offset(btf, linfo[i].line_off) ||
16370 		    !btf_name_by_offset(btf, linfo[i].file_name_off)) {
16371 			verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
16372 			err = -EINVAL;
16373 			goto err_free;
16374 		}
16375 
16376 		if (s != env->subprog_cnt) {
16377 			if (linfo[i].insn_off == sub[s].start) {
16378 				sub[s].linfo_idx = i;
16379 				s++;
16380 			} else if (sub[s].start < linfo[i].insn_off) {
16381 				verbose(env, "missing bpf_line_info for func#%u\n", s);
16382 				err = -EINVAL;
16383 				goto err_free;
16384 			}
16385 		}
16386 
16387 		prev_offset = linfo[i].insn_off;
16388 		bpfptr_add(&ulinfo, rec_size);
16389 	}
16390 
16391 	if (s != env->subprog_cnt) {
16392 		verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
16393 			env->subprog_cnt - s, s);
16394 		err = -EINVAL;
16395 		goto err_free;
16396 	}
16397 
16398 	prog->aux->linfo = linfo;
16399 	prog->aux->nr_linfo = nr_linfo;
16400 
16401 	return 0;
16402 
16403 err_free:
16404 	kvfree(linfo);
16405 	return err;
16406 }
16407 
16408 #define MIN_CORE_RELO_SIZE	sizeof(struct bpf_core_relo)
16409 #define MAX_CORE_RELO_SIZE	MAX_FUNCINFO_REC_SIZE
16410 
16411 static int check_core_relo(struct bpf_verifier_env *env,
16412 			   const union bpf_attr *attr,
16413 			   bpfptr_t uattr)
16414 {
16415 	u32 i, nr_core_relo, ncopy, expected_size, rec_size;
16416 	struct bpf_core_relo core_relo = {};
16417 	struct bpf_prog *prog = env->prog;
16418 	const struct btf *btf = prog->aux->btf;
16419 	struct bpf_core_ctx ctx = {
16420 		.log = &env->log,
16421 		.btf = btf,
16422 	};
16423 	bpfptr_t u_core_relo;
16424 	int err;
16425 
16426 	nr_core_relo = attr->core_relo_cnt;
16427 	if (!nr_core_relo)
16428 		return 0;
16429 	if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo))
16430 		return -EINVAL;
16431 
16432 	rec_size = attr->core_relo_rec_size;
16433 	if (rec_size < MIN_CORE_RELO_SIZE ||
16434 	    rec_size > MAX_CORE_RELO_SIZE ||
16435 	    rec_size % sizeof(u32))
16436 		return -EINVAL;
16437 
16438 	u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel);
16439 	expected_size = sizeof(struct bpf_core_relo);
16440 	ncopy = min_t(u32, expected_size, rec_size);
16441 
16442 	/* Unlike func_info and line_info, copy and apply each CO-RE
16443 	 * relocation record one at a time.
16444 	 */
16445 	for (i = 0; i < nr_core_relo; i++) {
16446 		/* future proofing when sizeof(bpf_core_relo) changes */
16447 		err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size);
16448 		if (err) {
16449 			if (err == -E2BIG) {
16450 				verbose(env, "nonzero tailing record in core_relo");
16451 				if (copy_to_bpfptr_offset(uattr,
16452 							  offsetof(union bpf_attr, core_relo_rec_size),
16453 							  &expected_size, sizeof(expected_size)))
16454 					err = -EFAULT;
16455 			}
16456 			break;
16457 		}
16458 
16459 		if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) {
16460 			err = -EFAULT;
16461 			break;
16462 		}
16463 
16464 		if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) {
16465 			verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",
16466 				i, core_relo.insn_off, prog->len);
16467 			err = -EINVAL;
16468 			break;
16469 		}
16470 
16471 		err = bpf_core_apply(&ctx, &core_relo, i,
16472 				     &prog->insnsi[core_relo.insn_off / 8]);
16473 		if (err)
16474 			break;
16475 		bpfptr_add(&u_core_relo, rec_size);
16476 	}
16477 	return err;
16478 }
16479 
16480 static int check_btf_info_early(struct bpf_verifier_env *env,
16481 				const union bpf_attr *attr,
16482 				bpfptr_t uattr)
16483 {
16484 	struct btf *btf;
16485 	int err;
16486 
16487 	if (!attr->func_info_cnt && !attr->line_info_cnt) {
16488 		if (check_abnormal_return(env))
16489 			return -EINVAL;
16490 		return 0;
16491 	}
16492 
16493 	btf = btf_get_by_fd(attr->prog_btf_fd);
16494 	if (IS_ERR(btf))
16495 		return PTR_ERR(btf);
16496 	if (btf_is_kernel(btf)) {
16497 		btf_put(btf);
16498 		return -EACCES;
16499 	}
16500 	env->prog->aux->btf = btf;
16501 
16502 	err = check_btf_func_early(env, attr, uattr);
16503 	if (err)
16504 		return err;
16505 	return 0;
16506 }
16507 
16508 static int check_btf_info(struct bpf_verifier_env *env,
16509 			  const union bpf_attr *attr,
16510 			  bpfptr_t uattr)
16511 {
16512 	int err;
16513 
16514 	if (!attr->func_info_cnt && !attr->line_info_cnt) {
16515 		if (check_abnormal_return(env))
16516 			return -EINVAL;
16517 		return 0;
16518 	}
16519 
16520 	err = check_btf_func(env, attr, uattr);
16521 	if (err)
16522 		return err;
16523 
16524 	err = check_btf_line(env, attr, uattr);
16525 	if (err)
16526 		return err;
16527 
16528 	err = check_core_relo(env, attr, uattr);
16529 	if (err)
16530 		return err;
16531 
16532 	return 0;
16533 }
16534 
16535 /* check %cur's range satisfies %old's */
16536 static bool range_within(const struct bpf_reg_state *old,
16537 			 const struct bpf_reg_state *cur)
16538 {
16539 	return old->umin_value <= cur->umin_value &&
16540 	       old->umax_value >= cur->umax_value &&
16541 	       old->smin_value <= cur->smin_value &&
16542 	       old->smax_value >= cur->smax_value &&
16543 	       old->u32_min_value <= cur->u32_min_value &&
16544 	       old->u32_max_value >= cur->u32_max_value &&
16545 	       old->s32_min_value <= cur->s32_min_value &&
16546 	       old->s32_max_value >= cur->s32_max_value;
16547 }
16548 
16549 /* If in the old state two registers had the same id, then they need to have
16550  * the same id in the new state as well.  But that id could be different from
16551  * the old state, so we need to track the mapping from old to new ids.
16552  * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent
16553  * regs with old id 5 must also have new id 9 for the new state to be safe.  But
16554  * regs with a different old id could still have new id 9, we don't care about
16555  * that.
16556  * So we look through our idmap to see if this old id has been seen before.  If
16557  * so, we require the new id to match; otherwise, we add the id pair to the map.
16558  */
16559 static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap)
16560 {
16561 	struct bpf_id_pair *map = idmap->map;
16562 	unsigned int i;
16563 
16564 	/* either both IDs should be set or both should be zero */
16565 	if (!!old_id != !!cur_id)
16566 		return false;
16567 
16568 	if (old_id == 0) /* cur_id == 0 as well */
16569 		return true;
16570 
16571 	for (i = 0; i < BPF_ID_MAP_SIZE; i++) {
16572 		if (!map[i].old) {
16573 			/* Reached an empty slot; haven't seen this id before */
16574 			map[i].old = old_id;
16575 			map[i].cur = cur_id;
16576 			return true;
16577 		}
16578 		if (map[i].old == old_id)
16579 			return map[i].cur == cur_id;
16580 		if (map[i].cur == cur_id)
16581 			return false;
16582 	}
16583 	/* We ran out of idmap slots, which should be impossible */
16584 	WARN_ON_ONCE(1);
16585 	return false;
16586 }
16587 
16588 /* Similar to check_ids(), but allocate a unique temporary ID
16589  * for 'old_id' or 'cur_id' of zero.
16590  * This makes pairs like '0 vs unique ID', 'unique ID vs 0' valid.
16591  */
16592 static bool check_scalar_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap)
16593 {
16594 	old_id = old_id ? old_id : ++idmap->tmp_id_gen;
16595 	cur_id = cur_id ? cur_id : ++idmap->tmp_id_gen;
16596 
16597 	return check_ids(old_id, cur_id, idmap);
16598 }
16599 
16600 static void clean_func_state(struct bpf_verifier_env *env,
16601 			     struct bpf_func_state *st)
16602 {
16603 	enum bpf_reg_liveness live;
16604 	int i, j;
16605 
16606 	for (i = 0; i < BPF_REG_FP; i++) {
16607 		live = st->regs[i].live;
16608 		/* liveness must not touch this register anymore */
16609 		st->regs[i].live |= REG_LIVE_DONE;
16610 		if (!(live & REG_LIVE_READ))
16611 			/* since the register is unused, clear its state
16612 			 * to make further comparison simpler
16613 			 */
16614 			__mark_reg_not_init(env, &st->regs[i]);
16615 	}
16616 
16617 	for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
16618 		live = st->stack[i].spilled_ptr.live;
16619 		/* liveness must not touch this stack slot anymore */
16620 		st->stack[i].spilled_ptr.live |= REG_LIVE_DONE;
16621 		if (!(live & REG_LIVE_READ)) {
16622 			__mark_reg_not_init(env, &st->stack[i].spilled_ptr);
16623 			for (j = 0; j < BPF_REG_SIZE; j++)
16624 				st->stack[i].slot_type[j] = STACK_INVALID;
16625 		}
16626 	}
16627 }
16628 
16629 static void clean_verifier_state(struct bpf_verifier_env *env,
16630 				 struct bpf_verifier_state *st)
16631 {
16632 	int i;
16633 
16634 	if (st->frame[0]->regs[0].live & REG_LIVE_DONE)
16635 		/* all regs in this state in all frames were already marked */
16636 		return;
16637 
16638 	for (i = 0; i <= st->curframe; i++)
16639 		clean_func_state(env, st->frame[i]);
16640 }
16641 
16642 /* the parentage chains form a tree.
16643  * the verifier states are added to state lists at given insn and
16644  * pushed into state stack for future exploration.
16645  * when the verifier reaches bpf_exit insn some of the verifer states
16646  * stored in the state lists have their final liveness state already,
16647  * but a lot of states will get revised from liveness point of view when
16648  * the verifier explores other branches.
16649  * Example:
16650  * 1: r0 = 1
16651  * 2: if r1 == 100 goto pc+1
16652  * 3: r0 = 2
16653  * 4: exit
16654  * when the verifier reaches exit insn the register r0 in the state list of
16655  * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch
16656  * of insn 2 and goes exploring further. At the insn 4 it will walk the
16657  * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ.
16658  *
16659  * Since the verifier pushes the branch states as it sees them while exploring
16660  * the program the condition of walking the branch instruction for the second
16661  * time means that all states below this branch were already explored and
16662  * their final liveness marks are already propagated.
16663  * Hence when the verifier completes the search of state list in is_state_visited()
16664  * we can call this clean_live_states() function to mark all liveness states
16665  * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state'
16666  * will not be used.
16667  * This function also clears the registers and stack for states that !READ
16668  * to simplify state merging.
16669  *
16670  * Important note here that walking the same branch instruction in the callee
16671  * doesn't meant that the states are DONE. The verifier has to compare
16672  * the callsites
16673  */
16674 static void clean_live_states(struct bpf_verifier_env *env, int insn,
16675 			      struct bpf_verifier_state *cur)
16676 {
16677 	struct bpf_verifier_state_list *sl;
16678 
16679 	sl = *explored_state(env, insn);
16680 	while (sl) {
16681 		if (sl->state.branches)
16682 			goto next;
16683 		if (sl->state.insn_idx != insn ||
16684 		    !same_callsites(&sl->state, cur))
16685 			goto next;
16686 		clean_verifier_state(env, &sl->state);
16687 next:
16688 		sl = sl->next;
16689 	}
16690 }
16691 
16692 static bool regs_exact(const struct bpf_reg_state *rold,
16693 		       const struct bpf_reg_state *rcur,
16694 		       struct bpf_idmap *idmap)
16695 {
16696 	return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
16697 	       check_ids(rold->id, rcur->id, idmap) &&
16698 	       check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap);
16699 }
16700 
16701 enum exact_level {
16702 	NOT_EXACT,
16703 	EXACT,
16704 	RANGE_WITHIN
16705 };
16706 
16707 /* Returns true if (rold safe implies rcur safe) */
16708 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
16709 		    struct bpf_reg_state *rcur, struct bpf_idmap *idmap,
16710 		    enum exact_level exact)
16711 {
16712 	if (exact == EXACT)
16713 		return regs_exact(rold, rcur, idmap);
16714 
16715 	if (!(rold->live & REG_LIVE_READ) && exact == NOT_EXACT)
16716 		/* explored state didn't use this */
16717 		return true;
16718 	if (rold->type == NOT_INIT) {
16719 		if (exact == NOT_EXACT || rcur->type == NOT_INIT)
16720 			/* explored state can't have used this */
16721 			return true;
16722 	}
16723 
16724 	/* Enforce that register types have to match exactly, including their
16725 	 * modifiers (like PTR_MAYBE_NULL, MEM_RDONLY, etc), as a general
16726 	 * rule.
16727 	 *
16728 	 * One can make a point that using a pointer register as unbounded
16729 	 * SCALAR would be technically acceptable, but this could lead to
16730 	 * pointer leaks because scalars are allowed to leak while pointers
16731 	 * are not. We could make this safe in special cases if root is
16732 	 * calling us, but it's probably not worth the hassle.
16733 	 *
16734 	 * Also, register types that are *not* MAYBE_NULL could technically be
16735 	 * safe to use as their MAYBE_NULL variants (e.g., PTR_TO_MAP_VALUE
16736 	 * is safe to be used as PTR_TO_MAP_VALUE_OR_NULL, provided both point
16737 	 * to the same map).
16738 	 * However, if the old MAYBE_NULL register then got NULL checked,
16739 	 * doing so could have affected others with the same id, and we can't
16740 	 * check for that because we lost the id when we converted to
16741 	 * a non-MAYBE_NULL variant.
16742 	 * So, as a general rule we don't allow mixing MAYBE_NULL and
16743 	 * non-MAYBE_NULL registers as well.
16744 	 */
16745 	if (rold->type != rcur->type)
16746 		return false;
16747 
16748 	switch (base_type(rold->type)) {
16749 	case SCALAR_VALUE:
16750 		if (env->explore_alu_limits) {
16751 			/* explore_alu_limits disables tnum_in() and range_within()
16752 			 * logic and requires everything to be strict
16753 			 */
16754 			return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
16755 			       check_scalar_ids(rold->id, rcur->id, idmap);
16756 		}
16757 		if (!rold->precise && exact == NOT_EXACT)
16758 			return true;
16759 		if ((rold->id & BPF_ADD_CONST) != (rcur->id & BPF_ADD_CONST))
16760 			return false;
16761 		if ((rold->id & BPF_ADD_CONST) && (rold->off != rcur->off))
16762 			return false;
16763 		/* Why check_ids() for scalar registers?
16764 		 *
16765 		 * Consider the following BPF code:
16766 		 *   1: r6 = ... unbound scalar, ID=a ...
16767 		 *   2: r7 = ... unbound scalar, ID=b ...
16768 		 *   3: if (r6 > r7) goto +1
16769 		 *   4: r6 = r7
16770 		 *   5: if (r6 > X) goto ...
16771 		 *   6: ... memory operation using r7 ...
16772 		 *
16773 		 * First verification path is [1-6]:
16774 		 * - at (4) same bpf_reg_state::id (b) would be assigned to r6 and r7;
16775 		 * - at (5) r6 would be marked <= X, find_equal_scalars() would also mark
16776 		 *   r7 <= X, because r6 and r7 share same id.
16777 		 * Next verification path is [1-4, 6].
16778 		 *
16779 		 * Instruction (6) would be reached in two states:
16780 		 *   I.  r6{.id=b}, r7{.id=b} via path 1-6;
16781 		 *   II. r6{.id=a}, r7{.id=b} via path 1-4, 6.
16782 		 *
16783 		 * Use check_ids() to distinguish these states.
16784 		 * ---
16785 		 * Also verify that new value satisfies old value range knowledge.
16786 		 */
16787 		return range_within(rold, rcur) &&
16788 		       tnum_in(rold->var_off, rcur->var_off) &&
16789 		       check_scalar_ids(rold->id, rcur->id, idmap);
16790 	case PTR_TO_MAP_KEY:
16791 	case PTR_TO_MAP_VALUE:
16792 	case PTR_TO_MEM:
16793 	case PTR_TO_BUF:
16794 	case PTR_TO_TP_BUFFER:
16795 		/* If the new min/max/var_off satisfy the old ones and
16796 		 * everything else matches, we are OK.
16797 		 */
16798 		return memcmp(rold, rcur, offsetof(struct bpf_reg_state, var_off)) == 0 &&
16799 		       range_within(rold, rcur) &&
16800 		       tnum_in(rold->var_off, rcur->var_off) &&
16801 		       check_ids(rold->id, rcur->id, idmap) &&
16802 		       check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap);
16803 	case PTR_TO_PACKET_META:
16804 	case PTR_TO_PACKET:
16805 		/* We must have at least as much range as the old ptr
16806 		 * did, so that any accesses which were safe before are
16807 		 * still safe.  This is true even if old range < old off,
16808 		 * since someone could have accessed through (ptr - k), or
16809 		 * even done ptr -= k in a register, to get a safe access.
16810 		 */
16811 		if (rold->range > rcur->range)
16812 			return false;
16813 		/* If the offsets don't match, we can't trust our alignment;
16814 		 * nor can we be sure that we won't fall out of range.
16815 		 */
16816 		if (rold->off != rcur->off)
16817 			return false;
16818 		/* id relations must be preserved */
16819 		if (!check_ids(rold->id, rcur->id, idmap))
16820 			return false;
16821 		/* new val must satisfy old val knowledge */
16822 		return range_within(rold, rcur) &&
16823 		       tnum_in(rold->var_off, rcur->var_off);
16824 	case PTR_TO_STACK:
16825 		/* two stack pointers are equal only if they're pointing to
16826 		 * the same stack frame, since fp-8 in foo != fp-8 in bar
16827 		 */
16828 		return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno;
16829 	case PTR_TO_ARENA:
16830 		return true;
16831 	default:
16832 		return regs_exact(rold, rcur, idmap);
16833 	}
16834 }
16835 
16836 static struct bpf_reg_state unbound_reg;
16837 
16838 static __init int unbound_reg_init(void)
16839 {
16840 	__mark_reg_unknown_imprecise(&unbound_reg);
16841 	unbound_reg.live |= REG_LIVE_READ;
16842 	return 0;
16843 }
16844 late_initcall(unbound_reg_init);
16845 
16846 static bool is_stack_all_misc(struct bpf_verifier_env *env,
16847 			      struct bpf_stack_state *stack)
16848 {
16849 	u32 i;
16850 
16851 	for (i = 0; i < ARRAY_SIZE(stack->slot_type); ++i) {
16852 		if ((stack->slot_type[i] == STACK_MISC) ||
16853 		    (stack->slot_type[i] == STACK_INVALID && env->allow_uninit_stack))
16854 			continue;
16855 		return false;
16856 	}
16857 
16858 	return true;
16859 }
16860 
16861 static struct bpf_reg_state *scalar_reg_for_stack(struct bpf_verifier_env *env,
16862 						  struct bpf_stack_state *stack)
16863 {
16864 	if (is_spilled_scalar_reg64(stack))
16865 		return &stack->spilled_ptr;
16866 
16867 	if (is_stack_all_misc(env, stack))
16868 		return &unbound_reg;
16869 
16870 	return NULL;
16871 }
16872 
16873 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
16874 		      struct bpf_func_state *cur, struct bpf_idmap *idmap,
16875 		      enum exact_level exact)
16876 {
16877 	int i, spi;
16878 
16879 	/* walk slots of the explored stack and ignore any additional
16880 	 * slots in the current stack, since explored(safe) state
16881 	 * didn't use them
16882 	 */
16883 	for (i = 0; i < old->allocated_stack; i++) {
16884 		struct bpf_reg_state *old_reg, *cur_reg;
16885 
16886 		spi = i / BPF_REG_SIZE;
16887 
16888 		if (exact != NOT_EXACT &&
16889 		    (i >= cur->allocated_stack ||
16890 		     old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
16891 		     cur->stack[spi].slot_type[i % BPF_REG_SIZE]))
16892 			return false;
16893 
16894 		if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)
16895 		    && exact == NOT_EXACT) {
16896 			i += BPF_REG_SIZE - 1;
16897 			/* explored state didn't use this */
16898 			continue;
16899 		}
16900 
16901 		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID)
16902 			continue;
16903 
16904 		if (env->allow_uninit_stack &&
16905 		    old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC)
16906 			continue;
16907 
16908 		/* explored stack has more populated slots than current stack
16909 		 * and these slots were used
16910 		 */
16911 		if (i >= cur->allocated_stack)
16912 			return false;
16913 
16914 		/* 64-bit scalar spill vs all slots MISC and vice versa.
16915 		 * Load from all slots MISC produces unbound scalar.
16916 		 * Construct a fake register for such stack and call
16917 		 * regsafe() to ensure scalar ids are compared.
16918 		 */
16919 		old_reg = scalar_reg_for_stack(env, &old->stack[spi]);
16920 		cur_reg = scalar_reg_for_stack(env, &cur->stack[spi]);
16921 		if (old_reg && cur_reg) {
16922 			if (!regsafe(env, old_reg, cur_reg, idmap, exact))
16923 				return false;
16924 			i += BPF_REG_SIZE - 1;
16925 			continue;
16926 		}
16927 
16928 		/* if old state was safe with misc data in the stack
16929 		 * it will be safe with zero-initialized stack.
16930 		 * The opposite is not true
16931 		 */
16932 		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
16933 		    cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
16934 			continue;
16935 		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
16936 		    cur->stack[spi].slot_type[i % BPF_REG_SIZE])
16937 			/* Ex: old explored (safe) state has STACK_SPILL in
16938 			 * this stack slot, but current has STACK_MISC ->
16939 			 * this verifier states are not equivalent,
16940 			 * return false to continue verification of this path
16941 			 */
16942 			return false;
16943 		if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1)
16944 			continue;
16945 		/* Both old and cur are having same slot_type */
16946 		switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) {
16947 		case STACK_SPILL:
16948 			/* when explored and current stack slot are both storing
16949 			 * spilled registers, check that stored pointers types
16950 			 * are the same as well.
16951 			 * Ex: explored safe path could have stored
16952 			 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8}
16953 			 * but current path has stored:
16954 			 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16}
16955 			 * such verifier states are not equivalent.
16956 			 * return false to continue verification of this path
16957 			 */
16958 			if (!regsafe(env, &old->stack[spi].spilled_ptr,
16959 				     &cur->stack[spi].spilled_ptr, idmap, exact))
16960 				return false;
16961 			break;
16962 		case STACK_DYNPTR:
16963 			old_reg = &old->stack[spi].spilled_ptr;
16964 			cur_reg = &cur->stack[spi].spilled_ptr;
16965 			if (old_reg->dynptr.type != cur_reg->dynptr.type ||
16966 			    old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot ||
16967 			    !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap))
16968 				return false;
16969 			break;
16970 		case STACK_ITER:
16971 			old_reg = &old->stack[spi].spilled_ptr;
16972 			cur_reg = &cur->stack[spi].spilled_ptr;
16973 			/* iter.depth is not compared between states as it
16974 			 * doesn't matter for correctness and would otherwise
16975 			 * prevent convergence; we maintain it only to prevent
16976 			 * infinite loop check triggering, see
16977 			 * iter_active_depths_differ()
16978 			 */
16979 			if (old_reg->iter.btf != cur_reg->iter.btf ||
16980 			    old_reg->iter.btf_id != cur_reg->iter.btf_id ||
16981 			    old_reg->iter.state != cur_reg->iter.state ||
16982 			    /* ignore {old_reg,cur_reg}->iter.depth, see above */
16983 			    !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap))
16984 				return false;
16985 			break;
16986 		case STACK_MISC:
16987 		case STACK_ZERO:
16988 		case STACK_INVALID:
16989 			continue;
16990 		/* Ensure that new unhandled slot types return false by default */
16991 		default:
16992 			return false;
16993 		}
16994 	}
16995 	return true;
16996 }
16997 
16998 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur,
16999 		    struct bpf_idmap *idmap)
17000 {
17001 	int i;
17002 
17003 	if (old->acquired_refs != cur->acquired_refs)
17004 		return false;
17005 
17006 	for (i = 0; i < old->acquired_refs; i++) {
17007 		if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap))
17008 			return false;
17009 	}
17010 
17011 	return true;
17012 }
17013 
17014 /* compare two verifier states
17015  *
17016  * all states stored in state_list are known to be valid, since
17017  * verifier reached 'bpf_exit' instruction through them
17018  *
17019  * this function is called when verifier exploring different branches of
17020  * execution popped from the state stack. If it sees an old state that has
17021  * more strict register state and more strict stack state then this execution
17022  * branch doesn't need to be explored further, since verifier already
17023  * concluded that more strict state leads to valid finish.
17024  *
17025  * Therefore two states are equivalent if register state is more conservative
17026  * and explored stack state is more conservative than the current one.
17027  * Example:
17028  *       explored                   current
17029  * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC)
17030  * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC)
17031  *
17032  * In other words if current stack state (one being explored) has more
17033  * valid slots than old one that already passed validation, it means
17034  * the verifier can stop exploring and conclude that current state is valid too
17035  *
17036  * Similarly with registers. If explored state has register type as invalid
17037  * whereas register type in current state is meaningful, it means that
17038  * the current state will reach 'bpf_exit' instruction safely
17039  */
17040 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,
17041 			      struct bpf_func_state *cur, enum exact_level exact)
17042 {
17043 	int i;
17044 
17045 	if (old->callback_depth > cur->callback_depth)
17046 		return false;
17047 
17048 	for (i = 0; i < MAX_BPF_REG; i++)
17049 		if (!regsafe(env, &old->regs[i], &cur->regs[i],
17050 			     &env->idmap_scratch, exact))
17051 			return false;
17052 
17053 	if (!stacksafe(env, old, cur, &env->idmap_scratch, exact))
17054 		return false;
17055 
17056 	if (!refsafe(old, cur, &env->idmap_scratch))
17057 		return false;
17058 
17059 	return true;
17060 }
17061 
17062 static void reset_idmap_scratch(struct bpf_verifier_env *env)
17063 {
17064 	env->idmap_scratch.tmp_id_gen = env->id_gen;
17065 	memset(&env->idmap_scratch.map, 0, sizeof(env->idmap_scratch.map));
17066 }
17067 
17068 static bool states_equal(struct bpf_verifier_env *env,
17069 			 struct bpf_verifier_state *old,
17070 			 struct bpf_verifier_state *cur,
17071 			 enum exact_level exact)
17072 {
17073 	int i;
17074 
17075 	if (old->curframe != cur->curframe)
17076 		return false;
17077 
17078 	reset_idmap_scratch(env);
17079 
17080 	/* Verification state from speculative execution simulation
17081 	 * must never prune a non-speculative execution one.
17082 	 */
17083 	if (old->speculative && !cur->speculative)
17084 		return false;
17085 
17086 	if (old->active_lock.ptr != cur->active_lock.ptr)
17087 		return false;
17088 
17089 	/* Old and cur active_lock's have to be either both present
17090 	 * or both absent.
17091 	 */
17092 	if (!!old->active_lock.id != !!cur->active_lock.id)
17093 		return false;
17094 
17095 	if (old->active_lock.id &&
17096 	    !check_ids(old->active_lock.id, cur->active_lock.id, &env->idmap_scratch))
17097 		return false;
17098 
17099 	if (old->active_rcu_lock != cur->active_rcu_lock)
17100 		return false;
17101 
17102 	if (old->active_preempt_lock != cur->active_preempt_lock)
17103 		return false;
17104 
17105 	if (old->in_sleepable != cur->in_sleepable)
17106 		return false;
17107 
17108 	/* for states to be equal callsites have to be the same
17109 	 * and all frame states need to be equivalent
17110 	 */
17111 	for (i = 0; i <= old->curframe; i++) {
17112 		if (old->frame[i]->callsite != cur->frame[i]->callsite)
17113 			return false;
17114 		if (!func_states_equal(env, old->frame[i], cur->frame[i], exact))
17115 			return false;
17116 	}
17117 	return true;
17118 }
17119 
17120 /* Return 0 if no propagation happened. Return negative error code if error
17121  * happened. Otherwise, return the propagated bit.
17122  */
17123 static int propagate_liveness_reg(struct bpf_verifier_env *env,
17124 				  struct bpf_reg_state *reg,
17125 				  struct bpf_reg_state *parent_reg)
17126 {
17127 	u8 parent_flag = parent_reg->live & REG_LIVE_READ;
17128 	u8 flag = reg->live & REG_LIVE_READ;
17129 	int err;
17130 
17131 	/* When comes here, read flags of PARENT_REG or REG could be any of
17132 	 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need
17133 	 * of propagation if PARENT_REG has strongest REG_LIVE_READ64.
17134 	 */
17135 	if (parent_flag == REG_LIVE_READ64 ||
17136 	    /* Or if there is no read flag from REG. */
17137 	    !flag ||
17138 	    /* Or if the read flag from REG is the same as PARENT_REG. */
17139 	    parent_flag == flag)
17140 		return 0;
17141 
17142 	err = mark_reg_read(env, reg, parent_reg, flag);
17143 	if (err)
17144 		return err;
17145 
17146 	return flag;
17147 }
17148 
17149 /* A write screens off any subsequent reads; but write marks come from the
17150  * straight-line code between a state and its parent.  When we arrive at an
17151  * equivalent state (jump target or such) we didn't arrive by the straight-line
17152  * code, so read marks in the state must propagate to the parent regardless
17153  * of the state's write marks. That's what 'parent == state->parent' comparison
17154  * in mark_reg_read() is for.
17155  */
17156 static int propagate_liveness(struct bpf_verifier_env *env,
17157 			      const struct bpf_verifier_state *vstate,
17158 			      struct bpf_verifier_state *vparent)
17159 {
17160 	struct bpf_reg_state *state_reg, *parent_reg;
17161 	struct bpf_func_state *state, *parent;
17162 	int i, frame, err = 0;
17163 
17164 	if (vparent->curframe != vstate->curframe) {
17165 		WARN(1, "propagate_live: parent frame %d current frame %d\n",
17166 		     vparent->curframe, vstate->curframe);
17167 		return -EFAULT;
17168 	}
17169 	/* Propagate read liveness of registers... */
17170 	BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
17171 	for (frame = 0; frame <= vstate->curframe; frame++) {
17172 		parent = vparent->frame[frame];
17173 		state = vstate->frame[frame];
17174 		parent_reg = parent->regs;
17175 		state_reg = state->regs;
17176 		/* We don't need to worry about FP liveness, it's read-only */
17177 		for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) {
17178 			err = propagate_liveness_reg(env, &state_reg[i],
17179 						     &parent_reg[i]);
17180 			if (err < 0)
17181 				return err;
17182 			if (err == REG_LIVE_READ64)
17183 				mark_insn_zext(env, &parent_reg[i]);
17184 		}
17185 
17186 		/* Propagate stack slots. */
17187 		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE &&
17188 			    i < parent->allocated_stack / BPF_REG_SIZE; i++) {
17189 			parent_reg = &parent->stack[i].spilled_ptr;
17190 			state_reg = &state->stack[i].spilled_ptr;
17191 			err = propagate_liveness_reg(env, state_reg,
17192 						     parent_reg);
17193 			if (err < 0)
17194 				return err;
17195 		}
17196 	}
17197 	return 0;
17198 }
17199 
17200 /* find precise scalars in the previous equivalent state and
17201  * propagate them into the current state
17202  */
17203 static int propagate_precision(struct bpf_verifier_env *env,
17204 			       const struct bpf_verifier_state *old)
17205 {
17206 	struct bpf_reg_state *state_reg;
17207 	struct bpf_func_state *state;
17208 	int i, err = 0, fr;
17209 	bool first;
17210 
17211 	for (fr = old->curframe; fr >= 0; fr--) {
17212 		state = old->frame[fr];
17213 		state_reg = state->regs;
17214 		first = true;
17215 		for (i = 0; i < BPF_REG_FP; i++, state_reg++) {
17216 			if (state_reg->type != SCALAR_VALUE ||
17217 			    !state_reg->precise ||
17218 			    !(state_reg->live & REG_LIVE_READ))
17219 				continue;
17220 			if (env->log.level & BPF_LOG_LEVEL2) {
17221 				if (first)
17222 					verbose(env, "frame %d: propagating r%d", fr, i);
17223 				else
17224 					verbose(env, ",r%d", i);
17225 			}
17226 			bt_set_frame_reg(&env->bt, fr, i);
17227 			first = false;
17228 		}
17229 
17230 		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
17231 			if (!is_spilled_reg(&state->stack[i]))
17232 				continue;
17233 			state_reg = &state->stack[i].spilled_ptr;
17234 			if (state_reg->type != SCALAR_VALUE ||
17235 			    !state_reg->precise ||
17236 			    !(state_reg->live & REG_LIVE_READ))
17237 				continue;
17238 			if (env->log.level & BPF_LOG_LEVEL2) {
17239 				if (first)
17240 					verbose(env, "frame %d: propagating fp%d",
17241 						fr, (-i - 1) * BPF_REG_SIZE);
17242 				else
17243 					verbose(env, ",fp%d", (-i - 1) * BPF_REG_SIZE);
17244 			}
17245 			bt_set_frame_slot(&env->bt, fr, i);
17246 			first = false;
17247 		}
17248 		if (!first)
17249 			verbose(env, "\n");
17250 	}
17251 
17252 	err = mark_chain_precision_batch(env);
17253 	if (err < 0)
17254 		return err;
17255 
17256 	return 0;
17257 }
17258 
17259 static bool states_maybe_looping(struct bpf_verifier_state *old,
17260 				 struct bpf_verifier_state *cur)
17261 {
17262 	struct bpf_func_state *fold, *fcur;
17263 	int i, fr = cur->curframe;
17264 
17265 	if (old->curframe != fr)
17266 		return false;
17267 
17268 	fold = old->frame[fr];
17269 	fcur = cur->frame[fr];
17270 	for (i = 0; i < MAX_BPF_REG; i++)
17271 		if (memcmp(&fold->regs[i], &fcur->regs[i],
17272 			   offsetof(struct bpf_reg_state, parent)))
17273 			return false;
17274 	return true;
17275 }
17276 
17277 static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx)
17278 {
17279 	return env->insn_aux_data[insn_idx].is_iter_next;
17280 }
17281 
17282 /* is_state_visited() handles iter_next() (see process_iter_next_call() for
17283  * terminology) calls specially: as opposed to bounded BPF loops, it *expects*
17284  * states to match, which otherwise would look like an infinite loop. So while
17285  * iter_next() calls are taken care of, we still need to be careful and
17286  * prevent erroneous and too eager declaration of "ininite loop", when
17287  * iterators are involved.
17288  *
17289  * Here's a situation in pseudo-BPF assembly form:
17290  *
17291  *   0: again:                          ; set up iter_next() call args
17292  *   1:   r1 = &it                      ; <CHECKPOINT HERE>
17293  *   2:   call bpf_iter_num_next        ; this is iter_next() call
17294  *   3:   if r0 == 0 goto done
17295  *   4:   ... something useful here ...
17296  *   5:   goto again                    ; another iteration
17297  *   6: done:
17298  *   7:   r1 = &it
17299  *   8:   call bpf_iter_num_destroy     ; clean up iter state
17300  *   9:   exit
17301  *
17302  * This is a typical loop. Let's assume that we have a prune point at 1:,
17303  * before we get to `call bpf_iter_num_next` (e.g., because of that `goto
17304  * again`, assuming other heuristics don't get in a way).
17305  *
17306  * When we first time come to 1:, let's say we have some state X. We proceed
17307  * to 2:, fork states, enqueue ACTIVE, validate NULL case successfully, exit.
17308  * Now we come back to validate that forked ACTIVE state. We proceed through
17309  * 3-5, come to goto, jump to 1:. Let's assume our state didn't change, so we
17310  * are converging. But the problem is that we don't know that yet, as this
17311  * convergence has to happen at iter_next() call site only. So if nothing is
17312  * done, at 1: verifier will use bounded loop logic and declare infinite
17313  * looping (and would be *technically* correct, if not for iterator's
17314  * "eventual sticky NULL" contract, see process_iter_next_call()). But we
17315  * don't want that. So what we do in process_iter_next_call() when we go on
17316  * another ACTIVE iteration, we bump slot->iter.depth, to mark that it's
17317  * a different iteration. So when we suspect an infinite loop, we additionally
17318  * check if any of the *ACTIVE* iterator states depths differ. If yes, we
17319  * pretend we are not looping and wait for next iter_next() call.
17320  *
17321  * This only applies to ACTIVE state. In DRAINED state we don't expect to
17322  * loop, because that would actually mean infinite loop, as DRAINED state is
17323  * "sticky", and so we'll keep returning into the same instruction with the
17324  * same state (at least in one of possible code paths).
17325  *
17326  * This approach allows to keep infinite loop heuristic even in the face of
17327  * active iterator. E.g., C snippet below is and will be detected as
17328  * inifintely looping:
17329  *
17330  *   struct bpf_iter_num it;
17331  *   int *p, x;
17332  *
17333  *   bpf_iter_num_new(&it, 0, 10);
17334  *   while ((p = bpf_iter_num_next(&t))) {
17335  *       x = p;
17336  *       while (x--) {} // <<-- infinite loop here
17337  *   }
17338  *
17339  */
17340 static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf_verifier_state *cur)
17341 {
17342 	struct bpf_reg_state *slot, *cur_slot;
17343 	struct bpf_func_state *state;
17344 	int i, fr;
17345 
17346 	for (fr = old->curframe; fr >= 0; fr--) {
17347 		state = old->frame[fr];
17348 		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
17349 			if (state->stack[i].slot_type[0] != STACK_ITER)
17350 				continue;
17351 
17352 			slot = &state->stack[i].spilled_ptr;
17353 			if (slot->iter.state != BPF_ITER_STATE_ACTIVE)
17354 				continue;
17355 
17356 			cur_slot = &cur->frame[fr]->stack[i].spilled_ptr;
17357 			if (cur_slot->iter.depth != slot->iter.depth)
17358 				return true;
17359 		}
17360 	}
17361 	return false;
17362 }
17363 
17364 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
17365 {
17366 	struct bpf_verifier_state_list *new_sl;
17367 	struct bpf_verifier_state_list *sl, **pprev;
17368 	struct bpf_verifier_state *cur = env->cur_state, *new, *loop_entry;
17369 	int i, j, n, err, states_cnt = 0;
17370 	bool force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx);
17371 	bool add_new_state = force_new_state;
17372 	bool force_exact;
17373 
17374 	/* bpf progs typically have pruning point every 4 instructions
17375 	 * http://vger.kernel.org/bpfconf2019.html#session-1
17376 	 * Do not add new state for future pruning if the verifier hasn't seen
17377 	 * at least 2 jumps and at least 8 instructions.
17378 	 * This heuristics helps decrease 'total_states' and 'peak_states' metric.
17379 	 * In tests that amounts to up to 50% reduction into total verifier
17380 	 * memory consumption and 20% verifier time speedup.
17381 	 */
17382 	if (env->jmps_processed - env->prev_jmps_processed >= 2 &&
17383 	    env->insn_processed - env->prev_insn_processed >= 8)
17384 		add_new_state = true;
17385 
17386 	pprev = explored_state(env, insn_idx);
17387 	sl = *pprev;
17388 
17389 	clean_live_states(env, insn_idx, cur);
17390 
17391 	while (sl) {
17392 		states_cnt++;
17393 		if (sl->state.insn_idx != insn_idx)
17394 			goto next;
17395 
17396 		if (sl->state.branches) {
17397 			struct bpf_func_state *frame = sl->state.frame[sl->state.curframe];
17398 
17399 			if (frame->in_async_callback_fn &&
17400 			    frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) {
17401 				/* Different async_entry_cnt means that the verifier is
17402 				 * processing another entry into async callback.
17403 				 * Seeing the same state is not an indication of infinite
17404 				 * loop or infinite recursion.
17405 				 * But finding the same state doesn't mean that it's safe
17406 				 * to stop processing the current state. The previous state
17407 				 * hasn't yet reached bpf_exit, since state.branches > 0.
17408 				 * Checking in_async_callback_fn alone is not enough either.
17409 				 * Since the verifier still needs to catch infinite loops
17410 				 * inside async callbacks.
17411 				 */
17412 				goto skip_inf_loop_check;
17413 			}
17414 			/* BPF open-coded iterators loop detection is special.
17415 			 * states_maybe_looping() logic is too simplistic in detecting
17416 			 * states that *might* be equivalent, because it doesn't know
17417 			 * about ID remapping, so don't even perform it.
17418 			 * See process_iter_next_call() and iter_active_depths_differ()
17419 			 * for overview of the logic. When current and one of parent
17420 			 * states are detected as equivalent, it's a good thing: we prove
17421 			 * convergence and can stop simulating further iterations.
17422 			 * It's safe to assume that iterator loop will finish, taking into
17423 			 * account iter_next() contract of eventually returning
17424 			 * sticky NULL result.
17425 			 *
17426 			 * Note, that states have to be compared exactly in this case because
17427 			 * read and precision marks might not be finalized inside the loop.
17428 			 * E.g. as in the program below:
17429 			 *
17430 			 *     1. r7 = -16
17431 			 *     2. r6 = bpf_get_prandom_u32()
17432 			 *     3. while (bpf_iter_num_next(&fp[-8])) {
17433 			 *     4.   if (r6 != 42) {
17434 			 *     5.     r7 = -32
17435 			 *     6.     r6 = bpf_get_prandom_u32()
17436 			 *     7.     continue
17437 			 *     8.   }
17438 			 *     9.   r0 = r10
17439 			 *    10.   r0 += r7
17440 			 *    11.   r8 = *(u64 *)(r0 + 0)
17441 			 *    12.   r6 = bpf_get_prandom_u32()
17442 			 *    13. }
17443 			 *
17444 			 * Here verifier would first visit path 1-3, create a checkpoint at 3
17445 			 * with r7=-16, continue to 4-7,3. Existing checkpoint at 3 does
17446 			 * not have read or precision mark for r7 yet, thus inexact states
17447 			 * comparison would discard current state with r7=-32
17448 			 * => unsafe memory access at 11 would not be caught.
17449 			 */
17450 			if (is_iter_next_insn(env, insn_idx)) {
17451 				if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {
17452 					struct bpf_func_state *cur_frame;
17453 					struct bpf_reg_state *iter_state, *iter_reg;
17454 					int spi;
17455 
17456 					cur_frame = cur->frame[cur->curframe];
17457 					/* btf_check_iter_kfuncs() enforces that
17458 					 * iter state pointer is always the first arg
17459 					 */
17460 					iter_reg = &cur_frame->regs[BPF_REG_1];
17461 					/* current state is valid due to states_equal(),
17462 					 * so we can assume valid iter and reg state,
17463 					 * no need for extra (re-)validations
17464 					 */
17465 					spi = __get_spi(iter_reg->off + iter_reg->var_off.value);
17466 					iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr;
17467 					if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) {
17468 						update_loop_entry(cur, &sl->state);
17469 						goto hit;
17470 					}
17471 				}
17472 				goto skip_inf_loop_check;
17473 			}
17474 			if (is_may_goto_insn_at(env, insn_idx)) {
17475 				if (sl->state.may_goto_depth != cur->may_goto_depth &&
17476 				    states_equal(env, &sl->state, cur, RANGE_WITHIN)) {
17477 					update_loop_entry(cur, &sl->state);
17478 					goto hit;
17479 				}
17480 			}
17481 			if (calls_callback(env, insn_idx)) {
17482 				if (states_equal(env, &sl->state, cur, RANGE_WITHIN))
17483 					goto hit;
17484 				goto skip_inf_loop_check;
17485 			}
17486 			/* attempt to detect infinite loop to avoid unnecessary doomed work */
17487 			if (states_maybe_looping(&sl->state, cur) &&
17488 			    states_equal(env, &sl->state, cur, EXACT) &&
17489 			    !iter_active_depths_differ(&sl->state, cur) &&
17490 			    sl->state.may_goto_depth == cur->may_goto_depth &&
17491 			    sl->state.callback_unroll_depth == cur->callback_unroll_depth) {
17492 				verbose_linfo(env, insn_idx, "; ");
17493 				verbose(env, "infinite loop detected at insn %d\n", insn_idx);
17494 				verbose(env, "cur state:");
17495 				print_verifier_state(env, cur->frame[cur->curframe], true);
17496 				verbose(env, "old state:");
17497 				print_verifier_state(env, sl->state.frame[cur->curframe], true);
17498 				return -EINVAL;
17499 			}
17500 			/* if the verifier is processing a loop, avoid adding new state
17501 			 * too often, since different loop iterations have distinct
17502 			 * states and may not help future pruning.
17503 			 * This threshold shouldn't be too low to make sure that
17504 			 * a loop with large bound will be rejected quickly.
17505 			 * The most abusive loop will be:
17506 			 * r1 += 1
17507 			 * if r1 < 1000000 goto pc-2
17508 			 * 1M insn_procssed limit / 100 == 10k peak states.
17509 			 * This threshold shouldn't be too high either, since states
17510 			 * at the end of the loop are likely to be useful in pruning.
17511 			 */
17512 skip_inf_loop_check:
17513 			if (!force_new_state &&
17514 			    env->jmps_processed - env->prev_jmps_processed < 20 &&
17515 			    env->insn_processed - env->prev_insn_processed < 100)
17516 				add_new_state = false;
17517 			goto miss;
17518 		}
17519 		/* If sl->state is a part of a loop and this loop's entry is a part of
17520 		 * current verification path then states have to be compared exactly.
17521 		 * 'force_exact' is needed to catch the following case:
17522 		 *
17523 		 *                initial     Here state 'succ' was processed first,
17524 		 *                  |         it was eventually tracked to produce a
17525 		 *                  V         state identical to 'hdr'.
17526 		 *     .---------> hdr        All branches from 'succ' had been explored
17527 		 *     |            |         and thus 'succ' has its .branches == 0.
17528 		 *     |            V
17529 		 *     |    .------...        Suppose states 'cur' and 'succ' correspond
17530 		 *     |    |       |         to the same instruction + callsites.
17531 		 *     |    V       V         In such case it is necessary to check
17532 		 *     |   ...     ...        if 'succ' and 'cur' are states_equal().
17533 		 *     |    |       |         If 'succ' and 'cur' are a part of the
17534 		 *     |    V       V         same loop exact flag has to be set.
17535 		 *     |   succ <- cur        To check if that is the case, verify
17536 		 *     |    |                 if loop entry of 'succ' is in current
17537 		 *     |    V                 DFS path.
17538 		 *     |   ...
17539 		 *     |    |
17540 		 *     '----'
17541 		 *
17542 		 * Additional details are in the comment before get_loop_entry().
17543 		 */
17544 		loop_entry = get_loop_entry(&sl->state);
17545 		force_exact = loop_entry && loop_entry->branches > 0;
17546 		if (states_equal(env, &sl->state, cur, force_exact ? RANGE_WITHIN : NOT_EXACT)) {
17547 			if (force_exact)
17548 				update_loop_entry(cur, loop_entry);
17549 hit:
17550 			sl->hit_cnt++;
17551 			/* reached equivalent register/stack state,
17552 			 * prune the search.
17553 			 * Registers read by the continuation are read by us.
17554 			 * If we have any write marks in env->cur_state, they
17555 			 * will prevent corresponding reads in the continuation
17556 			 * from reaching our parent (an explored_state).  Our
17557 			 * own state will get the read marks recorded, but
17558 			 * they'll be immediately forgotten as we're pruning
17559 			 * this state and will pop a new one.
17560 			 */
17561 			err = propagate_liveness(env, &sl->state, cur);
17562 
17563 			/* if previous state reached the exit with precision and
17564 			 * current state is equivalent to it (except precision marks)
17565 			 * the precision needs to be propagated back in
17566 			 * the current state.
17567 			 */
17568 			if (is_jmp_point(env, env->insn_idx))
17569 				err = err ? : push_jmp_history(env, cur, 0);
17570 			err = err ? : propagate_precision(env, &sl->state);
17571 			if (err)
17572 				return err;
17573 			return 1;
17574 		}
17575 miss:
17576 		/* when new state is not going to be added do not increase miss count.
17577 		 * Otherwise several loop iterations will remove the state
17578 		 * recorded earlier. The goal of these heuristics is to have
17579 		 * states from some iterations of the loop (some in the beginning
17580 		 * and some at the end) to help pruning.
17581 		 */
17582 		if (add_new_state)
17583 			sl->miss_cnt++;
17584 		/* heuristic to determine whether this state is beneficial
17585 		 * to keep checking from state equivalence point of view.
17586 		 * Higher numbers increase max_states_per_insn and verification time,
17587 		 * but do not meaningfully decrease insn_processed.
17588 		 * 'n' controls how many times state could miss before eviction.
17589 		 * Use bigger 'n' for checkpoints because evicting checkpoint states
17590 		 * too early would hinder iterator convergence.
17591 		 */
17592 		n = is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3;
17593 		if (sl->miss_cnt > sl->hit_cnt * n + n) {
17594 			/* the state is unlikely to be useful. Remove it to
17595 			 * speed up verification
17596 			 */
17597 			*pprev = sl->next;
17598 			if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE &&
17599 			    !sl->state.used_as_loop_entry) {
17600 				u32 br = sl->state.branches;
17601 
17602 				WARN_ONCE(br,
17603 					  "BUG live_done but branches_to_explore %d\n",
17604 					  br);
17605 				free_verifier_state(&sl->state, false);
17606 				kfree(sl);
17607 				env->peak_states--;
17608 			} else {
17609 				/* cannot free this state, since parentage chain may
17610 				 * walk it later. Add it for free_list instead to
17611 				 * be freed at the end of verification
17612 				 */
17613 				sl->next = env->free_list;
17614 				env->free_list = sl;
17615 			}
17616 			sl = *pprev;
17617 			continue;
17618 		}
17619 next:
17620 		pprev = &sl->next;
17621 		sl = *pprev;
17622 	}
17623 
17624 	if (env->max_states_per_insn < states_cnt)
17625 		env->max_states_per_insn = states_cnt;
17626 
17627 	if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
17628 		return 0;
17629 
17630 	if (!add_new_state)
17631 		return 0;
17632 
17633 	/* There were no equivalent states, remember the current one.
17634 	 * Technically the current state is not proven to be safe yet,
17635 	 * but it will either reach outer most bpf_exit (which means it's safe)
17636 	 * or it will be rejected. When there are no loops the verifier won't be
17637 	 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx)
17638 	 * again on the way to bpf_exit.
17639 	 * When looping the sl->state.branches will be > 0 and this state
17640 	 * will not be considered for equivalence until branches == 0.
17641 	 */
17642 	new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
17643 	if (!new_sl)
17644 		return -ENOMEM;
17645 	env->total_states++;
17646 	env->peak_states++;
17647 	env->prev_jmps_processed = env->jmps_processed;
17648 	env->prev_insn_processed = env->insn_processed;
17649 
17650 	/* forget precise markings we inherited, see __mark_chain_precision */
17651 	if (env->bpf_capable)
17652 		mark_all_scalars_imprecise(env, cur);
17653 
17654 	/* add new state to the head of linked list */
17655 	new = &new_sl->state;
17656 	err = copy_verifier_state(new, cur);
17657 	if (err) {
17658 		free_verifier_state(new, false);
17659 		kfree(new_sl);
17660 		return err;
17661 	}
17662 	new->insn_idx = insn_idx;
17663 	WARN_ONCE(new->branches != 1,
17664 		  "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx);
17665 
17666 	cur->parent = new;
17667 	cur->first_insn_idx = insn_idx;
17668 	cur->dfs_depth = new->dfs_depth + 1;
17669 	clear_jmp_history(cur);
17670 	new_sl->next = *explored_state(env, insn_idx);
17671 	*explored_state(env, insn_idx) = new_sl;
17672 	/* connect new state to parentage chain. Current frame needs all
17673 	 * registers connected. Only r6 - r9 of the callers are alive (pushed
17674 	 * to the stack implicitly by JITs) so in callers' frames connect just
17675 	 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to
17676 	 * the state of the call instruction (with WRITTEN set), and r0 comes
17677 	 * from callee with its full parentage chain, anyway.
17678 	 */
17679 	/* clear write marks in current state: the writes we did are not writes
17680 	 * our child did, so they don't screen off its reads from us.
17681 	 * (There are no read marks in current state, because reads always mark
17682 	 * their parent and current state never has children yet.  Only
17683 	 * explored_states can get read marks.)
17684 	 */
17685 	for (j = 0; j <= cur->curframe; j++) {
17686 		for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
17687 			cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
17688 		for (i = 0; i < BPF_REG_FP; i++)
17689 			cur->frame[j]->regs[i].live = REG_LIVE_NONE;
17690 	}
17691 
17692 	/* all stack frames are accessible from callee, clear them all */
17693 	for (j = 0; j <= cur->curframe; j++) {
17694 		struct bpf_func_state *frame = cur->frame[j];
17695 		struct bpf_func_state *newframe = new->frame[j];
17696 
17697 		for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) {
17698 			frame->stack[i].spilled_ptr.live = REG_LIVE_NONE;
17699 			frame->stack[i].spilled_ptr.parent =
17700 						&newframe->stack[i].spilled_ptr;
17701 		}
17702 	}
17703 	return 0;
17704 }
17705 
17706 /* Return true if it's OK to have the same insn return a different type. */
17707 static bool reg_type_mismatch_ok(enum bpf_reg_type type)
17708 {
17709 	switch (base_type(type)) {
17710 	case PTR_TO_CTX:
17711 	case PTR_TO_SOCKET:
17712 	case PTR_TO_SOCK_COMMON:
17713 	case PTR_TO_TCP_SOCK:
17714 	case PTR_TO_XDP_SOCK:
17715 	case PTR_TO_BTF_ID:
17716 	case PTR_TO_ARENA:
17717 		return false;
17718 	default:
17719 		return true;
17720 	}
17721 }
17722 
17723 /* If an instruction was previously used with particular pointer types, then we
17724  * need to be careful to avoid cases such as the below, where it may be ok
17725  * for one branch accessing the pointer, but not ok for the other branch:
17726  *
17727  * R1 = sock_ptr
17728  * goto X;
17729  * ...
17730  * R1 = some_other_valid_ptr;
17731  * goto X;
17732  * ...
17733  * R2 = *(u32 *)(R1 + 0);
17734  */
17735 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
17736 {
17737 	return src != prev && (!reg_type_mismatch_ok(src) ||
17738 			       !reg_type_mismatch_ok(prev));
17739 }
17740 
17741 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
17742 			     bool allow_trust_mismatch)
17743 {
17744 	enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type;
17745 
17746 	if (*prev_type == NOT_INIT) {
17747 		/* Saw a valid insn
17748 		 * dst_reg = *(u32 *)(src_reg + off)
17749 		 * save type to validate intersecting paths
17750 		 */
17751 		*prev_type = type;
17752 	} else if (reg_type_mismatch(type, *prev_type)) {
17753 		/* Abuser program is trying to use the same insn
17754 		 * dst_reg = *(u32*) (src_reg + off)
17755 		 * with different pointer types:
17756 		 * src_reg == ctx in one branch and
17757 		 * src_reg == stack|map in some other branch.
17758 		 * Reject it.
17759 		 */
17760 		if (allow_trust_mismatch &&
17761 		    base_type(type) == PTR_TO_BTF_ID &&
17762 		    base_type(*prev_type) == PTR_TO_BTF_ID) {
17763 			/*
17764 			 * Have to support a use case when one path through
17765 			 * the program yields TRUSTED pointer while another
17766 			 * is UNTRUSTED. Fallback to UNTRUSTED to generate
17767 			 * BPF_PROBE_MEM/BPF_PROBE_MEMSX.
17768 			 */
17769 			*prev_type = PTR_TO_BTF_ID | PTR_UNTRUSTED;
17770 		} else {
17771 			verbose(env, "same insn cannot be used with different pointers\n");
17772 			return -EINVAL;
17773 		}
17774 	}
17775 
17776 	return 0;
17777 }
17778 
17779 static int do_check(struct bpf_verifier_env *env)
17780 {
17781 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
17782 	struct bpf_verifier_state *state = env->cur_state;
17783 	struct bpf_insn *insns = env->prog->insnsi;
17784 	struct bpf_reg_state *regs;
17785 	int insn_cnt = env->prog->len;
17786 	bool do_print_state = false;
17787 	int prev_insn_idx = -1;
17788 
17789 	for (;;) {
17790 		bool exception_exit = false;
17791 		struct bpf_insn *insn;
17792 		u8 class;
17793 		int err;
17794 
17795 		/* reset current history entry on each new instruction */
17796 		env->cur_hist_ent = NULL;
17797 
17798 		env->prev_insn_idx = prev_insn_idx;
17799 		if (env->insn_idx >= insn_cnt) {
17800 			verbose(env, "invalid insn idx %d insn_cnt %d\n",
17801 				env->insn_idx, insn_cnt);
17802 			return -EFAULT;
17803 		}
17804 
17805 		insn = &insns[env->insn_idx];
17806 		class = BPF_CLASS(insn->code);
17807 
17808 		if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
17809 			verbose(env,
17810 				"BPF program is too large. Processed %d insn\n",
17811 				env->insn_processed);
17812 			return -E2BIG;
17813 		}
17814 
17815 		state->last_insn_idx = env->prev_insn_idx;
17816 
17817 		if (is_prune_point(env, env->insn_idx)) {
17818 			err = is_state_visited(env, env->insn_idx);
17819 			if (err < 0)
17820 				return err;
17821 			if (err == 1) {
17822 				/* found equivalent state, can prune the search */
17823 				if (env->log.level & BPF_LOG_LEVEL) {
17824 					if (do_print_state)
17825 						verbose(env, "\nfrom %d to %d%s: safe\n",
17826 							env->prev_insn_idx, env->insn_idx,
17827 							env->cur_state->speculative ?
17828 							" (speculative execution)" : "");
17829 					else
17830 						verbose(env, "%d: safe\n", env->insn_idx);
17831 				}
17832 				goto process_bpf_exit;
17833 			}
17834 		}
17835 
17836 		if (is_jmp_point(env, env->insn_idx)) {
17837 			err = push_jmp_history(env, state, 0);
17838 			if (err)
17839 				return err;
17840 		}
17841 
17842 		if (signal_pending(current))
17843 			return -EAGAIN;
17844 
17845 		if (need_resched())
17846 			cond_resched();
17847 
17848 		if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {
17849 			verbose(env, "\nfrom %d to %d%s:",
17850 				env->prev_insn_idx, env->insn_idx,
17851 				env->cur_state->speculative ?
17852 				" (speculative execution)" : "");
17853 			print_verifier_state(env, state->frame[state->curframe], true);
17854 			do_print_state = false;
17855 		}
17856 
17857 		if (env->log.level & BPF_LOG_LEVEL) {
17858 			const struct bpf_insn_cbs cbs = {
17859 				.cb_call	= disasm_kfunc_name,
17860 				.cb_print	= verbose,
17861 				.private_data	= env,
17862 			};
17863 
17864 			if (verifier_state_scratched(env))
17865 				print_insn_state(env, state->frame[state->curframe]);
17866 
17867 			verbose_linfo(env, env->insn_idx, "; ");
17868 			env->prev_log_pos = env->log.end_pos;
17869 			verbose(env, "%d: ", env->insn_idx);
17870 			print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
17871 			env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;
17872 			env->prev_log_pos = env->log.end_pos;
17873 		}
17874 
17875 		if (bpf_prog_is_offloaded(env->prog->aux)) {
17876 			err = bpf_prog_offload_verify_insn(env, env->insn_idx,
17877 							   env->prev_insn_idx);
17878 			if (err)
17879 				return err;
17880 		}
17881 
17882 		regs = cur_regs(env);
17883 		sanitize_mark_insn_seen(env);
17884 		prev_insn_idx = env->insn_idx;
17885 
17886 		if (class == BPF_ALU || class == BPF_ALU64) {
17887 			err = check_alu_op(env, insn);
17888 			if (err)
17889 				return err;
17890 
17891 		} else if (class == BPF_LDX) {
17892 			enum bpf_reg_type src_reg_type;
17893 
17894 			/* check for reserved fields is already done */
17895 
17896 			/* check src operand */
17897 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
17898 			if (err)
17899 				return err;
17900 
17901 			err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
17902 			if (err)
17903 				return err;
17904 
17905 			src_reg_type = regs[insn->src_reg].type;
17906 
17907 			/* check that memory (src_reg + off) is readable,
17908 			 * the state of dst_reg will be updated by this func
17909 			 */
17910 			err = check_mem_access(env, env->insn_idx, insn->src_reg,
17911 					       insn->off, BPF_SIZE(insn->code),
17912 					       BPF_READ, insn->dst_reg, false,
17913 					       BPF_MODE(insn->code) == BPF_MEMSX);
17914 			err = err ?: save_aux_ptr_type(env, src_reg_type, true);
17915 			err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], "ldx");
17916 			if (err)
17917 				return err;
17918 		} else if (class == BPF_STX) {
17919 			enum bpf_reg_type dst_reg_type;
17920 
17921 			if (BPF_MODE(insn->code) == BPF_ATOMIC) {
17922 				err = check_atomic(env, env->insn_idx, insn);
17923 				if (err)
17924 					return err;
17925 				env->insn_idx++;
17926 				continue;
17927 			}
17928 
17929 			if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) {
17930 				verbose(env, "BPF_STX uses reserved fields\n");
17931 				return -EINVAL;
17932 			}
17933 
17934 			/* check src1 operand */
17935 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
17936 			if (err)
17937 				return err;
17938 			/* check src2 operand */
17939 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);
17940 			if (err)
17941 				return err;
17942 
17943 			dst_reg_type = regs[insn->dst_reg].type;
17944 
17945 			/* check that memory (dst_reg + off) is writeable */
17946 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,
17947 					       insn->off, BPF_SIZE(insn->code),
17948 					       BPF_WRITE, insn->src_reg, false, false);
17949 			if (err)
17950 				return err;
17951 
17952 			err = save_aux_ptr_type(env, dst_reg_type, false);
17953 			if (err)
17954 				return err;
17955 		} else if (class == BPF_ST) {
17956 			enum bpf_reg_type dst_reg_type;
17957 
17958 			if (BPF_MODE(insn->code) != BPF_MEM ||
17959 			    insn->src_reg != BPF_REG_0) {
17960 				verbose(env, "BPF_ST uses reserved fields\n");
17961 				return -EINVAL;
17962 			}
17963 			/* check src operand */
17964 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);
17965 			if (err)
17966 				return err;
17967 
17968 			dst_reg_type = regs[insn->dst_reg].type;
17969 
17970 			/* check that memory (dst_reg + off) is writeable */
17971 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,
17972 					       insn->off, BPF_SIZE(insn->code),
17973 					       BPF_WRITE, -1, false, false);
17974 			if (err)
17975 				return err;
17976 
17977 			err = save_aux_ptr_type(env, dst_reg_type, false);
17978 			if (err)
17979 				return err;
17980 		} else if (class == BPF_JMP || class == BPF_JMP32) {
17981 			u8 opcode = BPF_OP(insn->code);
17982 
17983 			env->jmps_processed++;
17984 			if (opcode == BPF_CALL) {
17985 				if (BPF_SRC(insn->code) != BPF_K ||
17986 				    (insn->src_reg != BPF_PSEUDO_KFUNC_CALL
17987 				     && insn->off != 0) ||
17988 				    (insn->src_reg != BPF_REG_0 &&
17989 				     insn->src_reg != BPF_PSEUDO_CALL &&
17990 				     insn->src_reg != BPF_PSEUDO_KFUNC_CALL) ||
17991 				    insn->dst_reg != BPF_REG_0 ||
17992 				    class == BPF_JMP32) {
17993 					verbose(env, "BPF_CALL uses reserved fields\n");
17994 					return -EINVAL;
17995 				}
17996 
17997 				if (env->cur_state->active_lock.ptr) {
17998 					if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) ||
17999 					    (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
18000 					     (insn->off != 0 || !is_bpf_graph_api_kfunc(insn->imm)))) {
18001 						verbose(env, "function calls are not allowed while holding a lock\n");
18002 						return -EINVAL;
18003 					}
18004 				}
18005 				if (insn->src_reg == BPF_PSEUDO_CALL) {
18006 					err = check_func_call(env, insn, &env->insn_idx);
18007 				} else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
18008 					err = check_kfunc_call(env, insn, &env->insn_idx);
18009 					if (!err && is_bpf_throw_kfunc(insn)) {
18010 						exception_exit = true;
18011 						goto process_bpf_exit_full;
18012 					}
18013 				} else {
18014 					err = check_helper_call(env, insn, &env->insn_idx);
18015 				}
18016 				if (err)
18017 					return err;
18018 
18019 				mark_reg_scratched(env, BPF_REG_0);
18020 			} else if (opcode == BPF_JA) {
18021 				if (BPF_SRC(insn->code) != BPF_K ||
18022 				    insn->src_reg != BPF_REG_0 ||
18023 				    insn->dst_reg != BPF_REG_0 ||
18024 				    (class == BPF_JMP && insn->imm != 0) ||
18025 				    (class == BPF_JMP32 && insn->off != 0)) {
18026 					verbose(env, "BPF_JA uses reserved fields\n");
18027 					return -EINVAL;
18028 				}
18029 
18030 				if (class == BPF_JMP)
18031 					env->insn_idx += insn->off + 1;
18032 				else
18033 					env->insn_idx += insn->imm + 1;
18034 				continue;
18035 
18036 			} else if (opcode == BPF_EXIT) {
18037 				if (BPF_SRC(insn->code) != BPF_K ||
18038 				    insn->imm != 0 ||
18039 				    insn->src_reg != BPF_REG_0 ||
18040 				    insn->dst_reg != BPF_REG_0 ||
18041 				    class == BPF_JMP32) {
18042 					verbose(env, "BPF_EXIT uses reserved fields\n");
18043 					return -EINVAL;
18044 				}
18045 process_bpf_exit_full:
18046 				if (env->cur_state->active_lock.ptr && !env->cur_state->curframe) {
18047 					verbose(env, "bpf_spin_unlock is missing\n");
18048 					return -EINVAL;
18049 				}
18050 
18051 				if (env->cur_state->active_rcu_lock && !env->cur_state->curframe) {
18052 					verbose(env, "bpf_rcu_read_unlock is missing\n");
18053 					return -EINVAL;
18054 				}
18055 
18056 				if (env->cur_state->active_preempt_lock && !env->cur_state->curframe) {
18057 					verbose(env, "%d bpf_preempt_enable%s missing\n",
18058 						env->cur_state->active_preempt_lock,
18059 						env->cur_state->active_preempt_lock == 1 ? " is" : "(s) are");
18060 					return -EINVAL;
18061 				}
18062 
18063 				/* We must do check_reference_leak here before
18064 				 * prepare_func_exit to handle the case when
18065 				 * state->curframe > 0, it may be a callback
18066 				 * function, for which reference_state must
18067 				 * match caller reference state when it exits.
18068 				 */
18069 				err = check_reference_leak(env, exception_exit);
18070 				if (err)
18071 					return err;
18072 
18073 				/* The side effect of the prepare_func_exit
18074 				 * which is being skipped is that it frees
18075 				 * bpf_func_state. Typically, process_bpf_exit
18076 				 * will only be hit with outermost exit.
18077 				 * copy_verifier_state in pop_stack will handle
18078 				 * freeing of any extra bpf_func_state left over
18079 				 * from not processing all nested function
18080 				 * exits. We also skip return code checks as
18081 				 * they are not needed for exceptional exits.
18082 				 */
18083 				if (exception_exit)
18084 					goto process_bpf_exit;
18085 
18086 				if (state->curframe) {
18087 					/* exit from nested function */
18088 					err = prepare_func_exit(env, &env->insn_idx);
18089 					if (err)
18090 						return err;
18091 					do_print_state = true;
18092 					continue;
18093 				}
18094 
18095 				err = check_return_code(env, BPF_REG_0, "R0");
18096 				if (err)
18097 					return err;
18098 process_bpf_exit:
18099 				mark_verifier_state_scratched(env);
18100 				update_branch_counts(env, env->cur_state);
18101 				err = pop_stack(env, &prev_insn_idx,
18102 						&env->insn_idx, pop_log);
18103 				if (err < 0) {
18104 					if (err != -ENOENT)
18105 						return err;
18106 					break;
18107 				} else {
18108 					do_print_state = true;
18109 					continue;
18110 				}
18111 			} else {
18112 				err = check_cond_jmp_op(env, insn, &env->insn_idx);
18113 				if (err)
18114 					return err;
18115 			}
18116 		} else if (class == BPF_LD) {
18117 			u8 mode = BPF_MODE(insn->code);
18118 
18119 			if (mode == BPF_ABS || mode == BPF_IND) {
18120 				err = check_ld_abs(env, insn);
18121 				if (err)
18122 					return err;
18123 
18124 			} else if (mode == BPF_IMM) {
18125 				err = check_ld_imm(env, insn);
18126 				if (err)
18127 					return err;
18128 
18129 				env->insn_idx++;
18130 				sanitize_mark_insn_seen(env);
18131 			} else {
18132 				verbose(env, "invalid BPF_LD mode\n");
18133 				return -EINVAL;
18134 			}
18135 		} else {
18136 			verbose(env, "unknown insn class %d\n", class);
18137 			return -EINVAL;
18138 		}
18139 
18140 		env->insn_idx++;
18141 	}
18142 
18143 	return 0;
18144 }
18145 
18146 static int find_btf_percpu_datasec(struct btf *btf)
18147 {
18148 	const struct btf_type *t;
18149 	const char *tname;
18150 	int i, n;
18151 
18152 	/*
18153 	 * Both vmlinux and module each have their own ".data..percpu"
18154 	 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF
18155 	 * types to look at only module's own BTF types.
18156 	 */
18157 	n = btf_nr_types(btf);
18158 	if (btf_is_module(btf))
18159 		i = btf_nr_types(btf_vmlinux);
18160 	else
18161 		i = 1;
18162 
18163 	for(; i < n; i++) {
18164 		t = btf_type_by_id(btf, i);
18165 		if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC)
18166 			continue;
18167 
18168 		tname = btf_name_by_offset(btf, t->name_off);
18169 		if (!strcmp(tname, ".data..percpu"))
18170 			return i;
18171 	}
18172 
18173 	return -ENOENT;
18174 }
18175 
18176 /* replace pseudo btf_id with kernel symbol address */
18177 static int check_pseudo_btf_id(struct bpf_verifier_env *env,
18178 			       struct bpf_insn *insn,
18179 			       struct bpf_insn_aux_data *aux)
18180 {
18181 	const struct btf_var_secinfo *vsi;
18182 	const struct btf_type *datasec;
18183 	struct btf_mod_pair *btf_mod;
18184 	const struct btf_type *t;
18185 	const char *sym_name;
18186 	bool percpu = false;
18187 	u32 type, id = insn->imm;
18188 	struct btf *btf;
18189 	s32 datasec_id;
18190 	u64 addr;
18191 	int i, btf_fd, err;
18192 
18193 	btf_fd = insn[1].imm;
18194 	if (btf_fd) {
18195 		btf = btf_get_by_fd(btf_fd);
18196 		if (IS_ERR(btf)) {
18197 			verbose(env, "invalid module BTF object FD specified.\n");
18198 			return -EINVAL;
18199 		}
18200 	} else {
18201 		if (!btf_vmlinux) {
18202 			verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
18203 			return -EINVAL;
18204 		}
18205 		btf = btf_vmlinux;
18206 		btf_get(btf);
18207 	}
18208 
18209 	t = btf_type_by_id(btf, id);
18210 	if (!t) {
18211 		verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
18212 		err = -ENOENT;
18213 		goto err_put;
18214 	}
18215 
18216 	if (!btf_type_is_var(t) && !btf_type_is_func(t)) {
18217 		verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id);
18218 		err = -EINVAL;
18219 		goto err_put;
18220 	}
18221 
18222 	sym_name = btf_name_by_offset(btf, t->name_off);
18223 	addr = kallsyms_lookup_name(sym_name);
18224 	if (!addr) {
18225 		verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
18226 			sym_name);
18227 		err = -ENOENT;
18228 		goto err_put;
18229 	}
18230 	insn[0].imm = (u32)addr;
18231 	insn[1].imm = addr >> 32;
18232 
18233 	if (btf_type_is_func(t)) {
18234 		aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
18235 		aux->btf_var.mem_size = 0;
18236 		goto check_btf;
18237 	}
18238 
18239 	datasec_id = find_btf_percpu_datasec(btf);
18240 	if (datasec_id > 0) {
18241 		datasec = btf_type_by_id(btf, datasec_id);
18242 		for_each_vsi(i, datasec, vsi) {
18243 			if (vsi->type == id) {
18244 				percpu = true;
18245 				break;
18246 			}
18247 		}
18248 	}
18249 
18250 	type = t->type;
18251 	t = btf_type_skip_modifiers(btf, type, NULL);
18252 	if (percpu) {
18253 		aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU;
18254 		aux->btf_var.btf = btf;
18255 		aux->btf_var.btf_id = type;
18256 	} else if (!btf_type_is_struct(t)) {
18257 		const struct btf_type *ret;
18258 		const char *tname;
18259 		u32 tsize;
18260 
18261 		/* resolve the type size of ksym. */
18262 		ret = btf_resolve_size(btf, t, &tsize);
18263 		if (IS_ERR(ret)) {
18264 			tname = btf_name_by_offset(btf, t->name_off);
18265 			verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
18266 				tname, PTR_ERR(ret));
18267 			err = -EINVAL;
18268 			goto err_put;
18269 		}
18270 		aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
18271 		aux->btf_var.mem_size = tsize;
18272 	} else {
18273 		aux->btf_var.reg_type = PTR_TO_BTF_ID;
18274 		aux->btf_var.btf = btf;
18275 		aux->btf_var.btf_id = type;
18276 	}
18277 check_btf:
18278 	/* check whether we recorded this BTF (and maybe module) already */
18279 	for (i = 0; i < env->used_btf_cnt; i++) {
18280 		if (env->used_btfs[i].btf == btf) {
18281 			btf_put(btf);
18282 			return 0;
18283 		}
18284 	}
18285 
18286 	if (env->used_btf_cnt >= MAX_USED_BTFS) {
18287 		err = -E2BIG;
18288 		goto err_put;
18289 	}
18290 
18291 	btf_mod = &env->used_btfs[env->used_btf_cnt];
18292 	btf_mod->btf = btf;
18293 	btf_mod->module = NULL;
18294 
18295 	/* if we reference variables from kernel module, bump its refcount */
18296 	if (btf_is_module(btf)) {
18297 		btf_mod->module = btf_try_get_module(btf);
18298 		if (!btf_mod->module) {
18299 			err = -ENXIO;
18300 			goto err_put;
18301 		}
18302 	}
18303 
18304 	env->used_btf_cnt++;
18305 
18306 	return 0;
18307 err_put:
18308 	btf_put(btf);
18309 	return err;
18310 }
18311 
18312 static bool is_tracing_prog_type(enum bpf_prog_type type)
18313 {
18314 	switch (type) {
18315 	case BPF_PROG_TYPE_KPROBE:
18316 	case BPF_PROG_TYPE_TRACEPOINT:
18317 	case BPF_PROG_TYPE_PERF_EVENT:
18318 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
18319 	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
18320 		return true;
18321 	default:
18322 		return false;
18323 	}
18324 }
18325 
18326 static int check_map_prog_compatibility(struct bpf_verifier_env *env,
18327 					struct bpf_map *map,
18328 					struct bpf_prog *prog)
18329 
18330 {
18331 	enum bpf_prog_type prog_type = resolve_prog_type(prog);
18332 
18333 	if (btf_record_has_field(map->record, BPF_LIST_HEAD) ||
18334 	    btf_record_has_field(map->record, BPF_RB_ROOT)) {
18335 		if (is_tracing_prog_type(prog_type)) {
18336 			verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n");
18337 			return -EINVAL;
18338 		}
18339 	}
18340 
18341 	if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
18342 		if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) {
18343 			verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");
18344 			return -EINVAL;
18345 		}
18346 
18347 		if (is_tracing_prog_type(prog_type)) {
18348 			verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
18349 			return -EINVAL;
18350 		}
18351 	}
18352 
18353 	if (btf_record_has_field(map->record, BPF_TIMER)) {
18354 		if (is_tracing_prog_type(prog_type)) {
18355 			verbose(env, "tracing progs cannot use bpf_timer yet\n");
18356 			return -EINVAL;
18357 		}
18358 	}
18359 
18360 	if (btf_record_has_field(map->record, BPF_WORKQUEUE)) {
18361 		if (is_tracing_prog_type(prog_type)) {
18362 			verbose(env, "tracing progs cannot use bpf_wq yet\n");
18363 			return -EINVAL;
18364 		}
18365 	}
18366 
18367 	if ((bpf_prog_is_offloaded(prog->aux) || bpf_map_is_offloaded(map)) &&
18368 	    !bpf_offload_prog_map_match(prog, map)) {
18369 		verbose(env, "offload device mismatch between prog and map\n");
18370 		return -EINVAL;
18371 	}
18372 
18373 	if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
18374 		verbose(env, "bpf_struct_ops map cannot be used in prog\n");
18375 		return -EINVAL;
18376 	}
18377 
18378 	if (prog->sleepable)
18379 		switch (map->map_type) {
18380 		case BPF_MAP_TYPE_HASH:
18381 		case BPF_MAP_TYPE_LRU_HASH:
18382 		case BPF_MAP_TYPE_ARRAY:
18383 		case BPF_MAP_TYPE_PERCPU_HASH:
18384 		case BPF_MAP_TYPE_PERCPU_ARRAY:
18385 		case BPF_MAP_TYPE_LRU_PERCPU_HASH:
18386 		case BPF_MAP_TYPE_ARRAY_OF_MAPS:
18387 		case BPF_MAP_TYPE_HASH_OF_MAPS:
18388 		case BPF_MAP_TYPE_RINGBUF:
18389 		case BPF_MAP_TYPE_USER_RINGBUF:
18390 		case BPF_MAP_TYPE_INODE_STORAGE:
18391 		case BPF_MAP_TYPE_SK_STORAGE:
18392 		case BPF_MAP_TYPE_TASK_STORAGE:
18393 		case BPF_MAP_TYPE_CGRP_STORAGE:
18394 		case BPF_MAP_TYPE_QUEUE:
18395 		case BPF_MAP_TYPE_STACK:
18396 		case BPF_MAP_TYPE_ARENA:
18397 			break;
18398 		default:
18399 			verbose(env,
18400 				"Sleepable programs can only use array, hash, ringbuf and local storage maps\n");
18401 			return -EINVAL;
18402 		}
18403 
18404 	return 0;
18405 }
18406 
18407 static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
18408 {
18409 	return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
18410 		map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
18411 }
18412 
18413 /* find and rewrite pseudo imm in ld_imm64 instructions:
18414  *
18415  * 1. if it accesses map FD, replace it with actual map pointer.
18416  * 2. if it accesses btf_id of a VAR, replace it with pointer to the var.
18417  *
18418  * NOTE: btf_vmlinux is required for converting pseudo btf_id.
18419  */
18420 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
18421 {
18422 	struct bpf_insn *insn = env->prog->insnsi;
18423 	int insn_cnt = env->prog->len;
18424 	int i, j, err;
18425 
18426 	err = bpf_prog_calc_tag(env->prog);
18427 	if (err)
18428 		return err;
18429 
18430 	for (i = 0; i < insn_cnt; i++, insn++) {
18431 		if (BPF_CLASS(insn->code) == BPF_LDX &&
18432 		    ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) ||
18433 		    insn->imm != 0)) {
18434 			verbose(env, "BPF_LDX uses reserved fields\n");
18435 			return -EINVAL;
18436 		}
18437 
18438 		if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
18439 			struct bpf_insn_aux_data *aux;
18440 			struct bpf_map *map;
18441 			struct fd f;
18442 			u64 addr;
18443 			u32 fd;
18444 
18445 			if (i == insn_cnt - 1 || insn[1].code != 0 ||
18446 			    insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
18447 			    insn[1].off != 0) {
18448 				verbose(env, "invalid bpf_ld_imm64 insn\n");
18449 				return -EINVAL;
18450 			}
18451 
18452 			if (insn[0].src_reg == 0)
18453 				/* valid generic load 64-bit imm */
18454 				goto next_insn;
18455 
18456 			if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
18457 				aux = &env->insn_aux_data[i];
18458 				err = check_pseudo_btf_id(env, insn, aux);
18459 				if (err)
18460 					return err;
18461 				goto next_insn;
18462 			}
18463 
18464 			if (insn[0].src_reg == BPF_PSEUDO_FUNC) {
18465 				aux = &env->insn_aux_data[i];
18466 				aux->ptr_type = PTR_TO_FUNC;
18467 				goto next_insn;
18468 			}
18469 
18470 			/* In final convert_pseudo_ld_imm64() step, this is
18471 			 * converted into regular 64-bit imm load insn.
18472 			 */
18473 			switch (insn[0].src_reg) {
18474 			case BPF_PSEUDO_MAP_VALUE:
18475 			case BPF_PSEUDO_MAP_IDX_VALUE:
18476 				break;
18477 			case BPF_PSEUDO_MAP_FD:
18478 			case BPF_PSEUDO_MAP_IDX:
18479 				if (insn[1].imm == 0)
18480 					break;
18481 				fallthrough;
18482 			default:
18483 				verbose(env, "unrecognized bpf_ld_imm64 insn\n");
18484 				return -EINVAL;
18485 			}
18486 
18487 			switch (insn[0].src_reg) {
18488 			case BPF_PSEUDO_MAP_IDX_VALUE:
18489 			case BPF_PSEUDO_MAP_IDX:
18490 				if (bpfptr_is_null(env->fd_array)) {
18491 					verbose(env, "fd_idx without fd_array is invalid\n");
18492 					return -EPROTO;
18493 				}
18494 				if (copy_from_bpfptr_offset(&fd, env->fd_array,
18495 							    insn[0].imm * sizeof(fd),
18496 							    sizeof(fd)))
18497 					return -EFAULT;
18498 				break;
18499 			default:
18500 				fd = insn[0].imm;
18501 				break;
18502 			}
18503 
18504 			f = fdget(fd);
18505 			map = __bpf_map_get(f);
18506 			if (IS_ERR(map)) {
18507 				verbose(env, "fd %d is not pointing to valid bpf_map\n", fd);
18508 				return PTR_ERR(map);
18509 			}
18510 
18511 			err = check_map_prog_compatibility(env, map, env->prog);
18512 			if (err) {
18513 				fdput(f);
18514 				return err;
18515 			}
18516 
18517 			aux = &env->insn_aux_data[i];
18518 			if (insn[0].src_reg == BPF_PSEUDO_MAP_FD ||
18519 			    insn[0].src_reg == BPF_PSEUDO_MAP_IDX) {
18520 				addr = (unsigned long)map;
18521 			} else {
18522 				u32 off = insn[1].imm;
18523 
18524 				if (off >= BPF_MAX_VAR_OFF) {
18525 					verbose(env, "direct value offset of %u is not allowed\n", off);
18526 					fdput(f);
18527 					return -EINVAL;
18528 				}
18529 
18530 				if (!map->ops->map_direct_value_addr) {
18531 					verbose(env, "no direct value access support for this map type\n");
18532 					fdput(f);
18533 					return -EINVAL;
18534 				}
18535 
18536 				err = map->ops->map_direct_value_addr(map, &addr, off);
18537 				if (err) {
18538 					verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",
18539 						map->value_size, off);
18540 					fdput(f);
18541 					return err;
18542 				}
18543 
18544 				aux->map_off = off;
18545 				addr += off;
18546 			}
18547 
18548 			insn[0].imm = (u32)addr;
18549 			insn[1].imm = addr >> 32;
18550 
18551 			/* check whether we recorded this map already */
18552 			for (j = 0; j < env->used_map_cnt; j++) {
18553 				if (env->used_maps[j] == map) {
18554 					aux->map_index = j;
18555 					fdput(f);
18556 					goto next_insn;
18557 				}
18558 			}
18559 
18560 			if (env->used_map_cnt >= MAX_USED_MAPS) {
18561 				verbose(env, "The total number of maps per program has reached the limit of %u\n",
18562 					MAX_USED_MAPS);
18563 				fdput(f);
18564 				return -E2BIG;
18565 			}
18566 
18567 			if (env->prog->sleepable)
18568 				atomic64_inc(&map->sleepable_refcnt);
18569 			/* hold the map. If the program is rejected by verifier,
18570 			 * the map will be released by release_maps() or it
18571 			 * will be used by the valid program until it's unloaded
18572 			 * and all maps are released in bpf_free_used_maps()
18573 			 */
18574 			bpf_map_inc(map);
18575 
18576 			aux->map_index = env->used_map_cnt;
18577 			env->used_maps[env->used_map_cnt++] = map;
18578 
18579 			if (bpf_map_is_cgroup_storage(map) &&
18580 			    bpf_cgroup_storage_assign(env->prog->aux, map)) {
18581 				verbose(env, "only one cgroup storage of each type is allowed\n");
18582 				fdput(f);
18583 				return -EBUSY;
18584 			}
18585 			if (map->map_type == BPF_MAP_TYPE_ARENA) {
18586 				if (env->prog->aux->arena) {
18587 					verbose(env, "Only one arena per program\n");
18588 					fdput(f);
18589 					return -EBUSY;
18590 				}
18591 				if (!env->allow_ptr_leaks || !env->bpf_capable) {
18592 					verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n");
18593 					fdput(f);
18594 					return -EPERM;
18595 				}
18596 				if (!env->prog->jit_requested) {
18597 					verbose(env, "JIT is required to use arena\n");
18598 					fdput(f);
18599 					return -EOPNOTSUPP;
18600 				}
18601 				if (!bpf_jit_supports_arena()) {
18602 					verbose(env, "JIT doesn't support arena\n");
18603 					fdput(f);
18604 					return -EOPNOTSUPP;
18605 				}
18606 				env->prog->aux->arena = (void *)map;
18607 				if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) {
18608 					verbose(env, "arena's user address must be set via map_extra or mmap()\n");
18609 					fdput(f);
18610 					return -EINVAL;
18611 				}
18612 			}
18613 
18614 			fdput(f);
18615 next_insn:
18616 			insn++;
18617 			i++;
18618 			continue;
18619 		}
18620 
18621 		/* Basic sanity check before we invest more work here. */
18622 		if (!bpf_opcode_in_insntable(insn->code)) {
18623 			verbose(env, "unknown opcode %02x\n", insn->code);
18624 			return -EINVAL;
18625 		}
18626 	}
18627 
18628 	/* now all pseudo BPF_LD_IMM64 instructions load valid
18629 	 * 'struct bpf_map *' into a register instead of user map_fd.
18630 	 * These pointers will be used later by verifier to validate map access.
18631 	 */
18632 	return 0;
18633 }
18634 
18635 /* drop refcnt of maps used by the rejected program */
18636 static void release_maps(struct bpf_verifier_env *env)
18637 {
18638 	__bpf_free_used_maps(env->prog->aux, env->used_maps,
18639 			     env->used_map_cnt);
18640 }
18641 
18642 /* drop refcnt of maps used by the rejected program */
18643 static void release_btfs(struct bpf_verifier_env *env)
18644 {
18645 	__bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt);
18646 }
18647 
18648 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */
18649 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
18650 {
18651 	struct bpf_insn *insn = env->prog->insnsi;
18652 	int insn_cnt = env->prog->len;
18653 	int i;
18654 
18655 	for (i = 0; i < insn_cnt; i++, insn++) {
18656 		if (insn->code != (BPF_LD | BPF_IMM | BPF_DW))
18657 			continue;
18658 		if (insn->src_reg == BPF_PSEUDO_FUNC)
18659 			continue;
18660 		insn->src_reg = 0;
18661 	}
18662 }
18663 
18664 /* single env->prog->insni[off] instruction was replaced with the range
18665  * insni[off, off + cnt).  Adjust corresponding insn_aux_data by copying
18666  * [0, off) and [off, end) to new locations, so the patched range stays zero
18667  */
18668 static void adjust_insn_aux_data(struct bpf_verifier_env *env,
18669 				 struct bpf_insn_aux_data *new_data,
18670 				 struct bpf_prog *new_prog, u32 off, u32 cnt)
18671 {
18672 	struct bpf_insn_aux_data *old_data = env->insn_aux_data;
18673 	struct bpf_insn *insn = new_prog->insnsi;
18674 	u32 old_seen = old_data[off].seen;
18675 	u32 prog_len;
18676 	int i;
18677 
18678 	/* aux info at OFF always needs adjustment, no matter fast path
18679 	 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the
18680 	 * original insn at old prog.
18681 	 */
18682 	old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1);
18683 
18684 	if (cnt == 1)
18685 		return;
18686 	prog_len = new_prog->len;
18687 
18688 	memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
18689 	memcpy(new_data + off + cnt - 1, old_data + off,
18690 	       sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
18691 	for (i = off; i < off + cnt - 1; i++) {
18692 		/* Expand insni[off]'s seen count to the patched range. */
18693 		new_data[i].seen = old_seen;
18694 		new_data[i].zext_dst = insn_has_def32(env, insn + i);
18695 	}
18696 	env->insn_aux_data = new_data;
18697 	vfree(old_data);
18698 }
18699 
18700 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
18701 {
18702 	int i;
18703 
18704 	if (len == 1)
18705 		return;
18706 	/* NOTE: fake 'exit' subprog should be updated as well. */
18707 	for (i = 0; i <= env->subprog_cnt; i++) {
18708 		if (env->subprog_info[i].start <= off)
18709 			continue;
18710 		env->subprog_info[i].start += len - 1;
18711 	}
18712 }
18713 
18714 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
18715 {
18716 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
18717 	int i, sz = prog->aux->size_poke_tab;
18718 	struct bpf_jit_poke_descriptor *desc;
18719 
18720 	for (i = 0; i < sz; i++) {
18721 		desc = &tab[i];
18722 		if (desc->insn_idx <= off)
18723 			continue;
18724 		desc->insn_idx += len - 1;
18725 	}
18726 }
18727 
18728 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
18729 					    const struct bpf_insn *patch, u32 len)
18730 {
18731 	struct bpf_prog *new_prog;
18732 	struct bpf_insn_aux_data *new_data = NULL;
18733 
18734 	if (len > 1) {
18735 		new_data = vzalloc(array_size(env->prog->len + len - 1,
18736 					      sizeof(struct bpf_insn_aux_data)));
18737 		if (!new_data)
18738 			return NULL;
18739 	}
18740 
18741 	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
18742 	if (IS_ERR(new_prog)) {
18743 		if (PTR_ERR(new_prog) == -ERANGE)
18744 			verbose(env,
18745 				"insn %d cannot be patched due to 16-bit range\n",
18746 				env->insn_aux_data[off].orig_idx);
18747 		vfree(new_data);
18748 		return NULL;
18749 	}
18750 	adjust_insn_aux_data(env, new_data, new_prog, off, len);
18751 	adjust_subprog_starts(env, off, len);
18752 	adjust_poke_descs(new_prog, off, len);
18753 	return new_prog;
18754 }
18755 
18756 /*
18757  * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the
18758  * jump offset by 'delta'.
18759  */
18760 static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta)
18761 {
18762 	struct bpf_insn *insn = prog->insnsi;
18763 	u32 insn_cnt = prog->len, i;
18764 	s32 imm;
18765 	s16 off;
18766 
18767 	for (i = 0; i < insn_cnt; i++, insn++) {
18768 		u8 code = insn->code;
18769 
18770 		if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) ||
18771 		    BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT)
18772 			continue;
18773 
18774 		if (insn->code == (BPF_JMP32 | BPF_JA)) {
18775 			if (i + 1 + insn->imm != tgt_idx)
18776 				continue;
18777 			if (check_add_overflow(insn->imm, delta, &imm))
18778 				return -ERANGE;
18779 			insn->imm = imm;
18780 		} else {
18781 			if (i + 1 + insn->off != tgt_idx)
18782 				continue;
18783 			if (check_add_overflow(insn->off, delta, &off))
18784 				return -ERANGE;
18785 			insn->off = off;
18786 		}
18787 	}
18788 	return 0;
18789 }
18790 
18791 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
18792 					      u32 off, u32 cnt)
18793 {
18794 	int i, j;
18795 
18796 	/* find first prog starting at or after off (first to remove) */
18797 	for (i = 0; i < env->subprog_cnt; i++)
18798 		if (env->subprog_info[i].start >= off)
18799 			break;
18800 	/* find first prog starting at or after off + cnt (first to stay) */
18801 	for (j = i; j < env->subprog_cnt; j++)
18802 		if (env->subprog_info[j].start >= off + cnt)
18803 			break;
18804 	/* if j doesn't start exactly at off + cnt, we are just removing
18805 	 * the front of previous prog
18806 	 */
18807 	if (env->subprog_info[j].start != off + cnt)
18808 		j--;
18809 
18810 	if (j > i) {
18811 		struct bpf_prog_aux *aux = env->prog->aux;
18812 		int move;
18813 
18814 		/* move fake 'exit' subprog as well */
18815 		move = env->subprog_cnt + 1 - j;
18816 
18817 		memmove(env->subprog_info + i,
18818 			env->subprog_info + j,
18819 			sizeof(*env->subprog_info) * move);
18820 		env->subprog_cnt -= j - i;
18821 
18822 		/* remove func_info */
18823 		if (aux->func_info) {
18824 			move = aux->func_info_cnt - j;
18825 
18826 			memmove(aux->func_info + i,
18827 				aux->func_info + j,
18828 				sizeof(*aux->func_info) * move);
18829 			aux->func_info_cnt -= j - i;
18830 			/* func_info->insn_off is set after all code rewrites,
18831 			 * in adjust_btf_func() - no need to adjust
18832 			 */
18833 		}
18834 	} else {
18835 		/* convert i from "first prog to remove" to "first to adjust" */
18836 		if (env->subprog_info[i].start == off)
18837 			i++;
18838 	}
18839 
18840 	/* update fake 'exit' subprog as well */
18841 	for (; i <= env->subprog_cnt; i++)
18842 		env->subprog_info[i].start -= cnt;
18843 
18844 	return 0;
18845 }
18846 
18847 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
18848 				      u32 cnt)
18849 {
18850 	struct bpf_prog *prog = env->prog;
18851 	u32 i, l_off, l_cnt, nr_linfo;
18852 	struct bpf_line_info *linfo;
18853 
18854 	nr_linfo = prog->aux->nr_linfo;
18855 	if (!nr_linfo)
18856 		return 0;
18857 
18858 	linfo = prog->aux->linfo;
18859 
18860 	/* find first line info to remove, count lines to be removed */
18861 	for (i = 0; i < nr_linfo; i++)
18862 		if (linfo[i].insn_off >= off)
18863 			break;
18864 
18865 	l_off = i;
18866 	l_cnt = 0;
18867 	for (; i < nr_linfo; i++)
18868 		if (linfo[i].insn_off < off + cnt)
18869 			l_cnt++;
18870 		else
18871 			break;
18872 
18873 	/* First live insn doesn't match first live linfo, it needs to "inherit"
18874 	 * last removed linfo.  prog is already modified, so prog->len == off
18875 	 * means no live instructions after (tail of the program was removed).
18876 	 */
18877 	if (prog->len != off && l_cnt &&
18878 	    (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
18879 		l_cnt--;
18880 		linfo[--i].insn_off = off + cnt;
18881 	}
18882 
18883 	/* remove the line info which refer to the removed instructions */
18884 	if (l_cnt) {
18885 		memmove(linfo + l_off, linfo + i,
18886 			sizeof(*linfo) * (nr_linfo - i));
18887 
18888 		prog->aux->nr_linfo -= l_cnt;
18889 		nr_linfo = prog->aux->nr_linfo;
18890 	}
18891 
18892 	/* pull all linfo[i].insn_off >= off + cnt in by cnt */
18893 	for (i = l_off; i < nr_linfo; i++)
18894 		linfo[i].insn_off -= cnt;
18895 
18896 	/* fix up all subprogs (incl. 'exit') which start >= off */
18897 	for (i = 0; i <= env->subprog_cnt; i++)
18898 		if (env->subprog_info[i].linfo_idx > l_off) {
18899 			/* program may have started in the removed region but
18900 			 * may not be fully removed
18901 			 */
18902 			if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)
18903 				env->subprog_info[i].linfo_idx -= l_cnt;
18904 			else
18905 				env->subprog_info[i].linfo_idx = l_off;
18906 		}
18907 
18908 	return 0;
18909 }
18910 
18911 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
18912 {
18913 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
18914 	unsigned int orig_prog_len = env->prog->len;
18915 	int err;
18916 
18917 	if (bpf_prog_is_offloaded(env->prog->aux))
18918 		bpf_prog_offload_remove_insns(env, off, cnt);
18919 
18920 	err = bpf_remove_insns(env->prog, off, cnt);
18921 	if (err)
18922 		return err;
18923 
18924 	err = adjust_subprog_starts_after_remove(env, off, cnt);
18925 	if (err)
18926 		return err;
18927 
18928 	err = bpf_adj_linfo_after_remove(env, off, cnt);
18929 	if (err)
18930 		return err;
18931 
18932 	memmove(aux_data + off,	aux_data + off + cnt,
18933 		sizeof(*aux_data) * (orig_prog_len - off - cnt));
18934 
18935 	return 0;
18936 }
18937 
18938 /* The verifier does more data flow analysis than llvm and will not
18939  * explore branches that are dead at run time. Malicious programs can
18940  * have dead code too. Therefore replace all dead at-run-time code
18941  * with 'ja -1'.
18942  *
18943  * Just nops are not optimal, e.g. if they would sit at the end of the
18944  * program and through another bug we would manage to jump there, then
18945  * we'd execute beyond program memory otherwise. Returning exception
18946  * code also wouldn't work since we can have subprogs where the dead
18947  * code could be located.
18948  */
18949 static void sanitize_dead_code(struct bpf_verifier_env *env)
18950 {
18951 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
18952 	struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
18953 	struct bpf_insn *insn = env->prog->insnsi;
18954 	const int insn_cnt = env->prog->len;
18955 	int i;
18956 
18957 	for (i = 0; i < insn_cnt; i++) {
18958 		if (aux_data[i].seen)
18959 			continue;
18960 		memcpy(insn + i, &trap, sizeof(trap));
18961 		aux_data[i].zext_dst = false;
18962 	}
18963 }
18964 
18965 static bool insn_is_cond_jump(u8 code)
18966 {
18967 	u8 op;
18968 
18969 	op = BPF_OP(code);
18970 	if (BPF_CLASS(code) == BPF_JMP32)
18971 		return op != BPF_JA;
18972 
18973 	if (BPF_CLASS(code) != BPF_JMP)
18974 		return false;
18975 
18976 	return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
18977 }
18978 
18979 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
18980 {
18981 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
18982 	struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
18983 	struct bpf_insn *insn = env->prog->insnsi;
18984 	const int insn_cnt = env->prog->len;
18985 	int i;
18986 
18987 	for (i = 0; i < insn_cnt; i++, insn++) {
18988 		if (!insn_is_cond_jump(insn->code))
18989 			continue;
18990 
18991 		if (!aux_data[i + 1].seen)
18992 			ja.off = insn->off;
18993 		else if (!aux_data[i + 1 + insn->off].seen)
18994 			ja.off = 0;
18995 		else
18996 			continue;
18997 
18998 		if (bpf_prog_is_offloaded(env->prog->aux))
18999 			bpf_prog_offload_replace_insn(env, i, &ja);
19000 
19001 		memcpy(insn, &ja, sizeof(ja));
19002 	}
19003 }
19004 
19005 static int opt_remove_dead_code(struct bpf_verifier_env *env)
19006 {
19007 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
19008 	int insn_cnt = env->prog->len;
19009 	int i, err;
19010 
19011 	for (i = 0; i < insn_cnt; i++) {
19012 		int j;
19013 
19014 		j = 0;
19015 		while (i + j < insn_cnt && !aux_data[i + j].seen)
19016 			j++;
19017 		if (!j)
19018 			continue;
19019 
19020 		err = verifier_remove_insns(env, i, j);
19021 		if (err)
19022 			return err;
19023 		insn_cnt = env->prog->len;
19024 	}
19025 
19026 	return 0;
19027 }
19028 
19029 static int opt_remove_nops(struct bpf_verifier_env *env)
19030 {
19031 	const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
19032 	struct bpf_insn *insn = env->prog->insnsi;
19033 	int insn_cnt = env->prog->len;
19034 	int i, err;
19035 
19036 	for (i = 0; i < insn_cnt; i++) {
19037 		if (memcmp(&insn[i], &ja, sizeof(ja)))
19038 			continue;
19039 
19040 		err = verifier_remove_insns(env, i, 1);
19041 		if (err)
19042 			return err;
19043 		insn_cnt--;
19044 		i--;
19045 	}
19046 
19047 	return 0;
19048 }
19049 
19050 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,
19051 					 const union bpf_attr *attr)
19052 {
19053 	struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4];
19054 	struct bpf_insn_aux_data *aux = env->insn_aux_data;
19055 	int i, patch_len, delta = 0, len = env->prog->len;
19056 	struct bpf_insn *insns = env->prog->insnsi;
19057 	struct bpf_prog *new_prog;
19058 	bool rnd_hi32;
19059 
19060 	rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32;
19061 	zext_patch[1] = BPF_ZEXT_REG(0);
19062 	rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0);
19063 	rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32);
19064 	rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX);
19065 	for (i = 0; i < len; i++) {
19066 		int adj_idx = i + delta;
19067 		struct bpf_insn insn;
19068 		int load_reg;
19069 
19070 		insn = insns[adj_idx];
19071 		load_reg = insn_def_regno(&insn);
19072 		if (!aux[adj_idx].zext_dst) {
19073 			u8 code, class;
19074 			u32 imm_rnd;
19075 
19076 			if (!rnd_hi32)
19077 				continue;
19078 
19079 			code = insn.code;
19080 			class = BPF_CLASS(code);
19081 			if (load_reg == -1)
19082 				continue;
19083 
19084 			/* NOTE: arg "reg" (the fourth one) is only used for
19085 			 *       BPF_STX + SRC_OP, so it is safe to pass NULL
19086 			 *       here.
19087 			 */
19088 			if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) {
19089 				if (class == BPF_LD &&
19090 				    BPF_MODE(code) == BPF_IMM)
19091 					i++;
19092 				continue;
19093 			}
19094 
19095 			/* ctx load could be transformed into wider load. */
19096 			if (class == BPF_LDX &&
19097 			    aux[adj_idx].ptr_type == PTR_TO_CTX)
19098 				continue;
19099 
19100 			imm_rnd = get_random_u32();
19101 			rnd_hi32_patch[0] = insn;
19102 			rnd_hi32_patch[1].imm = imm_rnd;
19103 			rnd_hi32_patch[3].dst_reg = load_reg;
19104 			patch = rnd_hi32_patch;
19105 			patch_len = 4;
19106 			goto apply_patch_buffer;
19107 		}
19108 
19109 		/* Add in an zero-extend instruction if a) the JIT has requested
19110 		 * it or b) it's a CMPXCHG.
19111 		 *
19112 		 * The latter is because: BPF_CMPXCHG always loads a value into
19113 		 * R0, therefore always zero-extends. However some archs'
19114 		 * equivalent instruction only does this load when the
19115 		 * comparison is successful. This detail of CMPXCHG is
19116 		 * orthogonal to the general zero-extension behaviour of the
19117 		 * CPU, so it's treated independently of bpf_jit_needs_zext.
19118 		 */
19119 		if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn))
19120 			continue;
19121 
19122 		/* Zero-extension is done by the caller. */
19123 		if (bpf_pseudo_kfunc_call(&insn))
19124 			continue;
19125 
19126 		if (WARN_ON(load_reg == -1)) {
19127 			verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n");
19128 			return -EFAULT;
19129 		}
19130 
19131 		zext_patch[0] = insn;
19132 		zext_patch[1].dst_reg = load_reg;
19133 		zext_patch[1].src_reg = load_reg;
19134 		patch = zext_patch;
19135 		patch_len = 2;
19136 apply_patch_buffer:
19137 		new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);
19138 		if (!new_prog)
19139 			return -ENOMEM;
19140 		env->prog = new_prog;
19141 		insns = new_prog->insnsi;
19142 		aux = env->insn_aux_data;
19143 		delta += patch_len - 1;
19144 	}
19145 
19146 	return 0;
19147 }
19148 
19149 /* convert load instructions that access fields of a context type into a
19150  * sequence of instructions that access fields of the underlying structure:
19151  *     struct __sk_buff    -> struct sk_buff
19152  *     struct bpf_sock_ops -> struct sock
19153  */
19154 static int convert_ctx_accesses(struct bpf_verifier_env *env)
19155 {
19156 	const struct bpf_verifier_ops *ops = env->ops;
19157 	int i, cnt, size, ctx_field_size, delta = 0;
19158 	const int insn_cnt = env->prog->len;
19159 	struct bpf_insn insn_buf[16], *insn;
19160 	u32 target_size, size_default, off;
19161 	struct bpf_prog *new_prog;
19162 	enum bpf_access_type type;
19163 	bool is_narrower_load;
19164 
19165 	if (ops->gen_prologue || env->seen_direct_write) {
19166 		if (!ops->gen_prologue) {
19167 			verbose(env, "bpf verifier is misconfigured\n");
19168 			return -EINVAL;
19169 		}
19170 		cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
19171 					env->prog);
19172 		if (cnt >= ARRAY_SIZE(insn_buf)) {
19173 			verbose(env, "bpf verifier is misconfigured\n");
19174 			return -EINVAL;
19175 		} else if (cnt) {
19176 			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
19177 			if (!new_prog)
19178 				return -ENOMEM;
19179 
19180 			env->prog = new_prog;
19181 			delta += cnt - 1;
19182 		}
19183 	}
19184 
19185 	if (bpf_prog_is_offloaded(env->prog->aux))
19186 		return 0;
19187 
19188 	insn = env->prog->insnsi + delta;
19189 
19190 	for (i = 0; i < insn_cnt; i++, insn++) {
19191 		bpf_convert_ctx_access_t convert_ctx_access;
19192 		u8 mode;
19193 
19194 		if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
19195 		    insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
19196 		    insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
19197 		    insn->code == (BPF_LDX | BPF_MEM | BPF_DW) ||
19198 		    insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) ||
19199 		    insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) ||
19200 		    insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) {
19201 			type = BPF_READ;
19202 		} else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
19203 			   insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
19204 			   insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
19205 			   insn->code == (BPF_STX | BPF_MEM | BPF_DW) ||
19206 			   insn->code == (BPF_ST | BPF_MEM | BPF_B) ||
19207 			   insn->code == (BPF_ST | BPF_MEM | BPF_H) ||
19208 			   insn->code == (BPF_ST | BPF_MEM | BPF_W) ||
19209 			   insn->code == (BPF_ST | BPF_MEM | BPF_DW)) {
19210 			type = BPF_WRITE;
19211 		} else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) ||
19212 			    insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) &&
19213 			   env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {
19214 			insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code);
19215 			env->prog->aux->num_exentries++;
19216 			continue;
19217 		} else {
19218 			continue;
19219 		}
19220 
19221 		if (type == BPF_WRITE &&
19222 		    env->insn_aux_data[i + delta].sanitize_stack_spill) {
19223 			struct bpf_insn patch[] = {
19224 				*insn,
19225 				BPF_ST_NOSPEC(),
19226 			};
19227 
19228 			cnt = ARRAY_SIZE(patch);
19229 			new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
19230 			if (!new_prog)
19231 				return -ENOMEM;
19232 
19233 			delta    += cnt - 1;
19234 			env->prog = new_prog;
19235 			insn      = new_prog->insnsi + i + delta;
19236 			continue;
19237 		}
19238 
19239 		switch ((int)env->insn_aux_data[i + delta].ptr_type) {
19240 		case PTR_TO_CTX:
19241 			if (!ops->convert_ctx_access)
19242 				continue;
19243 			convert_ctx_access = ops->convert_ctx_access;
19244 			break;
19245 		case PTR_TO_SOCKET:
19246 		case PTR_TO_SOCK_COMMON:
19247 			convert_ctx_access = bpf_sock_convert_ctx_access;
19248 			break;
19249 		case PTR_TO_TCP_SOCK:
19250 			convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
19251 			break;
19252 		case PTR_TO_XDP_SOCK:
19253 			convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
19254 			break;
19255 		case PTR_TO_BTF_ID:
19256 		case PTR_TO_BTF_ID | PTR_UNTRUSTED:
19257 		/* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike
19258 		 * PTR_TO_BTF_ID, and an active ref_obj_id, but the same cannot
19259 		 * be said once it is marked PTR_UNTRUSTED, hence we must handle
19260 		 * any faults for loads into such types. BPF_WRITE is disallowed
19261 		 * for this case.
19262 		 */
19263 		case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED:
19264 			if (type == BPF_READ) {
19265 				if (BPF_MODE(insn->code) == BPF_MEM)
19266 					insn->code = BPF_LDX | BPF_PROBE_MEM |
19267 						     BPF_SIZE((insn)->code);
19268 				else
19269 					insn->code = BPF_LDX | BPF_PROBE_MEMSX |
19270 						     BPF_SIZE((insn)->code);
19271 				env->prog->aux->num_exentries++;
19272 			}
19273 			continue;
19274 		case PTR_TO_ARENA:
19275 			if (BPF_MODE(insn->code) == BPF_MEMSX) {
19276 				verbose(env, "sign extending loads from arena are not supported yet\n");
19277 				return -EOPNOTSUPP;
19278 			}
19279 			insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code);
19280 			env->prog->aux->num_exentries++;
19281 			continue;
19282 		default:
19283 			continue;
19284 		}
19285 
19286 		ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
19287 		size = BPF_LDST_BYTES(insn);
19288 		mode = BPF_MODE(insn->code);
19289 
19290 		/* If the read access is a narrower load of the field,
19291 		 * convert to a 4/8-byte load, to minimum program type specific
19292 		 * convert_ctx_access changes. If conversion is successful,
19293 		 * we will apply proper mask to the result.
19294 		 */
19295 		is_narrower_load = size < ctx_field_size;
19296 		size_default = bpf_ctx_off_adjust_machine(ctx_field_size);
19297 		off = insn->off;
19298 		if (is_narrower_load) {
19299 			u8 size_code;
19300 
19301 			if (type == BPF_WRITE) {
19302 				verbose(env, "bpf verifier narrow ctx access misconfigured\n");
19303 				return -EINVAL;
19304 			}
19305 
19306 			size_code = BPF_H;
19307 			if (ctx_field_size == 4)
19308 				size_code = BPF_W;
19309 			else if (ctx_field_size == 8)
19310 				size_code = BPF_DW;
19311 
19312 			insn->off = off & ~(size_default - 1);
19313 			insn->code = BPF_LDX | BPF_MEM | size_code;
19314 		}
19315 
19316 		target_size = 0;
19317 		cnt = convert_ctx_access(type, insn, insn_buf, env->prog,
19318 					 &target_size);
19319 		if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) ||
19320 		    (ctx_field_size && !target_size)) {
19321 			verbose(env, "bpf verifier is misconfigured\n");
19322 			return -EINVAL;
19323 		}
19324 
19325 		if (is_narrower_load && size < target_size) {
19326 			u8 shift = bpf_ctx_narrow_access_offset(
19327 				off, size, size_default) * 8;
19328 			if (shift && cnt + 1 >= ARRAY_SIZE(insn_buf)) {
19329 				verbose(env, "bpf verifier narrow ctx load misconfigured\n");
19330 				return -EINVAL;
19331 			}
19332 			if (ctx_field_size <= 4) {
19333 				if (shift)
19334 					insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH,
19335 									insn->dst_reg,
19336 									shift);
19337 				insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
19338 								(1 << size * 8) - 1);
19339 			} else {
19340 				if (shift)
19341 					insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH,
19342 									insn->dst_reg,
19343 									shift);
19344 				insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
19345 								(1ULL << size * 8) - 1);
19346 			}
19347 		}
19348 		if (mode == BPF_MEMSX)
19349 			insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X,
19350 						       insn->dst_reg, insn->dst_reg,
19351 						       size * 8, 0);
19352 
19353 		new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
19354 		if (!new_prog)
19355 			return -ENOMEM;
19356 
19357 		delta += cnt - 1;
19358 
19359 		/* keep walking new program and skip insns we just inserted */
19360 		env->prog = new_prog;
19361 		insn      = new_prog->insnsi + i + delta;
19362 	}
19363 
19364 	return 0;
19365 }
19366 
19367 static int jit_subprogs(struct bpf_verifier_env *env)
19368 {
19369 	struct bpf_prog *prog = env->prog, **func, *tmp;
19370 	int i, j, subprog_start, subprog_end = 0, len, subprog;
19371 	struct bpf_map *map_ptr;
19372 	struct bpf_insn *insn;
19373 	void *old_bpf_func;
19374 	int err, num_exentries;
19375 
19376 	if (env->subprog_cnt <= 1)
19377 		return 0;
19378 
19379 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
19380 		if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
19381 			continue;
19382 
19383 		/* Upon error here we cannot fall back to interpreter but
19384 		 * need a hard reject of the program. Thus -EFAULT is
19385 		 * propagated in any case.
19386 		 */
19387 		subprog = find_subprog(env, i + insn->imm + 1);
19388 		if (subprog < 0) {
19389 			WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
19390 				  i + insn->imm + 1);
19391 			return -EFAULT;
19392 		}
19393 		/* temporarily remember subprog id inside insn instead of
19394 		 * aux_data, since next loop will split up all insns into funcs
19395 		 */
19396 		insn->off = subprog;
19397 		/* remember original imm in case JIT fails and fallback
19398 		 * to interpreter will be needed
19399 		 */
19400 		env->insn_aux_data[i].call_imm = insn->imm;
19401 		/* point imm to __bpf_call_base+1 from JITs point of view */
19402 		insn->imm = 1;
19403 		if (bpf_pseudo_func(insn)) {
19404 #if defined(MODULES_VADDR)
19405 			u64 addr = MODULES_VADDR;
19406 #else
19407 			u64 addr = VMALLOC_START;
19408 #endif
19409 			/* jit (e.g. x86_64) may emit fewer instructions
19410 			 * if it learns a u32 imm is the same as a u64 imm.
19411 			 * Set close enough to possible prog address.
19412 			 */
19413 			insn[0].imm = (u32)addr;
19414 			insn[1].imm = addr >> 32;
19415 		}
19416 	}
19417 
19418 	err = bpf_prog_alloc_jited_linfo(prog);
19419 	if (err)
19420 		goto out_undo_insn;
19421 
19422 	err = -ENOMEM;
19423 	func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
19424 	if (!func)
19425 		goto out_undo_insn;
19426 
19427 	for (i = 0; i < env->subprog_cnt; i++) {
19428 		subprog_start = subprog_end;
19429 		subprog_end = env->subprog_info[i + 1].start;
19430 
19431 		len = subprog_end - subprog_start;
19432 		/* bpf_prog_run() doesn't call subprogs directly,
19433 		 * hence main prog stats include the runtime of subprogs.
19434 		 * subprogs don't have IDs and not reachable via prog_get_next_id
19435 		 * func[i]->stats will never be accessed and stays NULL
19436 		 */
19437 		func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
19438 		if (!func[i])
19439 			goto out_free;
19440 		memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
19441 		       len * sizeof(struct bpf_insn));
19442 		func[i]->type = prog->type;
19443 		func[i]->len = len;
19444 		if (bpf_prog_calc_tag(func[i]))
19445 			goto out_free;
19446 		func[i]->is_func = 1;
19447 		func[i]->sleepable = prog->sleepable;
19448 		func[i]->aux->func_idx = i;
19449 		/* Below members will be freed only at prog->aux */
19450 		func[i]->aux->btf = prog->aux->btf;
19451 		func[i]->aux->func_info = prog->aux->func_info;
19452 		func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;
19453 		func[i]->aux->poke_tab = prog->aux->poke_tab;
19454 		func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;
19455 
19456 		for (j = 0; j < prog->aux->size_poke_tab; j++) {
19457 			struct bpf_jit_poke_descriptor *poke;
19458 
19459 			poke = &prog->aux->poke_tab[j];
19460 			if (poke->insn_idx < subprog_end &&
19461 			    poke->insn_idx >= subprog_start)
19462 				poke->aux = func[i]->aux;
19463 		}
19464 
19465 		func[i]->aux->name[0] = 'F';
19466 		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
19467 		func[i]->jit_requested = 1;
19468 		func[i]->blinding_requested = prog->blinding_requested;
19469 		func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
19470 		func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab;
19471 		func[i]->aux->linfo = prog->aux->linfo;
19472 		func[i]->aux->nr_linfo = prog->aux->nr_linfo;
19473 		func[i]->aux->jited_linfo = prog->aux->jited_linfo;
19474 		func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
19475 		func[i]->aux->arena = prog->aux->arena;
19476 		num_exentries = 0;
19477 		insn = func[i]->insnsi;
19478 		for (j = 0; j < func[i]->len; j++, insn++) {
19479 			if (BPF_CLASS(insn->code) == BPF_LDX &&
19480 			    (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
19481 			     BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
19482 			     BPF_MODE(insn->code) == BPF_PROBE_MEMSX))
19483 				num_exentries++;
19484 			if ((BPF_CLASS(insn->code) == BPF_STX ||
19485 			     BPF_CLASS(insn->code) == BPF_ST) &&
19486 			     BPF_MODE(insn->code) == BPF_PROBE_MEM32)
19487 				num_exentries++;
19488 			if (BPF_CLASS(insn->code) == BPF_STX &&
19489 			     BPF_MODE(insn->code) == BPF_PROBE_ATOMIC)
19490 				num_exentries++;
19491 		}
19492 		func[i]->aux->num_exentries = num_exentries;
19493 		func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
19494 		func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;
19495 		if (!i)
19496 			func[i]->aux->exception_boundary = env->seen_exception;
19497 		func[i] = bpf_int_jit_compile(func[i]);
19498 		if (!func[i]->jited) {
19499 			err = -ENOTSUPP;
19500 			goto out_free;
19501 		}
19502 		cond_resched();
19503 	}
19504 
19505 	/* at this point all bpf functions were successfully JITed
19506 	 * now populate all bpf_calls with correct addresses and
19507 	 * run last pass of JIT
19508 	 */
19509 	for (i = 0; i < env->subprog_cnt; i++) {
19510 		insn = func[i]->insnsi;
19511 		for (j = 0; j < func[i]->len; j++, insn++) {
19512 			if (bpf_pseudo_func(insn)) {
19513 				subprog = insn->off;
19514 				insn[0].imm = (u32)(long)func[subprog]->bpf_func;
19515 				insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32;
19516 				continue;
19517 			}
19518 			if (!bpf_pseudo_call(insn))
19519 				continue;
19520 			subprog = insn->off;
19521 			insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func);
19522 		}
19523 
19524 		/* we use the aux data to keep a list of the start addresses
19525 		 * of the JITed images for each function in the program
19526 		 *
19527 		 * for some architectures, such as powerpc64, the imm field
19528 		 * might not be large enough to hold the offset of the start
19529 		 * address of the callee's JITed image from __bpf_call_base
19530 		 *
19531 		 * in such cases, we can lookup the start address of a callee
19532 		 * by using its subprog id, available from the off field of
19533 		 * the call instruction, as an index for this list
19534 		 */
19535 		func[i]->aux->func = func;
19536 		func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;
19537 		func[i]->aux->real_func_cnt = env->subprog_cnt;
19538 	}
19539 	for (i = 0; i < env->subprog_cnt; i++) {
19540 		old_bpf_func = func[i]->bpf_func;
19541 		tmp = bpf_int_jit_compile(func[i]);
19542 		if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
19543 			verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
19544 			err = -ENOTSUPP;
19545 			goto out_free;
19546 		}
19547 		cond_resched();
19548 	}
19549 
19550 	/* finally lock prog and jit images for all functions and
19551 	 * populate kallsysm. Begin at the first subprogram, since
19552 	 * bpf_prog_load will add the kallsyms for the main program.
19553 	 */
19554 	for (i = 1; i < env->subprog_cnt; i++) {
19555 		err = bpf_prog_lock_ro(func[i]);
19556 		if (err)
19557 			goto out_free;
19558 	}
19559 
19560 	for (i = 1; i < env->subprog_cnt; i++)
19561 		bpf_prog_kallsyms_add(func[i]);
19562 
19563 	/* Last step: make now unused interpreter insns from main
19564 	 * prog consistent for later dump requests, so they can
19565 	 * later look the same as if they were interpreted only.
19566 	 */
19567 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
19568 		if (bpf_pseudo_func(insn)) {
19569 			insn[0].imm = env->insn_aux_data[i].call_imm;
19570 			insn[1].imm = insn->off;
19571 			insn->off = 0;
19572 			continue;
19573 		}
19574 		if (!bpf_pseudo_call(insn))
19575 			continue;
19576 		insn->off = env->insn_aux_data[i].call_imm;
19577 		subprog = find_subprog(env, i + insn->off + 1);
19578 		insn->imm = subprog;
19579 	}
19580 
19581 	prog->jited = 1;
19582 	prog->bpf_func = func[0]->bpf_func;
19583 	prog->jited_len = func[0]->jited_len;
19584 	prog->aux->extable = func[0]->aux->extable;
19585 	prog->aux->num_exentries = func[0]->aux->num_exentries;
19586 	prog->aux->func = func;
19587 	prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;
19588 	prog->aux->real_func_cnt = env->subprog_cnt;
19589 	prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;
19590 	prog->aux->exception_boundary = func[0]->aux->exception_boundary;
19591 	bpf_prog_jit_attempt_done(prog);
19592 	return 0;
19593 out_free:
19594 	/* We failed JIT'ing, so at this point we need to unregister poke
19595 	 * descriptors from subprogs, so that kernel is not attempting to
19596 	 * patch it anymore as we're freeing the subprog JIT memory.
19597 	 */
19598 	for (i = 0; i < prog->aux->size_poke_tab; i++) {
19599 		map_ptr = prog->aux->poke_tab[i].tail_call.map;
19600 		map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);
19601 	}
19602 	/* At this point we're guaranteed that poke descriptors are not
19603 	 * live anymore. We can just unlink its descriptor table as it's
19604 	 * released with the main prog.
19605 	 */
19606 	for (i = 0; i < env->subprog_cnt; i++) {
19607 		if (!func[i])
19608 			continue;
19609 		func[i]->aux->poke_tab = NULL;
19610 		bpf_jit_free(func[i]);
19611 	}
19612 	kfree(func);
19613 out_undo_insn:
19614 	/* cleanup main prog to be interpreted */
19615 	prog->jit_requested = 0;
19616 	prog->blinding_requested = 0;
19617 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
19618 		if (!bpf_pseudo_call(insn))
19619 			continue;
19620 		insn->off = 0;
19621 		insn->imm = env->insn_aux_data[i].call_imm;
19622 	}
19623 	bpf_prog_jit_attempt_done(prog);
19624 	return err;
19625 }
19626 
19627 static int fixup_call_args(struct bpf_verifier_env *env)
19628 {
19629 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
19630 	struct bpf_prog *prog = env->prog;
19631 	struct bpf_insn *insn = prog->insnsi;
19632 	bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
19633 	int i, depth;
19634 #endif
19635 	int err = 0;
19636 
19637 	if (env->prog->jit_requested &&
19638 	    !bpf_prog_is_offloaded(env->prog->aux)) {
19639 		err = jit_subprogs(env);
19640 		if (err == 0)
19641 			return 0;
19642 		if (err == -EFAULT)
19643 			return err;
19644 	}
19645 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
19646 	if (has_kfunc_call) {
19647 		verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
19648 		return -EINVAL;
19649 	}
19650 	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
19651 		/* When JIT fails the progs with bpf2bpf calls and tail_calls
19652 		 * have to be rejected, since interpreter doesn't support them yet.
19653 		 */
19654 		verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
19655 		return -EINVAL;
19656 	}
19657 	for (i = 0; i < prog->len; i++, insn++) {
19658 		if (bpf_pseudo_func(insn)) {
19659 			/* When JIT fails the progs with callback calls
19660 			 * have to be rejected, since interpreter doesn't support them yet.
19661 			 */
19662 			verbose(env, "callbacks are not allowed in non-JITed programs\n");
19663 			return -EINVAL;
19664 		}
19665 
19666 		if (!bpf_pseudo_call(insn))
19667 			continue;
19668 		depth = get_callee_stack_depth(env, insn, i);
19669 		if (depth < 0)
19670 			return depth;
19671 		bpf_patch_call_args(insn, depth);
19672 	}
19673 	err = 0;
19674 #endif
19675 	return err;
19676 }
19677 
19678 /* replace a generic kfunc with a specialized version if necessary */
19679 static void specialize_kfunc(struct bpf_verifier_env *env,
19680 			     u32 func_id, u16 offset, unsigned long *addr)
19681 {
19682 	struct bpf_prog *prog = env->prog;
19683 	bool seen_direct_write;
19684 	void *xdp_kfunc;
19685 	bool is_rdonly;
19686 
19687 	if (bpf_dev_bound_kfunc_id(func_id)) {
19688 		xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id);
19689 		if (xdp_kfunc) {
19690 			*addr = (unsigned long)xdp_kfunc;
19691 			return;
19692 		}
19693 		/* fallback to default kfunc when not supported by netdev */
19694 	}
19695 
19696 	if (offset)
19697 		return;
19698 
19699 	if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
19700 		seen_direct_write = env->seen_direct_write;
19701 		is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
19702 
19703 		if (is_rdonly)
19704 			*addr = (unsigned long)bpf_dynptr_from_skb_rdonly;
19705 
19706 		/* restore env->seen_direct_write to its original value, since
19707 		 * may_access_direct_pkt_data mutates it
19708 		 */
19709 		env->seen_direct_write = seen_direct_write;
19710 	}
19711 }
19712 
19713 static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
19714 					    u16 struct_meta_reg,
19715 					    u16 node_offset_reg,
19716 					    struct bpf_insn *insn,
19717 					    struct bpf_insn *insn_buf,
19718 					    int *cnt)
19719 {
19720 	struct btf_struct_meta *kptr_struct_meta = insn_aux->kptr_struct_meta;
19721 	struct bpf_insn addr[2] = { BPF_LD_IMM64(struct_meta_reg, (long)kptr_struct_meta) };
19722 
19723 	insn_buf[0] = addr[0];
19724 	insn_buf[1] = addr[1];
19725 	insn_buf[2] = BPF_MOV64_IMM(node_offset_reg, insn_aux->insert_off);
19726 	insn_buf[3] = *insn;
19727 	*cnt = 4;
19728 }
19729 
19730 static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
19731 			    struct bpf_insn *insn_buf, int insn_idx, int *cnt)
19732 {
19733 	const struct bpf_kfunc_desc *desc;
19734 
19735 	if (!insn->imm) {
19736 		verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
19737 		return -EINVAL;
19738 	}
19739 
19740 	*cnt = 0;
19741 
19742 	/* insn->imm has the btf func_id. Replace it with an offset relative to
19743 	 * __bpf_call_base, unless the JIT needs to call functions that are
19744 	 * further than 32 bits away (bpf_jit_supports_far_kfunc_call()).
19745 	 */
19746 	desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
19747 	if (!desc) {
19748 		verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
19749 			insn->imm);
19750 		return -EFAULT;
19751 	}
19752 
19753 	if (!bpf_jit_supports_far_kfunc_call())
19754 		insn->imm = BPF_CALL_IMM(desc->addr);
19755 	if (insn->off)
19756 		return 0;
19757 	if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl] ||
19758 	    desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
19759 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
19760 		struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };
19761 		u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size;
19762 
19763 		if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl] && kptr_struct_meta) {
19764 			verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n",
19765 				insn_idx);
19766 			return -EFAULT;
19767 		}
19768 
19769 		insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size);
19770 		insn_buf[1] = addr[0];
19771 		insn_buf[2] = addr[1];
19772 		insn_buf[3] = *insn;
19773 		*cnt = 4;
19774 	} else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl] ||
19775 		   desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] ||
19776 		   desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {
19777 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
19778 		struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };
19779 
19780 		if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] && kptr_struct_meta) {
19781 			verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n",
19782 				insn_idx);
19783 			return -EFAULT;
19784 		}
19785 
19786 		if (desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] &&
19787 		    !kptr_struct_meta) {
19788 			verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n",
19789 				insn_idx);
19790 			return -EFAULT;
19791 		}
19792 
19793 		insn_buf[0] = addr[0];
19794 		insn_buf[1] = addr[1];
19795 		insn_buf[2] = *insn;
19796 		*cnt = 3;
19797 	} else if (desc->func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
19798 		   desc->func_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
19799 		   desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
19800 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
19801 		int struct_meta_reg = BPF_REG_3;
19802 		int node_offset_reg = BPF_REG_4;
19803 
19804 		/* rbtree_add has extra 'less' arg, so args-to-fixup are in diff regs */
19805 		if (desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
19806 			struct_meta_reg = BPF_REG_4;
19807 			node_offset_reg = BPF_REG_5;
19808 		}
19809 
19810 		if (!kptr_struct_meta) {
19811 			verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n",
19812 				insn_idx);
19813 			return -EFAULT;
19814 		}
19815 
19816 		__fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg,
19817 						node_offset_reg, insn, insn_buf, cnt);
19818 	} else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
19819 		   desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
19820 		insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
19821 		*cnt = 1;
19822 	} else if (is_bpf_wq_set_callback_impl_kfunc(desc->func_id)) {
19823 		struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(BPF_REG_4, (long)env->prog->aux) };
19824 
19825 		insn_buf[0] = ld_addrs[0];
19826 		insn_buf[1] = ld_addrs[1];
19827 		insn_buf[2] = *insn;
19828 		*cnt = 3;
19829 	}
19830 	return 0;
19831 }
19832 
19833 /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */
19834 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len)
19835 {
19836 	struct bpf_subprog_info *info = env->subprog_info;
19837 	int cnt = env->subprog_cnt;
19838 	struct bpf_prog *prog;
19839 
19840 	/* We only reserve one slot for hidden subprogs in subprog_info. */
19841 	if (env->hidden_subprog_cnt) {
19842 		verbose(env, "verifier internal error: only one hidden subprog supported\n");
19843 		return -EFAULT;
19844 	}
19845 	/* We're not patching any existing instruction, just appending the new
19846 	 * ones for the hidden subprog. Hence all of the adjustment operations
19847 	 * in bpf_patch_insn_data are no-ops.
19848 	 */
19849 	prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len);
19850 	if (!prog)
19851 		return -ENOMEM;
19852 	env->prog = prog;
19853 	info[cnt + 1].start = info[cnt].start;
19854 	info[cnt].start = prog->len - len + 1;
19855 	env->subprog_cnt++;
19856 	env->hidden_subprog_cnt++;
19857 	return 0;
19858 }
19859 
19860 /* Do various post-verification rewrites in a single program pass.
19861  * These rewrites simplify JIT and interpreter implementations.
19862  */
19863 static int do_misc_fixups(struct bpf_verifier_env *env)
19864 {
19865 	struct bpf_prog *prog = env->prog;
19866 	enum bpf_attach_type eatype = prog->expected_attach_type;
19867 	enum bpf_prog_type prog_type = resolve_prog_type(prog);
19868 	struct bpf_insn *insn = prog->insnsi;
19869 	const struct bpf_func_proto *fn;
19870 	const int insn_cnt = prog->len;
19871 	const struct bpf_map_ops *ops;
19872 	struct bpf_insn_aux_data *aux;
19873 	struct bpf_insn insn_buf[16];
19874 	struct bpf_prog *new_prog;
19875 	struct bpf_map *map_ptr;
19876 	int i, ret, cnt, delta = 0, cur_subprog = 0;
19877 	struct bpf_subprog_info *subprogs = env->subprog_info;
19878 	u16 stack_depth = subprogs[cur_subprog].stack_depth;
19879 	u16 stack_depth_extra = 0;
19880 
19881 	if (env->seen_exception && !env->exception_callback_subprog) {
19882 		struct bpf_insn patch[] = {
19883 			env->prog->insnsi[insn_cnt - 1],
19884 			BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
19885 			BPF_EXIT_INSN(),
19886 		};
19887 
19888 		ret = add_hidden_subprog(env, patch, ARRAY_SIZE(patch));
19889 		if (ret < 0)
19890 			return ret;
19891 		prog = env->prog;
19892 		insn = prog->insnsi;
19893 
19894 		env->exception_callback_subprog = env->subprog_cnt - 1;
19895 		/* Don't update insn_cnt, as add_hidden_subprog always appends insns */
19896 		mark_subprog_exc_cb(env, env->exception_callback_subprog);
19897 	}
19898 
19899 	for (i = 0; i < insn_cnt;) {
19900 		if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->imm) {
19901 			if ((insn->off == BPF_ADDR_SPACE_CAST && insn->imm == 1) ||
19902 			    (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) {
19903 				/* convert to 32-bit mov that clears upper 32-bit */
19904 				insn->code = BPF_ALU | BPF_MOV | BPF_X;
19905 				/* clear off and imm, so it's a normal 'wX = wY' from JIT pov */
19906 				insn->off = 0;
19907 				insn->imm = 0;
19908 			} /* cast from as(0) to as(1) should be handled by JIT */
19909 			goto next_insn;
19910 		}
19911 
19912 		if (env->insn_aux_data[i + delta].needs_zext)
19913 			/* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */
19914 			insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code);
19915 
19916 		/* Make divide-by-zero exceptions impossible. */
19917 		if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
19918 		    insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
19919 		    insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
19920 		    insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
19921 			bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
19922 			bool isdiv = BPF_OP(insn->code) == BPF_DIV;
19923 			struct bpf_insn *patchlet;
19924 			struct bpf_insn chk_and_div[] = {
19925 				/* [R,W]x div 0 -> 0 */
19926 				BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
19927 					     BPF_JNE | BPF_K, insn->src_reg,
19928 					     0, 2, 0),
19929 				BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
19930 				BPF_JMP_IMM(BPF_JA, 0, 0, 1),
19931 				*insn,
19932 			};
19933 			struct bpf_insn chk_and_mod[] = {
19934 				/* [R,W]x mod 0 -> [R,W]x */
19935 				BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
19936 					     BPF_JEQ | BPF_K, insn->src_reg,
19937 					     0, 1 + (is64 ? 0 : 1), 0),
19938 				*insn,
19939 				BPF_JMP_IMM(BPF_JA, 0, 0, 1),
19940 				BPF_MOV32_REG(insn->dst_reg, insn->dst_reg),
19941 			};
19942 
19943 			patchlet = isdiv ? chk_and_div : chk_and_mod;
19944 			cnt = isdiv ? ARRAY_SIZE(chk_and_div) :
19945 				      ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0);
19946 
19947 			new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
19948 			if (!new_prog)
19949 				return -ENOMEM;
19950 
19951 			delta    += cnt - 1;
19952 			env->prog = prog = new_prog;
19953 			insn      = new_prog->insnsi + i + delta;
19954 			goto next_insn;
19955 		}
19956 
19957 		/* Make it impossible to de-reference a userspace address */
19958 		if (BPF_CLASS(insn->code) == BPF_LDX &&
19959 		    (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
19960 		     BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) {
19961 			struct bpf_insn *patch = &insn_buf[0];
19962 			u64 uaddress_limit = bpf_arch_uaddress_limit();
19963 
19964 			if (!uaddress_limit)
19965 				goto next_insn;
19966 
19967 			*patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg);
19968 			if (insn->off)
19969 				*patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off);
19970 			*patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32);
19971 			*patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2);
19972 			*patch++ = *insn;
19973 			*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
19974 			*patch++ = BPF_MOV64_IMM(insn->dst_reg, 0);
19975 
19976 			cnt = patch - insn_buf;
19977 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
19978 			if (!new_prog)
19979 				return -ENOMEM;
19980 
19981 			delta    += cnt - 1;
19982 			env->prog = prog = new_prog;
19983 			insn      = new_prog->insnsi + i + delta;
19984 			goto next_insn;
19985 		}
19986 
19987 		/* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */
19988 		if (BPF_CLASS(insn->code) == BPF_LD &&
19989 		    (BPF_MODE(insn->code) == BPF_ABS ||
19990 		     BPF_MODE(insn->code) == BPF_IND)) {
19991 			cnt = env->ops->gen_ld_abs(insn, insn_buf);
19992 			if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
19993 				verbose(env, "bpf verifier is misconfigured\n");
19994 				return -EINVAL;
19995 			}
19996 
19997 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
19998 			if (!new_prog)
19999 				return -ENOMEM;
20000 
20001 			delta    += cnt - 1;
20002 			env->prog = prog = new_prog;
20003 			insn      = new_prog->insnsi + i + delta;
20004 			goto next_insn;
20005 		}
20006 
20007 		/* Rewrite pointer arithmetic to mitigate speculation attacks. */
20008 		if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
20009 		    insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
20010 			const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
20011 			const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
20012 			struct bpf_insn *patch = &insn_buf[0];
20013 			bool issrc, isneg, isimm;
20014 			u32 off_reg;
20015 
20016 			aux = &env->insn_aux_data[i + delta];
20017 			if (!aux->alu_state ||
20018 			    aux->alu_state == BPF_ALU_NON_POINTER)
20019 				goto next_insn;
20020 
20021 			isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
20022 			issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
20023 				BPF_ALU_SANITIZE_SRC;
20024 			isimm = aux->alu_state & BPF_ALU_IMMEDIATE;
20025 
20026 			off_reg = issrc ? insn->src_reg : insn->dst_reg;
20027 			if (isimm) {
20028 				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
20029 			} else {
20030 				if (isneg)
20031 					*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
20032 				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
20033 				*patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
20034 				*patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
20035 				*patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
20036 				*patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
20037 				*patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg);
20038 			}
20039 			if (!issrc)
20040 				*patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg);
20041 			insn->src_reg = BPF_REG_AX;
20042 			if (isneg)
20043 				insn->code = insn->code == code_add ?
20044 					     code_sub : code_add;
20045 			*patch++ = *insn;
20046 			if (issrc && isneg && !isimm)
20047 				*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
20048 			cnt = patch - insn_buf;
20049 
20050 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20051 			if (!new_prog)
20052 				return -ENOMEM;
20053 
20054 			delta    += cnt - 1;
20055 			env->prog = prog = new_prog;
20056 			insn      = new_prog->insnsi + i + delta;
20057 			goto next_insn;
20058 		}
20059 
20060 		if (is_may_goto_insn(insn)) {
20061 			int stack_off = -stack_depth - 8;
20062 
20063 			stack_depth_extra = 8;
20064 			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off);
20065 			if (insn->off >= 0)
20066 				insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2);
20067 			else
20068 				insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1);
20069 			insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1);
20070 			insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off);
20071 			cnt = 4;
20072 
20073 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20074 			if (!new_prog)
20075 				return -ENOMEM;
20076 
20077 			delta += cnt - 1;
20078 			env->prog = prog = new_prog;
20079 			insn = new_prog->insnsi + i + delta;
20080 			goto next_insn;
20081 		}
20082 
20083 		if (insn->code != (BPF_JMP | BPF_CALL))
20084 			goto next_insn;
20085 		if (insn->src_reg == BPF_PSEUDO_CALL)
20086 			goto next_insn;
20087 		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
20088 			ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);
20089 			if (ret)
20090 				return ret;
20091 			if (cnt == 0)
20092 				goto next_insn;
20093 
20094 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20095 			if (!new_prog)
20096 				return -ENOMEM;
20097 
20098 			delta	 += cnt - 1;
20099 			env->prog = prog = new_prog;
20100 			insn	  = new_prog->insnsi + i + delta;
20101 			goto next_insn;
20102 		}
20103 
20104 		/* Skip inlining the helper call if the JIT does it. */
20105 		if (bpf_jit_inlines_helper_call(insn->imm))
20106 			goto next_insn;
20107 
20108 		if (insn->imm == BPF_FUNC_get_route_realm)
20109 			prog->dst_needed = 1;
20110 		if (insn->imm == BPF_FUNC_get_prandom_u32)
20111 			bpf_user_rnd_init_once();
20112 		if (insn->imm == BPF_FUNC_override_return)
20113 			prog->kprobe_override = 1;
20114 		if (insn->imm == BPF_FUNC_tail_call) {
20115 			/* If we tail call into other programs, we
20116 			 * cannot make any assumptions since they can
20117 			 * be replaced dynamically during runtime in
20118 			 * the program array.
20119 			 */
20120 			prog->cb_access = 1;
20121 			if (!allow_tail_call_in_subprogs(env))
20122 				prog->aux->stack_depth = MAX_BPF_STACK;
20123 			prog->aux->max_pkt_offset = MAX_PACKET_OFF;
20124 
20125 			/* mark bpf_tail_call as different opcode to avoid
20126 			 * conditional branch in the interpreter for every normal
20127 			 * call and to prevent accidental JITing by JIT compiler
20128 			 * that doesn't support bpf_tail_call yet
20129 			 */
20130 			insn->imm = 0;
20131 			insn->code = BPF_JMP | BPF_TAIL_CALL;
20132 
20133 			aux = &env->insn_aux_data[i + delta];
20134 			if (env->bpf_capable && !prog->blinding_requested &&
20135 			    prog->jit_requested &&
20136 			    !bpf_map_key_poisoned(aux) &&
20137 			    !bpf_map_ptr_poisoned(aux) &&
20138 			    !bpf_map_ptr_unpriv(aux)) {
20139 				struct bpf_jit_poke_descriptor desc = {
20140 					.reason = BPF_POKE_REASON_TAIL_CALL,
20141 					.tail_call.map = aux->map_ptr_state.map_ptr,
20142 					.tail_call.key = bpf_map_key_immediate(aux),
20143 					.insn_idx = i + delta,
20144 				};
20145 
20146 				ret = bpf_jit_add_poke_descriptor(prog, &desc);
20147 				if (ret < 0) {
20148 					verbose(env, "adding tail call poke descriptor failed\n");
20149 					return ret;
20150 				}
20151 
20152 				insn->imm = ret + 1;
20153 				goto next_insn;
20154 			}
20155 
20156 			if (!bpf_map_ptr_unpriv(aux))
20157 				goto next_insn;
20158 
20159 			/* instead of changing every JIT dealing with tail_call
20160 			 * emit two extra insns:
20161 			 * if (index >= max_entries) goto out;
20162 			 * index &= array->index_mask;
20163 			 * to avoid out-of-bounds cpu speculation
20164 			 */
20165 			if (bpf_map_ptr_poisoned(aux)) {
20166 				verbose(env, "tail_call abusing map_ptr\n");
20167 				return -EINVAL;
20168 			}
20169 
20170 			map_ptr = aux->map_ptr_state.map_ptr;
20171 			insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
20172 						  map_ptr->max_entries, 2);
20173 			insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
20174 						    container_of(map_ptr,
20175 								 struct bpf_array,
20176 								 map)->index_mask);
20177 			insn_buf[2] = *insn;
20178 			cnt = 3;
20179 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20180 			if (!new_prog)
20181 				return -ENOMEM;
20182 
20183 			delta    += cnt - 1;
20184 			env->prog = prog = new_prog;
20185 			insn      = new_prog->insnsi + i + delta;
20186 			goto next_insn;
20187 		}
20188 
20189 		if (insn->imm == BPF_FUNC_timer_set_callback) {
20190 			/* The verifier will process callback_fn as many times as necessary
20191 			 * with different maps and the register states prepared by
20192 			 * set_timer_callback_state will be accurate.
20193 			 *
20194 			 * The following use case is valid:
20195 			 *   map1 is shared by prog1, prog2, prog3.
20196 			 *   prog1 calls bpf_timer_init for some map1 elements
20197 			 *   prog2 calls bpf_timer_set_callback for some map1 elements.
20198 			 *     Those that were not bpf_timer_init-ed will return -EINVAL.
20199 			 *   prog3 calls bpf_timer_start for some map1 elements.
20200 			 *     Those that were not both bpf_timer_init-ed and
20201 			 *     bpf_timer_set_callback-ed will return -EINVAL.
20202 			 */
20203 			struct bpf_insn ld_addrs[2] = {
20204 				BPF_LD_IMM64(BPF_REG_3, (long)prog->aux),
20205 			};
20206 
20207 			insn_buf[0] = ld_addrs[0];
20208 			insn_buf[1] = ld_addrs[1];
20209 			insn_buf[2] = *insn;
20210 			cnt = 3;
20211 
20212 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20213 			if (!new_prog)
20214 				return -ENOMEM;
20215 
20216 			delta    += cnt - 1;
20217 			env->prog = prog = new_prog;
20218 			insn      = new_prog->insnsi + i + delta;
20219 			goto patch_call_imm;
20220 		}
20221 
20222 		if (is_storage_get_function(insn->imm)) {
20223 			if (!in_sleepable(env) ||
20224 			    env->insn_aux_data[i + delta].storage_get_func_atomic)
20225 				insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC);
20226 			else
20227 				insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL);
20228 			insn_buf[1] = *insn;
20229 			cnt = 2;
20230 
20231 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20232 			if (!new_prog)
20233 				return -ENOMEM;
20234 
20235 			delta += cnt - 1;
20236 			env->prog = prog = new_prog;
20237 			insn = new_prog->insnsi + i + delta;
20238 			goto patch_call_imm;
20239 		}
20240 
20241 		/* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */
20242 		if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {
20243 			/* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data,
20244 			 * bpf_mem_alloc() returns a ptr to the percpu data ptr.
20245 			 */
20246 			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
20247 			insn_buf[1] = *insn;
20248 			cnt = 2;
20249 
20250 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20251 			if (!new_prog)
20252 				return -ENOMEM;
20253 
20254 			delta += cnt - 1;
20255 			env->prog = prog = new_prog;
20256 			insn = new_prog->insnsi + i + delta;
20257 			goto patch_call_imm;
20258 		}
20259 
20260 		/* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup
20261 		 * and other inlining handlers are currently limited to 64 bit
20262 		 * only.
20263 		 */
20264 		if (prog->jit_requested && BITS_PER_LONG == 64 &&
20265 		    (insn->imm == BPF_FUNC_map_lookup_elem ||
20266 		     insn->imm == BPF_FUNC_map_update_elem ||
20267 		     insn->imm == BPF_FUNC_map_delete_elem ||
20268 		     insn->imm == BPF_FUNC_map_push_elem   ||
20269 		     insn->imm == BPF_FUNC_map_pop_elem    ||
20270 		     insn->imm == BPF_FUNC_map_peek_elem   ||
20271 		     insn->imm == BPF_FUNC_redirect_map    ||
20272 		     insn->imm == BPF_FUNC_for_each_map_elem ||
20273 		     insn->imm == BPF_FUNC_map_lookup_percpu_elem)) {
20274 			aux = &env->insn_aux_data[i + delta];
20275 			if (bpf_map_ptr_poisoned(aux))
20276 				goto patch_call_imm;
20277 
20278 			map_ptr = aux->map_ptr_state.map_ptr;
20279 			ops = map_ptr->ops;
20280 			if (insn->imm == BPF_FUNC_map_lookup_elem &&
20281 			    ops->map_gen_lookup) {
20282 				cnt = ops->map_gen_lookup(map_ptr, insn_buf);
20283 				if (cnt == -EOPNOTSUPP)
20284 					goto patch_map_ops_generic;
20285 				if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) {
20286 					verbose(env, "bpf verifier is misconfigured\n");
20287 					return -EINVAL;
20288 				}
20289 
20290 				new_prog = bpf_patch_insn_data(env, i + delta,
20291 							       insn_buf, cnt);
20292 				if (!new_prog)
20293 					return -ENOMEM;
20294 
20295 				delta    += cnt - 1;
20296 				env->prog = prog = new_prog;
20297 				insn      = new_prog->insnsi + i + delta;
20298 				goto next_insn;
20299 			}
20300 
20301 			BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
20302 				     (void *(*)(struct bpf_map *map, void *key))NULL));
20303 			BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
20304 				     (long (*)(struct bpf_map *map, void *key))NULL));
20305 			BUILD_BUG_ON(!__same_type(ops->map_update_elem,
20306 				     (long (*)(struct bpf_map *map, void *key, void *value,
20307 					      u64 flags))NULL));
20308 			BUILD_BUG_ON(!__same_type(ops->map_push_elem,
20309 				     (long (*)(struct bpf_map *map, void *value,
20310 					      u64 flags))NULL));
20311 			BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
20312 				     (long (*)(struct bpf_map *map, void *value))NULL));
20313 			BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
20314 				     (long (*)(struct bpf_map *map, void *value))NULL));
20315 			BUILD_BUG_ON(!__same_type(ops->map_redirect,
20316 				     (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL));
20317 			BUILD_BUG_ON(!__same_type(ops->map_for_each_callback,
20318 				     (long (*)(struct bpf_map *map,
20319 					      bpf_callback_t callback_fn,
20320 					      void *callback_ctx,
20321 					      u64 flags))NULL));
20322 			BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem,
20323 				     (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL));
20324 
20325 patch_map_ops_generic:
20326 			switch (insn->imm) {
20327 			case BPF_FUNC_map_lookup_elem:
20328 				insn->imm = BPF_CALL_IMM(ops->map_lookup_elem);
20329 				goto next_insn;
20330 			case BPF_FUNC_map_update_elem:
20331 				insn->imm = BPF_CALL_IMM(ops->map_update_elem);
20332 				goto next_insn;
20333 			case BPF_FUNC_map_delete_elem:
20334 				insn->imm = BPF_CALL_IMM(ops->map_delete_elem);
20335 				goto next_insn;
20336 			case BPF_FUNC_map_push_elem:
20337 				insn->imm = BPF_CALL_IMM(ops->map_push_elem);
20338 				goto next_insn;
20339 			case BPF_FUNC_map_pop_elem:
20340 				insn->imm = BPF_CALL_IMM(ops->map_pop_elem);
20341 				goto next_insn;
20342 			case BPF_FUNC_map_peek_elem:
20343 				insn->imm = BPF_CALL_IMM(ops->map_peek_elem);
20344 				goto next_insn;
20345 			case BPF_FUNC_redirect_map:
20346 				insn->imm = BPF_CALL_IMM(ops->map_redirect);
20347 				goto next_insn;
20348 			case BPF_FUNC_for_each_map_elem:
20349 				insn->imm = BPF_CALL_IMM(ops->map_for_each_callback);
20350 				goto next_insn;
20351 			case BPF_FUNC_map_lookup_percpu_elem:
20352 				insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem);
20353 				goto next_insn;
20354 			}
20355 
20356 			goto patch_call_imm;
20357 		}
20358 
20359 		/* Implement bpf_jiffies64 inline. */
20360 		if (prog->jit_requested && BITS_PER_LONG == 64 &&
20361 		    insn->imm == BPF_FUNC_jiffies64) {
20362 			struct bpf_insn ld_jiffies_addr[2] = {
20363 				BPF_LD_IMM64(BPF_REG_0,
20364 					     (unsigned long)&jiffies),
20365 			};
20366 
20367 			insn_buf[0] = ld_jiffies_addr[0];
20368 			insn_buf[1] = ld_jiffies_addr[1];
20369 			insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0,
20370 						  BPF_REG_0, 0);
20371 			cnt = 3;
20372 
20373 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
20374 						       cnt);
20375 			if (!new_prog)
20376 				return -ENOMEM;
20377 
20378 			delta    += cnt - 1;
20379 			env->prog = prog = new_prog;
20380 			insn      = new_prog->insnsi + i + delta;
20381 			goto next_insn;
20382 		}
20383 
20384 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
20385 		/* Implement bpf_get_smp_processor_id() inline. */
20386 		if (insn->imm == BPF_FUNC_get_smp_processor_id &&
20387 		    prog->jit_requested && bpf_jit_supports_percpu_insn()) {
20388 			/* BPF_FUNC_get_smp_processor_id inlining is an
20389 			 * optimization, so if pcpu_hot.cpu_number is ever
20390 			 * changed in some incompatible and hard to support
20391 			 * way, it's fine to back out this inlining logic
20392 			 */
20393 			insn_buf[0] = BPF_MOV32_IMM(BPF_REG_0, (u32)(unsigned long)&pcpu_hot.cpu_number);
20394 			insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
20395 			insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0);
20396 			cnt = 3;
20397 
20398 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20399 			if (!new_prog)
20400 				return -ENOMEM;
20401 
20402 			delta    += cnt - 1;
20403 			env->prog = prog = new_prog;
20404 			insn      = new_prog->insnsi + i + delta;
20405 			goto next_insn;
20406 		}
20407 #endif
20408 		/* Implement bpf_get_func_arg inline. */
20409 		if (prog_type == BPF_PROG_TYPE_TRACING &&
20410 		    insn->imm == BPF_FUNC_get_func_arg) {
20411 			/* Load nr_args from ctx - 8 */
20412 			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
20413 			insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6);
20414 			insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3);
20415 			insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1);
20416 			insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0);
20417 			insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
20418 			insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0);
20419 			insn_buf[7] = BPF_JMP_A(1);
20420 			insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
20421 			cnt = 9;
20422 
20423 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20424 			if (!new_prog)
20425 				return -ENOMEM;
20426 
20427 			delta    += cnt - 1;
20428 			env->prog = prog = new_prog;
20429 			insn      = new_prog->insnsi + i + delta;
20430 			goto next_insn;
20431 		}
20432 
20433 		/* Implement bpf_get_func_ret inline. */
20434 		if (prog_type == BPF_PROG_TYPE_TRACING &&
20435 		    insn->imm == BPF_FUNC_get_func_ret) {
20436 			if (eatype == BPF_TRACE_FEXIT ||
20437 			    eatype == BPF_MODIFY_RETURN) {
20438 				/* Load nr_args from ctx - 8 */
20439 				insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
20440 				insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
20441 				insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);
20442 				insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
20443 				insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0);
20444 				insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0);
20445 				cnt = 6;
20446 			} else {
20447 				insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP);
20448 				cnt = 1;
20449 			}
20450 
20451 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20452 			if (!new_prog)
20453 				return -ENOMEM;
20454 
20455 			delta    += cnt - 1;
20456 			env->prog = prog = new_prog;
20457 			insn      = new_prog->insnsi + i + delta;
20458 			goto next_insn;
20459 		}
20460 
20461 		/* Implement get_func_arg_cnt inline. */
20462 		if (prog_type == BPF_PROG_TYPE_TRACING &&
20463 		    insn->imm == BPF_FUNC_get_func_arg_cnt) {
20464 			/* Load nr_args from ctx - 8 */
20465 			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
20466 
20467 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
20468 			if (!new_prog)
20469 				return -ENOMEM;
20470 
20471 			env->prog = prog = new_prog;
20472 			insn      = new_prog->insnsi + i + delta;
20473 			goto next_insn;
20474 		}
20475 
20476 		/* Implement bpf_get_func_ip inline. */
20477 		if (prog_type == BPF_PROG_TYPE_TRACING &&
20478 		    insn->imm == BPF_FUNC_get_func_ip) {
20479 			/* Load IP address from ctx - 16 */
20480 			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
20481 
20482 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
20483 			if (!new_prog)
20484 				return -ENOMEM;
20485 
20486 			env->prog = prog = new_prog;
20487 			insn      = new_prog->insnsi + i + delta;
20488 			goto next_insn;
20489 		}
20490 
20491 		/* Implement bpf_get_branch_snapshot inline. */
20492 		if (IS_ENABLED(CONFIG_PERF_EVENTS) &&
20493 		    prog->jit_requested && BITS_PER_LONG == 64 &&
20494 		    insn->imm == BPF_FUNC_get_branch_snapshot) {
20495 			/* We are dealing with the following func protos:
20496 			 * u64 bpf_get_branch_snapshot(void *buf, u32 size, u64 flags);
20497 			 * int perf_snapshot_branch_stack(struct perf_branch_entry *entries, u32 cnt);
20498 			 */
20499 			const u32 br_entry_size = sizeof(struct perf_branch_entry);
20500 
20501 			/* struct perf_branch_entry is part of UAPI and is
20502 			 * used as an array element, so extremely unlikely to
20503 			 * ever grow or shrink
20504 			 */
20505 			BUILD_BUG_ON(br_entry_size != 24);
20506 
20507 			/* if (unlikely(flags)) return -EINVAL */
20508 			insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7);
20509 
20510 			/* Transform size (bytes) into number of entries (cnt = size / 24).
20511 			 * But to avoid expensive division instruction, we implement
20512 			 * divide-by-3 through multiplication, followed by further
20513 			 * division by 8 through 3-bit right shift.
20514 			 * Refer to book "Hacker's Delight, 2nd ed." by Henry S. Warren, Jr.,
20515 			 * p. 227, chapter "Unsigned Division by 3" for details and proofs.
20516 			 *
20517 			 * N / 3 <=> M * N / 2^33, where M = (2^33 + 1) / 3 = 0xaaaaaaab.
20518 			 */
20519 			insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab);
20520 			insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0);
20521 			insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36);
20522 
20523 			/* call perf_snapshot_branch_stack implementation */
20524 			insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack));
20525 			/* if (entry_cnt == 0) return -ENOENT */
20526 			insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4);
20527 			/* return entry_cnt * sizeof(struct perf_branch_entry) */
20528 			insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size);
20529 			insn_buf[7] = BPF_JMP_A(3);
20530 			/* return -EINVAL; */
20531 			insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
20532 			insn_buf[9] = BPF_JMP_A(1);
20533 			/* return -ENOENT; */
20534 			insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT);
20535 			cnt = 11;
20536 
20537 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20538 			if (!new_prog)
20539 				return -ENOMEM;
20540 
20541 			delta    += cnt - 1;
20542 			env->prog = prog = new_prog;
20543 			insn      = new_prog->insnsi + i + delta;
20544 			continue;
20545 		}
20546 
20547 		/* Implement bpf_kptr_xchg inline */
20548 		if (prog->jit_requested && BITS_PER_LONG == 64 &&
20549 		    insn->imm == BPF_FUNC_kptr_xchg &&
20550 		    bpf_jit_supports_ptr_xchg()) {
20551 			insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2);
20552 			insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0);
20553 			cnt = 2;
20554 
20555 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
20556 			if (!new_prog)
20557 				return -ENOMEM;
20558 
20559 			delta    += cnt - 1;
20560 			env->prog = prog = new_prog;
20561 			insn      = new_prog->insnsi + i + delta;
20562 			goto next_insn;
20563 		}
20564 patch_call_imm:
20565 		fn = env->ops->get_func_proto(insn->imm, env->prog);
20566 		/* all functions that have prototype and verifier allowed
20567 		 * programs to call them, must be real in-kernel functions
20568 		 */
20569 		if (!fn->func) {
20570 			verbose(env,
20571 				"kernel subsystem misconfigured func %s#%d\n",
20572 				func_id_name(insn->imm), insn->imm);
20573 			return -EFAULT;
20574 		}
20575 		insn->imm = fn->func - __bpf_call_base;
20576 next_insn:
20577 		if (subprogs[cur_subprog + 1].start == i + delta + 1) {
20578 			subprogs[cur_subprog].stack_depth += stack_depth_extra;
20579 			subprogs[cur_subprog].stack_extra = stack_depth_extra;
20580 			cur_subprog++;
20581 			stack_depth = subprogs[cur_subprog].stack_depth;
20582 			stack_depth_extra = 0;
20583 		}
20584 		i++;
20585 		insn++;
20586 	}
20587 
20588 	env->prog->aux->stack_depth = subprogs[0].stack_depth;
20589 	for (i = 0; i < env->subprog_cnt; i++) {
20590 		int subprog_start = subprogs[i].start;
20591 		int stack_slots = subprogs[i].stack_extra / 8;
20592 
20593 		if (!stack_slots)
20594 			continue;
20595 		if (stack_slots > 1) {
20596 			verbose(env, "verifier bug: stack_slots supports may_goto only\n");
20597 			return -EFAULT;
20598 		}
20599 
20600 		/* Add ST insn to subprog prologue to init extra stack */
20601 		insn_buf[0] = BPF_ST_MEM(BPF_DW, BPF_REG_FP,
20602 					 -subprogs[i].stack_depth, BPF_MAX_LOOPS);
20603 		/* Copy first actual insn to preserve it */
20604 		insn_buf[1] = env->prog->insnsi[subprog_start];
20605 
20606 		new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, 2);
20607 		if (!new_prog)
20608 			return -ENOMEM;
20609 		env->prog = prog = new_prog;
20610 		/*
20611 		 * If may_goto is a first insn of a prog there could be a jmp
20612 		 * insn that points to it, hence adjust all such jmps to point
20613 		 * to insn after BPF_ST that inits may_goto count.
20614 		 * Adjustment will succeed because bpf_patch_insn_data() didn't fail.
20615 		 */
20616 		WARN_ON(adjust_jmp_off(env->prog, subprog_start, 1));
20617 	}
20618 
20619 	/* Since poke tab is now finalized, publish aux to tracker. */
20620 	for (i = 0; i < prog->aux->size_poke_tab; i++) {
20621 		map_ptr = prog->aux->poke_tab[i].tail_call.map;
20622 		if (!map_ptr->ops->map_poke_track ||
20623 		    !map_ptr->ops->map_poke_untrack ||
20624 		    !map_ptr->ops->map_poke_run) {
20625 			verbose(env, "bpf verifier is misconfigured\n");
20626 			return -EINVAL;
20627 		}
20628 
20629 		ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);
20630 		if (ret < 0) {
20631 			verbose(env, "tracking tail call prog failed\n");
20632 			return ret;
20633 		}
20634 	}
20635 
20636 	sort_kfunc_descs_by_imm_off(env->prog);
20637 
20638 	return 0;
20639 }
20640 
20641 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env,
20642 					int position,
20643 					s32 stack_base,
20644 					u32 callback_subprogno,
20645 					u32 *cnt)
20646 {
20647 	s32 r6_offset = stack_base + 0 * BPF_REG_SIZE;
20648 	s32 r7_offset = stack_base + 1 * BPF_REG_SIZE;
20649 	s32 r8_offset = stack_base + 2 * BPF_REG_SIZE;
20650 	int reg_loop_max = BPF_REG_6;
20651 	int reg_loop_cnt = BPF_REG_7;
20652 	int reg_loop_ctx = BPF_REG_8;
20653 
20654 	struct bpf_prog *new_prog;
20655 	u32 callback_start;
20656 	u32 call_insn_offset;
20657 	s32 callback_offset;
20658 
20659 	/* This represents an inlined version of bpf_iter.c:bpf_loop,
20660 	 * be careful to modify this code in sync.
20661 	 */
20662 	struct bpf_insn insn_buf[] = {
20663 		/* Return error and jump to the end of the patch if
20664 		 * expected number of iterations is too big.
20665 		 */
20666 		BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2),
20667 		BPF_MOV32_IMM(BPF_REG_0, -E2BIG),
20668 		BPF_JMP_IMM(BPF_JA, 0, 0, 16),
20669 		/* spill R6, R7, R8 to use these as loop vars */
20670 		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset),
20671 		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset),
20672 		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset),
20673 		/* initialize loop vars */
20674 		BPF_MOV64_REG(reg_loop_max, BPF_REG_1),
20675 		BPF_MOV32_IMM(reg_loop_cnt, 0),
20676 		BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3),
20677 		/* loop header,
20678 		 * if reg_loop_cnt >= reg_loop_max skip the loop body
20679 		 */
20680 		BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5),
20681 		/* callback call,
20682 		 * correct callback offset would be set after patching
20683 		 */
20684 		BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt),
20685 		BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx),
20686 		BPF_CALL_REL(0),
20687 		/* increment loop counter */
20688 		BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1),
20689 		/* jump to loop header if callback returned 0 */
20690 		BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6),
20691 		/* return value of bpf_loop,
20692 		 * set R0 to the number of iterations
20693 		 */
20694 		BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt),
20695 		/* restore original values of R6, R7, R8 */
20696 		BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset),
20697 		BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset),
20698 		BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset),
20699 	};
20700 
20701 	*cnt = ARRAY_SIZE(insn_buf);
20702 	new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt);
20703 	if (!new_prog)
20704 		return new_prog;
20705 
20706 	/* callback start is known only after patching */
20707 	callback_start = env->subprog_info[callback_subprogno].start;
20708 	/* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */
20709 	call_insn_offset = position + 12;
20710 	callback_offset = callback_start - call_insn_offset - 1;
20711 	new_prog->insnsi[call_insn_offset].imm = callback_offset;
20712 
20713 	return new_prog;
20714 }
20715 
20716 static bool is_bpf_loop_call(struct bpf_insn *insn)
20717 {
20718 	return insn->code == (BPF_JMP | BPF_CALL) &&
20719 		insn->src_reg == 0 &&
20720 		insn->imm == BPF_FUNC_loop;
20721 }
20722 
20723 /* For all sub-programs in the program (including main) check
20724  * insn_aux_data to see if there are bpf_loop calls that require
20725  * inlining. If such calls are found the calls are replaced with a
20726  * sequence of instructions produced by `inline_bpf_loop` function and
20727  * subprog stack_depth is increased by the size of 3 registers.
20728  * This stack space is used to spill values of the R6, R7, R8.  These
20729  * registers are used to store the loop bound, counter and context
20730  * variables.
20731  */
20732 static int optimize_bpf_loop(struct bpf_verifier_env *env)
20733 {
20734 	struct bpf_subprog_info *subprogs = env->subprog_info;
20735 	int i, cur_subprog = 0, cnt, delta = 0;
20736 	struct bpf_insn *insn = env->prog->insnsi;
20737 	int insn_cnt = env->prog->len;
20738 	u16 stack_depth = subprogs[cur_subprog].stack_depth;
20739 	u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth;
20740 	u16 stack_depth_extra = 0;
20741 
20742 	for (i = 0; i < insn_cnt; i++, insn++) {
20743 		struct bpf_loop_inline_state *inline_state =
20744 			&env->insn_aux_data[i + delta].loop_inline_state;
20745 
20746 		if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) {
20747 			struct bpf_prog *new_prog;
20748 
20749 			stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup;
20750 			new_prog = inline_bpf_loop(env,
20751 						   i + delta,
20752 						   -(stack_depth + stack_depth_extra),
20753 						   inline_state->callback_subprogno,
20754 						   &cnt);
20755 			if (!new_prog)
20756 				return -ENOMEM;
20757 
20758 			delta     += cnt - 1;
20759 			env->prog  = new_prog;
20760 			insn       = new_prog->insnsi + i + delta;
20761 		}
20762 
20763 		if (subprogs[cur_subprog + 1].start == i + delta + 1) {
20764 			subprogs[cur_subprog].stack_depth += stack_depth_extra;
20765 			cur_subprog++;
20766 			stack_depth = subprogs[cur_subprog].stack_depth;
20767 			stack_depth_roundup = round_up(stack_depth, 8) - stack_depth;
20768 			stack_depth_extra = 0;
20769 		}
20770 	}
20771 
20772 	env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
20773 
20774 	return 0;
20775 }
20776 
20777 static void free_states(struct bpf_verifier_env *env)
20778 {
20779 	struct bpf_verifier_state_list *sl, *sln;
20780 	int i;
20781 
20782 	sl = env->free_list;
20783 	while (sl) {
20784 		sln = sl->next;
20785 		free_verifier_state(&sl->state, false);
20786 		kfree(sl);
20787 		sl = sln;
20788 	}
20789 	env->free_list = NULL;
20790 
20791 	if (!env->explored_states)
20792 		return;
20793 
20794 	for (i = 0; i < state_htab_size(env); i++) {
20795 		sl = env->explored_states[i];
20796 
20797 		while (sl) {
20798 			sln = sl->next;
20799 			free_verifier_state(&sl->state, false);
20800 			kfree(sl);
20801 			sl = sln;
20802 		}
20803 		env->explored_states[i] = NULL;
20804 	}
20805 }
20806 
20807 static int do_check_common(struct bpf_verifier_env *env, int subprog)
20808 {
20809 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
20810 	struct bpf_subprog_info *sub = subprog_info(env, subprog);
20811 	struct bpf_verifier_state *state;
20812 	struct bpf_reg_state *regs;
20813 	int ret, i;
20814 
20815 	env->prev_linfo = NULL;
20816 	env->pass_cnt++;
20817 
20818 	state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL);
20819 	if (!state)
20820 		return -ENOMEM;
20821 	state->curframe = 0;
20822 	state->speculative = false;
20823 	state->branches = 1;
20824 	state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
20825 	if (!state->frame[0]) {
20826 		kfree(state);
20827 		return -ENOMEM;
20828 	}
20829 	env->cur_state = state;
20830 	init_func_state(env, state->frame[0],
20831 			BPF_MAIN_FUNC /* callsite */,
20832 			0 /* frameno */,
20833 			subprog);
20834 	state->first_insn_idx = env->subprog_info[subprog].start;
20835 	state->last_insn_idx = -1;
20836 
20837 	regs = state->frame[state->curframe]->regs;
20838 	if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {
20839 		const char *sub_name = subprog_name(env, subprog);
20840 		struct bpf_subprog_arg_info *arg;
20841 		struct bpf_reg_state *reg;
20842 
20843 		verbose(env, "Validating %s() func#%d...\n", sub_name, subprog);
20844 		ret = btf_prepare_func_args(env, subprog);
20845 		if (ret)
20846 			goto out;
20847 
20848 		if (subprog_is_exc_cb(env, subprog)) {
20849 			state->frame[0]->in_exception_callback_fn = true;
20850 			/* We have already ensured that the callback returns an integer, just
20851 			 * like all global subprogs. We need to determine it only has a single
20852 			 * scalar argument.
20853 			 */
20854 			if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) {
20855 				verbose(env, "exception cb only supports single integer argument\n");
20856 				ret = -EINVAL;
20857 				goto out;
20858 			}
20859 		}
20860 		for (i = BPF_REG_1; i <= sub->arg_cnt; i++) {
20861 			arg = &sub->args[i - BPF_REG_1];
20862 			reg = &regs[i];
20863 
20864 			if (arg->arg_type == ARG_PTR_TO_CTX) {
20865 				reg->type = PTR_TO_CTX;
20866 				mark_reg_known_zero(env, regs, i);
20867 			} else if (arg->arg_type == ARG_ANYTHING) {
20868 				reg->type = SCALAR_VALUE;
20869 				mark_reg_unknown(env, regs, i);
20870 			} else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) {
20871 				/* assume unspecial LOCAL dynptr type */
20872 				__mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen);
20873 			} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
20874 				reg->type = PTR_TO_MEM;
20875 				if (arg->arg_type & PTR_MAYBE_NULL)
20876 					reg->type |= PTR_MAYBE_NULL;
20877 				mark_reg_known_zero(env, regs, i);
20878 				reg->mem_size = arg->mem_size;
20879 				reg->id = ++env->id_gen;
20880 			} else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
20881 				reg->type = PTR_TO_BTF_ID;
20882 				if (arg->arg_type & PTR_MAYBE_NULL)
20883 					reg->type |= PTR_MAYBE_NULL;
20884 				if (arg->arg_type & PTR_UNTRUSTED)
20885 					reg->type |= PTR_UNTRUSTED;
20886 				if (arg->arg_type & PTR_TRUSTED)
20887 					reg->type |= PTR_TRUSTED;
20888 				mark_reg_known_zero(env, regs, i);
20889 				reg->btf = bpf_get_btf_vmlinux(); /* can't fail at this point */
20890 				reg->btf_id = arg->btf_id;
20891 				reg->id = ++env->id_gen;
20892 			} else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) {
20893 				/* caller can pass either PTR_TO_ARENA or SCALAR */
20894 				mark_reg_unknown(env, regs, i);
20895 			} else {
20896 				WARN_ONCE(1, "BUG: unhandled arg#%d type %d\n",
20897 					  i - BPF_REG_1, arg->arg_type);
20898 				ret = -EFAULT;
20899 				goto out;
20900 			}
20901 		}
20902 	} else {
20903 		/* if main BPF program has associated BTF info, validate that
20904 		 * it's matching expected signature, and otherwise mark BTF
20905 		 * info for main program as unreliable
20906 		 */
20907 		if (env->prog->aux->func_info_aux) {
20908 			ret = btf_prepare_func_args(env, 0);
20909 			if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX)
20910 				env->prog->aux->func_info_aux[0].unreliable = true;
20911 		}
20912 
20913 		/* 1st arg to a function */
20914 		regs[BPF_REG_1].type = PTR_TO_CTX;
20915 		mark_reg_known_zero(env, regs, BPF_REG_1);
20916 	}
20917 
20918 	ret = do_check(env);
20919 out:
20920 	/* check for NULL is necessary, since cur_state can be freed inside
20921 	 * do_check() under memory pressure.
20922 	 */
20923 	if (env->cur_state) {
20924 		free_verifier_state(env->cur_state, true);
20925 		env->cur_state = NULL;
20926 	}
20927 	while (!pop_stack(env, NULL, NULL, false));
20928 	if (!ret && pop_log)
20929 		bpf_vlog_reset(&env->log, 0);
20930 	free_states(env);
20931 	return ret;
20932 }
20933 
20934 /* Lazily verify all global functions based on their BTF, if they are called
20935  * from main BPF program or any of subprograms transitively.
20936  * BPF global subprogs called from dead code are not validated.
20937  * All callable global functions must pass verification.
20938  * Otherwise the whole program is rejected.
20939  * Consider:
20940  * int bar(int);
20941  * int foo(int f)
20942  * {
20943  *    return bar(f);
20944  * }
20945  * int bar(int b)
20946  * {
20947  *    ...
20948  * }
20949  * foo() will be verified first for R1=any_scalar_value. During verification it
20950  * will be assumed that bar() already verified successfully and call to bar()
20951  * from foo() will be checked for type match only. Later bar() will be verified
20952  * independently to check that it's safe for R1=any_scalar_value.
20953  */
20954 static int do_check_subprogs(struct bpf_verifier_env *env)
20955 {
20956 	struct bpf_prog_aux *aux = env->prog->aux;
20957 	struct bpf_func_info_aux *sub_aux;
20958 	int i, ret, new_cnt;
20959 
20960 	if (!aux->func_info)
20961 		return 0;
20962 
20963 	/* exception callback is presumed to be always called */
20964 	if (env->exception_callback_subprog)
20965 		subprog_aux(env, env->exception_callback_subprog)->called = true;
20966 
20967 again:
20968 	new_cnt = 0;
20969 	for (i = 1; i < env->subprog_cnt; i++) {
20970 		if (!subprog_is_global(env, i))
20971 			continue;
20972 
20973 		sub_aux = subprog_aux(env, i);
20974 		if (!sub_aux->called || sub_aux->verified)
20975 			continue;
20976 
20977 		env->insn_idx = env->subprog_info[i].start;
20978 		WARN_ON_ONCE(env->insn_idx == 0);
20979 		ret = do_check_common(env, i);
20980 		if (ret) {
20981 			return ret;
20982 		} else if (env->log.level & BPF_LOG_LEVEL) {
20983 			verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n",
20984 				i, subprog_name(env, i));
20985 		}
20986 
20987 		/* We verified new global subprog, it might have called some
20988 		 * more global subprogs that we haven't verified yet, so we
20989 		 * need to do another pass over subprogs to verify those.
20990 		 */
20991 		sub_aux->verified = true;
20992 		new_cnt++;
20993 	}
20994 
20995 	/* We can't loop forever as we verify at least one global subprog on
20996 	 * each pass.
20997 	 */
20998 	if (new_cnt)
20999 		goto again;
21000 
21001 	return 0;
21002 }
21003 
21004 static int do_check_main(struct bpf_verifier_env *env)
21005 {
21006 	int ret;
21007 
21008 	env->insn_idx = 0;
21009 	ret = do_check_common(env, 0);
21010 	if (!ret)
21011 		env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
21012 	return ret;
21013 }
21014 
21015 
21016 static void print_verification_stats(struct bpf_verifier_env *env)
21017 {
21018 	int i;
21019 
21020 	if (env->log.level & BPF_LOG_STATS) {
21021 		verbose(env, "verification time %lld usec\n",
21022 			div_u64(env->verification_time, 1000));
21023 		verbose(env, "stack depth ");
21024 		for (i = 0; i < env->subprog_cnt; i++) {
21025 			u32 depth = env->subprog_info[i].stack_depth;
21026 
21027 			verbose(env, "%d", depth);
21028 			if (i + 1 < env->subprog_cnt)
21029 				verbose(env, "+");
21030 		}
21031 		verbose(env, "\n");
21032 	}
21033 	verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
21034 		"total_states %d peak_states %d mark_read %d\n",
21035 		env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,
21036 		env->max_states_per_insn, env->total_states,
21037 		env->peak_states, env->longest_mark_read_walk);
21038 }
21039 
21040 static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
21041 {
21042 	const struct btf_type *t, *func_proto;
21043 	const struct bpf_struct_ops_desc *st_ops_desc;
21044 	const struct bpf_struct_ops *st_ops;
21045 	const struct btf_member *member;
21046 	struct bpf_prog *prog = env->prog;
21047 	u32 btf_id, member_idx;
21048 	struct btf *btf;
21049 	const char *mname;
21050 
21051 	if (!prog->gpl_compatible) {
21052 		verbose(env, "struct ops programs must have a GPL compatible license\n");
21053 		return -EINVAL;
21054 	}
21055 
21056 	if (!prog->aux->attach_btf_id)
21057 		return -ENOTSUPP;
21058 
21059 	btf = prog->aux->attach_btf;
21060 	if (btf_is_module(btf)) {
21061 		/* Make sure st_ops is valid through the lifetime of env */
21062 		env->attach_btf_mod = btf_try_get_module(btf);
21063 		if (!env->attach_btf_mod) {
21064 			verbose(env, "struct_ops module %s is not found\n",
21065 				btf_get_name(btf));
21066 			return -ENOTSUPP;
21067 		}
21068 	}
21069 
21070 	btf_id = prog->aux->attach_btf_id;
21071 	st_ops_desc = bpf_struct_ops_find(btf, btf_id);
21072 	if (!st_ops_desc) {
21073 		verbose(env, "attach_btf_id %u is not a supported struct\n",
21074 			btf_id);
21075 		return -ENOTSUPP;
21076 	}
21077 	st_ops = st_ops_desc->st_ops;
21078 
21079 	t = st_ops_desc->type;
21080 	member_idx = prog->expected_attach_type;
21081 	if (member_idx >= btf_type_vlen(t)) {
21082 		verbose(env, "attach to invalid member idx %u of struct %s\n",
21083 			member_idx, st_ops->name);
21084 		return -EINVAL;
21085 	}
21086 
21087 	member = &btf_type_member(t)[member_idx];
21088 	mname = btf_name_by_offset(btf, member->name_off);
21089 	func_proto = btf_type_resolve_func_ptr(btf, member->type,
21090 					       NULL);
21091 	if (!func_proto) {
21092 		verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",
21093 			mname, member_idx, st_ops->name);
21094 		return -EINVAL;
21095 	}
21096 
21097 	if (st_ops->check_member) {
21098 		int err = st_ops->check_member(t, member, prog);
21099 
21100 		if (err) {
21101 			verbose(env, "attach to unsupported member %s of struct %s\n",
21102 				mname, st_ops->name);
21103 			return err;
21104 		}
21105 	}
21106 
21107 	/* btf_ctx_access() used this to provide argument type info */
21108 	prog->aux->ctx_arg_info =
21109 		st_ops_desc->arg_info[member_idx].info;
21110 	prog->aux->ctx_arg_info_size =
21111 		st_ops_desc->arg_info[member_idx].cnt;
21112 
21113 	prog->aux->attach_func_proto = func_proto;
21114 	prog->aux->attach_func_name = mname;
21115 	env->ops = st_ops->verifier_ops;
21116 
21117 	return 0;
21118 }
21119 #define SECURITY_PREFIX "security_"
21120 
21121 static int check_attach_modify_return(unsigned long addr, const char *func_name)
21122 {
21123 	if (within_error_injection_list(addr) ||
21124 	    !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
21125 		return 0;
21126 
21127 	return -EINVAL;
21128 }
21129 
21130 /* list of non-sleepable functions that are otherwise on
21131  * ALLOW_ERROR_INJECTION list
21132  */
21133 BTF_SET_START(btf_non_sleepable_error_inject)
21134 /* Three functions below can be called from sleepable and non-sleepable context.
21135  * Assume non-sleepable from bpf safety point of view.
21136  */
21137 BTF_ID(func, __filemap_add_folio)
21138 #ifdef CONFIG_FAIL_PAGE_ALLOC
21139 BTF_ID(func, should_fail_alloc_page)
21140 #endif
21141 #ifdef CONFIG_FAILSLAB
21142 BTF_ID(func, should_failslab)
21143 #endif
21144 BTF_SET_END(btf_non_sleepable_error_inject)
21145 
21146 static int check_non_sleepable_error_inject(u32 btf_id)
21147 {
21148 	return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
21149 }
21150 
21151 int bpf_check_attach_target(struct bpf_verifier_log *log,
21152 			    const struct bpf_prog *prog,
21153 			    const struct bpf_prog *tgt_prog,
21154 			    u32 btf_id,
21155 			    struct bpf_attach_target_info *tgt_info)
21156 {
21157 	bool prog_extension = prog->type == BPF_PROG_TYPE_EXT;
21158 	bool prog_tracing = prog->type == BPF_PROG_TYPE_TRACING;
21159 	char trace_symbol[KSYM_SYMBOL_LEN];
21160 	const char prefix[] = "btf_trace_";
21161 	struct bpf_raw_event_map *btp;
21162 	int ret = 0, subprog = -1, i;
21163 	const struct btf_type *t;
21164 	bool conservative = true;
21165 	const char *tname, *fname;
21166 	struct btf *btf;
21167 	long addr = 0;
21168 	struct module *mod = NULL;
21169 
21170 	if (!btf_id) {
21171 		bpf_log(log, "Tracing programs must provide btf_id\n");
21172 		return -EINVAL;
21173 	}
21174 	btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf;
21175 	if (!btf) {
21176 		bpf_log(log,
21177 			"FENTRY/FEXIT program can only be attached to another program annotated with BTF\n");
21178 		return -EINVAL;
21179 	}
21180 	t = btf_type_by_id(btf, btf_id);
21181 	if (!t) {
21182 		bpf_log(log, "attach_btf_id %u is invalid\n", btf_id);
21183 		return -EINVAL;
21184 	}
21185 	tname = btf_name_by_offset(btf, t->name_off);
21186 	if (!tname) {
21187 		bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id);
21188 		return -EINVAL;
21189 	}
21190 	if (tgt_prog) {
21191 		struct bpf_prog_aux *aux = tgt_prog->aux;
21192 
21193 		if (bpf_prog_is_dev_bound(prog->aux) &&
21194 		    !bpf_prog_dev_bound_match(prog, tgt_prog)) {
21195 			bpf_log(log, "Target program bound device mismatch");
21196 			return -EINVAL;
21197 		}
21198 
21199 		for (i = 0; i < aux->func_info_cnt; i++)
21200 			if (aux->func_info[i].type_id == btf_id) {
21201 				subprog = i;
21202 				break;
21203 			}
21204 		if (subprog == -1) {
21205 			bpf_log(log, "Subprog %s doesn't exist\n", tname);
21206 			return -EINVAL;
21207 		}
21208 		if (aux->func && aux->func[subprog]->aux->exception_cb) {
21209 			bpf_log(log,
21210 				"%s programs cannot attach to exception callback\n",
21211 				prog_extension ? "Extension" : "FENTRY/FEXIT");
21212 			return -EINVAL;
21213 		}
21214 		conservative = aux->func_info_aux[subprog].unreliable;
21215 		if (prog_extension) {
21216 			if (conservative) {
21217 				bpf_log(log,
21218 					"Cannot replace static functions\n");
21219 				return -EINVAL;
21220 			}
21221 			if (!prog->jit_requested) {
21222 				bpf_log(log,
21223 					"Extension programs should be JITed\n");
21224 				return -EINVAL;
21225 			}
21226 		}
21227 		if (!tgt_prog->jited) {
21228 			bpf_log(log, "Can attach to only JITed progs\n");
21229 			return -EINVAL;
21230 		}
21231 		if (prog_tracing) {
21232 			if (aux->attach_tracing_prog) {
21233 				/*
21234 				 * Target program is an fentry/fexit which is already attached
21235 				 * to another tracing program. More levels of nesting
21236 				 * attachment are not allowed.
21237 				 */
21238 				bpf_log(log, "Cannot nest tracing program attach more than once\n");
21239 				return -EINVAL;
21240 			}
21241 		} else if (tgt_prog->type == prog->type) {
21242 			/*
21243 			 * To avoid potential call chain cycles, prevent attaching of a
21244 			 * program extension to another extension. It's ok to attach
21245 			 * fentry/fexit to extension program.
21246 			 */
21247 			bpf_log(log, "Cannot recursively attach\n");
21248 			return -EINVAL;
21249 		}
21250 		if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&
21251 		    prog_extension &&
21252 		    (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY ||
21253 		     tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) {
21254 			/* Program extensions can extend all program types
21255 			 * except fentry/fexit. The reason is the following.
21256 			 * The fentry/fexit programs are used for performance
21257 			 * analysis, stats and can be attached to any program
21258 			 * type. When extension program is replacing XDP function
21259 			 * it is necessary to allow performance analysis of all
21260 			 * functions. Both original XDP program and its program
21261 			 * extension. Hence attaching fentry/fexit to
21262 			 * BPF_PROG_TYPE_EXT is allowed. If extending of
21263 			 * fentry/fexit was allowed it would be possible to create
21264 			 * long call chain fentry->extension->fentry->extension
21265 			 * beyond reasonable stack size. Hence extending fentry
21266 			 * is not allowed.
21267 			 */
21268 			bpf_log(log, "Cannot extend fentry/fexit\n");
21269 			return -EINVAL;
21270 		}
21271 	} else {
21272 		if (prog_extension) {
21273 			bpf_log(log, "Cannot replace kernel functions\n");
21274 			return -EINVAL;
21275 		}
21276 	}
21277 
21278 	switch (prog->expected_attach_type) {
21279 	case BPF_TRACE_RAW_TP:
21280 		if (tgt_prog) {
21281 			bpf_log(log,
21282 				"Only FENTRY/FEXIT progs are attachable to another BPF prog\n");
21283 			return -EINVAL;
21284 		}
21285 		if (!btf_type_is_typedef(t)) {
21286 			bpf_log(log, "attach_btf_id %u is not a typedef\n",
21287 				btf_id);
21288 			return -EINVAL;
21289 		}
21290 		if (strncmp(prefix, tname, sizeof(prefix) - 1)) {
21291 			bpf_log(log, "attach_btf_id %u points to wrong type name %s\n",
21292 				btf_id, tname);
21293 			return -EINVAL;
21294 		}
21295 		tname += sizeof(prefix) - 1;
21296 
21297 		/* The func_proto of "btf_trace_##tname" is generated from typedef without argument
21298 		 * names. Thus using bpf_raw_event_map to get argument names.
21299 		 */
21300 		btp = bpf_get_raw_tracepoint(tname);
21301 		if (!btp)
21302 			return -EINVAL;
21303 		fname = kallsyms_lookup((unsigned long)btp->bpf_func, NULL, NULL, NULL,
21304 					trace_symbol);
21305 		bpf_put_raw_tracepoint(btp);
21306 
21307 		if (fname)
21308 			ret = btf_find_by_name_kind(btf, fname, BTF_KIND_FUNC);
21309 
21310 		if (!fname || ret < 0) {
21311 			bpf_log(log, "Cannot find btf of tracepoint template, fall back to %s%s.\n",
21312 				prefix, tname);
21313 			t = btf_type_by_id(btf, t->type);
21314 			if (!btf_type_is_ptr(t))
21315 				/* should never happen in valid vmlinux build */
21316 				return -EINVAL;
21317 		} else {
21318 			t = btf_type_by_id(btf, ret);
21319 			if (!btf_type_is_func(t))
21320 				/* should never happen in valid vmlinux build */
21321 				return -EINVAL;
21322 		}
21323 
21324 		t = btf_type_by_id(btf, t->type);
21325 		if (!btf_type_is_func_proto(t))
21326 			/* should never happen in valid vmlinux build */
21327 			return -EINVAL;
21328 
21329 		break;
21330 	case BPF_TRACE_ITER:
21331 		if (!btf_type_is_func(t)) {
21332 			bpf_log(log, "attach_btf_id %u is not a function\n",
21333 				btf_id);
21334 			return -EINVAL;
21335 		}
21336 		t = btf_type_by_id(btf, t->type);
21337 		if (!btf_type_is_func_proto(t))
21338 			return -EINVAL;
21339 		ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
21340 		if (ret)
21341 			return ret;
21342 		break;
21343 	default:
21344 		if (!prog_extension)
21345 			return -EINVAL;
21346 		fallthrough;
21347 	case BPF_MODIFY_RETURN:
21348 	case BPF_LSM_MAC:
21349 	case BPF_LSM_CGROUP:
21350 	case BPF_TRACE_FENTRY:
21351 	case BPF_TRACE_FEXIT:
21352 		if (!btf_type_is_func(t)) {
21353 			bpf_log(log, "attach_btf_id %u is not a function\n",
21354 				btf_id);
21355 			return -EINVAL;
21356 		}
21357 		if (prog_extension &&
21358 		    btf_check_type_match(log, prog, btf, t))
21359 			return -EINVAL;
21360 		t = btf_type_by_id(btf, t->type);
21361 		if (!btf_type_is_func_proto(t))
21362 			return -EINVAL;
21363 
21364 		if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
21365 		    (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type ||
21366 		     prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type))
21367 			return -EINVAL;
21368 
21369 		if (tgt_prog && conservative)
21370 			t = NULL;
21371 
21372 		ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
21373 		if (ret < 0)
21374 			return ret;
21375 
21376 		if (tgt_prog) {
21377 			if (subprog == 0)
21378 				addr = (long) tgt_prog->bpf_func;
21379 			else
21380 				addr = (long) tgt_prog->aux->func[subprog]->bpf_func;
21381 		} else {
21382 			if (btf_is_module(btf)) {
21383 				mod = btf_try_get_module(btf);
21384 				if (mod)
21385 					addr = find_kallsyms_symbol_value(mod, tname);
21386 				else
21387 					addr = 0;
21388 			} else {
21389 				addr = kallsyms_lookup_name(tname);
21390 			}
21391 			if (!addr) {
21392 				module_put(mod);
21393 				bpf_log(log,
21394 					"The address of function %s cannot be found\n",
21395 					tname);
21396 				return -ENOENT;
21397 			}
21398 		}
21399 
21400 		if (prog->sleepable) {
21401 			ret = -EINVAL;
21402 			switch (prog->type) {
21403 			case BPF_PROG_TYPE_TRACING:
21404 
21405 				/* fentry/fexit/fmod_ret progs can be sleepable if they are
21406 				 * attached to ALLOW_ERROR_INJECTION and are not in denylist.
21407 				 */
21408 				if (!check_non_sleepable_error_inject(btf_id) &&
21409 				    within_error_injection_list(addr))
21410 					ret = 0;
21411 				/* fentry/fexit/fmod_ret progs can also be sleepable if they are
21412 				 * in the fmodret id set with the KF_SLEEPABLE flag.
21413 				 */
21414 				else {
21415 					u32 *flags = btf_kfunc_is_modify_return(btf, btf_id,
21416 										prog);
21417 
21418 					if (flags && (*flags & KF_SLEEPABLE))
21419 						ret = 0;
21420 				}
21421 				break;
21422 			case BPF_PROG_TYPE_LSM:
21423 				/* LSM progs check that they are attached to bpf_lsm_*() funcs.
21424 				 * Only some of them are sleepable.
21425 				 */
21426 				if (bpf_lsm_is_sleepable_hook(btf_id))
21427 					ret = 0;
21428 				break;
21429 			default:
21430 				break;
21431 			}
21432 			if (ret) {
21433 				module_put(mod);
21434 				bpf_log(log, "%s is not sleepable\n", tname);
21435 				return ret;
21436 			}
21437 		} else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
21438 			if (tgt_prog) {
21439 				module_put(mod);
21440 				bpf_log(log, "can't modify return codes of BPF programs\n");
21441 				return -EINVAL;
21442 			}
21443 			ret = -EINVAL;
21444 			if (btf_kfunc_is_modify_return(btf, btf_id, prog) ||
21445 			    !check_attach_modify_return(addr, tname))
21446 				ret = 0;
21447 			if (ret) {
21448 				module_put(mod);
21449 				bpf_log(log, "%s() is not modifiable\n", tname);
21450 				return ret;
21451 			}
21452 		}
21453 
21454 		break;
21455 	}
21456 	tgt_info->tgt_addr = addr;
21457 	tgt_info->tgt_name = tname;
21458 	tgt_info->tgt_type = t;
21459 	tgt_info->tgt_mod = mod;
21460 	return 0;
21461 }
21462 
21463 BTF_SET_START(btf_id_deny)
21464 BTF_ID_UNUSED
21465 #ifdef CONFIG_SMP
21466 BTF_ID(func, migrate_disable)
21467 BTF_ID(func, migrate_enable)
21468 #endif
21469 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU
21470 BTF_ID(func, rcu_read_unlock_strict)
21471 #endif
21472 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE)
21473 BTF_ID(func, preempt_count_add)
21474 BTF_ID(func, preempt_count_sub)
21475 #endif
21476 #ifdef CONFIG_PREEMPT_RCU
21477 BTF_ID(func, __rcu_read_lock)
21478 BTF_ID(func, __rcu_read_unlock)
21479 #endif
21480 BTF_SET_END(btf_id_deny)
21481 
21482 static bool can_be_sleepable(struct bpf_prog *prog)
21483 {
21484 	if (prog->type == BPF_PROG_TYPE_TRACING) {
21485 		switch (prog->expected_attach_type) {
21486 		case BPF_TRACE_FENTRY:
21487 		case BPF_TRACE_FEXIT:
21488 		case BPF_MODIFY_RETURN:
21489 		case BPF_TRACE_ITER:
21490 			return true;
21491 		default:
21492 			return false;
21493 		}
21494 	}
21495 	return prog->type == BPF_PROG_TYPE_LSM ||
21496 	       prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ ||
21497 	       prog->type == BPF_PROG_TYPE_STRUCT_OPS;
21498 }
21499 
21500 static int check_attach_btf_id(struct bpf_verifier_env *env)
21501 {
21502 	struct bpf_prog *prog = env->prog;
21503 	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
21504 	struct bpf_attach_target_info tgt_info = {};
21505 	u32 btf_id = prog->aux->attach_btf_id;
21506 	struct bpf_trampoline *tr;
21507 	int ret;
21508 	u64 key;
21509 
21510 	if (prog->type == BPF_PROG_TYPE_SYSCALL) {
21511 		if (prog->sleepable)
21512 			/* attach_btf_id checked to be zero already */
21513 			return 0;
21514 		verbose(env, "Syscall programs can only be sleepable\n");
21515 		return -EINVAL;
21516 	}
21517 
21518 	if (prog->sleepable && !can_be_sleepable(prog)) {
21519 		verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n");
21520 		return -EINVAL;
21521 	}
21522 
21523 	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
21524 		return check_struct_ops_btf_id(env);
21525 
21526 	if (prog->type != BPF_PROG_TYPE_TRACING &&
21527 	    prog->type != BPF_PROG_TYPE_LSM &&
21528 	    prog->type != BPF_PROG_TYPE_EXT)
21529 		return 0;
21530 
21531 	ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);
21532 	if (ret)
21533 		return ret;
21534 
21535 	if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) {
21536 		/* to make freplace equivalent to their targets, they need to
21537 		 * inherit env->ops and expected_attach_type for the rest of the
21538 		 * verification
21539 		 */
21540 		env->ops = bpf_verifier_ops[tgt_prog->type];
21541 		prog->expected_attach_type = tgt_prog->expected_attach_type;
21542 	}
21543 
21544 	/* store info about the attachment target that will be used later */
21545 	prog->aux->attach_func_proto = tgt_info.tgt_type;
21546 	prog->aux->attach_func_name = tgt_info.tgt_name;
21547 	prog->aux->mod = tgt_info.tgt_mod;
21548 
21549 	if (tgt_prog) {
21550 		prog->aux->saved_dst_prog_type = tgt_prog->type;
21551 		prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type;
21552 	}
21553 
21554 	if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
21555 		prog->aux->attach_btf_trace = true;
21556 		return 0;
21557 	} else if (prog->expected_attach_type == BPF_TRACE_ITER) {
21558 		if (!bpf_iter_prog_supported(prog))
21559 			return -EINVAL;
21560 		return 0;
21561 	}
21562 
21563 	if (prog->type == BPF_PROG_TYPE_LSM) {
21564 		ret = bpf_lsm_verify_prog(&env->log, prog);
21565 		if (ret < 0)
21566 			return ret;
21567 	} else if (prog->type == BPF_PROG_TYPE_TRACING &&
21568 		   btf_id_set_contains(&btf_id_deny, btf_id)) {
21569 		return -EINVAL;
21570 	}
21571 
21572 	key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
21573 	tr = bpf_trampoline_get(key, &tgt_info);
21574 	if (!tr)
21575 		return -ENOMEM;
21576 
21577 	if (tgt_prog && tgt_prog->aux->tail_call_reachable)
21578 		tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX;
21579 
21580 	prog->aux->dst_trampoline = tr;
21581 	return 0;
21582 }
21583 
21584 struct btf *bpf_get_btf_vmlinux(void)
21585 {
21586 	if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
21587 		mutex_lock(&bpf_verifier_lock);
21588 		if (!btf_vmlinux)
21589 			btf_vmlinux = btf_parse_vmlinux();
21590 		mutex_unlock(&bpf_verifier_lock);
21591 	}
21592 	return btf_vmlinux;
21593 }
21594 
21595 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size)
21596 {
21597 	u64 start_time = ktime_get_ns();
21598 	struct bpf_verifier_env *env;
21599 	int i, len, ret = -EINVAL, err;
21600 	u32 log_true_size;
21601 	bool is_priv;
21602 
21603 	/* no program is valid */
21604 	if (ARRAY_SIZE(bpf_verifier_ops) == 0)
21605 		return -EINVAL;
21606 
21607 	/* 'struct bpf_verifier_env' can be global, but since it's not small,
21608 	 * allocate/free it every time bpf_check() is called
21609 	 */
21610 	env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
21611 	if (!env)
21612 		return -ENOMEM;
21613 
21614 	env->bt.env = env;
21615 
21616 	len = (*prog)->len;
21617 	env->insn_aux_data =
21618 		vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
21619 	ret = -ENOMEM;
21620 	if (!env->insn_aux_data)
21621 		goto err_free_env;
21622 	for (i = 0; i < len; i++)
21623 		env->insn_aux_data[i].orig_idx = i;
21624 	env->prog = *prog;
21625 	env->ops = bpf_verifier_ops[env->prog->type];
21626 	env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
21627 
21628 	env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token);
21629 	env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token);
21630 	env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token);
21631 	env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token);
21632 	env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF);
21633 
21634 	bpf_get_btf_vmlinux();
21635 
21636 	/* grab the mutex to protect few globals used by verifier */
21637 	if (!is_priv)
21638 		mutex_lock(&bpf_verifier_lock);
21639 
21640 	/* user could have requested verbose verifier output
21641 	 * and supplied buffer to store the verification trace
21642 	 */
21643 	ret = bpf_vlog_init(&env->log, attr->log_level,
21644 			    (char __user *) (unsigned long) attr->log_buf,
21645 			    attr->log_size);
21646 	if (ret)
21647 		goto err_unlock;
21648 
21649 	mark_verifier_state_clean(env);
21650 
21651 	if (IS_ERR(btf_vmlinux)) {
21652 		/* Either gcc or pahole or kernel are broken. */
21653 		verbose(env, "in-kernel BTF is malformed\n");
21654 		ret = PTR_ERR(btf_vmlinux);
21655 		goto skip_full_check;
21656 	}
21657 
21658 	env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
21659 	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
21660 		env->strict_alignment = true;
21661 	if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
21662 		env->strict_alignment = false;
21663 
21664 	if (is_priv)
21665 		env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
21666 	env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;
21667 
21668 	env->explored_states = kvcalloc(state_htab_size(env),
21669 				       sizeof(struct bpf_verifier_state_list *),
21670 				       GFP_USER);
21671 	ret = -ENOMEM;
21672 	if (!env->explored_states)
21673 		goto skip_full_check;
21674 
21675 	ret = check_btf_info_early(env, attr, uattr);
21676 	if (ret < 0)
21677 		goto skip_full_check;
21678 
21679 	ret = add_subprog_and_kfunc(env);
21680 	if (ret < 0)
21681 		goto skip_full_check;
21682 
21683 	ret = check_subprogs(env);
21684 	if (ret < 0)
21685 		goto skip_full_check;
21686 
21687 	ret = check_btf_info(env, attr, uattr);
21688 	if (ret < 0)
21689 		goto skip_full_check;
21690 
21691 	ret = check_attach_btf_id(env);
21692 	if (ret)
21693 		goto skip_full_check;
21694 
21695 	ret = resolve_pseudo_ldimm64(env);
21696 	if (ret < 0)
21697 		goto skip_full_check;
21698 
21699 	if (bpf_prog_is_offloaded(env->prog->aux)) {
21700 		ret = bpf_prog_offload_verifier_prep(env->prog);
21701 		if (ret)
21702 			goto skip_full_check;
21703 	}
21704 
21705 	ret = check_cfg(env);
21706 	if (ret < 0)
21707 		goto skip_full_check;
21708 
21709 	ret = do_check_main(env);
21710 	ret = ret ?: do_check_subprogs(env);
21711 
21712 	if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux))
21713 		ret = bpf_prog_offload_finalize(env);
21714 
21715 skip_full_check:
21716 	kvfree(env->explored_states);
21717 
21718 	if (ret == 0)
21719 		ret = check_max_stack_depth(env);
21720 
21721 	/* instruction rewrites happen after this point */
21722 	if (ret == 0)
21723 		ret = optimize_bpf_loop(env);
21724 
21725 	if (is_priv) {
21726 		if (ret == 0)
21727 			opt_hard_wire_dead_code_branches(env);
21728 		if (ret == 0)
21729 			ret = opt_remove_dead_code(env);
21730 		if (ret == 0)
21731 			ret = opt_remove_nops(env);
21732 	} else {
21733 		if (ret == 0)
21734 			sanitize_dead_code(env);
21735 	}
21736 
21737 	if (ret == 0)
21738 		/* program is valid, convert *(u32*)(ctx + off) accesses */
21739 		ret = convert_ctx_accesses(env);
21740 
21741 	if (ret == 0)
21742 		ret = do_misc_fixups(env);
21743 
21744 	/* do 32-bit optimization after insn patching has done so those patched
21745 	 * insns could be handled correctly.
21746 	 */
21747 	if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) {
21748 		ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);
21749 		env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret
21750 								     : false;
21751 	}
21752 
21753 	if (ret == 0)
21754 		ret = fixup_call_args(env);
21755 
21756 	env->verification_time = ktime_get_ns() - start_time;
21757 	print_verification_stats(env);
21758 	env->prog->aux->verified_insns = env->insn_processed;
21759 
21760 	/* preserve original error even if log finalization is successful */
21761 	err = bpf_vlog_finalize(&env->log, &log_true_size);
21762 	if (err)
21763 		ret = err;
21764 
21765 	if (uattr_size >= offsetofend(union bpf_attr, log_true_size) &&
21766 	    copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, log_true_size),
21767 				  &log_true_size, sizeof(log_true_size))) {
21768 		ret = -EFAULT;
21769 		goto err_release_maps;
21770 	}
21771 
21772 	if (ret)
21773 		goto err_release_maps;
21774 
21775 	if (env->used_map_cnt) {
21776 		/* if program passed verifier, update used_maps in bpf_prog_info */
21777 		env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
21778 							  sizeof(env->used_maps[0]),
21779 							  GFP_KERNEL);
21780 
21781 		if (!env->prog->aux->used_maps) {
21782 			ret = -ENOMEM;
21783 			goto err_release_maps;
21784 		}
21785 
21786 		memcpy(env->prog->aux->used_maps, env->used_maps,
21787 		       sizeof(env->used_maps[0]) * env->used_map_cnt);
21788 		env->prog->aux->used_map_cnt = env->used_map_cnt;
21789 	}
21790 	if (env->used_btf_cnt) {
21791 		/* if program passed verifier, update used_btfs in bpf_prog_aux */
21792 		env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt,
21793 							  sizeof(env->used_btfs[0]),
21794 							  GFP_KERNEL);
21795 		if (!env->prog->aux->used_btfs) {
21796 			ret = -ENOMEM;
21797 			goto err_release_maps;
21798 		}
21799 
21800 		memcpy(env->prog->aux->used_btfs, env->used_btfs,
21801 		       sizeof(env->used_btfs[0]) * env->used_btf_cnt);
21802 		env->prog->aux->used_btf_cnt = env->used_btf_cnt;
21803 	}
21804 	if (env->used_map_cnt || env->used_btf_cnt) {
21805 		/* program is valid. Convert pseudo bpf_ld_imm64 into generic
21806 		 * bpf_ld_imm64 instructions
21807 		 */
21808 		convert_pseudo_ld_imm64(env);
21809 	}
21810 
21811 	adjust_btf_func(env);
21812 
21813 err_release_maps:
21814 	if (!env->prog->aux->used_maps)
21815 		/* if we didn't copy map pointers into bpf_prog_info, release
21816 		 * them now. Otherwise free_used_maps() will release them.
21817 		 */
21818 		release_maps(env);
21819 	if (!env->prog->aux->used_btfs)
21820 		release_btfs(env);
21821 
21822 	/* extension progs temporarily inherit the attach_type of their targets
21823 	   for verification purposes, so set it back to zero before returning
21824 	 */
21825 	if (env->prog->type == BPF_PROG_TYPE_EXT)
21826 		env->prog->expected_attach_type = 0;
21827 
21828 	*prog = env->prog;
21829 
21830 	module_put(env->attach_btf_mod);
21831 err_unlock:
21832 	if (!is_priv)
21833 		mutex_unlock(&bpf_verifier_lock);
21834 	vfree(env->insn_aux_data);
21835 err_free_env:
21836 	kfree(env);
21837 	return ret;
21838 }
21839