xref: /linux/kernel/bpf/verifier.c (revision 18f6c47d1097b703814e97c79239ce9dafb160f8)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3  * Copyright (c) 2016 Facebook
4  * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
5  */
6 #include <uapi/linux/btf.h>
7 #include <linux/bpf-cgroup.h>
8 #include <linux/count_zeros.h>
9 #include <linux/kernel.h>
10 #include <linux/types.h>
11 #include <linux/slab.h>
12 #include <linux/bpf.h>
13 #include <linux/btf.h>
14 #include <linux/bpf_verifier.h>
15 #include <linux/filter.h>
16 #include <net/netlink.h>
17 #include <linux/file.h>
18 #include <linux/vmalloc.h>
19 #include <linux/stringify.h>
20 #include <linux/bsearch.h>
21 #include <linux/sort.h>
22 #include <linux/perf_event.h>
23 #include <linux/ctype.h>
24 #include <linux/error-injection.h>
25 #include <linux/bpf_lsm.h>
26 #include <linux/security.h>
27 #include <linux/verification.h>
28 #include <linux/keyctl.h>
29 #include <linux/btf_ids.h>
30 #include <linux/poison.h>
31 #include <linux/module.h>
32 #include <linux/cpumask.h>
33 #include <linux/cnum.h>
34 #include <linux/gcd.h>
35 #include <linux/bpf_mem_alloc.h>
36 #include <net/xdp.h>
37 #include <linux/trace_events.h>
38 #include <linux/kallsyms.h>
39 
40 #include "diagnostics.h"
41 #include "disasm.h"
42 
43 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
44 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
45 	[_id] = & _name ## _verifier_ops,
46 #define BPF_MAP_TYPE(_id, _ops)
47 #define BPF_LINK_TYPE(_id, _name)
48 #include <linux/bpf_types.h>
49 #undef BPF_PROG_TYPE
50 #undef BPF_MAP_TYPE
51 #undef BPF_LINK_TYPE
52 };
53 
54 enum bpf_features {
55 	BPF_FEAT_RDONLY_CAST_TO_VOID = 0,
56 	BPF_FEAT_STREAMS	     = 1,
57 	__MAX_BPF_FEAT,
58 };
59 
60 struct bpf_mem_alloc bpf_global_percpu_ma;
61 static bool bpf_global_percpu_ma_set;
62 
63 /* bpf_check() is a static code analyzer that walks eBPF program
64  * instruction by instruction and updates register/stack state.
65  * All paths of conditional branches are analyzed until 'bpf_exit' insn.
66  *
67  * The first pass is depth-first-search to check that the program is a DAG.
68  * It rejects the following programs:
69  * - larger than BPF_MAXINSNS insns
70  * - if loop is present (detected via back-edge)
71  * - unreachable insns exist (shouldn't be a forest. program = one function)
72  * - out of bounds or malformed jumps
73  * The second pass is all possible path descent from the 1st insn.
74  * Since it's analyzing all paths through the program, the length of the
75  * analysis is limited to 64k insn, which may be hit even if total number of
76  * insn is less then 4K, but there are too many branches that change stack/regs.
77  * Number of 'branches to be analyzed' is limited to 1k
78  *
79  * On entry to each instruction, each register has a type, and the instruction
80  * changes the types of the registers depending on instruction semantics.
81  * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is
82  * copied to R1.
83  *
84  * All registers are 64-bit.
85  * R0 - return register
86  * R1-R5 argument passing registers
87  * R6-R9 callee saved registers
88  * R10 - frame pointer read-only
89  *
90  * At the start of BPF program the register R1 contains a pointer to bpf_context
91  * and has type PTR_TO_CTX.
92  *
93  * Verifier tracks arithmetic operations on pointers in case:
94  *    BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
95  *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20),
96  * 1st insn copies R10 (which has FRAME_PTR) type into R1
97  * and 2nd arithmetic instruction is pattern matched to recognize
98  * that it wants to construct a pointer to some element within stack.
99  * So after 2nd insn, the register R1 has type PTR_TO_STACK
100  * (and -20 constant is saved for further stack bounds checking).
101  * Meaning that this reg is a pointer to stack plus known immediate constant.
102  *
103  * Most of the time the registers have SCALAR_VALUE type, which
104  * means the register has some value, but it's not a valid pointer.
105  * (like pointer plus pointer becomes SCALAR_VALUE type)
106  *
107  * When verifier sees load or store instructions the type of base register
108  * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are
109  * four pointer types recognized by check_mem_access() function.
110  *
111  * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value'
112  * and the range of [ptr, ptr + map's value_size) is accessible.
113  *
114  * registers used to pass values to function calls are checked against
115  * function argument constraints.
116  *
117  * ARG_PTR_TO_MAP_KEY is one of such argument constraints.
118  * It means that the register type passed to this function must be
119  * PTR_TO_STACK and it will be used inside the function as
120  * 'pointer to map element key'
121  *
122  * For example the argument constraints for bpf_map_lookup_elem():
123  *   .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
124  *   .arg1_type = ARG_CONST_MAP_PTR,
125  *   .arg2_type = ARG_PTR_TO_MAP_KEY,
126  *
127  * ret_type says that this function returns 'pointer to map elem value or null'
128  * function expects 1st argument to be a const pointer to 'struct bpf_map' and
129  * 2nd argument should be a pointer to stack, which will be used inside
130  * the helper function as a pointer to map element key.
131  *
132  * On the kernel side the helper function looks like:
133  * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
134  * {
135  *    struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;
136  *    void *key = (void *) (unsigned long) r2;
137  *    void *value;
138  *
139  *    here kernel can access 'key' and 'map' pointers safely, knowing that
140  *    [key, key + map->key_size) bytes are valid and were initialized on
141  *    the stack of eBPF program.
142  * }
143  *
144  * Corresponding eBPF program may look like:
145  *    BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),  // after this insn R2 type is FRAME_PTR
146  *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK
147  *    BPF_LD_MAP_FD(BPF_REG_1, map_fd),      // after this insn R1 type is CONST_PTR_TO_MAP
148  *    BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
149  * here verifier looks at prototype of map_lookup_elem() and sees:
150  * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok,
151  * Now verifier knows that this map has key of R1->map_ptr->key_size bytes
152  *
153  * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far,
154  * Now verifier checks that [R2, R2 + map's key_size) are within stack limits
155  * and were initialized prior to this call.
156  * If it's ok, then verifier allows this BPF_CALL insn and looks at
157  * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets
158  * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function
159  * returns either pointer to map value or NULL.
160  *
161  * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off'
162  * insn, the register holding that pointer in the true branch changes state to
163  * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false
164  * branch. See check_cond_jmp_op().
165  *
166  * After the call R0 is set to return type of the function and registers R1-R5
167  * are set to NOT_INIT to indicate that they are no longer readable.
168  *
169  * The following reference types represent a potential reference to a kernel
170  * resource which, after first being allocated, must be checked and freed by
171  * the BPF program:
172  * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET
173  *
174  * When the verifier sees a helper call return a reference type, it allocates a
175  * pointer id for the reference and stores it in the current function state.
176  * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into
177  * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type
178  * passes through a NULL-check conditional. For the branch wherein the state is
179  * changed to CONST_IMM, the verifier releases the reference.
180  *
181  * For each helper function that allocates a reference, such as
182  * bpf_sk_lookup_tcp(), there is a corresponding release function, such as
183  * bpf_sk_release(). When a reference type passes into the release function,
184  * the verifier also releases the reference. If any unchecked or unreleased
185  * reference remains at the end of the program, the verifier rejects it.
186  */
187 
188 /* verifier_state + insn_idx are pushed to stack when branch is encountered */
189 struct bpf_verifier_stack_elem {
190 	/* verifier state is 'st'
191 	 * before processing instruction 'insn_idx'
192 	 * and after processing instruction 'prev_insn_idx'
193 	 */
194 	struct bpf_verifier_state st;
195 	int insn_idx;
196 	int prev_insn_idx;
197 	struct bpf_verifier_stack_elem *next;
198 	/* length of verifier log at the time this state was pushed on stack */
199 	u32 log_pos;
200 	u64 diag_log_pos;
201 };
202 
203 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ	8192
204 #define BPF_COMPLEXITY_LIMIT_STATES	64
205 
206 #define BPF_GLOBAL_PERCPU_MA_MAX_SIZE  512
207 
208 #define BPF_PRIV_STACK_MIN_SIZE		64
209 
210 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id);
211 static int __release_reference_nomark(struct bpf_verifier_state *state, int id);
212 static int release_reference_nomark(struct bpf_verifier_env *env, int id);
213 static int release_reference(struct bpf_verifier_env *env, int id);
214 static int invalidate_reference(struct bpf_verifier_env *env, int id);
215 static u32 reg_lifetime_id(struct bpf_verifier_env *env,
216 			   const struct bpf_reg_state *reg);
217 static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
218 static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env);
219 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);
220 static bool is_tracing_prog_type(enum bpf_prog_type type);
221 static int ref_set_non_owning(struct bpf_verifier_env *env,
222 			      struct bpf_reg_state *reg);
223 static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg);
224 static inline bool in_sleepable_context(struct bpf_verifier_env *env);
225 static bool in_rcu_callback(struct bpf_verifier_env *env);
226 static const char *non_sleepable_context_description(struct bpf_verifier_env *env);
227 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg);
228 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg);
229 
230 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
231 			      struct bpf_map *map,
232 			      bool unpriv, bool poison)
233 {
234 	unpriv |= bpf_map_ptr_unpriv(aux);
235 	aux->map_ptr_state.unpriv = unpriv;
236 	aux->map_ptr_state.poison = poison;
237 	aux->map_ptr_state.map_ptr = map;
238 }
239 
240 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state)
241 {
242 	bool poisoned = bpf_map_key_poisoned(aux);
243 
244 	aux->map_key_state = state | BPF_MAP_KEY_SEEN |
245 			     (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
246 }
247 
248 static void update_ref_obj(struct ref_obj_desc *ref_obj, struct bpf_reg_state *reg)
249 {
250 	ref_obj->id = reg->id;
251 	ref_obj->parent_id = reg->parent_id;
252 	ref_obj->cnt++;
253 }
254 
255 static int validate_ref_obj(struct bpf_verifier_env *env, struct ref_obj_desc *ref_obj)
256 {
257 	if (ref_obj->cnt > 1) {
258 		verifier_bug(env, "function expects only one referenced object but got %d\n",
259 			     ref_obj->cnt);
260 		return -EFAULT;
261 	}
262 
263 	return 0;
264 }
265 
266 struct bpf_kfunc_meta {
267 	struct btf *btf;
268 	const struct btf_type *proto;
269 	const char *name;
270 	const u32 *flags;
271 	s32 id;
272 };
273 
274 struct btf *btf_vmlinux;
275 
276 typedef struct argno {
277 	int argno;
278 } argno_t;
279 
280 static argno_t argno_from_reg(u32 regno)
281 {
282 	return (argno_t){ .argno = regno };
283 }
284 
285 static argno_t argno_from_arg(u32 arg)
286 {
287 	return (argno_t){ .argno = -arg };
288 }
289 
290 static int reg_from_argno(argno_t a)
291 {
292 	if (a.argno >= 0)
293 		return a.argno;
294 	if (a.argno >= -MAX_BPF_FUNC_REG_ARGS)
295 		return -a.argno;
296 	return -1;
297 }
298 
299 static int arg_from_argno(argno_t a)
300 {
301 	if (a.argno < 0)
302 		return -a.argno;
303 	return -1;
304 }
305 
306 static int arg_idx_from_argno(argno_t a)
307 {
308 	return arg_from_argno(a) - 1;
309 }
310 
311 /* Normalize helper register numbers and kfunc argument numbers to ABI slots. */
312 static u32 arg_slot_from_argno(argno_t a)
313 {
314 	return abs(a.argno) - 1;
315 }
316 
317 static const char *btf_type_name(const struct btf *btf, u32 id)
318 {
319 	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
320 }
321 
322 static DEFINE_MUTEX(bpf_verifier_lock);
323 static DEFINE_MUTEX(btf_vmlinux_lock);
324 static DEFINE_MUTEX(bpf_percpu_ma_lock);
325 
326 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
327 {
328 	struct bpf_verifier_env *env = private_data;
329 	va_list args;
330 
331 	if (!bpf_verifier_log_needed(&env->log))
332 		return;
333 
334 	va_start(args, fmt);
335 	bpf_verifier_vlog(&env->log, fmt, args);
336 	va_end(args);
337 }
338 
339 static void verbose_invalid_scalar(struct bpf_verifier_env *env,
340 				   struct bpf_reg_state *reg,
341 				   struct bpf_retval_range range, const char *ctx,
342 				   const char *reg_name)
343 {
344 	bool unknown = true;
345 
346 	verbose(env, "%s the register %s has", ctx, reg_name);
347 	if (reg_smin(reg) > S64_MIN) {
348 		verbose(env, " smin=%lld", reg_smin(reg));
349 		unknown = false;
350 	}
351 	if (reg_smax(reg) < S64_MAX) {
352 		verbose(env, " smax=%lld", reg_smax(reg));
353 		unknown = false;
354 	}
355 	if (unknown)
356 		verbose(env, " unknown scalar value");
357 	verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval);
358 }
359 
360 static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)
361 {
362 	enum bpf_reg_type type;
363 
364 	type = reg->type;
365 	if (type_may_be_null(type))
366 		return false;
367 
368 	/*
369 	 * The types below guarantee a non-NULL base, an unbounded offset can
370 	 * still wrap base + offset to zero.
371 	 */
372 	if (reg_smin(reg) <= -BPF_MAX_VAR_OFF || reg_smax(reg) >= BPF_MAX_VAR_OFF)
373 		return false;
374 
375 	type = base_type(type);
376 	return type == PTR_TO_SOCKET ||
377 		type == PTR_TO_TCP_SOCK ||
378 		type == PTR_TO_XDP_SOCK ||
379 		type == PTR_TO_BUF ||
380 		type == PTR_TO_MAP_VALUE ||
381 		type == PTR_TO_MAP_KEY ||
382 		type == PTR_TO_SOCK_COMMON ||
383 		(type == PTR_TO_BTF_ID && is_trusted_reg(env, reg)) ||
384 		(type == PTR_TO_MEM && !(reg->type & PTR_UNTRUSTED)) ||
385 		type == CONST_PTR_TO_MAP;
386 }
387 
388 static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg)
389 {
390 	struct btf_record *rec = NULL;
391 	struct btf_struct_meta *meta;
392 
393 	if (reg->type == PTR_TO_MAP_VALUE) {
394 		rec = reg->map_ptr->record;
395 	} else if (type_is_ptr_alloc_obj(reg->type)) {
396 		meta = btf_find_struct_meta(reg->btf, reg->btf_id);
397 		if (meta)
398 			rec = meta->record;
399 	}
400 	return rec;
401 }
402 
403 bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog)
404 {
405 	struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;
406 
407 	return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;
408 }
409 
410 static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog)
411 {
412 	const struct btf_type *func, *func_proto;
413 	const struct btf *btf = env->prog->aux->btf;
414 	u32 btf_id;
415 
416 	if (!btf || !env->prog->aux->func_info)
417 		return NULL;
418 
419 	btf_id = env->prog->aux->func_info[subprog].type_id;
420 
421 	/* Both already validated by prepare_btf_func() at prog load. */
422 	func = btf_type_by_id(btf, btf_id);
423 	func_proto = btf_type_by_id(btf, func->type);
424 
425 	return btf_type_skip_modifiers(btf, func_proto->type, NULL);
426 }
427 
428 static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog)
429 {
430 	const struct btf_type *type = subprog_ret_type(env, subprog);
431 
432 	return type && btf_type_is_void(type);
433 }
434 
435 static u32 ret_regs_cnt(u32 size)
436 {
437 	return size > 8 && size <= 16 ? 2 : 1;
438 }
439 
440 /* Registers holding a function return value, in order. See ret_regs_cnt(). */
441 static const int ret_regs[] = { BPF_REG_0, BPF_REG_2 };
442 
443 static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env)
444 {
445 	const struct btf *btf = env->prog->aux->btf;
446 	const struct btf_type *type;
447 	int subprog;
448 	u32 size;
449 
450 	if (!env->prog->jit_requested || bpf_prog_is_offloaded(env->prog->aux))
451 		return 0;
452 
453 	/*
454 	 * Skip the main program: its return value is the program's exit code,
455 	 * read out of R0, so it never uses the register pair. An extension does
456 	 * have a real prototype for subprog 0, but btf_check_func_type_match()
457 	 * refuses to replace a function returning more than 8 bytes.
458 	 */
459 	for (subprog = 1; subprog < env->subprog_cnt; subprog++) {
460 		type = subprog_ret_type(env, subprog);
461 		if (!type || btf_type_is_void(type))
462 			continue;
463 		if (verifier_bug_if(IS_ERR(btf_resolve_size(btf, type, &size)), env,
464 				    "cannot size return type of subprog %d", subprog))
465 			return -EFAULT;
466 		if (ret_regs_cnt(size) > 1) {
467 			subprog_info(env, subprog)->ret_reg_pair = true;
468 			env->prog->jit_required = 1;
469 		}
470 	}
471 
472 	return 0;
473 }
474 
475 const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog)
476 {
477 	struct bpf_func_info *info;
478 
479 	if (!env->prog->aux->func_info)
480 		return "";
481 
482 	info = &env->prog->aux->func_info[subprog];
483 	return btf_type_name(env->prog->aux->btf, info->type_id);
484 }
485 
486 void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog)
487 {
488 	struct bpf_subprog_info *info = subprog_info(env, subprog);
489 
490 	info->is_cb = true;
491 	info->is_async_cb = true;
492 	info->is_exception_cb = true;
493 }
494 
495 static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog)
496 {
497 	return subprog_info(env, subprog)->is_exception_cb;
498 }
499 
500 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
501 {
502 	return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK);
503 }
504 
505 static bool type_is_rdonly_mem(u32 type)
506 {
507 	return type & MEM_RDONLY;
508 }
509 
510 static bool is_acquire_function(enum bpf_func_id func_id,
511 				const struct bpf_map *map)
512 {
513 	enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC;
514 
515 	if (func_id == BPF_FUNC_sk_lookup_tcp ||
516 	    func_id == BPF_FUNC_sk_lookup_udp ||
517 	    func_id == BPF_FUNC_skc_lookup_tcp ||
518 	    func_id == BPF_FUNC_ringbuf_reserve ||
519 	    func_id == BPF_FUNC_kptr_xchg)
520 		return true;
521 
522 	if (func_id == BPF_FUNC_map_lookup_elem &&
523 	    (map_type == BPF_MAP_TYPE_SOCKMAP ||
524 	     map_type == BPF_MAP_TYPE_SOCKHASH))
525 		return true;
526 
527 	return false;
528 }
529 
530 static bool is_ptr_cast_function(enum bpf_func_id func_id)
531 {
532 	return func_id == BPF_FUNC_tcp_sock ||
533 		func_id == BPF_FUNC_sk_fullsock ||
534 		func_id == BPF_FUNC_skc_to_tcp_sock ||
535 		func_id == BPF_FUNC_skc_to_tcp6_sock ||
536 		func_id == BPF_FUNC_skc_to_udp6_sock ||
537 		func_id == BPF_FUNC_skc_to_mptcp_sock ||
538 		func_id == BPF_FUNC_skc_to_tcp_timewait_sock ||
539 		func_id == BPF_FUNC_skc_to_tcp_request_sock;
540 }
541 
542 static bool is_sync_callback_calling_kfunc(u32 btf_id);
543 static bool is_async_callback_calling_kfunc(u32 btf_id);
544 static bool is_call_rcu_kfunc(u32 btf_id);
545 static bool is_callback_calling_kfunc(u32 btf_id);
546 
547 static bool is_bpf_wq_set_callback_kfunc(u32 btf_id);
548 static bool is_task_work_add_kfunc(u32 func_id);
549 
550 static bool is_sync_callback_calling_function(enum bpf_func_id func_id)
551 {
552 	return func_id == BPF_FUNC_for_each_map_elem ||
553 	       func_id == BPF_FUNC_find_vma ||
554 	       func_id == BPF_FUNC_loop ||
555 	       func_id == BPF_FUNC_user_ringbuf_drain;
556 }
557 
558 static bool is_async_callback_calling_function(enum bpf_func_id func_id)
559 {
560 	return func_id == BPF_FUNC_timer_set_callback;
561 }
562 
563 static bool is_callback_calling_function(enum bpf_func_id func_id)
564 {
565 	return is_sync_callback_calling_function(func_id) ||
566 	       is_async_callback_calling_function(func_id);
567 }
568 
569 bool bpf_is_sync_callback_calling_insn(struct bpf_insn *insn)
570 {
571 	return (bpf_helper_call(insn) && is_sync_callback_calling_function(insn->imm)) ||
572 	       (bpf_pseudo_kfunc_call(insn) && is_sync_callback_calling_kfunc(insn->imm));
573 }
574 
575 bool bpf_is_async_callback_calling_insn(struct bpf_insn *insn)
576 {
577 	return (bpf_helper_call(insn) && is_async_callback_calling_function(insn->imm)) ||
578 	       (bpf_pseudo_kfunc_call(insn) && is_async_callback_calling_kfunc(insn->imm));
579 }
580 
581 static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn *insn)
582 {
583 	/* bpf_timer callbacks are never sleepable. */
584 	if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback)
585 		return false;
586 
587 	/* bpf_call_rcu and bpf_call_rcu_tasks_trace callbacks are never sleepable. */
588 	if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && is_call_rcu_kfunc(insn->imm))
589 		return false;
590 
591 	/* bpf_wq and bpf_task_work callbacks are always sleepable. */
592 	if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
593 	    (is_bpf_wq_set_callback_kfunc(insn->imm) || is_task_work_add_kfunc(insn->imm)))
594 		return true;
595 
596 	verifier_bug(env, "unhandled async callback in is_async_cb_sleepable");
597 	return false;
598 }
599 
600 /* From here on the maps of the program count it as a sleepable user. */
601 static void mark_sleepable_async_cb(struct bpf_verifier_env *env)
602 {
603 	int i;
604 
605 	if (bpf_prog_runs_sleepable(env->prog))
606 		return;
607 
608 	env->prog->aux->sleepable_async_cb = true;
609 	for (i = 0; i < env->used_map_cnt; i++)
610 		atomic64_inc(&env->used_maps[i]->sleepable_refcnt);
611 }
612 
613 bool bpf_is_may_goto_insn(struct bpf_insn *insn)
614 {
615 	return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO;
616 }
617 
618 static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots)
619 {
620 	int allocated_slots = bpf_stack_nr_slots(state);
621 
622 	/*
623 	 * We need to check that slots between [spi - nr_slots + 1, spi] are
624 	 * within [0, allocated_stack).
625 	 *
626 	 * Please note that the spi grows downwards. For example, a dynptr
627 	 * takes the size of two stack slots; the first slot will be at
628 	 * spi and the second slot will be at spi - 1.
629 	 */
630 	return spi - nr_slots + 1 >= 0 && spi < allocated_slots;
631 }
632 
633 static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
634 			          const char *obj_kind, int nr_slots)
635 {
636 	int off, spi;
637 
638 	if (!tnum_is_const(reg->var_off)) {
639 		verbose(env, "%s has to be at a constant offset\n", obj_kind);
640 		return -EINVAL;
641 	}
642 
643 	off = reg->var_off.value;
644 	if (off >= 0 || off % BPF_REG_SIZE) {
645 		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
646 		return -EINVAL;
647 	}
648 
649 	spi = bpf_get_spi(off);
650 	if (spi + 1 < nr_slots) {
651 		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
652 		return -EINVAL;
653 	}
654 
655 	if (!is_spi_bounds_valid(bpf_func(env, reg), spi, nr_slots))
656 		return -ERANGE;
657 	return spi;
658 }
659 
660 static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
661 {
662 	return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS);
663 }
664 
665 static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots)
666 {
667 	return stack_slot_obj_get_spi(env, reg, "iter", nr_slots);
668 }
669 
670 static int irq_flag_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
671 {
672 	return stack_slot_obj_get_spi(env, reg, "irq_flag", 1);
673 }
674 
675 static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
676 {
677 	switch (arg_type & DYNPTR_TYPE_FLAG_MASK) {
678 	case DYNPTR_TYPE_LOCAL:
679 		return BPF_DYNPTR_TYPE_LOCAL;
680 	case DYNPTR_TYPE_RINGBUF:
681 		return BPF_DYNPTR_TYPE_RINGBUF;
682 	case DYNPTR_TYPE_SKB:
683 		return BPF_DYNPTR_TYPE_SKB;
684 	case DYNPTR_TYPE_XDP:
685 		return BPF_DYNPTR_TYPE_XDP;
686 	case DYNPTR_TYPE_SKB_META:
687 		return BPF_DYNPTR_TYPE_SKB_META;
688 	case DYNPTR_TYPE_FILE:
689 		return BPF_DYNPTR_TYPE_FILE;
690 	default:
691 		return BPF_DYNPTR_TYPE_INVALID;
692 	}
693 }
694 
695 static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type)
696 {
697 	switch (type) {
698 	case BPF_DYNPTR_TYPE_LOCAL:
699 		return DYNPTR_TYPE_LOCAL;
700 	case BPF_DYNPTR_TYPE_RINGBUF:
701 		return DYNPTR_TYPE_RINGBUF;
702 	case BPF_DYNPTR_TYPE_SKB:
703 		return DYNPTR_TYPE_SKB;
704 	case BPF_DYNPTR_TYPE_XDP:
705 		return DYNPTR_TYPE_XDP;
706 	case BPF_DYNPTR_TYPE_SKB_META:
707 		return DYNPTR_TYPE_SKB_META;
708 	case BPF_DYNPTR_TYPE_FILE:
709 		return DYNPTR_TYPE_FILE;
710 	default:
711 		return 0;
712 	}
713 }
714 
715 static bool dynptr_type_referenced(enum bpf_dynptr_type type)
716 {
717 	return type == BPF_DYNPTR_TYPE_RINGBUF || type == BPF_DYNPTR_TYPE_FILE;
718 }
719 
720 static void __mark_dynptr_reg(struct bpf_reg_state *reg,
721 			      enum bpf_dynptr_type type,
722 			      bool first_slot, int id, int parent_id);
723 
724 static void mark_dynptr_stack_regs(struct bpf_verifier_env *env,
725 				   struct bpf_reg_state *sreg1,
726 				   struct bpf_reg_state *sreg2,
727 				   enum bpf_dynptr_type type, int parent_id)
728 {
729 	int id = ++env->id_gen;
730 
731 	__mark_dynptr_reg(sreg1, type, true, id, parent_id);
732 	__mark_dynptr_reg(sreg2, type, false, id, parent_id);
733 }
734 
735 static void mark_dynptr_cb_reg(struct bpf_verifier_env *env,
736 			       struct bpf_reg_state *reg,
737 			       enum bpf_dynptr_type type)
738 {
739 	__mark_dynptr_reg(reg, type, true, ++env->id_gen, 0);
740 }
741 
742 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
743 				        struct bpf_func_state *state, int spi);
744 
745 static int destroy_dynptrs_in_stack_slots(struct bpf_verifier_env *env,
746 					  struct bpf_func_state *state,
747 					  int first_spi, int last_spi)
748 {
749 	int spi, err;
750 
751 	for (spi = first_spi; spi <= last_spi; spi++) {
752 		err = destroy_if_dynptr_stack_slot(env, state, spi);
753 		if (err)
754 			return err;
755 	}
756 
757 	return 0;
758 }
759 
760 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
761 				   enum bpf_arg_type arg_type, int insn_idx,
762 				   struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr)
763 {
764 	struct bpf_func_state *state = bpf_func(env, reg);
765 	int spi, i, err, parent_id = 0;
766 	enum bpf_dynptr_type type;
767 
768 	spi = dynptr_get_spi(env, reg);
769 	if (spi < 0)
770 		return spi;
771 
772 	/* We cannot assume both spi and spi - 1 belong to the same dynptr,
773 	 * hence we need to destroy dynptrs in both slots,
774 	 * to ensure that for the following example:
775 	 *	[d1][d1][d2][d2]
776 	 * spi    3   2   1   0
777 	 * So marking spi = 2 should lead to destruction of both d1 and d2. In
778 	 * case they do belong to same dynptr, second call won't see slot_type
779 	 * as STACK_DYNPTR and will simply skip destruction.
780 	 */
781 	err = destroy_dynptrs_in_stack_slots(env, state, spi - 1, spi);
782 	if (err)
783 		return err;
784 
785 	for (i = 0; i < BPF_REG_SIZE; i++) {
786 		bpf_stack_slot(state, spi)->slot_type[i] = STACK_DYNPTR;
787 		bpf_stack_slot(state, spi - 1)->slot_type[i] = STACK_DYNPTR;
788 	}
789 
790 	type = arg_to_dynptr_type(arg_type);
791 	if (type == BPF_DYNPTR_TYPE_INVALID)
792 		return -EINVAL;
793 
794 	if (dynptr->type == BPF_DYNPTR_TYPE_INVALID) { /* dynptr constructors */
795 		err = validate_ref_obj(env, ref_obj);
796 		if (err)
797 			return err;
798 
799 		/* All non-clone constructors take their backing object in R1. */
800 		parent_id = reg_lifetime_id(env, &cur_regs(env)[BPF_REG_1]);
801 
802 		if (dynptr_type_referenced(type)) {
803 			int id;
804 
805 			/*
806 			 * Create an intermediate reference that tracks the referenced
807 			 * object for the referenced dynptr. Freeing a referenced dynptr
808 			 * through helpers/kfuncs will invalidate all clones.
809 			 */
810 			id = acquire_reference(env, insn_idx, parent_id);
811 			if (id < 0)
812 				return id;
813 
814 			parent_id = id;
815 		}
816 	} else { /* bpf_dynptr_clone() */
817 		parent_id = dynptr->parent_id;
818 	}
819 
820 	mark_dynptr_stack_regs(env, &bpf_stack_slot(state, spi)->spilled_ptr,
821 			       &bpf_stack_slot(state, spi - 1)->spilled_ptr, type, parent_id);
822 
823 	return 0;
824 }
825 
826 static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_stack_state *stack)
827 {
828 	int i;
829 
830 	for (i = 0; i < BPF_REG_SIZE; i++) {
831 		stack[0].slot_type[i] = STACK_INVALID;
832 		stack[1].slot_type[i] = STACK_INVALID;
833 	}
834 
835 	bpf_mark_reg_not_init(env, &stack[0].spilled_ptr);
836 	bpf_mark_reg_not_init(env, &stack[1].spilled_ptr);
837 }
838 
839 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
840 {
841 	struct bpf_func_state *state = bpf_func(env, reg);
842 	int spi;
843 
844 	spi = dynptr_get_spi(env, reg);
845 	if (spi < 0)
846 		return spi;
847 
848 	/*
849 	 * For referenced dynptr, release the parent ref which cascades to
850 	 * all clones and derived slices. For non-referenced dynptr, only
851 	 * the dynptr and slices derived from it will be invalidated.
852 	 */
853 	reg = &bpf_stack_slot(state, spi)->spilled_ptr;
854 	return release_reference(env, dynptr_type_referenced(reg->dynptr.type)
855 				      ? reg->parent_id
856 				      : reg->id);
857 }
858 
859 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
860 			       struct bpf_reg_state *reg);
861 
862 static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
863 {
864 	if (!env->allow_ptr_leaks)
865 		bpf_mark_reg_not_init(env, reg);
866 	else
867 		__mark_reg_unknown(env, reg);
868 }
869 
870 static int dynptr_ref_cnt(struct bpf_verifier_env *env, int v_parent_id)
871 {
872 	struct bpf_stack_state *stack;
873 	struct bpf_func_state *state;
874 	struct bpf_reg_state *reg;
875 	int ref_cnt = 0;
876 
877 	bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, 1 << STACK_DYNPTR, ({
878 		if (!stack || stack->slot_type[0] != STACK_DYNPTR)
879 			continue;
880 		if (!stack->spilled_ptr.dynptr.first_slot)
881 			continue;
882 		if (stack->spilled_ptr.parent_id == v_parent_id)
883 			ref_cnt++;
884 	}));
885 
886 	return ref_cnt;
887 }
888 
889 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
890 				        struct bpf_func_state *state, int spi)
891 {
892 	struct bpf_stack_state *slot = bpf_stack_slot(state, spi);
893 	int err = 0;
894 
895 	/* We always ensure that STACK_DYNPTR is never set partially,
896 	 * hence just checking for slot_type[0] is enough. This is
897 	 * different for STACK_SPILL, where it may be only set for
898 	 * 1 byte, so code has to use is_spilled_reg.
899 	 */
900 	if (slot->slot_type[0] != STACK_DYNPTR)
901 		return 0;
902 
903 	/* Reposition spi to first slot */
904 	if (!slot->spilled_ptr.dynptr.first_slot) {
905 		spi = spi + 1;
906 		slot = bpf_stack_slot(state, spi);
907 	}
908 
909 	/*
910 	 * A referenced dynptr can be overwritten only if there is at
911 	 * least one other dynptr sharing the same virtual ref parent,
912 	 * ensuring the reference can still be properly released.
913 	 */
914 	if (dynptr_type_referenced(slot->spilled_ptr.dynptr.type) &&
915 	    dynptr_ref_cnt(env, slot->spilled_ptr.parent_id) <= 1) {
916 		verbose(env, "cannot overwrite referenced dynptr\n");
917 		bpf_diag_res(
918 			env, env->insn_idx, "referenced dynptr overwrite",
919 			"This stack slot contains a dynptr that owns or protects a referenced resource. Overwriting the last dynptr for that resource would lose the verifier-tracked release path.",
920 			"Release or clone the dynptr so another live dynptr still tracks the referenced resource before overwriting this stack slot.");
921 		return -EINVAL;
922 	}
923 
924 	/* Invalidate the dynptr and any derived slices */
925 	err = release_reference(env, slot->spilled_ptr.id);
926 	if (!err) {
927 		mark_stack_slot_scratched(env, spi);
928 		mark_stack_slot_scratched(env, spi - 1);
929 	}
930 
931 	return err;
932 }
933 
934 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
935 {
936 	int spi;
937 
938 	if (reg->type == CONST_PTR_TO_DYNPTR)
939 		return false;
940 
941 	spi = dynptr_get_spi(env, reg);
942 
943 	/* -ERANGE (i.e. spi not falling into allocated stack slots) isn't an
944 	 * error because this just means the stack state hasn't been updated yet.
945 	 * We will do check_mem_access to check and update stack bounds later.
946 	 */
947 	if (spi < 0 && spi != -ERANGE)
948 		return false;
949 
950 	/* We don't need to check if the stack slots are marked by previous
951 	 * dynptr initializations because we allow overwriting existing unreferenced
952 	 * STACK_DYNPTR slots, see mark_stack_slots_dynptr which calls
953 	 * destroy_if_dynptr_stack_slot to ensure dynptr objects at the slots we are
954 	 * touching are completely destructed before we reinitialize them for a new
955 	 * one. For referenced ones, destroy_if_dynptr_stack_slot returns an error early
956 	 * instead of delaying it until the end where the user will get "Unreleased
957 	 * reference" error.
958 	 */
959 	return true;
960 }
961 
962 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
963 {
964 	struct bpf_func_state *state = bpf_func(env, reg);
965 	struct bpf_stack_state *slot;
966 	int i, spi;
967 
968 	/* This already represents first slot of initialized bpf_dynptr.
969 	 *
970 	 * CONST_PTR_TO_DYNPTR already has fixed and var_off as 0 due to
971 	 * check_func_arg_reg_off's logic, so we don't need to check its
972 	 * offset and alignment.
973 	 */
974 	if (reg->type == CONST_PTR_TO_DYNPTR)
975 		return true;
976 
977 	spi = dynptr_get_spi(env, reg);
978 	if (spi < 0)
979 		return false;
980 	slot = bpf_stack_slot(state, spi);
981 	if (!slot->spilled_ptr.dynptr.first_slot)
982 		return false;
983 
984 	for (i = 0; i < BPF_REG_SIZE; i++) {
985 		if (slot->slot_type[i] != STACK_DYNPTR ||
986 		    bpf_stack_slot(state, spi - 1)->slot_type[i] != STACK_DYNPTR)
987 			return false;
988 	}
989 
990 	return true;
991 }
992 
993 static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
994 {
995 	struct bpf_func_state *state;
996 	int spi;
997 
998 	if (reg->type == CONST_PTR_TO_DYNPTR)
999 		return reg->dynptr.type;
1000 
1001 	spi = dynptr_get_spi(env, reg);
1002 	if (spi < 0)
1003 		return BPF_DYNPTR_TYPE_INVALID;
1004 	state = bpf_func(env, reg);
1005 	return bpf_stack_slot(state, spi)->spilled_ptr.dynptr.type;
1006 }
1007 
1008 static bool dynptr_write_changes_pkt_data(enum bpf_dynptr_type type)
1009 {
1010 	return type == BPF_DYNPTR_TYPE_SKB || type == BPF_DYNPTR_TYPE_SKB_META;
1011 }
1012 
1013 /* The data of the other types is contiguous: bpf_dynptr_slice() does not use the buffer. */
1014 static bool dynptr_slice_may_be_buffer(enum bpf_dynptr_type type)
1015 {
1016 	return type != BPF_DYNPTR_TYPE_LOCAL && type != BPF_DYNPTR_TYPE_RINGBUF &&
1017 	       type != BPF_DYNPTR_TYPE_SKB_META;
1018 }
1019 
1020 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
1021 				    enum bpf_arg_type arg_type)
1022 {
1023 	/*
1024 	 * ARG_PTR_TO_DYNPTR without a type flag takes any type of dynptr.
1025 	 * Test the flags rather than the whole arg_type, which may carry
1026 	 * unrelated ones such as PTR_MAYBE_NULL.
1027 	 */
1028 	if (!(arg_type & DYNPTR_TYPE_FLAG_MASK))
1029 		return true;
1030 
1031 	return dynptr_reg_type(env, reg) == arg_to_dynptr_type(arg_type);
1032 }
1033 
1034 static void __mark_reg_known_zero(struct bpf_reg_state *reg);
1035 
1036 static bool in_rcu_cs(struct bpf_verifier_env *env);
1037 
1038 static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta);
1039 
1040 static int mark_stack_slots_iter(struct bpf_verifier_env *env,
1041 				 struct bpf_call_arg_meta *meta,
1042 				 struct bpf_reg_state *reg, int insn_idx,
1043 				 struct btf *btf, u32 btf_id, int nr_slots)
1044 {
1045 	struct bpf_func_state *state = bpf_func(env, reg);
1046 	int spi, i, j, id;
1047 
1048 	spi = iter_get_spi(env, reg, nr_slots);
1049 	if (spi < 0)
1050 		return spi;
1051 
1052 	id = acquire_reference(env, insn_idx, 0);
1053 	if (id < 0)
1054 		return id;
1055 
1056 	for (i = 0; i < nr_slots; i++) {
1057 		struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i);
1058 		struct bpf_reg_state *st = &slot->spilled_ptr;
1059 
1060 		__mark_reg_known_zero(st);
1061 		st->type = PTR_TO_STACK; /* we don't have dedicated reg type */
1062 		if (is_kfunc_rcu_protected(meta)) {
1063 			if (in_rcu_cs(env))
1064 				st->type |= MEM_RCU;
1065 			else
1066 				st->type |= PTR_UNTRUSTED;
1067 		}
1068 		st->id = i == 0 ? id : 0;
1069 		st->iter.btf = btf;
1070 		st->iter.btf_id = btf_id;
1071 		st->iter.state = BPF_ITER_STATE_ACTIVE;
1072 		st->iter.depth = 0;
1073 
1074 		for (j = 0; j < BPF_REG_SIZE; j++)
1075 			slot->slot_type[j] = STACK_ITER;
1076 
1077 		mark_stack_slot_scratched(env, spi - i);
1078 	}
1079 
1080 	return 0;
1081 }
1082 
1083 static int unmark_stack_slots_iter(struct bpf_verifier_env *env,
1084 				   struct bpf_reg_state *reg, int nr_slots)
1085 {
1086 	struct bpf_func_state *state = bpf_func(env, reg);
1087 	int spi, i, j, err;
1088 
1089 	spi = iter_get_spi(env, reg, nr_slots);
1090 	if (spi < 0)
1091 		return spi;
1092 
1093 	for (i = 0; i < nr_slots; i++) {
1094 		struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i);
1095 		struct bpf_reg_state *st = &slot->spilled_ptr;
1096 
1097 		if (i == 0) {
1098 			err = release_reference(env, st->id);
1099 			if (err)
1100 				return err;
1101 		}
1102 
1103 		bpf_mark_reg_not_init(env, st);
1104 
1105 		for (j = 0; j < BPF_REG_SIZE; j++)
1106 			slot->slot_type[j] = STACK_INVALID;
1107 
1108 		mark_stack_slot_scratched(env, spi - i);
1109 	}
1110 
1111 	return 0;
1112 }
1113 
1114 static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env,
1115 				     struct bpf_reg_state *reg, int nr_slots)
1116 {
1117 	struct bpf_func_state *state = bpf_func(env, reg);
1118 	int spi, i, j;
1119 
1120 	/* For -ERANGE (i.e. spi not falling into allocated stack slots), we
1121 	 * will do check_mem_access to check and update stack bounds later, so
1122 	 * return true for that case.
1123 	 */
1124 	spi = iter_get_spi(env, reg, nr_slots);
1125 	if (spi == -ERANGE)
1126 		return true;
1127 	if (spi < 0)
1128 		return false;
1129 
1130 	for (i = 0; i < nr_slots; i++) {
1131 		struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i);
1132 
1133 		for (j = 0; j < BPF_REG_SIZE; j++)
1134 			if (slot->slot_type[j] == STACK_ITER)
1135 				return false;
1136 	}
1137 
1138 	return true;
1139 }
1140 
1141 static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
1142 				   struct btf *btf, u32 btf_id, int nr_slots)
1143 {
1144 	struct bpf_func_state *state = bpf_func(env, reg);
1145 	int spi, i, j;
1146 
1147 	spi = iter_get_spi(env, reg, nr_slots);
1148 	if (spi < 0)
1149 		return -EINVAL;
1150 
1151 	for (i = 0; i < nr_slots; i++) {
1152 		struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i);
1153 		struct bpf_reg_state *st = &slot->spilled_ptr;
1154 
1155 		if (st->type & PTR_UNTRUSTED)
1156 			return -EPROTO;
1157 		/* only main (first) slot has id set */
1158 		if (i == 0 && !st->id)
1159 			return -EINVAL;
1160 		if (i != 0 && st->id)
1161 			return -EINVAL;
1162 		if (st->iter.btf != btf || st->iter.btf_id != btf_id)
1163 			return -EINVAL;
1164 
1165 		for (j = 0; j < BPF_REG_SIZE; j++)
1166 			if (slot->slot_type[j] != STACK_ITER)
1167 				return -EINVAL;
1168 	}
1169 
1170 	return 0;
1171 }
1172 
1173 static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx);
1174 static int release_irq_state(struct bpf_verifier_env *env, int id);
1175 
1176 static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env,
1177 				     struct bpf_call_arg_meta *meta,
1178 				     struct bpf_reg_state *reg, int insn_idx,
1179 				     int kfunc_class)
1180 {
1181 	struct bpf_func_state *state = bpf_func(env, reg);
1182 	struct bpf_stack_state *slot;
1183 	struct bpf_reg_state *st;
1184 	int spi, i, id;
1185 
1186 	spi = irq_flag_get_spi(env, reg);
1187 	if (spi < 0)
1188 		return spi;
1189 
1190 	id = acquire_irq_state(env, insn_idx);
1191 	if (id < 0)
1192 		return id;
1193 
1194 	slot = bpf_stack_slot(state, spi);
1195 	st = &slot->spilled_ptr;
1196 
1197 	__mark_reg_known_zero(st);
1198 	st->type = PTR_TO_STACK; /* we don't have dedicated reg type */
1199 	st->id = id;
1200 	st->irq.kfunc_class = kfunc_class;
1201 
1202 	for (i = 0; i < BPF_REG_SIZE; i++)
1203 		slot->slot_type[i] = STACK_IRQ_FLAG;
1204 
1205 	mark_stack_slot_scratched(env, spi);
1206 	return 0;
1207 }
1208 
1209 static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
1210 				      int kfunc_class)
1211 {
1212 	struct bpf_func_state *state = bpf_func(env, reg);
1213 	struct bpf_stack_state *slot;
1214 	struct bpf_reg_state *st;
1215 	int spi, i, err;
1216 
1217 	spi = irq_flag_get_spi(env, reg);
1218 	if (spi < 0)
1219 		return spi;
1220 
1221 	slot = bpf_stack_slot(state, spi);
1222 	st = &slot->spilled_ptr;
1223 
1224 	if (st->irq.kfunc_class != kfunc_class) {
1225 		const char *flag_kfunc = st->irq.kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock";
1226 		const char *used_kfunc = kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock";
1227 		const char *reason;
1228 
1229 		verbose(env, "irq flag acquired by %s kfuncs cannot be restored with %s kfuncs\n",
1230 			flag_kfunc, used_kfunc);
1231 		reason = bpf_diag_fmt(env,
1232 				      "This IRQ flag was saved by %s IRQ kfuncs, but the restore call "
1233 			"belongs to the %s IRQ kfunc family. Save and restore operations "
1234 			"must use the same family.",
1235 			flag_kfunc, used_kfunc);
1236 		bpf_diag_irq(env, env->insn_idx, "IRQ flag restore mismatch", reason,
1237 			     "Restore the flag with the matching IRQ restore kfunc for the save "
1238 			     "operation that created it.",
1239 			     bpf_diag_irq_depth(env->cur_state));
1240 		return -EINVAL;
1241 	}
1242 
1243 	err = release_irq_state(env, st->id);
1244 	WARN_ON_ONCE(err && err != -EACCES);
1245 	if (err) {
1246 		int insn_idx = 0;
1247 
1248 		for (int i = 0; i < env->cur_state->acquired_refs; i++) {
1249 			if (env->cur_state->refs[i].id == env->cur_state->active_irq_id) {
1250 				insn_idx = env->cur_state->refs[i].insn_idx;
1251 				break;
1252 			}
1253 		}
1254 
1255 		verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n",
1256 			env->cur_state->active_irq_id, insn_idx);
1257 		bpf_diag_irq(env, env->insn_idx, "IRQ flag restore out of order",
1258 			     "IRQ-disabled regions must be restored in last-in, first-out order, "
1259 			     "but this restore does not match the currently active IRQ flag.",
1260 			     "Restore nested IRQ flags in the reverse order they were saved.",
1261 			     bpf_diag_irq_depth(env->cur_state));
1262 		return err;
1263 	}
1264 
1265 	bpf_mark_reg_not_init(env, st);
1266 
1267 	for (i = 0; i < BPF_REG_SIZE; i++)
1268 		slot->slot_type[i] = STACK_INVALID;
1269 
1270 	mark_stack_slot_scratched(env, spi);
1271 	return 0;
1272 }
1273 
1274 static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
1275 {
1276 	struct bpf_func_state *state = bpf_func(env, reg);
1277 	struct bpf_stack_state *slot;
1278 	int spi, i;
1279 
1280 	/* For -ERANGE (i.e. spi not falling into allocated stack slots), we
1281 	 * will do check_mem_access to check and update stack bounds later, so
1282 	 * return true for that case.
1283 	 */
1284 	spi = irq_flag_get_spi(env, reg);
1285 	if (spi == -ERANGE)
1286 		return true;
1287 	if (spi < 0)
1288 		return false;
1289 
1290 	slot = bpf_stack_slot(state, spi);
1291 
1292 	for (i = 0; i < BPF_REG_SIZE; i++)
1293 		if (slot->slot_type[i] == STACK_IRQ_FLAG)
1294 			return false;
1295 	return true;
1296 }
1297 
1298 static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
1299 {
1300 	struct bpf_func_state *state = bpf_func(env, reg);
1301 	struct bpf_stack_state *slot;
1302 	struct bpf_reg_state *st;
1303 	int spi, i;
1304 
1305 	spi = irq_flag_get_spi(env, reg);
1306 	if (spi < 0)
1307 		return -EINVAL;
1308 
1309 	slot = bpf_stack_slot(state, spi);
1310 	st = &slot->spilled_ptr;
1311 
1312 	if (!st->id)
1313 		return -EINVAL;
1314 
1315 	for (i = 0; i < BPF_REG_SIZE; i++)
1316 		if (slot->slot_type[i] != STACK_IRQ_FLAG)
1317 			return -EINVAL;
1318 	return 0;
1319 }
1320 
1321 /* Check if given stack slot is "special":
1322  *   - spilled register state (STACK_SPILL);
1323  *   - dynptr state (STACK_DYNPTR);
1324  *   - iter state (STACK_ITER).
1325  *   - irq flag state (STACK_IRQ_FLAG)
1326  */
1327 static bool is_stack_slot_special(const struct bpf_stack_state *stack)
1328 {
1329 	enum bpf_stack_slot_type type = stack->slot_type[BPF_REG_SIZE - 1];
1330 
1331 	switch (type) {
1332 	case STACK_SPILL:
1333 	case STACK_DYNPTR:
1334 	case STACK_ITER:
1335 	case STACK_IRQ_FLAG:
1336 		return true;
1337 	case STACK_INVALID:
1338 	case STACK_POISON:
1339 	case STACK_MISC:
1340 	case STACK_ZERO:
1341 		return false;
1342 	default:
1343 		WARN_ONCE(1, "unknown stack slot type %d\n", type);
1344 		return true;
1345 	}
1346 }
1347 
1348 /* The reg state of a pointer or a bounded scalar was saved when
1349  * it was spilled to the stack.
1350  */
1351 
1352 /*
1353  * Mark stack slot as STACK_MISC, unless it is already:
1354  * - STACK_INVALID, in which case they are equivalent.
1355  * - STACK_ZERO, in which case we preserve more precise STACK_ZERO.
1356  * - STACK_POISON, which truly forbids access to the slot.
1357  * Regardless of allow_ptr_leaks setting (i.e., privileged or unprivileged
1358  * mode), we won't promote STACK_INVALID to STACK_MISC. In privileged case it is
1359  * unnecessary as both are considered equivalent when loading data and pruning,
1360  * in case of unprivileged mode it will be incorrect to allow reads of invalid
1361  * slots.
1362  */
1363 static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype)
1364 {
1365 	if (*stype == STACK_ZERO)
1366 		return;
1367 	if (*stype == STACK_INVALID || *stype == STACK_POISON)
1368 		return;
1369 	*stype = STACK_MISC;
1370 }
1371 
1372 static void scrub_spilled_slot(u8 *stype)
1373 {
1374 	if (*stype != STACK_INVALID && *stype != STACK_POISON)
1375 		*stype = STACK_MISC;
1376 }
1377 
1378 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too
1379  * small to hold src. This is different from krealloc since we don't want to preserve
1380  * the contents of dst.
1381  *
1382  * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could
1383  * not be allocated.
1384  */
1385 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags)
1386 {
1387 	size_t alloc_bytes;
1388 	void *orig = dst;
1389 	size_t bytes;
1390 
1391 	if (ZERO_OR_NULL_PTR(src))
1392 		goto out;
1393 
1394 	if (unlikely(check_mul_overflow(n, size, &bytes)))
1395 		return NULL;
1396 
1397 	alloc_bytes = max(ksize(orig), kmalloc_size_roundup(bytes));
1398 	dst = krealloc(orig, alloc_bytes, flags);
1399 	if (!dst) {
1400 		kfree(orig);
1401 		return NULL;
1402 	}
1403 
1404 	memcpy(dst, src, bytes);
1405 out:
1406 	return dst ? dst : ZERO_SIZE_PTR;
1407 }
1408 
1409 /* resize an array from old_n items to new_n items. the array is reallocated if it's too
1410  * small to hold new_n items. new items are zeroed out if the array grows.
1411  *
1412  * Contrary to krealloc_array, does not free arr if new_n is zero.
1413  */
1414 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size)
1415 {
1416 	size_t alloc_size;
1417 	void *new_arr;
1418 
1419 	if (!new_n || old_n == new_n)
1420 		goto out;
1421 
1422 	alloc_size = kmalloc_size_roundup(size_mul(new_n, size));
1423 	new_arr = krealloc(arr, alloc_size, GFP_KERNEL_ACCOUNT);
1424 	if (!new_arr) {
1425 		kfree(arr);
1426 		return NULL;
1427 	}
1428 	arr = new_arr;
1429 
1430 	if (new_n > old_n)
1431 		memset(arr + old_n * size, 0, (new_n - old_n) * size);
1432 
1433 out:
1434 	return arr ? arr : ZERO_SIZE_PTR;
1435 }
1436 
1437 static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf_verifier_state *src)
1438 {
1439 	dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs,
1440 			       sizeof(struct bpf_reference_state), GFP_KERNEL_ACCOUNT);
1441 	if (!dst->refs)
1442 		return -ENOMEM;
1443 
1444 	dst->acquired_refs = src->acquired_refs;
1445 	dst->active_locks = src->active_locks;
1446 	dst->active_preempt_locks = src->active_preempt_locks;
1447 	dst->active_rcu_locks = src->active_rcu_locks;
1448 	dst->active_irq_id = src->active_irq_id;
1449 	dst->active_lock_id = src->active_lock_id;
1450 	dst->active_lock_ptr = src->active_lock_ptr;
1451 	return 0;
1452 }
1453 
1454 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
1455 {
1456 	size_t n = bpf_stack_nr_slots(src);
1457 
1458 	dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state),
1459 				GFP_KERNEL_ACCOUNT);
1460 	if (!dst->stack)
1461 		return -ENOMEM;
1462 
1463 	dst->allocated_stack = src->allocated_stack;
1464 
1465 	/* copy stack args state */
1466 	n = src->out_stack_arg_cnt;
1467 	if (n) {
1468 		dst->stack_arg_regs = copy_array(dst->stack_arg_regs, src->stack_arg_regs, n,
1469 						 sizeof(struct bpf_reg_state),
1470 						 GFP_KERNEL_ACCOUNT);
1471 		if (!dst->stack_arg_regs)
1472 			return -ENOMEM;
1473 	}
1474 
1475 	dst->out_stack_arg_cnt = src->out_stack_arg_cnt;
1476 	return 0;
1477 }
1478 
1479 static int resize_reference_state(struct bpf_verifier_state *state, size_t n)
1480 {
1481 	state->refs = realloc_array(state->refs, state->acquired_refs, n,
1482 				    sizeof(struct bpf_reference_state));
1483 	if (!state->refs)
1484 		return -ENOMEM;
1485 
1486 	state->acquired_refs = n;
1487 	return 0;
1488 }
1489 
1490 /* Possibly update state->allocated_stack to be at least size bytes. Also
1491  * possibly update the function's high-water mark in its bpf_subprog_info.
1492  */
1493 int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size)
1494 {
1495 	size_t old_n = bpf_stack_nr_slots(state), n;
1496 
1497 	/* The stack size is always a multiple of BPF_REG_SIZE. */
1498 	size = round_up(size, BPF_REG_SIZE);
1499 	n = size / BPF_REG_SIZE;
1500 
1501 	if (old_n >= n)
1502 		return 0;
1503 
1504 	state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state));
1505 	if (!state->stack)
1506 		return -ENOMEM;
1507 
1508 	state->allocated_stack = size;
1509 
1510 	/* update known max for given subprogram */
1511 	if (env->subprog_info[state->subprogno].stack_depth < size)
1512 		env->subprog_info[state->subprogno].stack_depth = size;
1513 
1514 	return 0;
1515 }
1516 
1517 static int grow_stack_arg_slots(struct bpf_verifier_env *env,
1518 				struct bpf_func_state *state, int cnt)
1519 {
1520 	size_t old_n = state->out_stack_arg_cnt;
1521 
1522 	if (old_n >= cnt)
1523 		return 0;
1524 
1525 	state->stack_arg_regs = realloc_array(state->stack_arg_regs, old_n, cnt,
1526 					      sizeof(struct bpf_reg_state));
1527 	if (!state->stack_arg_regs)
1528 		return -ENOMEM;
1529 
1530 	state->out_stack_arg_cnt = cnt;
1531 	return 0;
1532 }
1533 
1534 /* Acquire a pointer id from the env and update the state->refs to include
1535  * this new pointer reference.
1536  * On success, returns a valid pointer id to associate with the register
1537  * On failure, returns a negative errno.
1538  */
1539 static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
1540 {
1541 	struct bpf_verifier_state *state = env->cur_state;
1542 	int new_ofs = state->acquired_refs;
1543 	int err;
1544 
1545 	err = resize_reference_state(state, state->acquired_refs + 1);
1546 	if (err)
1547 		return NULL;
1548 	state->refs[new_ofs].insn_idx = insn_idx;
1549 
1550 	return &state->refs[new_ofs];
1551 }
1552 
1553 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id)
1554 {
1555 	struct bpf_reference_state *s;
1556 
1557 	s = acquire_reference_state(env, insn_idx);
1558 	if (!s)
1559 		return -ENOMEM;
1560 	s->type = REF_TYPE_PTR;
1561 	s->id = ++env->id_gen;
1562 	s->parent_id = parent_id;
1563 	bpf_diag_record_ref_acquire(env, insn_idx, s->id);
1564 	return s->id;
1565 }
1566 
1567 static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type,
1568 			      int id, void *ptr)
1569 {
1570 	struct bpf_verifier_state *state = env->cur_state;
1571 	struct bpf_reference_state *s;
1572 
1573 	s = acquire_reference_state(env, insn_idx);
1574 	if (!s)
1575 		return -ENOMEM;
1576 	s->type = type;
1577 	s->id = id;
1578 	s->ptr = ptr;
1579 
1580 	state->active_locks++;
1581 	state->active_lock_id = id;
1582 	state->active_lock_ptr = ptr;
1583 	bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_LOCK, true,
1584 				state->active_locks);
1585 	return 0;
1586 }
1587 
1588 static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx)
1589 {
1590 	struct bpf_verifier_state *state = env->cur_state;
1591 	struct bpf_reference_state *s;
1592 
1593 	s = acquire_reference_state(env, insn_idx);
1594 	if (!s)
1595 		return -ENOMEM;
1596 	s->type = REF_TYPE_IRQ;
1597 	s->id = ++env->id_gen;
1598 
1599 	state->active_irq_id = s->id;
1600 	bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_IRQ, true,
1601 				bpf_diag_irq_depth(state));
1602 	return s->id;
1603 }
1604 
1605 static void release_reference_state(struct bpf_verifier_state *state, int idx)
1606 {
1607 	int last_idx;
1608 	size_t rem;
1609 
1610 	/* IRQ state requires the relative ordering of elements remaining the
1611 	 * same, since it relies on the refs array to behave as a stack, so that
1612 	 * it can detect out-of-order IRQ restore. Hence use memmove to shift
1613 	 * the array instead of swapping the final element into the deleted idx.
1614 	 */
1615 	last_idx = state->acquired_refs - 1;
1616 	rem = state->acquired_refs - idx - 1;
1617 	if (last_idx && idx != last_idx)
1618 		memmove(&state->refs[idx], &state->refs[idx + 1], sizeof(*state->refs) * rem);
1619 	memset(&state->refs[last_idx], 0, sizeof(*state->refs));
1620 	state->acquired_refs--;
1621 	return;
1622 }
1623 
1624 static bool find_reference_state(struct bpf_verifier_state *state, int id)
1625 {
1626 	int i;
1627 
1628 	for (i = 0; i < state->acquired_refs; i++) {
1629 		if (state->refs[i].type != REF_TYPE_PTR)
1630 			continue;
1631 		if (state->refs[i].id == id)
1632 			return true;
1633 	}
1634 
1635 	return false;
1636 }
1637 
1638 static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)
1639 {
1640 	return find_reference_state(env->cur_state, reg->id);
1641 }
1642 
1643 static u32 reg_lifetime_id(struct bpf_verifier_env *env,
1644 			   const struct bpf_reg_state *reg)
1645 {
1646 	return reg_is_referenced(env, reg) ? reg->id : reg->parent_id;
1647 }
1648 
1649 static int release_lock_state(struct bpf_verifier_env *env, int type, int id, void *ptr)
1650 {
1651 	struct bpf_verifier_state *state = env->cur_state;
1652 	void *prev_ptr = NULL;
1653 	u32 prev_id = 0;
1654 	int i;
1655 
1656 	for (i = 0; i < state->acquired_refs; i++) {
1657 		if (state->refs[i].type == type && state->refs[i].id == id &&
1658 		    state->refs[i].ptr == ptr) {
1659 			release_reference_state(state, i);
1660 			state->active_locks--;
1661 			/* Reassign active lock (id, ptr). */
1662 			state->active_lock_id = prev_id;
1663 			state->active_lock_ptr = prev_ptr;
1664 			bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_LOCK,
1665 						false, state->active_locks);
1666 			return 0;
1667 		}
1668 		if (state->refs[i].type & REF_TYPE_LOCK_MASK) {
1669 			prev_id = state->refs[i].id;
1670 			prev_ptr = state->refs[i].ptr;
1671 		}
1672 	}
1673 	return -EINVAL;
1674 }
1675 
1676 static int release_irq_state(struct bpf_verifier_env *env, int id)
1677 {
1678 	struct bpf_verifier_state *state = env->cur_state;
1679 	u32 prev_id = 0;
1680 	int i;
1681 
1682 	if (id != state->active_irq_id)
1683 		return -EACCES;
1684 
1685 	for (i = 0; i < state->acquired_refs; i++) {
1686 		if (state->refs[i].type != REF_TYPE_IRQ)
1687 			continue;
1688 		if (state->refs[i].id == id) {
1689 			release_reference_state(state, i);
1690 			state->active_irq_id = prev_id;
1691 			bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_IRQ,
1692 						false, bpf_diag_irq_depth(state));
1693 			return 0;
1694 		} else {
1695 			prev_id = state->refs[i].id;
1696 		}
1697 	}
1698 	return -EINVAL;
1699 }
1700 
1701 static struct bpf_reference_state *find_lock_state(struct bpf_verifier_state *state, enum ref_state_type type,
1702 						   int id, void *ptr)
1703 {
1704 	int i;
1705 
1706 	for (i = 0; i < state->acquired_refs; i++) {
1707 		struct bpf_reference_state *s = &state->refs[i];
1708 
1709 		if (!(s->type & type))
1710 			continue;
1711 
1712 		if (s->id == id && s->ptr == ptr)
1713 			return s;
1714 	}
1715 	return NULL;
1716 }
1717 
1718 static void free_func_state(struct bpf_func_state *state)
1719 {
1720 	if (!state)
1721 		return;
1722 	kfree(state->loop_stack);
1723 	kfree(state->stack_arg_regs);
1724 	kfree(state->stack);
1725 	kfree(state);
1726 }
1727 
1728 void bpf_clear_jmp_history(struct bpf_verifier_state *state)
1729 {
1730 	kfree(state->jmp_history);
1731 	state->jmp_history = NULL;
1732 	state->jmp_history_cnt = 0;
1733 }
1734 
1735 void bpf_free_verifier_state(struct bpf_verifier_state *state,
1736 			    bool free_self)
1737 {
1738 	int i;
1739 
1740 	for (i = 0; i <= state->curframe; i++) {
1741 		free_func_state(state->frame[i]);
1742 		state->frame[i] = NULL;
1743 	}
1744 	kfree(state->refs);
1745 	bpf_clear_jmp_history(state);
1746 	if (free_self)
1747 		kfree(state);
1748 }
1749 
1750 /* copy verifier state from src to dst growing dst stack space
1751  * when necessary to accommodate larger src stack
1752  */
1753 static int copy_func_state(struct bpf_func_state *dst,
1754 			   const struct bpf_func_state *src)
1755 {
1756 	memcpy(dst, src, offsetof(struct bpf_func_state, stack));
1757 	/* Instruction accounting is path-local, not part of verifier state. */
1758 	dst->insns_subtotal = 0;
1759 	dst->loop_stack = copy_array(dst->loop_stack, src->loop_stack, src->loop_stack_cnt,
1760 				     sizeof(*src->loop_stack), GFP_KERNEL_ACCOUNT);
1761 	if (!dst->loop_stack)
1762 		return -ENOMEM;
1763 	return copy_stack_state(dst, src);
1764 }
1765 
1766 int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state,
1767 			   const struct bpf_verifier_state *src)
1768 {
1769 	struct bpf_func_state *dst;
1770 	int i, err;
1771 
1772 	dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history,
1773 					  src->jmp_history_cnt, sizeof(*dst_state->jmp_history),
1774 					  GFP_KERNEL_ACCOUNT);
1775 	if (!dst_state->jmp_history)
1776 		return -ENOMEM;
1777 	dst_state->jmp_history_cnt = src->jmp_history_cnt;
1778 
1779 	/* if dst has more stack frames then src frame, free them, this is also
1780 	 * necessary in case of exceptional exits using bpf_throw.
1781 	 */
1782 	for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
1783 		free_func_state(dst_state->frame[i]);
1784 		dst_state->frame[i] = NULL;
1785 	}
1786 	err = copy_reference_state(dst_state, src);
1787 	if (err)
1788 		return err;
1789 	dst_state->speculative = src->speculative;
1790 	dst_state->in_sleepable = src->in_sleepable;
1791 	dst_state->unwound = src->unwound;
1792 	dst_state->curframe = src->curframe;
1793 	dst_state->branches = src->branches;
1794 	dst_state->parent = src->parent;
1795 	dst_state->first_insn_idx = src->first_insn_idx;
1796 	dst_state->last_insn_idx = src->last_insn_idx;
1797 	dst_state->dfs_depth = src->dfs_depth;
1798 	dst_state->callback_unroll_depth = src->callback_unroll_depth;
1799 	dst_state->may_goto_depth = src->may_goto_depth;
1800 	dst_state->equal_state = src->equal_state;
1801 	for (i = 0; i <= src->curframe; i++) {
1802 		dst = dst_state->frame[i];
1803 		if (!dst) {
1804 			dst = kzalloc_obj(*dst, GFP_KERNEL_ACCOUNT);
1805 			if (!dst)
1806 				return -ENOMEM;
1807 			dst_state->frame[i] = dst;
1808 		}
1809 		err = copy_func_state(dst, src->frame[i]);
1810 		if (err)
1811 			return err;
1812 	}
1813 	return 0;
1814 }
1815 
1816 static u32 state_htab_size(struct bpf_verifier_env *env)
1817 {
1818 	return env->prog->len;
1819 }
1820 
1821 struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx)
1822 {
1823 	struct bpf_verifier_state *cur = env->cur_state;
1824 	struct bpf_func_state *state = cur->frame[cur->curframe];
1825 
1826 	return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];
1827 }
1828 
1829 static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_state *b)
1830 {
1831 	int fr;
1832 
1833 	if (a->curframe != b->curframe)
1834 		return false;
1835 
1836 	for (fr = a->curframe; fr >= 0; fr--)
1837 		if (a->frame[fr]->callsite != b->frame[fr]->callsite)
1838 			return false;
1839 
1840 	return true;
1841 }
1842 
1843 void bpf_free_backedges(struct bpf_scc_visit *visit)
1844 {
1845 	struct bpf_scc_backedge *backedge, *next;
1846 
1847 	for (backedge = visit->backedges; backedge; backedge = next) {
1848 		bpf_free_verifier_state(&backedge->state, false);
1849 		next = backedge->next;
1850 		kfree(backedge);
1851 	}
1852 	visit->backedges = NULL;
1853 }
1854 
1855 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
1856 		     int *insn_idx, bool pop_log)
1857 {
1858 	struct bpf_verifier_state *cur = env->cur_state;
1859 	struct bpf_verifier_stack_elem *elem, *head = env->head;
1860 	int err;
1861 
1862 	if (env->head == NULL)
1863 		return -ENOENT;
1864 
1865 	if (cur) {
1866 		err = bpf_copy_verifier_state(cur, &head->st);
1867 		if (err)
1868 			return err;
1869 		bpf_diag_event_log_restore(env, head->diag_log_pos);
1870 	}
1871 	if (pop_log)
1872 		bpf_vlog_reset(&env->log, head->log_pos);
1873 	if (insn_idx)
1874 		*insn_idx = head->insn_idx;
1875 	if (prev_insn_idx)
1876 		*prev_insn_idx = head->prev_insn_idx;
1877 	elem = head->next;
1878 	bpf_free_verifier_state(&head->st, false);
1879 	kfree(head);
1880 	env->head = elem;
1881 	env->stack_size--;
1882 	return 0;
1883 }
1884 
1885 static bool error_recoverable_with_nospec(int err)
1886 {
1887 	/* Should only return true for non-fatal errors that are allowed to
1888 	 * occur during speculative verification. For these we can insert a
1889 	 * nospec and the program might still be accepted. Do not include
1890 	 * something like ENOMEM because it is likely to re-occur for the next
1891 	 * architectural path once it has been recovered-from in all speculative
1892 	 * paths.
1893 	 */
1894 	return err == -EPERM || err == -EACCES || err == -EINVAL;
1895 }
1896 
1897 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
1898 					     int insn_idx, int prev_insn_idx,
1899 					     bool speculative)
1900 {
1901 	struct bpf_verifier_state *cur = env->cur_state;
1902 	struct bpf_verifier_stack_elem *elem;
1903 	int err;
1904 
1905 	elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT);
1906 	if (!elem)
1907 		return ERR_PTR(-ENOMEM);
1908 
1909 	elem->insn_idx = insn_idx;
1910 	elem->prev_insn_idx = prev_insn_idx;
1911 	elem->next = env->head;
1912 	elem->log_pos = env->log.end_pos;
1913 	elem->diag_log_pos = bpf_diag_event_log_save(env);
1914 	env->head = elem;
1915 	env->stack_size++;
1916 	err = bpf_copy_verifier_state(&elem->st, cur);
1917 	if (err)
1918 		return ERR_PTR(-ENOMEM);
1919 	elem->st.speculative |= speculative;
1920 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
1921 		verbose(env, "The sequence of %d jumps is too complex.\n",
1922 			env->stack_size);
1923 		return ERR_PTR(-E2BIG);
1924 	}
1925 	if (elem->st.parent) {
1926 		++elem->st.parent->branches;
1927 		/* WARN_ON(branches > 2) technically makes sense here,
1928 		 * but
1929 		 * 1. speculative states will bump 'branches' for non-branch
1930 		 * instructions
1931 		 * 2. is_state_visited() heuristics may decide not to create
1932 		 * a new state for a sequence of branches and all such current
1933 		 * and cloned states will be pointing to a single parent state
1934 		 * which might have large 'branches' count.
1935 		 */
1936 	}
1937 	return &elem->st;
1938 }
1939 
1940 static const char *reg_arg_name(struct bpf_verifier_env *env, argno_t argno)
1941 {
1942 	char *buf = env->tmp_arg_name;
1943 	int len = sizeof(env->tmp_arg_name);
1944 	int arg, regno = reg_from_argno(argno);
1945 
1946 	if (regno >= 0) {
1947 		snprintf(buf, len, "R%d", regno);
1948 	} else {
1949 		arg = arg_from_argno(argno);
1950 		snprintf(buf, len, "*(R11-%u)", (arg - MAX_BPF_FUNC_REG_ARGS) * BPF_REG_SIZE);
1951 	}
1952 
1953 	return buf;
1954 }
1955 
1956 static const int caller_saved[CALLER_SAVED_REGS] = {
1957 	BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
1958 };
1959 
1960 static void bpf_diag_record_caller_saved(struct bpf_verifier_env *env,
1961 					 struct bpf_reg_state *regs)
1962 {
1963 	int i;
1964 
1965 	for (i = 1; i < CALLER_SAVED_REGS; i++) {
1966 		bpf_diag_record_scrub(env, &regs[caller_saved[i]],
1967 				      BPF_DIAG_MOD_CALLER_SAVED);
1968 	}
1969 }
1970 
1971 static void reg_step_reset(struct bpf_reg_state *reg)
1972 {
1973 	reg->base = 0;
1974 	reg->step = 1;
1975 }
1976 
1977 static void scalar_step_add(struct bpf_reg_state *dst_reg,
1978 			    const struct bpf_reg_state *a,
1979 			    const struct bpf_reg_state *b)
1980 {
1981 	const struct bpf_reg_state *reg;
1982 	s64 amount, tmp;
1983 
1984 	if (tnum_is_const(b->var_off)) {
1985 		reg = a;
1986 		amount = (s64)b->var_off.value;
1987 	} else if (tnum_is_const(a->var_off)) {
1988 		reg = b;
1989 		amount = (s64)a->var_off.value;
1990 	} else {
1991 		reg_step_reset(dst_reg);
1992 		return;
1993 	}
1994 
1995 	/*
1996 	 * Let x be a possible signed value of the register,
1997 	 * d = amount, s = reg->step, and b = reg->base.
1998 	 * Then x mod s = b, with s > 0 and 0 <= b < s.
1999 	 * If the addition causes signed underflow, the resulting value
2000 	 * is x + d + 2**64.
2001 	 *
2002 	 * Taking the result modulo s and substituting x mod s = b gives:
2003 	 *   (x + d + 2**64) mod s = (b + d + (2**64 mod s)) mod s      (1)
2004 	 * Keeping reg->step = s and updating reg->base to (b + d) mod s
2005 	 * requires (1) to equal (b + d) mod s, which holds if and only
2006 	 * if 2**64 mod s = 0.
2007 	 * Signed overflow subtracts 2**64 and has the same requirement.
2008 	 *
2009 	 * Conservatively reset the stride if either wrap is possible.
2010 	 */
2011 	if (check_add_overflow(reg_smin(reg), amount, &tmp) ||
2012 	    check_add_overflow(reg_smax(reg), amount, &tmp)) {
2013 		reg_step_reset(dst_reg);
2014 		return;
2015 	}
2016 
2017 	dst_reg->base = (reg->base + imod(amount, reg->step)) % reg->step;
2018 	dst_reg->step = reg->step;
2019 }
2020 
2021 static void scalar_step_scale(struct bpf_reg_state *dst_reg, u64 amount)
2022 {
2023 	u16 step;
2024 	s64 tmp;
2025 
2026 	/*
2027 	 * Using the same notation as scalar_step_add(), with d > 0,
2028 	 * the scaled value satisfies (x * d) mod (s * d) = b * d.
2029 	 * If the multiplication wraps, its signed result can be represented
2030 	 * as x * d - n * 2**64 for some nonzero integer n.
2031 	 *
2032 	 * Taking the result modulo s * d gives:
2033 	 *   (x * d - n * 2**64) mod (s * d) =
2034 	 *   (b * d - (n * 2**64 mod (s * d))) mod (s * d)              (1)
2035 	 * Updating dst_reg->base to b * d and dst_reg->step to s * d
2036 	 * requires (1) to equal b * d, which holds if and only if
2037 	 * n * 2**64 mod (s * d) = 0.
2038 	 *
2039 	 * Conservatively reset the stride if a wrap is possible.
2040 	 */
2041 	if (amount == 0 || check_mul_overflow(dst_reg->step, amount, &step) ||
2042 	    check_mul_overflow(reg_smin(dst_reg), (s64)amount, &tmp) ||
2043 	    check_mul_overflow(reg_smax(dst_reg), (s64)amount, &tmp)) {
2044 		reg_step_reset(dst_reg);
2045 		return;
2046 	}
2047 	/*
2048 	 * The above established that `step = dst_reg->step * amount`
2049 	 * does not overflow, therefore the base < step invariant holds
2050 	 * after assignments below.
2051 	 */
2052 	dst_reg->base = dst_reg->base * amount;
2053 	dst_reg->step = step;
2054 }
2055 
2056 /*
2057  * Let x be a possible signed value of the register,
2058  * s = reg->step, and b = reg->base.
2059  * Then x mod s = b, with s > 0 and 0 <= b < s.
2060  * Truncating to N bits and then zero- or sign-extending x
2061  * is equivalent to computing x + k * 2**N for some integer k.
2062  * For example, sign-extending the low 8 bits of x = 255 can
2063  * be expressed as 255 - 2**8 = -1.
2064  *
2065  * Taking the result modulo s and substituting x mod s = b gives:
2066  *   (x + k * 2**N) mod s = (b + (k * 2**N mod s)) mod s
2067  * Keeping reg->base = b and reg->step = s requires this to equal b,
2068  * which holds if and only if k * 2**N mod s = 0.
2069  * Continuing the example with s = 3 and b = 0, we have 255 mod 3 = 0,
2070  * but (-1) mod 3 = 2, so the original base and step no longer hold.
2071  *
2072  * Conservatively reset the stride unless all possible values fit in
2073  * the range where the conversion leaves them unchanged.
2074  * Call before updating the register's bounds. Requires 0 < bits < 64.
2075  */
2076 static void reg_step_check_unsigned(struct bpf_reg_state *reg, u32 bits)
2077 {
2078 	if (reg_umax(reg) >= (1ULL << bits))
2079 		reg_step_reset(reg);
2080 }
2081 
2082 /* This helper doesn't clear reg->id */
2083 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)
2084 {
2085 	reg->var_off = tnum_const(imm);
2086 	reg->r64 = cnum64_from_urange(imm, imm);
2087 	reg->r32 = cnum32_from_urange((u32)imm, (u32)imm);
2088 	reg_step_reset(reg);
2089 }
2090 
2091 /* Mark the unknown part of a register (variable offset or scalar value) as
2092  * known to have the value @imm.
2093  */
2094 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
2095 {
2096 	/* Clear off and union(map_ptr, range) */
2097 	memset(((u8 *)reg) + sizeof(reg->type), 0,
2098 	       offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
2099 	reg->id = 0;
2100 	reg->parent_id = 0;
2101 	reg->map_uid = 0;
2102 	___mark_reg_known(reg, imm);
2103 }
2104 
2105 void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm)
2106 {
2107 	__mark_reg_known(reg, imm);
2108 	reg->type = SCALAR_VALUE;
2109 }
2110 
2111 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)
2112 {
2113 	reg->var_off = tnum_const_subreg(reg->var_off, imm);
2114 	reg->r32 = cnum32_from_urange((u32)imm, (u32)imm);
2115 }
2116 
2117 /* Mark the 'variable offset' part of a register as zero.  This should be
2118  * used only on registers holding a pointer type.
2119  */
2120 static void __mark_reg_known_zero(struct bpf_reg_state *reg)
2121 {
2122 	__mark_reg_known(reg, 0);
2123 }
2124 
2125 static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
2126 {
2127 	__mark_reg_known(reg, 0);
2128 	reg->type = SCALAR_VALUE;
2129 	/* all scalars are assumed imprecise initially (unless unprivileged,
2130 	 * in which case everything is forced to be precise)
2131 	 */
2132 	reg->precise = !env->bpf_capable;
2133 }
2134 
2135 static void mark_reg_known_zero(struct bpf_verifier_env *env,
2136 				struct bpf_reg_state *regs, u32 regno)
2137 {
2138 	__mark_reg_known_zero(regs + regno);
2139 }
2140 
2141 static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type,
2142 			      bool first_slot, int id, int parent_id)
2143 {
2144 	/* reg->type has no meaning for STACK_DYNPTR, but when we set reg for
2145 	 * callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply
2146 	 * set it unconditionally as it is ignored for STACK_DYNPTR anyway.
2147 	 */
2148 	__mark_reg_known_zero(reg);
2149 	reg->type = CONST_PTR_TO_DYNPTR;
2150 	/* Give each dynptr a unique id to uniquely associate slices to it. */
2151 	reg->id = id;
2152 	reg->parent_id = parent_id;
2153 	reg->dynptr.type = type;
2154 	reg->dynptr.first_slot = first_slot;
2155 }
2156 
2157 /*
2158  * Refine the return type of the bpf_map_lookup_elem() for special map types:
2159  * map-in-map, xskmap, sockmap and sockhash.
2160  */
2161 static void refine_map_lookup_value(struct bpf_reg_state *reg)
2162 {
2163 	enum bpf_type_flag maybe_null = reg->type & PTR_MAYBE_NULL;
2164 	const struct bpf_map *map = reg->map_ptr;
2165 
2166 	if (map->inner_map_meta) {
2167 		reg->type = CONST_PTR_TO_MAP | maybe_null;
2168 		reg->map_ptr = map->inner_map_meta;
2169 		/*
2170 		 * transfer reg's id which is unique for every map_lookup_elem
2171 		 * as UID of the inner map.
2172 		 */
2173 		reg->map_uid = reg->id;
2174 	} else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
2175 		reg->type = PTR_TO_XDP_SOCK | maybe_null;
2176 		reg->map_uid = 0;
2177 	} else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
2178 		   map->map_type == BPF_MAP_TYPE_SOCKHASH) {
2179 		reg->type = PTR_TO_SOCKET | maybe_null;
2180 		reg->map_uid = 0;
2181 	}
2182 }
2183 
2184 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
2185 {
2186 	reg->type &= ~PTR_MAYBE_NULL;
2187 }
2188 
2189 static void mark_reg_graph_node(struct bpf_reg_state *regs, u32 regno,
2190 				struct btf_field_graph_root *ds_head)
2191 {
2192 	__mark_reg_known(&regs[regno], ds_head->node_offset);
2193 	regs[regno].type = PTR_TO_BTF_ID | MEM_ALLOC;
2194 	regs[regno].btf = ds_head->btf;
2195 	regs[regno].btf_id = ds_head->value_btf_id;
2196 }
2197 
2198 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
2199 {
2200 	return type_is_pkt_pointer(reg->type);
2201 }
2202 
2203 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
2204 {
2205 	return reg_is_pkt_pointer(reg) ||
2206 	       reg->type == PTR_TO_PACKET_END;
2207 }
2208 
2209 static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg)
2210 {
2211 	return base_type(reg->type) == PTR_TO_MEM &&
2212 	       (reg->type &
2213 		(DYNPTR_TYPE_SKB | DYNPTR_TYPE_XDP | DYNPTR_TYPE_SKB_META));
2214 }
2215 
2216 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
2217 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
2218 				    enum bpf_reg_type which)
2219 {
2220 	/* The register can already have a range from prior markings.
2221 	 * This is fine as long as it hasn't been advanced from its
2222 	 * origin.
2223 	 */
2224 	return reg->type == which &&
2225 	       reg->id == 0 &&
2226 	       tnum_equals_const(reg->var_off, 0);
2227 }
2228 
2229 static void __mark_reg32_unbounded(struct bpf_reg_state *reg)
2230 {
2231 	reg->r32 = CNUM32_UNBOUNDED;
2232 }
2233 
2234 static void __mark_reg64_unbounded(struct bpf_reg_state *reg)
2235 {
2236 	reg->r64 = CNUM64_UNBOUNDED;
2237 }
2238 
2239 /* Reset the min/max bounds of a register */
2240 static void __mark_reg_unbounded(struct bpf_reg_state *reg)
2241 {
2242 	__mark_reg64_unbounded(reg);
2243 	__mark_reg32_unbounded(reg);
2244 }
2245 
2246 static void reset_reg64_and_tnum(struct bpf_reg_state *reg)
2247 {
2248 	__mark_reg64_unbounded(reg);
2249 	reg->var_off = tnum_unknown;
2250 }
2251 
2252 static void reset_reg32_and_tnum(struct bpf_reg_state *reg)
2253 {
2254 	__mark_reg32_unbounded(reg);
2255 	reg->var_off = tnum_unknown;
2256 }
2257 
2258 static struct cnum32 cnum32_from_tnum(struct tnum tnum)
2259 {
2260 	tnum = tnum_subreg(tnum);
2261 	if ((tnum.mask & S32_MIN) || (tnum.value & S32_MIN))
2262 		/* min signed is max(sign bit) | min(other bits) */
2263 		/* max signed is min(sign bit) | max(other bits) */
2264 		return cnum32_from_srange(tnum.value | (tnum.mask & S32_MIN),
2265 					  tnum.value | (tnum.mask & S32_MAX));
2266 	else
2267 		return cnum32_from_urange(tnum.value, (tnum.value | tnum.mask));
2268 }
2269 
2270 static struct cnum64 cnum64_from_tnum(struct tnum tnum)
2271 {
2272 	if ((tnum.mask & S64_MIN) || (tnum.value & S64_MIN))
2273 		/* min signed is max(sign bit) | min(other bits) */
2274 		/* max signed is min(sign bit) | max(other bits) */
2275 		return cnum64_from_srange(tnum.value | (tnum.mask & S64_MIN),
2276 					  tnum.value | (tnum.mask & S64_MAX));
2277 	else
2278 		return cnum64_from_urange(tnum.value, (tnum.value | tnum.mask));
2279 }
2280 
2281 static void __update_reg32_bounds(struct bpf_reg_state *reg)
2282 {
2283 	cnum32_intersect_with(&reg->r32, cnum32_from_tnum(reg->var_off));
2284 }
2285 
2286 static void __update_reg64_bounds(struct bpf_reg_state *reg)
2287 {
2288 	u64 tnum_next, tmax;
2289 	bool umin_in_tnum;
2290 
2291 	cnum64_intersect_with(&reg->r64, cnum64_from_tnum(reg->var_off));
2292 
2293 	/* Check if u64 and tnum overlap in a single value */
2294 	tnum_next = tnum_step(reg->var_off, reg_umin(reg));
2295 	umin_in_tnum = (reg_umin(reg) & ~reg->var_off.mask) == reg->var_off.value;
2296 	tmax = reg->var_off.value | reg->var_off.mask;
2297 	if (umin_in_tnum && tnum_next > reg_umax(reg)) {
2298 		/* The u64 range and the tnum only overlap in umin.
2299 		 * u64:  ---[xxxxxx]-----
2300 		 * tnum: --xx----------x-
2301 		 */
2302 		___mark_reg_known(reg, reg_umin(reg));
2303 	} else if (!umin_in_tnum && tnum_next == tmax) {
2304 		/* The u64 range and the tnum only overlap in the maximum value
2305 		 * represented by the tnum, called tmax.
2306 		 * u64:  ---[xxxxxx]-----
2307 		 * tnum: xx-----x--------
2308 		 */
2309 		___mark_reg_known(reg, tmax);
2310 	} else if (!umin_in_tnum && tnum_next <= reg_umax(reg) &&
2311 		   tnum_step(reg->var_off, tnum_next) > reg_umax(reg)) {
2312 		/* The u64 range and the tnum only overlap in between umin
2313 		 * (excluded) and umax.
2314 		 * u64:  ---[xxxxxx]-----
2315 		 * tnum: xx----x-------x-
2316 		 */
2317 		___mark_reg_known(reg, tnum_next);
2318 	}
2319 }
2320 
2321 static void __update_reg_bounds(struct bpf_reg_state *reg)
2322 {
2323 	__update_reg32_bounds(reg);
2324 	__update_reg64_bounds(reg);
2325 }
2326 
2327 static void deduce_bounds_32_from_64(struct bpf_reg_state *reg)
2328 {
2329 	cnum32_intersect_with(&reg->r32, cnum32_from_cnum64(reg->r64));
2330 }
2331 
2332 static void deduce_bounds_64_from_32(struct bpf_reg_state *reg)
2333 {
2334 	reg->r64 = cnum64_cnum32_intersect(reg->r64, reg->r32);
2335 }
2336 
2337 static void __reg_deduce_bounds(struct bpf_reg_state *reg)
2338 {
2339 	deduce_bounds_32_from_64(reg);
2340 	deduce_bounds_64_from_32(reg);
2341 }
2342 
2343 /* Attempts to improve var_off based on unsigned min/max information */
2344 static void __reg_bound_offset(struct bpf_reg_state *reg)
2345 {
2346 	struct tnum var64_off = tnum_intersect(reg->var_off,
2347 					       tnum_range(reg_umin(reg),
2348 							  reg_umax(reg)));
2349 	struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off),
2350 					       tnum_range(reg_u32_min(reg),
2351 							  reg_u32_max(reg)));
2352 	u32 trailing_zero_bits;
2353 	u16 base = reg->base;
2354 	u16 step = reg->step;
2355 
2356 	reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off);
2357 
2358 	if (base == 0)
2359 		trailing_zero_bits = count_trailing_zeros(step);
2360 	else
2361 		trailing_zero_bits = min(count_trailing_zeros(base),
2362 					 count_trailing_zeros(step));
2363 	reg->var_off = tnum_and(reg->var_off, tnum_const(~0ULL << trailing_zero_bits));
2364 }
2365 
2366 static void __update_bounds_from_step(struct bpf_reg_state *reg)
2367 {
2368 	reg->r64 = cnum64_intersect_linear(reg->r64, reg->base, reg->step);
2369 	deduce_bounds_32_from_64(reg);
2370 }
2371 
2372 static bool range_bounds_violation(struct bpf_reg_state *reg);
2373 
2374 static void reg_bounds_sync(struct bpf_reg_state *reg)
2375 {
2376 	/* If the input reg_state is invalid, we can exit early */
2377 	if (range_bounds_violation(reg))
2378 		return;
2379 	/* We might have learned new bounds from the var_off. */
2380 	__update_reg_bounds(reg);
2381 	/* We might have learned something about the sign bit. */
2382 	__reg_deduce_bounds(reg);
2383 	__reg_deduce_bounds(reg);
2384 	/* We might have learned some bits from the bounds. */
2385 	__reg_bound_offset(reg);
2386 	/* Intersecting with the old var_off might have improved our bounds
2387 	 * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
2388 	 * then new var_off is (0; 0x7f...fc) which improves our umax.
2389 	 */
2390 	__update_reg_bounds(reg);
2391 	/*
2392 	 * Both tnum and base+step define a sequence of points,
2393 	 * potentially w/o intersection, or intersecting after N
2394 	 * tnum_step()'s, where N can be large. On the other hand,
2395 	 * step+base *must* be in sync with r64 in order for
2396 	 * array traversal logic starting at reg_smin() to be sound.
2397 	 * Hence, place this as a final step in the reg_bounds_sync().
2398 	 */
2399 	__update_bounds_from_step(reg);
2400 }
2401 
2402 static bool const_tnum_range_mismatch(struct bpf_reg_state *reg)
2403 {
2404 	if (!tnum_is_const(reg->var_off))
2405 		return false;
2406 
2407 	return !cnum64_is_const(reg->r64) || reg->r64.base != reg->var_off.value;
2408 }
2409 
2410 static bool const_tnum_range_mismatch_32(struct bpf_reg_state *reg)
2411 {
2412 	if (!tnum_subreg_is_const(reg->var_off))
2413 		return false;
2414 
2415 	return !cnum32_is_const(reg->r32) || reg->r32.base != tnum_subreg(reg->var_off).value;
2416 }
2417 
2418 static bool range_bounds_violation(struct bpf_reg_state *reg)
2419 {
2420 	return cnum32_is_empty(reg->r32) || cnum64_is_empty(reg->r64);
2421 }
2422 
2423 static int reg_bounds_sanity_check(struct bpf_verifier_env *env,
2424 				   struct bpf_reg_state *reg, const char *ctx)
2425 {
2426 	const char *msg;
2427 
2428 	if (range_bounds_violation(reg)) {
2429 		msg = "range bounds violation";
2430 		goto out;
2431 	}
2432 
2433 	if (const_tnum_range_mismatch(reg)) {
2434 		msg = "const tnum out of sync with range bounds";
2435 		goto out;
2436 	}
2437 
2438 	if (const_tnum_range_mismatch_32(reg)) {
2439 		msg = "const subreg tnum out of sync with range bounds";
2440 		goto out;
2441 	}
2442 
2443 	if (reg->step == 0 || reg->base >= reg->step) {
2444 		msg = "invalid base/step";
2445 		goto out;
2446 	}
2447 
2448 	return 0;
2449 out:
2450 	verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s r64={.base=%#llx, .size=%#llx} "
2451 		     "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx) step=%u+%u",
2452 		     ctx, msg,
2453 		     reg->r64.base, reg->r64.size,
2454 		     reg->r32.base, reg->r32.size,
2455 		     reg->var_off.value, reg->var_off.mask,
2456 		     reg->base, reg->step);
2457 	if (env->test_reg_invariants)
2458 		return -EFAULT;
2459 	__mark_reg_unbounded(reg);
2460 	reg_step_reset(reg);
2461 	return 0;
2462 }
2463 
2464 /* Mark a register as having a completely unknown (scalar) value. */
2465 void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg)
2466 {
2467 	memset(reg, 0, sizeof(*reg));
2468 	reg->type = SCALAR_VALUE;
2469 	reg->var_off = tnum_unknown;
2470 	__mark_reg_unbounded(reg);
2471 	reg_step_reset(reg);
2472 }
2473 
2474 /* Mark a register as having a completely unknown (scalar) value,
2475  * initialize .precise as true when not bpf capable.
2476  */
2477 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
2478 			       struct bpf_reg_state *reg)
2479 {
2480 	bpf_mark_reg_unknown_imprecise(reg);
2481 	reg->precise = !env->bpf_capable;
2482 }
2483 
2484 static void mark_reg_unknown(struct bpf_verifier_env *env,
2485 			     struct bpf_reg_state *regs, u32 regno)
2486 {
2487 	__mark_reg_unknown(env, regs + regno);
2488 }
2489 
2490 static int __mark_reg_s32_range(struct bpf_verifier_env *env,
2491 				struct bpf_reg_state *regs,
2492 				u32 regno,
2493 				s32 s32_min,
2494 				s32 s32_max)
2495 {
2496 	struct bpf_reg_state *reg = regs + regno;
2497 
2498 	reg_set_srange32(reg,
2499 			 max_t(s32, reg_s32_min(reg), s32_min),
2500 			 min_t(s32, reg_s32_max(reg), s32_max));
2501 	reg_set_srange64(reg,
2502 			 max_t(s64, reg_smin(reg), s32_min),
2503 			 min_t(s64, reg_smax(reg), s32_max));
2504 
2505 	reg_bounds_sync(reg);
2506 
2507 	return reg_bounds_sanity_check(env, reg, "s32_range");
2508 }
2509 
2510 void bpf_mark_reg_not_init(const struct bpf_verifier_env *env,
2511 			   struct bpf_reg_state *reg)
2512 {
2513 	__mark_reg_unknown(env, reg);
2514 	reg->type = NOT_INIT;
2515 }
2516 
2517 static int mark_btf_ld_reg(struct bpf_verifier_env *env,
2518 			   struct bpf_reg_state *regs, u32 regno,
2519 			   enum bpf_reg_type reg_type,
2520 			   struct btf *btf, u32 btf_id,
2521 			   enum bpf_type_flag flag)
2522 {
2523 	switch (reg_type) {
2524 	case SCALAR_VALUE:
2525 		mark_reg_unknown(env, regs, regno);
2526 		return 0;
2527 	case PTR_TO_BTF_ID:
2528 		mark_reg_known_zero(env, regs, regno);
2529 		regs[regno].type = PTR_TO_BTF_ID | flag;
2530 		regs[regno].btf = btf;
2531 		regs[regno].btf_id = btf_id;
2532 		if (type_may_be_null(flag))
2533 			regs[regno].id = ++env->id_gen;
2534 		return 0;
2535 	case PTR_TO_MEM:
2536 		mark_reg_known_zero(env, regs, regno);
2537 		regs[regno].type = PTR_TO_MEM | flag;
2538 		regs[regno].mem_size = 0;
2539 		return 0;
2540 	default:
2541 		verifier_bug(env, "unexpected reg_type %d in %s\n", reg_type, __func__);
2542 		return -EFAULT;
2543 	}
2544 }
2545 
2546 static void init_reg_state(struct bpf_verifier_env *env,
2547 			   struct bpf_func_state *state)
2548 {
2549 	struct bpf_reg_state *regs = state->regs;
2550 	int i;
2551 
2552 	for (i = 0; i < MAX_BPF_REG; i++) {
2553 		bpf_mark_reg_not_init(env, &regs[i]);
2554 	}
2555 
2556 	/* frame pointer */
2557 	regs[BPF_REG_FP].type = PTR_TO_STACK;
2558 	mark_reg_known_zero(env, regs, BPF_REG_FP);
2559 	regs[BPF_REG_FP].frameno = state->frameno;
2560 }
2561 
2562 static struct bpf_retval_range retval_range(s32 minval, s32 maxval)
2563 {
2564 	/*
2565 	 * return_32bit is set to false by default and set explicitly
2566 	 * by the caller when necessary.
2567 	 */
2568 	return (struct bpf_retval_range){ minval, maxval, false };
2569 }
2570 
2571 static void init_func_state(struct bpf_verifier_env *env,
2572 			    struct bpf_func_state *state,
2573 			    int callsite, int frameno, int subprogno)
2574 {
2575 	state->callsite = callsite;
2576 	state->frameno = frameno;
2577 	bpf_diag_init_frame(env, state);
2578 	state->subprogno = subprogno;
2579 	state->callback_ret_range = retval_range(0, 0);
2580 	init_reg_state(env, state);
2581 	mark_verifier_state_scratched(env);
2582 }
2583 
2584 /* Similar to push_stack(), but for async callbacks */
2585 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
2586 						int insn_idx, int prev_insn_idx,
2587 						int subprog, bool is_sleepable)
2588 {
2589 	struct bpf_verifier_stack_elem *elem;
2590 	struct bpf_func_state *frame;
2591 
2592 	elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT);
2593 	if (!elem)
2594 		return ERR_PTR(-ENOMEM);
2595 
2596 	elem->insn_idx = insn_idx;
2597 	elem->prev_insn_idx = prev_insn_idx;
2598 	elem->next = env->head;
2599 	elem->log_pos = env->log.end_pos;
2600 	elem->diag_log_pos = bpf_diag_event_log_save(env);
2601 	env->head = elem;
2602 	env->stack_size++;
2603 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
2604 		verbose(env,
2605 			"The sequence of %d jumps is too complex for async cb.\n",
2606 			env->stack_size);
2607 		return ERR_PTR(-E2BIG);
2608 	}
2609 	/* Unlike push_stack() do not bpf_copy_verifier_state().
2610 	 * The caller state doesn't matter.
2611 	 * This is async callback. It starts in a fresh stack.
2612 	 * Initialize it similar to do_check_common().
2613 	 */
2614 	elem->st.branches = 1;
2615 	elem->st.in_sleepable = is_sleepable;
2616 	frame = kzalloc_obj(*frame, GFP_KERNEL_ACCOUNT);
2617 	if (!frame)
2618 		return ERR_PTR(-ENOMEM);
2619 	init_func_state(env, frame,
2620 			BPF_MAIN_FUNC /* callsite */,
2621 			0 /* frameno within this callchain */,
2622 			subprog /* subprog number within this prog */);
2623 	elem->st.frame[0] = frame;
2624 	return &elem->st;
2625 }
2626 
2627 /* Find subprogram that contains instruction at 'off' */
2628 struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off)
2629 {
2630 	struct bpf_subprog_info *vals = env->subprog_info;
2631 	int l, r, m;
2632 
2633 	if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0)
2634 		return NULL;
2635 
2636 	l = 0;
2637 	r = env->subprog_cnt - 1;
2638 	while (l < r) {
2639 		m = l + (r - l + 1) / 2;
2640 		if (vals[m].start <= off)
2641 			l = m;
2642 		else
2643 			r = m - 1;
2644 	}
2645 	return &vals[l];
2646 }
2647 
2648 /* Find subprogram that starts exactly at 'off' */
2649 int bpf_find_subprog(struct bpf_verifier_env *env, int off)
2650 {
2651 	struct bpf_subprog_info *p;
2652 
2653 	p = bpf_find_containing_subprog(env, off);
2654 	if (!p || p->start != off)
2655 		return -ENOENT;
2656 	return p - env->subprog_info;
2657 }
2658 
2659 static int add_subprog(struct bpf_verifier_env *env, int off)
2660 {
2661 	struct bpf_subprog_info *subprog = env->subprog_info;
2662 	int insn_cnt = env->prog->len;
2663 	int i;
2664 
2665 	if (off >= insn_cnt || off < 0) {
2666 		verbose(env, "call to invalid destination\n");
2667 		return -EINVAL;
2668 	}
2669 	i = bpf_find_subprog(env, off);
2670 	if (i >= 0)
2671 		return i;
2672 	if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
2673 		verbose(env, "too many subprograms\n");
2674 		return -E2BIG;
2675 	}
2676 	/*
2677 	 * Determine subprog starts. The end is one before the next starts.
2678 	 * The array stays sorted: subprogs that start after 'off' move up.
2679 	 */
2680 	for (i = env->subprog_cnt++; i > 0 && subprog[i - 1].start > off; i--)
2681 		subprog[i] = subprog[i - 1];
2682 	subprog[i] = (struct bpf_subprog_info){ .start = off, .exit_idx = U32_MAX };
2683 	return i;
2684 }
2685 
2686 static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env)
2687 {
2688 	struct bpf_prog_aux *aux = env->prog->aux;
2689 	struct btf *btf = aux->btf;
2690 	const struct btf_type *t;
2691 	u32 main_btf_id, id;
2692 	const char *name;
2693 	int ret, i;
2694 
2695 	/* Non-zero func_info_cnt implies valid btf */
2696 	if (!aux->func_info_cnt)
2697 		return 0;
2698 	main_btf_id = aux->func_info[0].type_id;
2699 
2700 	t = btf_type_by_id(btf, main_btf_id);
2701 	if (!t) {
2702 		verbose(env, "invalid btf id for main subprog in func_info\n");
2703 		return -EINVAL;
2704 	}
2705 
2706 	name = btf_find_decl_tag_value(btf, t, -1, "exception_callback:");
2707 	if (IS_ERR(name)) {
2708 		ret = PTR_ERR(name);
2709 		/* If there is no tag present, there is no exception callback */
2710 		if (ret == -ENOENT)
2711 			ret = 0;
2712 		else if (ret == -EEXIST)
2713 			verbose(env, "multiple exception callback tags for main subprog\n");
2714 		return ret;
2715 	}
2716 
2717 	ret = btf_find_by_name_kind(btf, name, BTF_KIND_FUNC);
2718 	if (ret < 0) {
2719 		verbose(env, "exception callback '%s' could not be found in BTF\n", name);
2720 		return ret;
2721 	}
2722 	id = ret;
2723 	t = btf_type_by_id(btf, id);
2724 	if (btf_func_linkage(t) != BTF_FUNC_GLOBAL) {
2725 		verbose(env, "exception callback '%s' must have global linkage\n", name);
2726 		return -EINVAL;
2727 	}
2728 	ret = 0;
2729 	for (i = 0; i < aux->func_info_cnt; i++) {
2730 		if (aux->func_info[i].type_id != id)
2731 			continue;
2732 		ret = aux->func_info[i].insn_off;
2733 		/* Further func_info and subprog checks will also happen
2734 		 * later, so assume this is the right insn_off for now.
2735 		 */
2736 		if (!ret) {
2737 			verbose(env, "invalid exception callback insn_off in func_info: 0\n");
2738 			ret = -EINVAL;
2739 		}
2740 	}
2741 	if (!ret) {
2742 		verbose(env, "exception callback type id not found in func_info\n");
2743 		ret = -EINVAL;
2744 	}
2745 	return ret;
2746 }
2747 
2748 #define MAX_KFUNC_BTFS	256
2749 
2750 struct bpf_kfunc_btf {
2751 	struct btf *btf;
2752 	struct module *module;
2753 	u16 offset;
2754 };
2755 
2756 struct bpf_kfunc_btf_tab {
2757 	struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS];
2758 	u32 nr_descs;
2759 };
2760 
2761 static int kfunc_desc_cmp_by_id_off(const void *a, const void *b)
2762 {
2763 	const struct bpf_kfunc_desc *d0 = a;
2764 	const struct bpf_kfunc_desc *d1 = b;
2765 
2766 	/* func_id is not greater than BTF_MAX_TYPE */
2767 	return d0->func_id - d1->func_id ?: d0->offset - d1->offset;
2768 }
2769 
2770 static int kfunc_btf_cmp_by_off(const void *a, const void *b)
2771 {
2772 	const struct bpf_kfunc_btf *d0 = a;
2773 	const struct bpf_kfunc_btf *d1 = b;
2774 
2775 	return d0->offset - d1->offset;
2776 }
2777 
2778 static struct bpf_kfunc_desc *
2779 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
2780 {
2781 	struct bpf_kfunc_desc desc = {
2782 		.func_id = func_id,
2783 		.offset = offset,
2784 	};
2785 	struct bpf_kfunc_desc_tab *tab;
2786 
2787 	tab = prog->aux->kfunc_tab;
2788 	return bsearch(&desc, tab->descs, tab->nr_base_descs,
2789 		       sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
2790 }
2791 
2792 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2793 		       u16 desc_idx, u8 **func_addr)
2794 {
2795 	struct bpf_kfunc_desc_tab *tab;
2796 	const struct bpf_kfunc_desc *desc;
2797 
2798 	tab = prog->aux->kfunc_tab;
2799 	if (desc_idx >= tab->nr_descs)
2800 		return -EFAULT;
2801 	desc = &tab->descs[desc_idx];
2802 	if (desc->func_id != func_id)
2803 		return -EFAULT;
2804 
2805 	*func_addr = (u8 *)desc->addr;
2806 	return 0;
2807 }
2808 
2809 #define BPF_FD_SLOT_BTF	1UL
2810 
2811 static void fd_slot_set_map(struct bpf_fd_array *slot, struct bpf_map *map)
2812 {
2813 	slot->val = (unsigned long)map;
2814 }
2815 
2816 static void fd_slot_set_btf(struct bpf_fd_array *slot, struct btf *btf)
2817 {
2818 	slot->val = (unsigned long)btf | BPF_FD_SLOT_BTF;
2819 }
2820 
2821 static struct bpf_map *fd_slot_map(struct bpf_fd_array slot)
2822 {
2823 	if (slot.val & BPF_FD_SLOT_BTF)
2824 		return NULL;
2825 	return (struct bpf_map *)slot.val;
2826 }
2827 
2828 static struct btf *fd_slot_btf(struct bpf_fd_array slot)
2829 {
2830 	if (!(slot.val & BPF_FD_SLOT_BTF))
2831 		return NULL;
2832 	return (struct btf *)(slot.val & ~BPF_FD_SLOT_BTF);
2833 }
2834 
2835 static struct btf *
2836 fd_array_get_btf_continuous(struct bpf_verifier_env *env, u32 idx)
2837 {
2838 	struct btf *btf;
2839 
2840 	if (idx >= env->fd_array_cnt) {
2841 		verbose(env, "kfunc fd_idx %u out of bounds, fd_array_cnt %u\n",
2842 			idx, env->fd_array_cnt);
2843 		return ERR_PTR(-EINVAL);
2844 	}
2845 	btf = fd_slot_btf(env->fd_array[idx]);
2846 	if (!btf) {
2847 		verbose(env, "kfunc fd_idx %u is not a module BTF\n", idx);
2848 		return ERR_PTR(-EINVAL);
2849 	}
2850 	btf_get(btf);
2851 	return btf;
2852 }
2853 
2854 static struct btf *
2855 fd_array_get_btf_sparse(struct bpf_verifier_env *env, u32 idx)
2856 {
2857 	struct btf *btf;
2858 	int btf_fd;
2859 
2860 	if (copy_from_bpfptr_offset(&btf_fd, env->fd_array_raw,
2861 				    (size_t)idx * sizeof(btf_fd), sizeof(btf_fd)))
2862 		return ERR_PTR(-EFAULT);
2863 	btf = btf_get_by_fd(btf_fd);
2864 	if (IS_ERR(btf)) {
2865 		verbose(env, "invalid module BTF fd specified\n");
2866 		return btf;
2867 	}
2868 	return btf;
2869 }
2870 
2871 static struct btf *fd_array_get_btf(struct bpf_verifier_env *env, u32 idx)
2872 {
2873 	if (env->signature) {
2874 		verbose(env, "signed program cannot bind any BTF\n");
2875 		return ERR_PTR(-EACCES);
2876 	}
2877 	if (env->fd_array)
2878 		return fd_array_get_btf_continuous(env, idx);
2879 	if (!bpfptr_is_null(env->fd_array_raw))
2880 		return fd_array_get_btf_sparse(env, idx);
2881 
2882 	verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
2883 	return ERR_PTR(-EPROTO);
2884 }
2885 
2886 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
2887 					 s16 offset)
2888 {
2889 	struct bpf_kfunc_btf kf_btf = { .offset = offset };
2890 	struct bpf_kfunc_btf_tab *tab;
2891 	struct bpf_kfunc_btf *b;
2892 	struct module *mod;
2893 	struct btf *btf;
2894 
2895 	tab = env->prog->aux->kfunc_btf_tab;
2896 	b = bsearch(&kf_btf, tab->descs, tab->nr_descs,
2897 		    sizeof(tab->descs[0]), kfunc_btf_cmp_by_off);
2898 	if (!b) {
2899 		if (tab->nr_descs == MAX_KFUNC_BTFS) {
2900 			verbose(env, "too many different module BTFs\n");
2901 			return ERR_PTR(-E2BIG);
2902 		}
2903 
2904 		btf = fd_array_get_btf(env, offset);
2905 		if (IS_ERR(btf))
2906 			return btf;
2907 		if (!btf_is_module(btf)) {
2908 			verbose(env, "BTF fd for kfunc is not a module BTF\n");
2909 			btf_put(btf);
2910 			return ERR_PTR(-EINVAL);
2911 		}
2912 
2913 		mod = btf_try_get_module(btf);
2914 		if (!mod) {
2915 			btf_put(btf);
2916 			return ERR_PTR(-ENXIO);
2917 		}
2918 
2919 		b = &tab->descs[tab->nr_descs++];
2920 		b->btf = btf;
2921 		b->module = mod;
2922 		b->offset = offset;
2923 
2924 		/* sort() reorders entries by value, so b may no longer point
2925 		 * to the right entry after this
2926 		 */
2927 		sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
2928 		     kfunc_btf_cmp_by_off, NULL);
2929 	} else {
2930 		btf = b->btf;
2931 	}
2932 
2933 	return btf;
2934 }
2935 
2936 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab)
2937 {
2938 	if (!tab)
2939 		return;
2940 
2941 	while (tab->nr_descs--) {
2942 		module_put(tab->descs[tab->nr_descs].module);
2943 		btf_put(tab->descs[tab->nr_descs].btf);
2944 	}
2945 	kfree(tab);
2946 }
2947 
2948 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)
2949 {
2950 	if (offset) {
2951 		if (offset < 0) {
2952 			/* In the future, this can be allowed to increase limit
2953 			 * of fd index into fd_array, interpreted as u16.
2954 			 */
2955 			verbose(env, "negative offset disallowed for kernel module function call\n");
2956 			return ERR_PTR(-EINVAL);
2957 		}
2958 
2959 		return __find_kfunc_desc_btf(env, offset);
2960 	}
2961 	return btf_vmlinux ?: ERR_PTR(-ENOENT);
2962 }
2963 
2964 static struct btf *find_kfunc_desc_btf_cached(struct bpf_verifier_env *env, s16 offset)
2965 {
2966 	struct bpf_kfunc_btf kf_btf = { .offset = offset };
2967 	struct bpf_kfunc_btf_tab *tab;
2968 	struct bpf_kfunc_btf *b;
2969 
2970 	if (!offset)
2971 		return btf_vmlinux ?: ERR_PTR(-ENOENT);
2972 	if (offset < 0)
2973 		return ERR_PTR(-EINVAL);
2974 
2975 	tab = env->prog->aux->kfunc_btf_tab;
2976 	if (!tab)
2977 		return ERR_PTR(-ENOENT);
2978 
2979 	b = bsearch(&kf_btf, tab->descs, tab->nr_descs,
2980 		    sizeof(tab->descs[0]), kfunc_btf_cmp_by_off);
2981 	return b ? b->btf : ERR_PTR(-ENOENT);
2982 }
2983 
2984 #define KF_IMPL_SUFFIX "_impl"
2985 
2986 static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_log *log,
2987 						    struct btf *btf,
2988 						    const char *func_name)
2989 {
2990 	const struct btf_type *func;
2991 	char buf[KSYM_NAME_LEN];
2992 	s32 impl_id;
2993 	int len;
2994 
2995 	len = snprintf(buf, sizeof(buf), "%s%s", func_name, KF_IMPL_SUFFIX);
2996 	if (len < 0 || len >= sizeof(buf)) {
2997 		bpf_log(log, "function name %s%s is too long\n",
2998 			func_name, KF_IMPL_SUFFIX);
2999 		return NULL;
3000 	}
3001 
3002 	impl_id = btf_find_by_name_kind(btf, buf, BTF_KIND_FUNC);
3003 	if (impl_id <= 0) {
3004 		bpf_log(log, "cannot find function %s in BTF\n", buf);
3005 		return NULL;
3006 	}
3007 
3008 	func = btf_type_by_id(btf, impl_id);
3009 
3010 	return btf_type_by_id(btf, func->type);
3011 }
3012 
3013 static int fetch_kfunc_meta(struct bpf_verifier_env *env,
3014 			    s32 func_id,
3015 			    s16 offset,
3016 			    struct bpf_kfunc_meta *kfunc)
3017 {
3018 	const struct btf_type *func, *func_proto;
3019 	const char *func_name;
3020 	u32 *kfunc_flags;
3021 	struct btf *btf;
3022 
3023 	if (func_id <= 0) {
3024 		verbose(env, "invalid kernel function btf_id %d\n", func_id);
3025 		return -EINVAL;
3026 	}
3027 
3028 	btf = find_kfunc_desc_btf(env, offset);
3029 	if (IS_ERR(btf)) {
3030 		verbose(env, "failed to find BTF for kernel function\n");
3031 		return PTR_ERR(btf);
3032 	}
3033 
3034 	/*
3035 	 * Note that kfunc_flags may be NULL at this point, which
3036 	 * means that we couldn't find func_id in any relevant
3037 	 * kfunc_id_set. This most likely indicates an invalid kfunc
3038 	 * call.  However we don't fail with an error here,
3039 	 * and let the caller decide what to do with NULL kfunc->flags.
3040 	 */
3041 	kfunc_flags = btf_kfunc_flags(btf, func_id, env->prog);
3042 
3043 	func = btf_type_by_id(btf, func_id);
3044 	if (!func || !btf_type_is_func(func)) {
3045 		verbose(env, "kernel btf_id %d is not a function\n", func_id);
3046 		return -EINVAL;
3047 	}
3048 
3049 	func_name = btf_name_by_offset(btf, func->name_off);
3050 
3051 	/*
3052 	 * An actual prototype of a kfunc with KF_IMPLICIT_ARGS flag
3053 	 * can be found through the counterpart _impl kfunc.
3054 	 */
3055 	if (kfunc_flags && (*kfunc_flags & KF_IMPLICIT_ARGS))
3056 		func_proto = find_kfunc_impl_proto(&env->log, btf, func_name);
3057 	else
3058 		func_proto = btf_type_by_id(btf, func->type);
3059 
3060 	if (!func_proto || !btf_type_is_func_proto(func_proto)) {
3061 		verbose(env, "kernel function btf_id %d does not have a valid func_proto\n",
3062 			func_id);
3063 		return -EINVAL;
3064 	}
3065 
3066 	memset(kfunc, 0, sizeof(*kfunc));
3067 	kfunc->btf = btf;
3068 	kfunc->id = func_id;
3069 	kfunc->name = func_name;
3070 	kfunc->proto = func_proto;
3071 	kfunc->flags = kfunc_flags;
3072 
3073 	return 0;
3074 }
3075 
3076 static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
3077 			       const struct btf_func_model *fm, struct bpf_func_proto *proto);
3078 
3079 int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
3080 {
3081 	struct bpf_call_arg_meta meta;
3082 	struct bpf_kfunc_btf_tab *btf_tab;
3083 	struct btf_func_model func_model;
3084 	struct bpf_kfunc_desc_tab *tab;
3085 	struct bpf_prog_aux *prog_aux;
3086 	struct bpf_kfunc_meta kfunc;
3087 	struct bpf_kfunc_desc *desc;
3088 	unsigned long addr;
3089 	int err;
3090 
3091 	prog_aux = env->prog->aux;
3092 	tab = prog_aux->kfunc_tab;
3093 	btf_tab = prog_aux->kfunc_btf_tab;
3094 	if (!tab) {
3095 		if (!btf_vmlinux) {
3096 			verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
3097 			return -ENOTSUPP;
3098 		}
3099 
3100 		if (!env->prog->jit_requested) {
3101 			verbose(env, "JIT is required for calling kernel function\n");
3102 			return -ENOTSUPP;
3103 		}
3104 
3105 		if (!bpf_jit_supports_kfunc_call()) {
3106 			verbose(env, "JIT does not support calling kernel function\n");
3107 			return -ENOTSUPP;
3108 		}
3109 
3110 		if (!env->prog->gpl_compatible) {
3111 			verbose(env, "cannot call kernel function from non-GPL compatible program\n");
3112 			return -EINVAL;
3113 		}
3114 
3115 		tab = kzalloc_obj(*tab, GFP_KERNEL_ACCOUNT);
3116 		if (!tab)
3117 			return -ENOMEM;
3118 		prog_aux->kfunc_tab = tab;
3119 	}
3120 
3121 	env->prog->jit_required = 1;
3122 
3123 	/* func_id == 0 is always invalid, but instead of returning an error, be
3124 	 * conservative and wait until the code elimination pass before returning
3125 	 * error, so that invalid calls that get pruned out can be in BPF programs
3126 	 * loaded from userspace.  It is also required that offset be untouched
3127 	 * for such calls.
3128 	 */
3129 	if (!func_id && !offset)
3130 		return 0;
3131 
3132 	if (!btf_tab && offset) {
3133 		btf_tab = kzalloc_obj(*btf_tab, GFP_KERNEL_ACCOUNT);
3134 		if (!btf_tab)
3135 			return -ENOMEM;
3136 		prog_aux->kfunc_btf_tab = btf_tab;
3137 	}
3138 
3139 	if (find_kfunc_desc(env->prog, func_id, offset))
3140 		return 0;
3141 
3142 	if (tab->nr_base_descs == MAX_KFUNC_DESCS) {
3143 		verbose(env, "too many different kernel function calls\n");
3144 		return -E2BIG;
3145 	}
3146 	if (tab->nr_descs != tab->nr_base_descs) {
3147 		verifier_bug(env, "unexpected specialized func desc found (%d descs, %d base descs)",
3148 			     tab->nr_descs, tab->nr_base_descs);
3149 		return -EFAULT;
3150 	}
3151 
3152 	err = fetch_kfunc_meta(env, func_id, offset, &kfunc);
3153 	if (err)
3154 		return err;
3155 
3156 	addr = kallsyms_lookup_name(kfunc.name);
3157 	if (!addr) {
3158 		verbose(env, "cannot find address for kernel function %s\n", kfunc.name);
3159 		return -EINVAL;
3160 	}
3161 
3162 	if (bpf_dev_bound_kfunc_id(func_id)) {
3163 		err = bpf_dev_bound_kfunc_check(&env->log, prog_aux);
3164 		if (err)
3165 			return err;
3166 	}
3167 
3168 	err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model);
3169 	if (err)
3170 		return err;
3171 	if (func_model.ret_size > 8) {
3172 		if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) {
3173 			verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n",
3174 				kfunc.name);
3175 			return -EOPNOTSUPP;
3176 		}
3177 		if (!bpf_jit_supports_kfunc_ret_reg_pair()) {
3178 			verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n",
3179 				kfunc.name);
3180 			return -EOPNOTSUPP;
3181 		}
3182 	}
3183 
3184 	memset(&meta, 0, sizeof(meta));
3185 	meta.btf = kfunc.btf;
3186 	meta.func_id = kfunc.id;
3187 	meta.func_proto = kfunc.proto;
3188 	meta.func_name = kfunc.name;
3189 	meta.kfunc_flags = kfunc.flags ? *kfunc.flags : 0;
3190 
3191 	tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), GFP_KERNEL_ACCOUNT);
3192 	if (!tab)
3193 		return -ENOMEM;
3194 	prog_aux->kfunc_tab = tab;
3195 
3196 	desc = &tab->descs[tab->nr_descs];
3197 	memset(desc, 0, sizeof(*desc));
3198 
3199 	err = gen_kfunc_arg_proto(env, &meta, &func_model, &desc->proto);
3200 	if (err)
3201 		return err;
3202 
3203 	desc->func_id = func_id;
3204 	desc->offset = offset;
3205 	desc->addr = addr;
3206 	desc->func_model = func_model;
3207 	tab->nr_descs++;
3208 	tab->nr_base_descs++;
3209 	sort(tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]),
3210 	     kfunc_desc_cmp_by_id_off, NULL);
3211 	return 0;
3212 }
3213 
3214 static int add_subprogs(struct bpf_verifier_env *env)
3215 {
3216 	struct bpf_subprog_info *subprog = env->subprog_info;
3217 	int i, ret, insn_cnt = env->prog->len, ex_cb_insn;
3218 	struct bpf_insn *insn = env->prog->insnsi;
3219 	const char *operation, *suggestion;
3220 
3221 	/* Add entry function. */
3222 	ret = add_subprog(env, 0);
3223 	if (ret)
3224 		return ret;
3225 
3226 	for (i = 0; i < insn_cnt; i++, insn++) {
3227 		if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
3228 			continue;
3229 
3230 		if (!env->bpf_capable) {
3231 			if (bpf_pseudo_func(insn)) {
3232 				operation = "BPF function reference";
3233 				suggestion = "Load this program with the required capability, or avoid BPF function references in unprivileged programs.";
3234 			} else {
3235 				operation = "BPF-to-BPF function call";
3236 				suggestion = "Load this program with the required capability, or avoid BPF-to-BPF function calls in unprivileged programs.";
3237 			}
3238 			verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
3239 			bpf_diag_policy(
3240 				env, i, operation,
3241 				"loading or calling other BPF functions requires CAP_BPF or CAP_SYS_ADMIN",
3242 				suggestion);
3243 			return -EPERM;
3244 		}
3245 
3246 		ret = add_subprog(env, i + insn->imm + 1);
3247 		if (ret < 0)
3248 			return ret;
3249 	}
3250 
3251 	/*
3252 	 * func_info describes all functions of the program. Those that are
3253 	 * referenced only from data, e.g. from a table of functions to be
3254 	 * called via callx, are not seen by the loop above. They are possible
3255 	 * callees that have to be known upfront as well.
3256 	 */
3257 	if (env->bpf_capable) {
3258 		struct bpf_prog_aux *aux = env->prog->aux;
3259 
3260 		for (i = 1; i < aux->func_info_cnt; i++) {
3261 			ret = add_subprog(env, aux->func_info[i].insn_off);
3262 			if (ret < 0)
3263 				return ret;
3264 		}
3265 	}
3266 
3267 	ret = bpf_find_exception_callback_insn_off(env);
3268 	if (ret < 0)
3269 		return ret;
3270 	ex_cb_insn = ret;
3271 
3272 	/* If ex_cb_insn > 0, this means that the main program has a subprog
3273 	 * marked using BTF decl tag to serve as the exception callback.
3274 	 */
3275 	if (ex_cb_insn) {
3276 		ret = add_subprog(env, ex_cb_insn);
3277 		if (ret < 0)
3278 			return ret;
3279 		for (i = 1; i < env->subprog_cnt; i++) {
3280 			if (env->subprog_info[i].start != ex_cb_insn)
3281 				continue;
3282 			env->exception_callback_subprog = i;
3283 			bpf_mark_subprog_exc_cb(env, i);
3284 			break;
3285 		}
3286 	}
3287 
3288 	/* Add a fake 'exit' subprog which could simplify subprog iteration
3289 	 * logic. 'subprog_cnt' should not be increased.
3290 	 */
3291 	subprog[env->subprog_cnt].start = insn_cnt;
3292 
3293 	if (env->log.level & BPF_LOG_LEVEL2)
3294 		for (i = 0; i < env->subprog_cnt; i++)
3295 			verbose(env, "func#%d @%d\n", i, subprog[i].start);
3296 
3297 	return 0;
3298 }
3299 
3300 static int add_kfuncs(struct bpf_verifier_env *env)
3301 {
3302 	struct bpf_insn *insn = env->prog->insnsi;
3303 	int i, ret, insn_cnt = env->prog->len;
3304 
3305 	for (i = 0; i < insn_cnt; i++, insn++) {
3306 		if (!bpf_pseudo_kfunc_call(insn))
3307 			continue;
3308 
3309 		if (!env->bpf_capable) {
3310 			verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
3311 			bpf_diag_policy(
3312 				env, i, "kernel function call",
3313 				"calling kernel functions requires CAP_BPF or CAP_SYS_ADMIN",
3314 				"Load this program with the required capability, or avoid kernel function calls in unprivileged programs.");
3315 			return -EPERM;
3316 		}
3317 
3318 		ret = bpf_add_kfunc_call(env, insn->imm, insn->off);
3319 		if (ret < 0)
3320 			return ret;
3321 	}
3322 
3323 	return 0;
3324 }
3325 
3326 static void find_subprog_properties(struct bpf_verifier_env *env)
3327 {
3328 	struct bpf_subprog_info *subprog = env->subprog_info;
3329 	struct bpf_insn *insn = env->prog->insnsi;
3330 	int cur_subprog;
3331 
3332 	for (cur_subprog = 0; cur_subprog < env->subprog_cnt; cur_subprog++) {
3333 		int i;
3334 
3335 		for (i = subprog[cur_subprog].start;
3336 		     i < subprog[cur_subprog + 1].start; i++) {
3337 			u8 code = insn[i].code;
3338 
3339 			if (code == (BPF_JMP | BPF_CALL) &&
3340 			    insn[i].src_reg == 0 &&
3341 			    insn[i].imm == BPF_FUNC_tail_call) {
3342 				subprog[cur_subprog].has_tail_call = true;
3343 				subprog[cur_subprog].tail_call_reachable = true;
3344 			}
3345 			if (BPF_CLASS(code) == BPF_LD &&
3346 			    (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND))
3347 				subprog[cur_subprog].has_ld_abs = true;
3348 			if (bpf_is_callx(&insn[i]))
3349 				env->has_callx = true;
3350 		}
3351 	}
3352 }
3353 
3354 static int check_subprogs(struct bpf_verifier_env *env)
3355 {
3356 	int i, subprog_start, subprog_end, off, cur_subprog = 0;
3357 	struct bpf_subprog_info *subprog = env->subprog_info;
3358 	struct bpf_insn *insn = env->prog->insnsi;
3359 	int insn_cnt = env->prog->len;
3360 
3361 	/* now check that all jumps are within the same subprog */
3362 	subprog_start = subprog[cur_subprog].start;
3363 	subprog_end = subprog[cur_subprog + 1].start;
3364 	for (i = 0; i < insn_cnt; i++) {
3365 		u8 code = insn[i].code;
3366 
3367 		if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
3368 			goto next;
3369 		if (BPF_OP(code) == BPF_CALL)
3370 			goto next;
3371 		if (BPF_OP(code) == BPF_EXIT) {
3372 			subprog[cur_subprog].exit_idx = i;
3373 			goto next;
3374 		}
3375 		if (insn_is_gotox(&insn[i]))
3376 			goto next;
3377 		off = i + bpf_jmp_offset(&insn[i]) + 1;
3378 		if (off < subprog_start || off >= subprog_end) {
3379 			verbose(env, "jump out of range from insn %d to %d\n", i, off);
3380 			bpf_diag_program_structure(
3381 				env, i, "jump out of range",
3382 				"Keep branch targets within the same subprogram, or use an explicit subprogram call.",
3383 				"Instruction %d jumps to instruction %d, but subprogram %d only contains instructions %d through %d. "
3384 				"A branch target must stay inside the same subprogram.",
3385 				i, off, cur_subprog, subprog_start, subprog_end - 1);
3386 			return -EINVAL;
3387 		}
3388 next:
3389 		if (i == subprog_end - 1) {
3390 			/*
3391 			 * To avoid fall-through from one subprog into another,
3392 			 * the last insn of the subprog should be either exit
3393 			 * or unconditional jump back or bpf_throw call.
3394 			 */
3395 			if (code != (BPF_JMP | BPF_EXIT) &&
3396 			    code != (BPF_JMP32 | BPF_JA) &&
3397 			    code != (BPF_JMP | BPF_JA) &&
3398 			    !insn_is_gotox(&insn[i])) {
3399 				verbose(env, "last insn is not an exit or jmp\n");
3400 				bpf_diag_program_structure(
3401 					env, i, "subprogram can fall through",
3402 					"End each subprogram with an exit or an explicit jump that keeps control flow inside the subprogram.",
3403 					"Subprogram %d reaches its last instruction %d without an exit or jump, so control could continue into the next subprogram.",
3404 					cur_subprog, i);
3405 				return -EINVAL;
3406 			}
3407 			subprog_start = subprog_end;
3408 			cur_subprog++;
3409 			if (cur_subprog < env->subprog_cnt)
3410 				subprog_end = subprog[cur_subprog + 1].start;
3411 		}
3412 	}
3413 	return 0;
3414 }
3415 
3416 /*
3417  * The callee of callx is known to the main verification pass only, which
3418  * records the 'caller' -> 'callee' edge of the call graph for the checks
3419  * that follow it: absence of recursion and the maximum stack depth.
3420  */
3421 static int record_callx_edge(struct bpf_verifier_env *env, int caller, int callee)
3422 {
3423 	u32 cnt = env->subprog_cnt;
3424 
3425 	if (!env->callx_edges) {
3426 		env->callx_edges = kvcalloc(BITS_TO_LONGS(cnt * cnt), sizeof(long),
3427 					    GFP_KERNEL_ACCOUNT);
3428 		if (!env->callx_edges)
3429 			return -ENOMEM;
3430 	}
3431 	__set_bit(caller * cnt + callee, env->callx_edges);
3432 	return 0;
3433 }
3434 
3435 /*
3436  * Return the first subprog with the number >= 'from' that 'caller' calls
3437  * via callx, or -1 when there is none.
3438  */
3439 static int next_callx_callee(struct bpf_verifier_env *env, int caller, int from)
3440 {
3441 	u32 cnt = env->subprog_cnt;
3442 	unsigned long bit, end = (caller + 1) * cnt;
3443 
3444 	if (!env->callx_edges || from >= cnt)
3445 		return -1;
3446 	bit = find_next_bit(env->callx_edges, end, caller * cnt + from);
3447 	return bit < end ? bit - caller * cnt : -1;
3448 }
3449 
3450 /*
3451  * 'call foo; unwind_goto pad' goes to the pad when foo left with unwind_exit.
3452  * JITs don't emit a branch for it. unwind_exit finds the pad by its return
3453  * address. So unwind_goto has to follow the call and nothing may jump to it.
3454  */
3455 static int check_unwind_insns(struct bpf_verifier_env *env)
3456 {
3457 	struct bpf_insn *insn = env->prog->insnsi;
3458 	bool seen = false;
3459 	int i;
3460 
3461 	for (i = 0; i < env->prog->len; i++, insn++) {
3462 		if (bpf_is_unwind_exit(insn)) {
3463 			bpf_find_containing_subprog(env, i)->can_unwind = true;
3464 			seen = true;
3465 		}
3466 		if (!bpf_is_unwind_goto(insn))
3467 			continue;
3468 		seen = true;
3469 		if (!i || !(bpf_pseudo_call(insn - 1) || bpf_is_callx(insn - 1))) {
3470 			verbose(env, "unwind_goto at insn %d doesn't follow a call of a subprog\n",
3471 				i);
3472 			return -EINVAL;
3473 		}
3474 		if (bpf_is_jump_target(env, i)) {
3475 			verbose(env, "jump to unwind_goto at insn %d\n", i);
3476 			return -EINVAL;
3477 		}
3478 	}
3479 	if (!seen)
3480 		return 0;
3481 	if (!bpf_jit_supports_unwind() || bpf_prog_is_offloaded(env->prog->aux)) {
3482 		verbose(env, "JIT does not support unwinding\n");
3483 		return -ENOTSUPP;
3484 	}
3485 	env->prog->jit_required = 1;
3486 	env->prog->aux->can_unwind = env->subprog_info[0].can_unwind;
3487 	return 0;
3488 }
3489 
3490 /*
3491  * Sort subprogs in topological order so that leaf subprogs come first and
3492  * their callers come later. This is a DFS post-order traversal of the call
3493  * graph. Scan only reachable instructions (those in the computed postorder) of
3494  * the current subprog to discover callees (direct subprogs and sync
3495  * callbacks).
3496  *
3497  * The callees of callx are not known before the main verification pass.
3498  * When callx is used the sort is repeated after it with the recorded callx
3499  * edges added to the call graph to reject recursion through indirect calls.
3500  */
3501 static int sort_subprogs_topo(struct bpf_verifier_env *env)
3502 {
3503 	struct bpf_subprog_info *si = env->subprog_info;
3504 	int *insn_postorder = env->cfg.insn_postorder;
3505 	struct bpf_insn *insn = env->prog->insnsi;
3506 	int cnt = env->subprog_cnt;
3507 	int *dfs_stack = NULL;
3508 	int top = 0, order = 0;
3509 	int i, ret = 0;
3510 	u8 *color = NULL;
3511 
3512 	color = kvzalloc_objs(*color, cnt, GFP_KERNEL_ACCOUNT);
3513 	dfs_stack = kvmalloc_objs(*dfs_stack, cnt, GFP_KERNEL_ACCOUNT);
3514 	if (!color || !dfs_stack) {
3515 		ret = -ENOMEM;
3516 		goto out;
3517 	}
3518 
3519 	/*
3520 	 * DFS post-order traversal.
3521 	 * Color values: 0 = unvisited, 1 = on stack, 2 = done.
3522 	 */
3523 	for (i = 0; i < cnt; i++) {
3524 		if (color[i])
3525 			continue;
3526 		color[i] = 1;
3527 		dfs_stack[top++] = i;
3528 
3529 		while (top > 0) {
3530 			int cur = dfs_stack[top - 1];
3531 			int po_start = si[cur].postorder_start;
3532 			int po_end = si[cur + 1].postorder_start;
3533 			bool pushed = false;
3534 			int j;
3535 
3536 			for (j = po_start; j < po_end; j++) {
3537 				int idx = insn_postorder[j];
3538 				int callee;
3539 
3540 				if (bpf_is_callx(&insn[idx])) {
3541 					/* find a callee that is not explored yet */
3542 					callee = -1;
3543 					do {
3544 						callee = next_callx_callee(env, cur, callee + 1);
3545 					} while (callee >= 0 && color[callee] == 2);
3546 					if (callee < 0)
3547 						continue;
3548 				} else if (bpf_pseudo_call(&insn[idx]) || bpf_pseudo_func(&insn[idx])) {
3549 					callee = bpf_find_subprog(env, idx + insn[idx].imm + 1);
3550 					if (callee < 0) {
3551 						ret = -EFAULT;
3552 						goto out;
3553 					}
3554 				} else {
3555 					continue;
3556 				}
3557 				if (color[callee] == 2)
3558 					continue;
3559 				if (color[callee] == 1) {
3560 					if (bpf_pseudo_func(&insn[idx]))
3561 						continue;
3562 					verbose(env, "recursive call from %s() to %s()\n",
3563 						bpf_subprog_name(env, cur),
3564 						bpf_subprog_name(env, callee));
3565 					bpf_diag_program_structure(
3566 						env, idx, "recursive subprogram call",
3567 						"Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.",
3568 						"This bpf2bpf call would make the subprogram call graph recursive. "
3569 						"The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis.");
3570 					ret = -EINVAL;
3571 					goto out;
3572 				}
3573 				color[callee] = 1;
3574 				dfs_stack[top++] = callee;
3575 				pushed = true;
3576 				break;
3577 			}
3578 
3579 			if (!pushed) {
3580 				color[cur] = 2;
3581 				env->subprog_topo_order[order++] = cur;
3582 				top--;
3583 			}
3584 		}
3585 	}
3586 
3587 	if (env->log.level & BPF_LOG_LEVEL2)
3588 		for (i = 0; i < cnt; i++)
3589 			verbose(env, "topo_order[%d] = %s\n",
3590 				i, bpf_subprog_name(env, env->subprog_topo_order[i]));
3591 out:
3592 	kvfree(dfs_stack);
3593 	kvfree(color);
3594 	return ret;
3595 }
3596 
3597 static void mark_stack_slots_scratched(struct bpf_verifier_env *env,
3598 				       int spi, int nr_slots)
3599 {
3600 	int i;
3601 
3602 	for (i = 0; i < nr_slots; i++)
3603 		mark_stack_slot_scratched(env, spi - i);
3604 }
3605 
3606 static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno,
3607 			   enum bpf_reg_arg_type t)
3608 {
3609 	struct bpf_reg_state *reg;
3610 
3611 	mark_reg_scratched(env, regno);
3612 
3613 	reg = &regs[regno];
3614 	if (t == SRC_OP) {
3615 		/* check whether register used as source operand can be read */
3616 		if (reg->type == NOT_INIT) {
3617 			verbose(env, "R%d !read_ok\n", regno);
3618 			bpf_diag_unreadable_reg(env, env->insn_idx, regno);
3619 			return -EACCES;
3620 		}
3621 		/* We don't need to worry about FP liveness because it's read-only */
3622 		if (regno == BPF_REG_FP)
3623 			return 0;
3624 
3625 		return 0;
3626 	} else {
3627 		/* check whether register used as dest operand can be written to */
3628 		if (regno == BPF_REG_FP) {
3629 			verbose(env, "frame pointer is read only\n");
3630 			return -EACCES;
3631 		}
3632 		if (t == DST_OP)
3633 			mark_reg_unknown(env, regs, regno);
3634 	}
3635 	return 0;
3636 }
3637 
3638 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
3639 			 enum bpf_reg_arg_type t)
3640 {
3641 	struct bpf_verifier_state *vstate = env->cur_state;
3642 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
3643 
3644 	return __check_reg_arg(env, state->regs, regno, t);
3645 }
3646 
3647 static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
3648 {
3649 	env->insn_aux_data[idx].indirect_target = true;
3650 }
3651 
3652 static void mark_non_stack_access(struct bpf_verifier_env *env, int idx)
3653 {
3654 	env->insn_aux_data[idx].non_stack_access = true;
3655 }
3656 
3657 /* Layout of one packed linked register in the jump history, see linked_regs_pack() */
3658 #define LR_FRAMENO_BITS	4
3659 #define LR_INDEX_BITS	11
3660 #define LR_FRAMENO_MASK	((1u << LR_FRAMENO_BITS) - 1)
3661 #define LR_IS_REG	BIT(LR_FRAMENO_BITS)
3662 #define LR_INDEX_OFF	(LR_FRAMENO_BITS + 1)
3663 #define LR_INDEX_MASK	((1u << LR_INDEX_BITS) - 1)
3664 #define LINKED_REGS_MAX	BPF_LINKED_REGS_MAX
3665 
3666 static_assert(MAX_CALL_FRAMES <= (1 << LR_FRAMENO_BITS));
3667 static_assert(MAX_BPF_REG <= (1 << LR_INDEX_BITS));
3668 static_assert(MAX_BPF_STACK_SLOTS <= (1 << LR_INDEX_BITS));
3669 static_assert(LR_INDEX_OFF + LR_INDEX_BITS <= 16);
3670 
3671 struct linked_reg {
3672 	u8 frameno;
3673 	union {
3674 		u16 spi;
3675 		u16 regno;
3676 	};
3677 	bool is_reg;
3678 };
3679 
3680 struct linked_regs {
3681 	int cnt;
3682 	struct linked_reg entries[LINKED_REGS_MAX];
3683 };
3684 
3685 static struct linked_reg *linked_regs_push(struct linked_regs *s)
3686 {
3687 	if (s->cnt < LINKED_REGS_MAX)
3688 		return &s->entries[s->cnt++];
3689 
3690 	return NULL;
3691 }
3692 
3693 /*
3694  * Pack linked registers for a jump history entry, one u16 each:
3695  * - 4 bits frameno
3696  * - 1 bit  is_reg
3697  * - 11 bits register or stack slot index
3698  */
3699 static void linked_regs_pack(const struct linked_regs *s, u16 *packed)
3700 {
3701 	int i;
3702 
3703 	for (i = 0; i < s->cnt; ++i) {
3704 		const struct linked_reg *e = &s->entries[i];
3705 
3706 		packed[i] = e->frameno | (e->is_reg ? LR_IS_REG : 0) | (e->spi << LR_INDEX_OFF);
3707 	}
3708 }
3709 
3710 static void linked_regs_unpack(const struct bpf_jmp_history_entry *hist, struct linked_regs *s)
3711 {
3712 	int i;
3713 
3714 	s->cnt = hist->linked_regs_cnt;
3715 	for (i = 0; i < s->cnt; ++i) {
3716 		struct linked_reg *e = &s->entries[i];
3717 		u16 packed = hist->linked_regs[i];
3718 
3719 		e->frameno = packed & LR_FRAMENO_MASK;
3720 		e->is_reg  = packed & LR_IS_REG;
3721 		e->spi     = (packed >> LR_INDEX_OFF) & LR_INDEX_MASK;
3722 	}
3723 }
3724 
3725 const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn)
3726 {
3727 	const struct btf_type *func;
3728 	struct btf *desc_btf;
3729 
3730 	if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
3731 		return NULL;
3732 
3733 	desc_btf = find_kfunc_desc_btf_cached(data, insn->off);
3734 	if (IS_ERR(desc_btf))
3735 		return "<error>";
3736 
3737 	func = btf_type_by_id(desc_btf, insn->imm);
3738 	if (!func || !btf_type_is_func(func))
3739 		return "<error>";
3740 	return btf_name_by_offset(desc_btf, func->name_off);
3741 }
3742 
3743 void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn)
3744 {
3745 	const struct bpf_insn_cbs cbs = {
3746 		.cb_call	= bpf_disasm_kfunc_name,
3747 		.cb_print	= verbose,
3748 		.private_data	= env,
3749 	};
3750 
3751 	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
3752 }
3753 
3754 /* If any register R in hist->linked_regs is marked as precise in bt,
3755  * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs.
3756  */
3757 void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist)
3758 {
3759 	struct linked_regs linked_regs;
3760 	bool some_precise = false;
3761 	int i;
3762 
3763 	if (!hist || !hist->linked_regs_cnt)
3764 		return;
3765 
3766 	linked_regs_unpack(hist, &linked_regs);
3767 	for (i = 0; i < linked_regs.cnt; ++i) {
3768 		struct linked_reg *e = &linked_regs.entries[i];
3769 
3770 		if ((e->is_reg && bt_is_frame_reg_set(bt, e->frameno, e->regno)) ||
3771 		    (!e->is_reg && bt_is_frame_slot_set(bt, e->frameno, e->spi))) {
3772 			some_precise = true;
3773 			break;
3774 		}
3775 	}
3776 
3777 	if (!some_precise)
3778 		return;
3779 
3780 	for (i = 0; i < linked_regs.cnt; ++i) {
3781 		struct linked_reg *e = &linked_regs.entries[i];
3782 
3783 		if (e->is_reg)
3784 			bpf_bt_set_frame_reg(bt, e->frameno, e->regno);
3785 		else
3786 			bpf_bt_set_frame_slot(bt, e->frameno, e->spi);
3787 	}
3788 }
3789 
3790 int mark_chain_precision(struct bpf_verifier_env *env, int regno)
3791 {
3792 	return bpf_mark_chain_precision(env, env->cur_state, regno, NULL);
3793 }
3794 
3795 /* mark_chain_precision_batch() assumes that env->bt is set in the caller to
3796  * desired reg and stack masks across all relevant frames
3797  */
3798 static int mark_chain_precision_batch(struct bpf_verifier_env *env,
3799 				      struct bpf_verifier_state *starting_state)
3800 {
3801 	return bpf_mark_chain_precision(env, starting_state, -1, NULL);
3802 }
3803 
3804 /* check if register is a constant scalar value */
3805 static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32)
3806 {
3807 	return reg->type == SCALAR_VALUE &&
3808 	       tnum_is_const(subreg32 ? tnum_subreg(reg->var_off) : reg->var_off);
3809 }
3810 
3811 /* assuming is_reg_const() is true, return constant value of a register */
3812 static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32)
3813 {
3814 	return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value;
3815 }
3816 
3817 static bool is_pointer_regtype(enum bpf_reg_type type)
3818 {
3819 	return type != SCALAR_VALUE && type != NOT_INIT;
3820 }
3821 
3822 static bool __is_pointer_value(bool allow_ptr_leaks,
3823 			       const struct bpf_reg_state *reg)
3824 {
3825 	if (allow_ptr_leaks)
3826 		return false;
3827 
3828 	return is_pointer_regtype(reg->type);
3829 }
3830 
3831 static void assign_scalar_id_before_mov(struct bpf_verifier_env *env,
3832 					struct bpf_reg_state *src_reg)
3833 {
3834 	if (src_reg->type != SCALAR_VALUE)
3835 		return;
3836 	/*
3837 	 * The verifier is processing rX = rY insn and
3838 	 * rY->id has special linked register already.
3839 	 * Cleared it, since multiple rX += const are not supported.
3840 	 */
3841 	if (src_reg->id & BPF_ADD_CONST)
3842 		clear_scalar_id(src_reg);
3843 	/*
3844 	 * Ensure that src_reg has a valid ID that will be copied to
3845 	 * dst_reg and then will be used by sync_linked_regs() to
3846 	 * propagate min/max range.
3847 	 */
3848 	if (!src_reg->id && !tnum_is_const(src_reg->var_off))
3849 		src_reg->id = ++env->id_gen;
3850 }
3851 
3852 static void save_register_state(struct bpf_verifier_env *env,
3853 				struct bpf_func_state *state,
3854 				int spi, struct bpf_reg_state *reg,
3855 				int size)
3856 {
3857 	struct bpf_stack_state *slot = bpf_stack_slot(state, spi);
3858 	int i;
3859 
3860 	bpf_diag_mod_begin(env, &slot->spilled_ptr, reg, BPF_DIAG_MOD_SPILL);
3861 	slot->spilled_ptr = *reg;
3862 
3863 	for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--)
3864 		slot->slot_type[i - 1] = STACK_SPILL;
3865 
3866 	/* size < 8 bytes spill */
3867 	for (; i; i--)
3868 		mark_stack_slot_misc(env, &slot->slot_type[i - 1]);
3869 
3870 	bpf_diag_mod_end(env);
3871 }
3872 
3873 static bool is_bpf_st_mem(struct bpf_insn *insn)
3874 {
3875 	return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM;
3876 }
3877 
3878 static int get_reg_width(struct bpf_reg_state *reg)
3879 {
3880 	return fls64(reg_umax(reg));
3881 }
3882 
3883 /* See comment for mark_fastcall_pattern_for_call() */
3884 static void check_fastcall_stack_contract(struct bpf_verifier_env *env,
3885 					  struct bpf_func_state *state, int insn_idx, int off)
3886 {
3887 	struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];
3888 	struct bpf_insn_aux_data *aux = env->insn_aux_data;
3889 	int i;
3890 
3891 	if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern)
3892 		return;
3893 	/* access to the region [max_stack_depth .. fastcall_stack_off)
3894 	 * from something that is not a part of the fastcall pattern,
3895 	 * disable fastcall rewrites for current subprogram by setting
3896 	 * fastcall_stack_off to a value smaller than any possible offset.
3897 	 */
3898 	subprog->fastcall_stack_off = S16_MIN;
3899 	/* reset fastcall aux flags within subprogram,
3900 	 * happens at most once per subprogram
3901 	 */
3902 	for (i = subprog->start; i < (subprog + 1)->start; ++i) {
3903 		aux[i].fastcall_spills_num = 0;
3904 		aux[i].fastcall_pattern = 0;
3905 	}
3906 }
3907 
3908 static void scrub_special_slot(struct bpf_func_state *state, int spi)
3909 {
3910 	struct bpf_stack_state *slot = bpf_stack_slot(state, spi);
3911 	int i;
3912 
3913 	/* regular write of data into stack destroys any spilled ptr */
3914 	slot->spilled_ptr.type = NOT_INIT;
3915 	/* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */
3916 	if (is_stack_slot_special(slot))
3917 		for (i = 0; i < BPF_REG_SIZE; i++)
3918 			scrub_spilled_slot(&slot->slot_type[i]);
3919 }
3920 
3921 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
3922  * stack boundary and alignment are checked in check_mem_access()
3923  */
3924 static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
3925 				       /* stack frame we're writing to */
3926 				       struct bpf_func_state *state,
3927 				       int off, int size, int value_regno,
3928 				       int insn_idx)
3929 {
3930 	struct bpf_func_state *cur; /* state of the current function */
3931 	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
3932 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
3933 	struct bpf_reg_state *reg = NULL;
3934 	int insn_flags = INSN_F_STACK_ACCESS;
3935 	int hist_spi = spi, hist_frame = state->frameno;
3936 	struct bpf_stack_state *ss = bpf_stack_slot(state, spi);
3937 
3938 	/*
3939 	 * caller checked that off % size == 0 and -env->stack_limit <= off < 0,
3940 	 * so it's aligned access and [off, off + size) are within stack limits
3941 	 */
3942 	if (!env->allow_ptr_leaks &&
3943 	    bpf_is_spilled_reg(ss) &&
3944 	    !bpf_is_spilled_scalar_reg(ss) &&
3945 	    size != BPF_REG_SIZE) {
3946 		const char *reason;
3947 
3948 		verbose(env, "attempt to corrupt spilled pointer on stack\n");
3949 		reason = bpf_diag_fmt(env,
3950 				      "This store writes %d bytes at stack offset %d into a stack slot that currently holds a spilled pointer. "
3951 			"Partial writes to spilled pointers are rejected because they can corrupt pointer metadata and leak kernel pointers.",
3952 			size, off);
3953 		bpf_diag_memory(
3954 			env, insn_idx, "stack spill corruption", reason,
3955 			"Write the full 8-byte spilled pointer slot, or use a separate stack slot for scalar data before overwriting only part of it.");
3956 		return -EACCES;
3957 	}
3958 
3959 	cur = env->cur_state->frame[env->cur_state->curframe];
3960 	if (value_regno >= 0)
3961 		reg = &cur->regs[value_regno];
3962 	if (!env->bypass_spec_v4) {
3963 		bool sanitize = reg && is_pointer_regtype(reg->type);
3964 
3965 		for (i = 0; i < size; i++) {
3966 			u8 type = ss->slot_type[(slot - i) %
3967 							      BPF_REG_SIZE];
3968 
3969 			if (type != STACK_MISC && type != STACK_ZERO) {
3970 				sanitize = true;
3971 				break;
3972 			}
3973 		}
3974 
3975 		if (sanitize)
3976 			env->insn_aux_data[insn_idx].nospec_result = true;
3977 	}
3978 
3979 	err = destroy_if_dynptr_stack_slot(env, state, spi);
3980 	if (err)
3981 		return err;
3982 
3983 	mark_stack_slot_scratched(env, spi);
3984 	if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {
3985 		bool reg_value_fits;
3986 
3987 		reg_value_fits = get_reg_width(reg) <= BITS_PER_BYTE * size;
3988 		/* Make sure that reg had an ID to build a relation on spill. */
3989 		if (reg_value_fits)
3990 			assign_scalar_id_before_mov(env, reg);
3991 		save_register_state(env, state, spi, reg, size);
3992 		/* Break the relation on a narrowing spill. */
3993 		if (!reg_value_fits)
3994 			clear_scalar_id(&ss->spilled_ptr);
3995 	} else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) &&
3996 		   env->bpf_capable) {
3997 		struct bpf_reg_state *tmp_reg = &env->fake_reg[0];
3998 
3999 		memset(tmp_reg, 0, sizeof(*tmp_reg));
4000 		__mark_reg_known(tmp_reg, insn->imm);
4001 		tmp_reg->type = SCALAR_VALUE;
4002 		save_register_state(env, state, spi, tmp_reg, size);
4003 	} else if (reg && is_pointer_regtype(reg->type)) {
4004 		/* register containing pointer is being spilled into stack */
4005 		if (size != BPF_REG_SIZE) {
4006 			verbose_linfo(env, insn_idx, "; ");
4007 			verbose(env, "invalid size of register spill\n");
4008 			return -EACCES;
4009 		}
4010 		if (state != cur && reg->type == PTR_TO_STACK) {
4011 			verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
4012 			return -EINVAL;
4013 		}
4014 		save_register_state(env, state, spi, reg, size);
4015 	} else {
4016 		u8 type = STACK_MISC;
4017 
4018 		if (bpf_is_spilled_reg(ss))
4019 			bpf_diag_record_scrub(env, &ss->spilled_ptr,
4020 					      BPF_DIAG_MOD_WRITE);
4021 		scrub_special_slot(state, spi);
4022 
4023 		/* when we zero initialize stack slots mark them as such */
4024 		if ((reg && bpf_register_is_null(reg)) ||
4025 		    (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) {
4026 			/* STACK_ZERO case happened because register spill
4027 			 * wasn't properly aligned at the stack slot boundary,
4028 			 * so it's not a register spill anymore; force
4029 			 * originating register to be precise to make
4030 			 * STACK_ZERO correct for subsequent states
4031 			 */
4032 			err = mark_chain_precision(env, value_regno);
4033 			if (err)
4034 				return err;
4035 			type = STACK_ZERO;
4036 		}
4037 
4038 		/* Mark slots affected by this stack write. */
4039 		for (i = 0; i < size; i++)
4040 			ss->slot_type[(slot - i) % BPF_REG_SIZE] = type;
4041 		insn_flags = 0; /* not a register spill */
4042 	}
4043 
4044 	if (insn_flags)
4045 		return bpf_push_jmp_history(env, env->cur_state, insn_flags,
4046 					    hist_spi, hist_frame, NULL, 0);
4047 	return 0;
4048 }
4049 
4050 /* Write the stack: 'stack[ptr_reg + off] = value_regno'. 'ptr_reg' is
4051  * known to contain a variable offset.
4052  * This function checks whether the write is permitted and conservatively
4053  * tracks the effects of the write, considering that each stack slot in the
4054  * dynamic range is potentially written to.
4055  *
4056  * 'value_regno' can be -1, meaning that an unknown value is being written to
4057  * the stack.
4058  *
4059  * Spilled pointers in range are not marked as written because we don't know
4060  * what's going to be actually written. This means that read propagation for
4061  * future reads cannot be terminated by this write.
4062  *
4063  * For privileged programs, uninitialized stack slots are considered
4064  * initialized by this write (even though we don't know exactly what offsets
4065  * are going to be written to). The idea is that we don't want the verifier to
4066  * reject future reads that access slots written to through variable offsets.
4067  */
4068 static int check_stack_write_var_off(struct bpf_verifier_env *env,
4069 				     /* func where register points to */
4070 				     struct bpf_func_state *state,
4071 				     struct bpf_reg_state *ptr_reg, int off, int size,
4072 				     int value_regno, int insn_idx)
4073 {
4074 	struct bpf_func_state *cur; /* state of the current function */
4075 	int min_off, max_off;
4076 	int i, err;
4077 	struct bpf_reg_state *value_reg = NULL;
4078 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
4079 	bool writing_zero = false;
4080 	/* set if the fact that we're writing a zero is used to let any
4081 	 * stack slots remain STACK_ZERO
4082 	 */
4083 	bool zero_used = false;
4084 
4085 	cur = env->cur_state->frame[env->cur_state->curframe];
4086 	min_off = reg_smin(ptr_reg) + off;
4087 	max_off = reg_smax(ptr_reg) + off + size;
4088 	if (value_regno >= 0)
4089 		value_reg = &cur->regs[value_regno];
4090 	if ((value_reg && bpf_register_is_null(value_reg)) ||
4091 	    (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0))
4092 		writing_zero = true;
4093 
4094 	err = destroy_dynptrs_in_stack_slots(env, state, bpf_get_spi(max_off - 1),
4095 					     bpf_get_spi(min_off));
4096 	if (err)
4097 		return err;
4098 
4099 	/* Variable offset writes destroy any spilled pointers in range. */
4100 	for (i = min_off; i < max_off; i++) {
4101 		struct bpf_stack_state *ss;
4102 		u8 new_type, *stype;
4103 		int slot, spi;
4104 
4105 		slot = -i - 1;
4106 		spi = slot / BPF_REG_SIZE;
4107 		ss = bpf_stack_slot(state, spi);
4108 		stype = &ss->slot_type[slot % BPF_REG_SIZE];
4109 		mark_stack_slot_scratched(env, spi);
4110 
4111 		if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) {
4112 			/* Reject the write if range we may write to has not
4113 			 * been initialized beforehand. If we didn't reject
4114 			 * here, the ptr status would be erased below (even
4115 			 * though not all slots are actually overwritten),
4116 			 * possibly opening the door to leaks.
4117 			 *
4118 			 * We do however catch STACK_INVALID case below, and
4119 			 * only allow reading possibly uninitialized memory
4120 			 * later for CAP_PERFMON, as the write may not happen to
4121 			 * that slot.
4122 			 */
4123 			verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
4124 				insn_idx, i);
4125 			return -EINVAL;
4126 		}
4127 
4128 		/* If writing_zero and the spi slot contains a spill of value 0,
4129 		 * maintain the spill type.
4130 		 */
4131 		if (writing_zero && *stype == STACK_SPILL && bpf_is_spilled_scalar_reg(ss)) {
4132 			struct bpf_reg_state *spill_reg = &ss->spilled_ptr;
4133 
4134 			if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) {
4135 				zero_used = true;
4136 				continue;
4137 			}
4138 		}
4139 
4140 		/*
4141 		 * Scrub slots if variable-offset stack write goes over spilled pointers.
4142 		 * Otherwise bpf_is_spilled_reg() may == true && spilled_ptr.type == NOT_INIT
4143 		 * and valid program is rejected by check_stack_read_fixed_off()
4144 		 * with obscure "invalid size of register fill" message.
4145 		 */
4146 		scrub_special_slot(state, spi);
4147 
4148 		/* Update the slot type. */
4149 		new_type = STACK_MISC;
4150 		if (writing_zero && *stype == STACK_ZERO) {
4151 			new_type = STACK_ZERO;
4152 			zero_used = true;
4153 		}
4154 		/* If the slot is STACK_INVALID, we check whether it's OK to
4155 		 * pretend that it will be initialized by this write. The slot
4156 		 * might not actually be written to, and so if we mark it as
4157 		 * initialized future reads might leak uninitialized memory.
4158 		 * For privileged programs, we will accept such reads to slots
4159 		 * that may or may not be written because, if we're reject
4160 		 * them, the error would be too confusing.
4161 		 * Conservatively, treat STACK_POISON in a similar way.
4162 		 */
4163 		if ((*stype == STACK_INVALID || *stype == STACK_POISON) &&
4164 		    !env->allow_uninit_stack) {
4165 			verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
4166 					insn_idx, i);
4167 			return -EINVAL;
4168 		}
4169 		*stype = new_type;
4170 	}
4171 	if (zero_used) {
4172 		/* backtracking doesn't work for STACK_ZERO yet. */
4173 		err = mark_chain_precision(env, value_regno);
4174 		if (err)
4175 			return err;
4176 	}
4177 	bpf_diag_record_scrub_stack(env, state, min_off, max_off,
4178 				    BPF_DIAG_MOD_VAR_WRITE);
4179 	return 0;
4180 }
4181 
4182 /* When register 'dst_regno' is assigned some values from stack[min_off,
4183  * max_off), we set the register's type according to the types of the
4184  * respective stack slots. If all the stack values are known to be zeros, then
4185  * so is the destination reg. Otherwise, the register is considered to be
4186  * SCALAR. This function does not deal with register filling; the caller must
4187  * ensure that all spilled registers in the stack range have been marked as
4188  * read.
4189  *
4190  * STACK_SPILL bytes backed by spilled scalar const zeroes are also considered
4191  * zero bytes. In that case, mark the contributing stack slots precise so
4192  * pruning cannot reuse a zero-spill state for a later non-zero spill state.
4193  *
4194  * Returns an error if precision backtracking fails.
4195  */
4196 static int mark_reg_stack_read(struct bpf_verifier_env *env,
4197 			       /* func where src register points to */
4198 			       struct bpf_func_state *ptr_state,
4199 			       int min_off, int max_off, int dst_regno)
4200 {
4201 	struct bpf_verifier_state *vstate = env->cur_state;
4202 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
4203 	int i, slot, spi;
4204 	u8 *stype;
4205 	int zeros = 0, zero_spills = 0;
4206 
4207 	for (i = min_off; i < max_off; i++) {
4208 		slot = -i - 1;
4209 		spi = slot / BPF_REG_SIZE;
4210 		mark_stack_slot_scratched(env, spi);
4211 		stype = bpf_stack_slot(ptr_state, spi)->slot_type;
4212 		if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) {
4213 			zeros++;
4214 			continue;
4215 		}
4216 		if (stype[slot % BPF_REG_SIZE] == STACK_SPILL &&
4217 		    bpf_register_is_null(&bpf_stack_slot(ptr_state, spi)->spilled_ptr)) {
4218 			zero_spills++;
4219 			zeros++;
4220 			continue;
4221 		}
4222 		break;
4223 	}
4224 	if (zeros == max_off - min_off) {
4225 		/* Any access_size read into register is zero extended,
4226 		 * so the whole register == const_zero.
4227 		 */
4228 		__mark_reg_const_zero(env, &state->regs[dst_regno]);
4229 		if (zero_spills) {
4230 			for (i = min_off; i < max_off; i++) {
4231 				slot = -i - 1;
4232 				spi = slot / BPF_REG_SIZE;
4233 				stype = bpf_stack_slot(ptr_state, spi)->slot_type;
4234 				if (stype[slot % BPF_REG_SIZE] == STACK_SPILL)
4235 					bpf_bt_set_frame_slot(&env->bt, ptr_state->frameno, spi);
4236 			}
4237 			return mark_chain_precision_batch(env, env->cur_state);
4238 		}
4239 	} else {
4240 		/* have read misc data from the stack */
4241 		mark_reg_unknown(env, state->regs, dst_regno);
4242 	}
4243 
4244 	return 0;
4245 }
4246 
4247 static void bpf_diag_stack_read_invalid(struct bpf_verifier_env *env, int off, int i, int size,
4248 					enum bpf_stack_slot_type type)
4249 {
4250 	const char *problem, *reason, *suggestion, *kind;
4251 
4252 	if (type == STACK_INVALID) {
4253 		reason = bpf_diag_fmt(
4254 			env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. "
4255 			"Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.",
4256 			size, off, i);
4257 		bpf_diag_memory(
4258 			env, env->insn_idx, "uninitialized stack read", reason,
4259 			"Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, "
4260 			"or load with CAP_PERFMON if uninitialized stack reads are intended.");
4261 		return;
4262 	}
4263 
4264 	switch (type) {
4265 	case STACK_DYNPTR:
4266 		kind = "dynptr";
4267 		suggestion = "Use dynptr helpers or kfuncs to access the object represented by the dynptr instead of reading the dynptr state directly.";
4268 		break;
4269 	case STACK_ITER:
4270 		kind = "iterator";
4271 		suggestion = "Use iterator kfuncs to advance or destroy the iterator instead of reading its state directly.";
4272 		break;
4273 	case STACK_IRQ_FLAG:
4274 		kind = "IRQ flag";
4275 		suggestion = "Pass the saved IRQ flag to the matching restore kfunc instead of reading its state directly.";
4276 		break;
4277 	default:
4278 		return;
4279 	}
4280 
4281 	problem = bpf_diag_fmt(env, "direct read of %s stack state", kind);
4282 	reason = bpf_diag_fmt(
4283 		env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range belongs to verifier-managed %s state. "
4284 		"This state has an opaque representation that BPF programs cannot read directly.",
4285 		size, off, i, kind);
4286 	bpf_diag_memory(env, env->insn_idx, problem, reason, suggestion);
4287 }
4288 
4289 /* Read the stack at 'off' and put the results into the register indicated by
4290  * 'dst_regno'. It handles reg filling if the addressed stack slot is a
4291  * spilled reg.
4292  *
4293  * 'dst_regno' can be -1, meaning that the read value is not going to a
4294  * register.
4295  *
4296  * The access is assumed to be within the current stack bounds.
4297  */
4298 static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
4299 				      /* func where src register points to */
4300 				      struct bpf_func_state *reg_state,
4301 				      int off, int size, int dst_regno)
4302 {
4303 	struct bpf_verifier_state *vstate = env->cur_state;
4304 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
4305 	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
4306 	struct bpf_reg_state *reg;
4307 	u8 *stype, type;
4308 	int err;
4309 	int insn_flags = INSN_F_STACK_ACCESS;
4310 	int hist_spi = spi, hist_frame = reg_state->frameno;
4311 	struct bpf_stack_state *ss = bpf_stack_slot(reg_state, spi);
4312 
4313 	stype = ss->slot_type;
4314 	reg = &ss->spilled_ptr;
4315 
4316 	mark_stack_slot_scratched(env, spi);
4317 
4318 	/*
4319 	 * Refine the in-progress load record's origin to the source stack slot.
4320 	 */
4321 	if (dst_regno >= 0)
4322 		bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE);
4323 
4324 	if (bpf_is_spilled_reg(ss)) {
4325 		u8 spill_size = bpf_spill_size(ss);
4326 
4327 		if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) {
4328 			if (reg->type != SCALAR_VALUE) {
4329 				verbose_linfo(env, env->insn_idx, "; ");
4330 				verbose(env, "invalid size of register fill\n");
4331 				return -EACCES;
4332 			}
4333 
4334 			if (size <= spill_size &&
4335 			    bpf_stack_narrow_access_ok(off, size, spill_size)) {
4336 				if (dst_regno < 0)
4337 					return 0;
4338 
4339 				if (env->bpf_capable && size == 4 && spill_size == 4 &&
4340 				    get_reg_width(reg) <= 32)
4341 					/* Ensure stack slot has an ID to build a relation
4342 					 * with the destination register on fill.
4343 					 */
4344 					assign_scalar_id_before_mov(env, reg);
4345 				state->regs[dst_regno] = *reg;
4346 
4347 				/* Break the relation on a narrowing fill.
4348 				 * coerce_reg_to_size will adjust the boundaries.
4349 				 */
4350 				if (get_reg_width(reg) > size * BITS_PER_BYTE)
4351 					clear_scalar_id(&state->regs[dst_regno]);
4352 			} else {
4353 				int spill_cnt = 0, zero_cnt = 0;
4354 
4355 				for (i = 0; i < size; i++) {
4356 					type = stype[(slot - i) % BPF_REG_SIZE];
4357 					if (type == STACK_SPILL) {
4358 						spill_cnt++;
4359 						continue;
4360 					}
4361 					if (type == STACK_MISC)
4362 						continue;
4363 					if (type == STACK_ZERO) {
4364 						zero_cnt++;
4365 						continue;
4366 					}
4367 					if (type == STACK_INVALID && env->allow_uninit_stack)
4368 						continue;
4369 					if (type == STACK_POISON) {
4370 						verbose(env, "reading from stack off %d+%d size %d, slot poisoned by dead code elimination\n",
4371 							off, i, size);
4372 					} else {
4373 						verbose(env, "invalid read from stack off %d+%d size %d\n",
4374 							off, i, size);
4375 						bpf_diag_stack_read_invalid(env, off, i, size, type);
4376 					}
4377 					return -EACCES;
4378 				}
4379 
4380 				if (dst_regno < 0)
4381 					return 0;
4382 
4383 				if (spill_cnt == size &&
4384 				    tnum_is_const(reg->var_off) && reg->var_off.value == 0) {
4385 					__mark_reg_const_zero(env, &state->regs[dst_regno]);
4386 					/* this IS register fill, so keep insn_flags */
4387 				} else if (zero_cnt == size) {
4388 					/* similarly to mark_reg_stack_read(), preserve zeroes */
4389 					__mark_reg_const_zero(env, &state->regs[dst_regno]);
4390 					insn_flags = 0; /* not restoring original register state */
4391 				} else {
4392 					err = mark_reg_stack_read(env, reg_state, off, off + size,
4393 								  dst_regno);
4394 					if (err)
4395 						return err;
4396 					insn_flags = 0; /* not restoring original register state */
4397 				}
4398 			}
4399 		} else if (dst_regno >= 0) {
4400 			/* restore register state from stack */
4401 			if (env->bpf_capable)
4402 				/* Ensure stack slot has an ID to build a relation
4403 				 * with the destination register on fill.
4404 				 */
4405 				assign_scalar_id_before_mov(env, reg);
4406 			state->regs[dst_regno] = *reg;
4407 			/* mark reg as written since spilled pointer state likely
4408 			 * has its liveness marks cleared by is_state_visited()
4409 			 * which resets stack/reg liveness for state transitions
4410 			 */
4411 		} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
4412 			/* If dst_regno==-1, the caller is asking us whether
4413 			 * it is acceptable to use this value as a SCALAR_VALUE
4414 			 * (e.g. for XADD).
4415 			 * We must not allow unprivileged callers to do that
4416 			 * with spilled pointers.
4417 			 */
4418 			verbose(env, "leaking pointer from stack off %d\n",
4419 				off);
4420 			return -EACCES;
4421 		}
4422 	} else {
4423 		for (i = 0; i < size; i++) {
4424 			type = stype[(slot - i) % BPF_REG_SIZE];
4425 			if (type == STACK_MISC)
4426 				continue;
4427 			if (type == STACK_ZERO)
4428 				continue;
4429 			if (type == STACK_INVALID && env->allow_uninit_stack)
4430 				continue;
4431 			if (type == STACK_POISON) {
4432 				verbose(env, "reading from stack off %d+%d size %d, slot poisoned by dead code elimination\n",
4433 					off, i, size);
4434 			} else {
4435 				verbose(env, "invalid read from stack off %d+%d size %d\n",
4436 					off, i, size);
4437 				bpf_diag_stack_read_invalid(env, off, i, size, type);
4438 			}
4439 			return -EACCES;
4440 		}
4441 		if (dst_regno >= 0) {
4442 			err = mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
4443 			if (err)
4444 				return err;
4445 		}
4446 		insn_flags = 0; /* we are not restoring spilled register */
4447 	}
4448 	if (insn_flags)
4449 		return bpf_push_jmp_history(env, env->cur_state, insn_flags,
4450 					    hist_spi, hist_frame, NULL, 0);
4451 	return 0;
4452 }
4453 
4454 enum bpf_access_src {
4455 	ACCESS_DIRECT = 1,  /* the access is performed by an instruction */
4456 	ACCESS_HELPER = 2,  /* the access is performed by a helper */
4457 };
4458 
4459 static int check_stack_range_initialized(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
4460 					 argno_t argno, int off, int access_size,
4461 					 bool zero_size_allowed,
4462 					 enum bpf_access_type type,
4463 					 struct bpf_call_arg_meta *meta);
4464 
4465 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
4466 {
4467 	return cur_regs(env) + regno;
4468 }
4469 
4470 /* Read the stack at 'reg + off' and put the result into the register
4471  * 'dst_regno'.
4472  * 'off' includes the pointer register's fixed offset(i.e. 'reg->off'),
4473  * but not its variable offset.
4474  * 'size' is assumed to be <= reg size and the access is assumed to be aligned.
4475  *
4476  * As opposed to check_stack_read_fixed_off, this function doesn't deal with
4477  * filling registers (i.e. reads of spilled register cannot be detected when
4478  * the offset is not fixed). We conservatively mark 'dst_regno' as containing
4479  * SCALAR_VALUE. That's why we assert that the 'reg' has a variable
4480  * offset; for a fixed offset check_stack_read_fixed_off should be used
4481  * instead.
4482  */
4483 static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
4484 				    argno_t ptr_argno, int off, int size, int dst_regno)
4485 {
4486 	struct bpf_func_state *ptr_state = bpf_func(env, reg);
4487 	int err;
4488 	int min_off, max_off;
4489 
4490 	/* Note that we pass a NULL meta, so raw access will not be permitted.
4491 	 */
4492 	err = check_stack_range_initialized(env, reg, ptr_argno, off, size,
4493 					    false, BPF_READ, NULL);
4494 	if (err)
4495 		return err;
4496 
4497 	min_off = reg_smin(reg) + off;
4498 	max_off = reg_smax(reg) + off;
4499 	err = mark_reg_stack_read(env, ptr_state, min_off, max_off + size,
4500 				  dst_regno);
4501 	if (err)
4502 		return err;
4503 	return 0;
4504 }
4505 
4506 /* check_stack_read dispatches to check_stack_read_fixed_off or
4507  * check_stack_read_var_off.
4508  *
4509  * The caller must ensure that the offset falls within the allocated stack
4510  * bounds.
4511  *
4512  * 'dst_regno' is a register which will receive the value from the stack. It
4513  * can be -1, meaning that the read value is not going to a register.
4514  */
4515 static int check_stack_read(struct bpf_verifier_env *env,
4516 			    struct bpf_reg_state *reg, argno_t ptr_argno, int off, int size,
4517 			    int dst_regno)
4518 {
4519 	struct bpf_func_state *state = bpf_func(env, reg);
4520 	int err;
4521 	/* Some accesses are only permitted with a static offset. */
4522 	bool var_off = !tnum_is_const(reg->var_off);
4523 
4524 	/* The offset is required to be static when reads don't go to a
4525 	 * register, in order to not leak pointers (see
4526 	 * check_stack_read_fixed_off).
4527 	 */
4528 	if (dst_regno < 0 && var_off) {
4529 		const char *reason;
4530 		char tn_buf[48];
4531 
4532 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
4533 		verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
4534 			tn_buf, off, size);
4535 		reason = bpf_diag_fmt(
4536 			env, "The instruction would access the stack through variable offset %s plus fixed offset %d and size %d. "
4537 			"This stack access requires a constant stack offset and a precise initialized range.",
4538 			tn_buf, off, size);
4539 		bpf_diag_memory(
4540 			env, env->insn_idx, "variable-offset stack access", reason,
4541 			"Use a fixed stack offset for the instruction, selecting the target stack slot on separate control-flow paths if necessary.");
4542 		return -EACCES;
4543 	}
4544 	/* Variable offset is prohibited for unprivileged mode for simplicity
4545 	 * since it requires corresponding support in Spectre masking for stack
4546 	 * ALU. See also retrieve_ptr_limit(). The check in
4547 	 * check_stack_access_for_ptr_arithmetic() called by
4548 	 * adjust_ptr_min_max_vals() prevents users from creating stack pointers
4549 	 * with variable offsets, therefore no check is required here. Further,
4550 	 * just checking it here would be insufficient as speculative stack
4551 	 * writes could still lead to unsafe speculative behaviour.
4552 	 */
4553 	if (!var_off) {
4554 		off += reg->var_off.value;
4555 		err = check_stack_read_fixed_off(env, state, off, size,
4556 						 dst_regno);
4557 	} else {
4558 		/* Variable offset stack reads need more conservative handling
4559 		 * than fixed offset ones. Note that dst_regno >= 0 on this
4560 		 * branch.
4561 		 */
4562 		err = check_stack_read_var_off(env, reg, ptr_argno, off, size,
4563 					       dst_regno);
4564 	}
4565 	return err;
4566 }
4567 
4568 /*
4569  * An access through a slice that may be the buffer on the stack, see bpf_reg_state::buf, is
4570  * not seen as an access to the stack. That's fine as long as nothing is tracked in the buffer.
4571  * The output of helpers and kfuncs is not tracked. Here an insn stores to [min_off, max_off)
4572  * of frame @frameno, a dynptr, an iterator or irq flags are made there, or the frame is gone.
4573  */
4574 static void invalidate_stack_slices(struct bpf_verifier_env *env, u32 frameno,
4575 				    int min_off, int max_off)
4576 {
4577 	struct bpf_func_state *state;
4578 	struct bpf_reg_state *reg;
4579 
4580 	if (!env->seen_stack_slice)
4581 		return;
4582 
4583 	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
4584 		if (base_type(reg->type) != PTR_TO_MEM || !reg->buf.size ||
4585 		    reg->buf.frameno != frameno || reg->buf.off >= max_off ||
4586 		    reg->buf.off + (int)reg->buf.size <= min_off)
4587 			continue;
4588 		bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_SLICE_BUF);
4589 		mark_reg_invalid(env, reg);
4590 	}));
4591 }
4592 
4593 /* check_stack_write dispatches to check_stack_write_fixed_off or
4594  * check_stack_write_var_off.
4595  *
4596  * 'reg' is the register used as a pointer into the stack.
4597  * 'value_regno' is the register whose value we're writing to the stack. It can
4598  * be -1, meaning that we're not writing from a register.
4599  *
4600  * The caller must ensure that the offset falls within the maximum stack size.
4601  */
4602 static int check_stack_write(struct bpf_verifier_env *env,
4603 			     struct bpf_reg_state *reg, int off, int size,
4604 			     int value_regno, int insn_idx)
4605 {
4606 	struct bpf_func_state *state = bpf_func(env, reg);
4607 	int min_off = reg_smin(reg) + off;
4608 	int max_off = reg_smax(reg) + off + size;
4609 	int err;
4610 
4611 	if (tnum_is_const(reg->var_off)) {
4612 		off += reg->var_off.value;
4613 		err = check_stack_write_fixed_off(env, state, off, size,
4614 						  value_regno, insn_idx);
4615 	} else {
4616 		/* Variable offset stack reads need more conservative handling
4617 		 * than fixed offset ones.
4618 		 */
4619 		err = check_stack_write_var_off(env, state,
4620 						reg, off, size,
4621 						value_regno, insn_idx);
4622 	}
4623 	/* After the write is modelled: the insn may store a slice to its own buffer. */
4624 	invalidate_stack_slices(env, state->frameno, min_off, max_off);
4625 	return err;
4626 }
4627 
4628 /*
4629  * Write a value to the outgoing stack arg area.
4630  * off is a negative offset from r11 (e.g. -8 for arg6, -16 for arg7).
4631  */
4632 static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_state *state,
4633 				 int off, struct bpf_reg_state *value_reg)
4634 {
4635 	int max_stack_arg_regs = MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS;
4636 	struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];
4637 	int spi = -off / BPF_REG_SIZE - 1;
4638 	struct bpf_reg_state *arg;
4639 	int err;
4640 
4641 	if (spi >= max_stack_arg_regs) {
4642 		verbose(env, "stack arg write offset %d exceeds max %d stack args\n",
4643 			off, max_stack_arg_regs);
4644 		return -EINVAL;
4645 	}
4646 
4647 	err = grow_stack_arg_slots(env, state, spi + 1);
4648 	if (err)
4649 		return err;
4650 
4651 	/* Track the max outgoing stack arg slot count. */
4652 	if (spi + 1 > subprog->max_out_stack_arg_cnt)
4653 		subprog->max_out_stack_arg_cnt = spi + 1;
4654 
4655 	arg = &state->stack_arg_regs[spi];
4656 	bpf_diag_mod_begin(env, arg, value_reg, BPF_DIAG_MOD_WRITE);
4657 
4658 	if (value_reg) {
4659 		state->stack_arg_regs[spi] = *value_reg;
4660 	} else {
4661 		/* BPF_ST: store immediate, treat as scalar */
4662 		arg->type = SCALAR_VALUE;
4663 		__mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm);
4664 	}
4665 	bpf_diag_mod_end(env);
4666 	state->no_stack_arg_load = true;
4667 	return bpf_push_jmp_history(env, env->cur_state,
4668 				    INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0);
4669 }
4670 
4671 /*
4672  * Read a value from the incoming stack arg area.
4673  * off is a positive offset from r11 (e.g. +8 for arg6, +16 for arg7).
4674  */
4675 static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_state *state,
4676 				int off, int dst_regno)
4677 {
4678 	struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];
4679 	struct bpf_verifier_state *vstate = env->cur_state;
4680 	int spi = off / BPF_REG_SIZE - 1;
4681 	struct bpf_func_state *caller, *cur;
4682 	struct bpf_reg_state *arg;
4683 
4684 	if (state->no_stack_arg_load) {
4685 		verbose(env, "r11 load must be before any r11 store or call insn\n");
4686 		return -EINVAL;
4687 	}
4688 
4689 	if (spi + 1 > bpf_in_stack_arg_cnt(subprog)) {
4690 		verbose(env, "invalid read from stack arg off %d depth %d\n",
4691 			off, bpf_in_stack_arg_cnt(subprog) * BPF_REG_SIZE);
4692 		return -EACCES;
4693 	}
4694 
4695 	caller = vstate->frame[vstate->curframe - 1];
4696 	arg = bpf_get_spilled_stack_arg(spi, caller);
4697 	if (!arg) {
4698 		verbose(env, "invalid read from uninitialized stack arg off %d\n", off);
4699 		return -EACCES;
4700 	}
4701 
4702 	cur = vstate->frame[vstate->curframe];
4703 	bpf_diag_mod_begin(env, &cur->regs[dst_regno], arg, BPF_DIAG_MOD_WRITE);
4704 	cur->regs[dst_regno] = *arg;
4705 	bpf_diag_mod_end(env);
4706 	return bpf_push_jmp_history(env, env->cur_state,
4707 				    INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0);
4708 }
4709 
4710 static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx)
4711 {
4712 	struct bpf_func_state *caller = cur_func(env);
4713 	int spi = arg_idx - MAX_BPF_FUNC_REG_ARGS;
4714 
4715 	bt_set_frame_stack_arg_slot(&env->bt, caller->frameno, spi);
4716 	return mark_chain_precision_batch(env, env->cur_state);
4717 }
4718 
4719 static int mark_arg_precision(struct bpf_verifier_env *env, argno_t argno)
4720 {
4721 	int regno = reg_from_argno(argno);
4722 
4723 	if (regno >= 0)
4724 		return mark_chain_precision(env, regno);
4725 	return mark_stack_arg_precision(env, arg_idx_from_argno(argno));
4726 }
4727 
4728 static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller,
4729 				     int nargs, const char *callee_name, const struct btf *btf,
4730 				     const struct btf_param *args)
4731 {
4732 	int i, spi;
4733 
4734 	for (i = MAX_BPF_FUNC_REG_ARGS; i < nargs; i++) {
4735 		spi = i - MAX_BPF_FUNC_REG_ARGS;
4736 		if (spi >= caller->out_stack_arg_cnt ||
4737 		    caller->stack_arg_regs[spi].type == NOT_INIT) {
4738 			const char *arg_name = NULL;
4739 
4740 			if (args && args[i].name_off)
4741 				arg_name = btf_name_by_offset(btf, args[i].name_off);
4742 			verbose(env, "callee expects %d args, stack arg%d is not initialized\n",
4743 				nargs, spi + 1);
4744 			bpf_diag_stack_arg_uninit(env, env->insn_idx, nargs, spi,
4745 						  callee_name, arg_name);
4746 			return -EFAULT;
4747 		}
4748 	}
4749 
4750 	return 0;
4751 }
4752 
4753 static struct bpf_reg_state *get_func_arg_reg(struct bpf_func_state *caller,
4754 					      struct bpf_reg_state *regs, int arg)
4755 {
4756 	if (arg < MAX_BPF_FUNC_REG_ARGS)
4757 		return &regs[arg + 1];
4758 
4759 	return &caller->stack_arg_regs[arg - MAX_BPF_FUNC_REG_ARGS];
4760 }
4761 
4762 static int check_map_access_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
4763 				 int off, int size, enum bpf_access_type type)
4764 {
4765 	struct bpf_map *map = reg->map_ptr;
4766 	u32 cap = bpf_map_flags_to_cap(map);
4767 
4768 	if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) {
4769 		verbose(env, "write into map forbidden, value_size=%d off=%lld size=%d\n",
4770 			map->value_size, reg_smin(reg) + off, size);
4771 		return -EACCES;
4772 	}
4773 
4774 	if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) {
4775 		verbose(env, "read from map forbidden, value_size=%d off=%lld size=%d\n",
4776 			map->value_size, reg_smin(reg) + off, size);
4777 		return -EACCES;
4778 	}
4779 
4780 	return 0;
4781 }
4782 
4783 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */
4784 static int __check_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
4785 			      int off, int size, u32 mem_size,
4786 			      bool zero_size_allowed)
4787 {
4788 	bool size_ok = size > 0 || (size == 0 && zero_size_allowed);
4789 
4790 	if (off >= 0 && size_ok && (u64)off + size <= mem_size)
4791 		return 0;
4792 
4793 	switch (reg->type) {
4794 	case PTR_TO_MAP_KEY:
4795 		verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",
4796 			mem_size, off, size);
4797 		break;
4798 	case PTR_TO_MAP_VALUE:
4799 		verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
4800 			mem_size, off, size);
4801 		break;
4802 	case PTR_TO_PACKET:
4803 	case PTR_TO_PACKET_META:
4804 	case PTR_TO_PACKET_END:
4805 		verbose(env, "invalid access to packet, off=%d size=%d, %s(id=%d,off=%d,r=%d)\n",
4806 			off, size, reg_arg_name(env, argno), reg->id, off, mem_size);
4807 		break;
4808 	case PTR_TO_CTX:
4809 		verbose(env, "invalid access to context, ctx_size=%d off=%d size=%d\n",
4810 			mem_size, off, size);
4811 		break;
4812 	case PTR_TO_MEM:
4813 	default:
4814 		verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",
4815 			mem_size, off, size);
4816 	}
4817 
4818 	return -EACCES;
4819 }
4820 
4821 /* check read/write into a memory region with possible variable offset */
4822 static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
4823 				   int off, int size, u32 mem_size,
4824 				   bool zero_size_allowed)
4825 {
4826 	const char *proof = "";
4827 	const char *start;
4828 	s64 max_start, max_end;
4829 	int err;
4830 
4831 	/* We may have adjusted the register pointing to memory region, so we
4832 	 * need to try adding each of min_value and max_value to off
4833 	 * to make sure our theoretical access will be safe.
4834 	 *
4835 	 * The minimum value is only important with signed
4836 	 * comparisons where we can't assume the floor of a
4837 	 * value is 0.  If we are using signed variables for our
4838 	 * index'es we need to make sure that whatever we use
4839 	 * will have a set floor within our range.
4840 	 */
4841 	if (reg_smin(reg) < 0 &&
4842 	    (reg_smin(reg) == S64_MIN ||
4843 	     (off + reg_smin(reg) != (s64)(s32)(off + reg_smin(reg))) ||
4844 	      reg_smin(reg) + off < 0)) {
4845 		verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n",
4846 			reg_arg_name(env, argno));
4847 		err = -EACCES;
4848 		if (bpf_diag_enabled(env)) {
4849 			start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off);
4850 			proof = bpf_diag_fmt(
4851 				env, "the minimal bound for a memory access is a negative value: %s",
4852 				start);
4853 		}
4854 		goto report_error;
4855 	}
4856 
4857 	err = __check_mem_access(env, reg, argno, reg_smin(reg) + off, size,
4858 				 mem_size, zero_size_allowed);
4859 	if (err) {
4860 		verbose(env, "%s min value is outside of the allowed memory range\n",
4861 			reg_arg_name(env, argno));
4862 		if (bpf_diag_enabled(env)) {
4863 			start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off);
4864 			proof = bpf_diag_fmt(
4865 				env, "the minimal bound for a memory access is %s and is outside of the object of size %u",
4866 				start, mem_size);
4867 		}
4868 		goto report_error;
4869 	}
4870 
4871 	/* If we haven't set a max value then we need to bail since we can't be
4872 	 * sure we won't do bad things.
4873 	 * If reg_umax(reg) + off could overflow, treat that as unbounded too.
4874 	 */
4875 	if (reg_umax(reg) >= BPF_MAX_VAR_OFF) {
4876 		verbose(env, "%s unbounded memory access, make sure to bounds check any such access\n",
4877 			reg_arg_name(env, argno));
4878 		err = -EACCES;
4879 		if (bpf_diag_enabled(env))
4880 			proof = bpf_diag_fmt(
4881 				env, "the maximal bound for a memory access is %llu and exceeds maximum allowed offset of %u",
4882 				reg_umax(reg), BPF_MAX_VAR_OFF);
4883 		goto report_error;
4884 	}
4885 
4886 	err = __check_mem_access(env, reg, argno, reg_umax(reg) + off, size,
4887 				 mem_size, zero_size_allowed);
4888 	if (err) {
4889 		verbose(env, "%s max value is outside of the allowed memory range\n",
4890 			reg_arg_name(env, argno));
4891 		if (bpf_diag_enabled(env)) {
4892 			max_start = (s64)reg_umax(reg) + off;
4893 			max_end = max_start + size;
4894 			proof = bpf_diag_fmt(
4895 				env, "the maximal bound for a memory access is %lld: start %lld + access_size %d, beyond object_size %u",
4896 				max_end, max_start, size, mem_size);
4897 		}
4898 		goto report_error;
4899 	}
4900 
4901 	return 0;
4902 
4903 report_error:
4904 	bpf_diag_mem_bounds(env, env->insn_idx, reg_from_argno(argno),
4905 			    reg_arg_name(env, argno), reg_type_str(env, reg->type), proof,
4906 			    off, size, mem_size, reg);
4907 	return err;
4908 }
4909 
4910 static int __check_ptr_off_reg(struct bpf_verifier_env *env,
4911 			       const struct bpf_reg_state *reg, argno_t argno,
4912 			       bool fixed_off_ok)
4913 {
4914 	/* Access to this pointer-typed register or passing it to a helper
4915 	 * is only allowed in its original, unmodified form.
4916 	 */
4917 
4918 	if (!tnum_is_const(reg->var_off)) {
4919 		char tn_buf[48];
4920 
4921 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
4922 		verbose(env, "variable %s access var_off=%s disallowed\n",
4923 			reg_type_str(env, reg->type), tn_buf);
4924 		return -EACCES;
4925 	}
4926 
4927 	if (reg_smin(reg) < 0) {
4928 		verbose(env, "negative offset %s ptr %s off=%lld disallowed\n",
4929 			reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value);
4930 		return -EACCES;
4931 	}
4932 
4933 	if (!fixed_off_ok && reg->var_off.value != 0) {
4934 		verbose(env, "dereference of modified %s ptr %s off=%lld disallowed\n",
4935 			reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value);
4936 		bpf_diag_invalid_deref(env, env->insn_idx, reg_from_argno(argno),
4937 				       reg_arg_name(env, argno), reg,
4938 					      BPF_DIAG_DEREF_MODIFIED_PTR, reg->var_off.value);
4939 		return -EACCES;
4940 	}
4941 
4942 	return 0;
4943 }
4944 
4945 static int check_ptr_off_reg(struct bpf_verifier_env *env,
4946 		             const struct bpf_reg_state *reg, int regno)
4947 {
4948 	return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false);
4949 }
4950 
4951 static int map_kptr_match_type(struct bpf_verifier_env *env,
4952 			       struct btf_field *kptr_field,
4953 			       struct bpf_reg_state *reg, u32 regno)
4954 {
4955 	const char *targ_name = btf_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id);
4956 	int perm_flags;
4957 	const char *reg_name = "";
4958 
4959 	if (base_type(reg->type) != PTR_TO_BTF_ID)
4960 		goto bad_type;
4961 
4962 	if (btf_is_kernel(reg->btf)) {
4963 		perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU;
4964 
4965 		/* Only unreferenced case accepts untrusted pointers */
4966 		if (kptr_field->type == BPF_KPTR_UNREF)
4967 			perm_flags |= PTR_UNTRUSTED;
4968 	} else {
4969 		perm_flags = PTR_MAYBE_NULL | MEM_ALLOC;
4970 		if (kptr_field->type == BPF_KPTR_PERCPU)
4971 			perm_flags |= MEM_PERCPU;
4972 	}
4973 
4974 	if (type_flag(reg->type) & ~perm_flags)
4975 		goto bad_type;
4976 
4977 	/*
4978 	 * A BPF_KPTR_PERCPU field is read back as MEM_PERCPU, so the value
4979 	 * stored in it must carry the same flag.
4980 	 */
4981 	if ((kptr_field->type == BPF_KPTR_PERCPU) != !!(reg->type & MEM_PERCPU))
4982 		goto bad_type;
4983 
4984 	/* We need to verify reg->type and reg->btf, before accessing reg->btf */
4985 	reg_name = btf_type_name(reg->btf, reg->btf_id);
4986 
4987 	/* For ref_ptr case, release function check should ensure we get one
4988 	 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the
4989 	 * normal store of unreferenced kptr, we must ensure var_off is zero.
4990 	 * Since ref_ptr cannot be accessed directly by BPF insns, check for
4991 	 * reg->id is not needed here.
4992 	 */
4993 	if (__check_ptr_off_reg(env, reg, argno_from_reg(regno), true))
4994 		return -EACCES;
4995 
4996 	/* A full type match is needed, as BTF can be vmlinux, module or prog BTF, and
4997 	 * we also need to take into account the reg->var_off.
4998 	 *
4999 	 * We want to support cases like:
5000 	 *
5001 	 * struct foo {
5002 	 *         struct bar br;
5003 	 *         struct baz bz;
5004 	 * };
5005 	 *
5006 	 * struct foo *v;
5007 	 * v = func();	      // PTR_TO_BTF_ID
5008 	 * val->foo = v;      // reg->var_off is zero, btf and btf_id match type
5009 	 * val->bar = &v->br; // reg->var_off is still zero, but we need to retry with
5010 	 *                    // first member type of struct after comparison fails
5011 	 * val->baz = &v->bz; // reg->var_off is non-zero, so struct needs to be walked
5012 	 *                    // to match type
5013 	 *
5014 	 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->var_off
5015 	 * is zero. We must also ensure that btf_struct_ids_match does not walk
5016 	 * the struct to match type against first member of struct, i.e. reject
5017 	 * second case from above. Hence, when type is BPF_KPTR_REF, we set
5018 	 * strict mode to true for type match.
5019 	 */
5020 	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value,
5021 				  kptr_field->kptr.btf, kptr_field->kptr.btf_id,
5022 				  kptr_field->type != BPF_KPTR_UNREF,
5023 				  !type_is_alloc(reg->type)))
5024 		goto bad_type;
5025 	return 0;
5026 bad_type:
5027 	verbose(env, "invalid kptr access, R%d type=%s%s ", regno,
5028 		reg_type_str(env, reg->type), reg_name);
5029 	verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name);
5030 	if (kptr_field->type == BPF_KPTR_UNREF)
5031 		verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED),
5032 			targ_name);
5033 	else
5034 		verbose(env, "\n");
5035 	return -EINVAL;
5036 }
5037 
5038 static bool in_sleepable(struct bpf_verifier_env *env)
5039 {
5040 	return env->cur_state->in_sleepable;
5041 }
5042 
5043 /* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock()
5044  * can dereference RCU protected pointers and result is PTR_TRUSTED.
5045  */
5046 static bool in_rcu_cs(struct bpf_verifier_env *env)
5047 {
5048 	return env->cur_state->active_rcu_locks ||
5049 	       env->cur_state->active_preempt_locks ||
5050 	       env->cur_state->active_locks ||
5051 	       env->cur_state->active_irq_id ||
5052 	       !in_sleepable(env);
5053 }
5054 
5055 /* Once GCC supports btf_type_tag the following mechanism will be replaced with tag check */
5056 BTF_SET_START(rcu_protected_types)
5057 #ifdef CONFIG_NET
5058 BTF_ID(struct, prog_test_ref_kfunc)
5059 #endif
5060 #ifdef CONFIG_CGROUPS
5061 BTF_ID(struct, cgroup)
5062 #endif
5063 #ifdef CONFIG_BPF_JIT
5064 BTF_ID(struct, bpf_cpumask)
5065 #endif
5066 BTF_ID(struct, task_struct)
5067 #ifdef CONFIG_BPF_CRYPTO
5068 BTF_ID(struct, bpf_crypto_ctx)
5069 #endif
5070 #ifdef CONFIG_INET
5071 BTF_ID(struct, bpf_ksock)
5072 #endif
5073 BTF_SET_END(rcu_protected_types)
5074 
5075 static bool rcu_protected_object(const struct btf *btf, u32 btf_id)
5076 {
5077 	if (!btf_is_kernel(btf))
5078 		return true;
5079 	return btf_id_set_contains(&rcu_protected_types, btf_id);
5080 }
5081 
5082 static struct btf_record *kptr_pointee_btf_record(struct btf_field *kptr_field)
5083 {
5084 	struct btf_struct_meta *meta;
5085 
5086 	if (btf_is_kernel(kptr_field->kptr.btf))
5087 		return NULL;
5088 
5089 	meta = btf_find_struct_meta(kptr_field->kptr.btf,
5090 				    kptr_field->kptr.btf_id);
5091 
5092 	return meta ? meta->record : NULL;
5093 }
5094 
5095 static bool rcu_safe_kptr(const struct btf_field *field)
5096 {
5097 	const struct btf_field_kptr *kptr = &field->kptr;
5098 
5099 	return field->type == BPF_KPTR_PERCPU ||
5100 	       (field->type == BPF_KPTR_REF && rcu_protected_object(kptr->btf, kptr->btf_id));
5101 }
5102 
5103 static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field)
5104 {
5105 	struct btf_record *rec;
5106 	u32 ret;
5107 
5108 	ret = PTR_MAYBE_NULL;
5109 	if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) {
5110 		ret |= MEM_RCU;
5111 		if (kptr_field->type == BPF_KPTR_PERCPU)
5112 			ret |= MEM_PERCPU;
5113 		else if (!btf_is_kernel(kptr_field->kptr.btf))
5114 			ret |= MEM_ALLOC;
5115 
5116 		rec = kptr_pointee_btf_record(kptr_field);
5117 		if (rec && btf_record_has_field(rec, BPF_GRAPH_NODE))
5118 			ret |= NON_OWN_REF;
5119 	} else {
5120 		ret |= PTR_UNTRUSTED;
5121 	}
5122 
5123 	return ret;
5124 }
5125 
5126 static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno,
5127 			    struct btf_field *field)
5128 {
5129 	struct bpf_reg_state *reg;
5130 	const struct btf_type *t;
5131 
5132 	t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);
5133 	mark_reg_known_zero(env, cur_regs(env), regno);
5134 	reg = reg_state(env, regno);
5135 	reg->type = PTR_TO_MEM | PTR_MAYBE_NULL;
5136 	reg->mem_size = t->size;
5137 	reg->id = ++env->id_gen;
5138 
5139 	return 0;
5140 }
5141 
5142 static int check_map_kptr_access(struct bpf_verifier_env *env,
5143 				 int value_regno, int insn_idx,
5144 				 struct btf_field *kptr_field)
5145 {
5146 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
5147 	int class = BPF_CLASS(insn->code);
5148 	struct bpf_reg_state *val_reg;
5149 	int ret;
5150 
5151 	/* Things we already checked for in check_map_access and caller:
5152 	 *  - Reject cases where variable offset may touch kptr
5153 	 *  - size of access (must be BPF_DW)
5154 	 *  - tnum_is_const(reg->var_off)
5155 	 *  - kptr_field->offset == off + reg->var_off.value
5156 	 */
5157 	/* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */
5158 	if (BPF_MODE(insn->code) != BPF_MEM) {
5159 		verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n");
5160 		return -EACCES;
5161 	}
5162 
5163 	/* We only allow loading referenced kptr, since it will be marked as
5164 	 * untrusted, similar to unreferenced kptr.
5165 	 */
5166 	if (class != BPF_LDX &&
5167 	    (kptr_field->type == BPF_KPTR_REF || kptr_field->type == BPF_KPTR_PERCPU)) {
5168 		verbose(env, "store to referenced kptr disallowed\n");
5169 		return -EACCES;
5170 	}
5171 	if (class != BPF_LDX && kptr_field->type == BPF_UPTR) {
5172 		verbose(env, "store to uptr disallowed\n");
5173 		return -EACCES;
5174 	}
5175 
5176 	if (class == BPF_LDX) {
5177 		if (kptr_field->type == BPF_UPTR)
5178 			return mark_uptr_ld_reg(env, value_regno, kptr_field);
5179 
5180 		/* We can simply mark the value_regno receiving the pointer
5181 		 * value from map as PTR_TO_BTF_ID, with the correct type.
5182 		 */
5183 		ret = mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID,
5184 				      kptr_field->kptr.btf, kptr_field->kptr.btf_id,
5185 				      btf_ld_kptr_type(env, kptr_field));
5186 		if (ret < 0)
5187 			return ret;
5188 	} else if (class == BPF_STX) {
5189 		val_reg = reg_state(env, value_regno);
5190 		if (bpf_register_is_null(val_reg)) {
5191 			/*
5192 			 * This store is valid only because the scalar is known to be
5193 			 * zero. Mark it precise so another scalar cannot be pruned
5194 			 * against this state.
5195 			 */
5196 			return mark_chain_precision(env, value_regno);
5197 		}
5198 		if (map_kptr_match_type(env, kptr_field, val_reg, value_regno))
5199 			return -EACCES;
5200 	} else if (class == BPF_ST) {
5201 		if (insn->imm) {
5202 			verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n",
5203 				kptr_field->offset);
5204 			return -EACCES;
5205 		}
5206 	} else {
5207 		verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n");
5208 		return -EACCES;
5209 	}
5210 	return 0;
5211 }
5212 
5213 /*
5214  * Return the size of the memory region accessible from a pointer to map value.
5215  * For INSN_ARRAY maps whole bpf_insn_array->ips array is accessible.
5216  */
5217 static u32 map_mem_size(const struct bpf_map *map)
5218 {
5219 	if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY)
5220 		return map->max_entries * sizeof(long);
5221 
5222 	return map->value_size;
5223 }
5224 
5225 /* check read/write into a map element with possible variable offset */
5226 static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
5227 			    int off, int size, bool zero_size_allowed,
5228 			    enum bpf_access_src src)
5229 {
5230 	struct bpf_map *map = reg->map_ptr;
5231 	u32 mem_size = map_mem_size(map);
5232 	struct btf_record *rec;
5233 	int err, i;
5234 
5235 	err = check_mem_region_access(env, reg, argno, off, size, mem_size, zero_size_allowed);
5236 	if (err)
5237 		return err;
5238 
5239 	if (IS_ERR_OR_NULL(map->record))
5240 		return 0;
5241 	rec = map->record;
5242 	for (i = 0; i < rec->cnt; i++) {
5243 		struct btf_field *field = &rec->fields[i];
5244 		u32 p = field->offset;
5245 
5246 		/* If any part of a field  can be touched by load/store, reject
5247 		 * this program. To check that [x1, x2) overlaps with [y1, y2),
5248 		 * it is sufficient to check x1 < y2 && y1 < x2.
5249 		 */
5250 		if (reg_smin(reg) + off < p + field->size &&
5251 		    p < reg_umax(reg) + off + size) {
5252 			switch (field->type) {
5253 			case BPF_KPTR_UNREF:
5254 			case BPF_KPTR_REF:
5255 			case BPF_KPTR_PERCPU:
5256 			case BPF_UPTR:
5257 				if (src != ACCESS_DIRECT) {
5258 					verbose(env, "%s cannot be accessed indirectly by helper\n",
5259 						btf_field_type_name(field->type));
5260 					return -EACCES;
5261 				}
5262 				if (!tnum_is_const(reg->var_off)) {
5263 					verbose(env, "%s access cannot have variable offset\n",
5264 						btf_field_type_name(field->type));
5265 					return -EACCES;
5266 				}
5267 				if (p != off + reg->var_off.value) {
5268 					verbose(env, "%s access misaligned expected=%u off=%llu\n",
5269 						btf_field_type_name(field->type),
5270 						p, off + reg->var_off.value);
5271 					return -EACCES;
5272 				}
5273 				if (size != bpf_size_to_bytes(BPF_DW)) {
5274 					verbose(env, "%s access size must be BPF_DW\n",
5275 						btf_field_type_name(field->type));
5276 					return -EACCES;
5277 				}
5278 				break;
5279 			default:
5280 				verbose(env, "%s cannot be accessed directly by load/store\n",
5281 					btf_field_type_name(field->type));
5282 				return -EACCES;
5283 			}
5284 		}
5285 	}
5286 	return 0;
5287 }
5288 
5289 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
5290 			       const struct bpf_call_arg_meta *meta,
5291 			       enum bpf_access_type t)
5292 {
5293 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
5294 
5295 	switch (prog_type) {
5296 	/* Program types only with direct read access go here! */
5297 	case BPF_PROG_TYPE_LWT_IN:
5298 	case BPF_PROG_TYPE_LWT_OUT:
5299 	case BPF_PROG_TYPE_LWT_SEG6LOCAL:
5300 	case BPF_PROG_TYPE_SK_REUSEPORT:
5301 	case BPF_PROG_TYPE_FLOW_DISSECTOR:
5302 	case BPF_PROG_TYPE_CGROUP_SKB:
5303 		if (t == BPF_WRITE)
5304 			return false;
5305 		fallthrough;
5306 
5307 	/* Program types with direct read + write access go here! */
5308 	case BPF_PROG_TYPE_SCHED_CLS:
5309 	case BPF_PROG_TYPE_SCHED_ACT:
5310 	case BPF_PROG_TYPE_XDP:
5311 	case BPF_PROG_TYPE_LWT_XMIT:
5312 	case BPF_PROG_TYPE_SK_SKB:
5313 	case BPF_PROG_TYPE_SK_MSG:
5314 		if (meta && !meta->btf && meta->func_id)
5315 			return meta->fn->pkt_access;
5316 
5317 		if (t == BPF_WRITE)
5318 			env->seen_direct_write = true;
5319 		return true;
5320 
5321 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
5322 		if (t == BPF_WRITE)
5323 			env->seen_direct_write = true;
5324 
5325 		return true;
5326 
5327 	default:
5328 		return false;
5329 	}
5330 }
5331 
5332 static int check_packet_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off,
5333 			       int size, bool zero_size_allowed)
5334 {
5335 	int err;
5336 
5337 	if (reg->range < 0) {
5338 		verbose(env, "%s offset is outside of the packet\n", reg_arg_name(env, argno));
5339 		return -EINVAL;
5340 	}
5341 
5342 	err = check_mem_region_access(env, reg, argno, off, size, reg->range, zero_size_allowed);
5343 	if (err)
5344 		return err;
5345 
5346 	/* __check_mem_access has made sure "off + size - 1" is within u16.
5347 	 * reg_umax(reg) can't be bigger than MAX_PACKET_OFF which is 0xffff,
5348 	 * otherwise find_good_pkt_pointers would have refused to set range info
5349 	 * that __check_mem_access would have rejected this pkt access.
5350 	 * Therefore, "off + reg_umax(reg) + size - 1" won't overflow u32.
5351 	 */
5352 	env->prog->aux->max_pkt_offset =
5353 		max_t(u32, env->prog->aux->max_pkt_offset,
5354 		      off + reg_umax(reg) + size - 1);
5355 
5356 	return 0;
5357 }
5358 
5359 static bool is_var_ctx_off_allowed(struct bpf_prog *prog)
5360 {
5361 	return resolve_prog_type(prog) == BPF_PROG_TYPE_SYSCALL;
5362 }
5363 
5364 /* check access to 'struct bpf_context' fields.  Supports fixed offsets only */
5365 static int __check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
5366 			      enum bpf_access_type t, struct bpf_insn_access_aux *info)
5367 {
5368 	if (env->ops->is_valid_access &&
5369 	    env->ops->is_valid_access(off, size, t, env->prog, info)) {
5370 		/* A non zero info.ctx_field_size indicates that this field is a
5371 		 * candidate for later verifier transformation to load the whole
5372 		 * field and then apply a mask when accessed with a narrower
5373 		 * access than actual ctx access size. A zero info.ctx_field_size
5374 		 * will only allow for whole field access and rejects any other
5375 		 * type of narrower access.
5376 		 */
5377 		if (base_type(info->reg_type) == PTR_TO_BTF_ID) {
5378 			if (info->ref_id &&
5379 			    !find_reference_state(env->cur_state, info->ref_id)) {
5380 				verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n",
5381 					off);
5382 				return -EACCES;
5383 			}
5384 		} else {
5385 			env->insn_aux_data[insn_idx].ctx_field_size = info->ctx_field_size;
5386 		}
5387 		/* remember the offset of last byte accessed in ctx */
5388 		if (env->prog->aux->max_ctx_offset < off + size)
5389 			env->prog->aux->max_ctx_offset = off + size;
5390 		return 0;
5391 	}
5392 
5393 	verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
5394 	return -EACCES;
5395 }
5396 
5397 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno,
5398 			    int off, int access_size, enum bpf_access_type t,
5399 			    struct bpf_insn_access_aux *info)
5400 {
5401 	/*
5402 	 * Program types that don't rewrite ctx accesses can safely
5403 	 * dereference ctx pointers with fixed offsets.
5404 	 */
5405 	bool var_off_ok = is_var_ctx_off_allowed(env->prog);
5406 	bool fixed_off_ok = !env->ops->convert_ctx_access;
5407 	int err;
5408 
5409 	if (var_off_ok)
5410 		err = check_mem_region_access(env, reg, argno, off, access_size, U16_MAX, false);
5411 	else
5412 		err = __check_ptr_off_reg(env, reg, argno, fixed_off_ok);
5413 	if (err)
5414 		return err;
5415 	off += reg_umax(reg);
5416 
5417 	err = __check_ctx_access(env, insn_idx, off, access_size, t, info);
5418 	if (err)
5419 		verbose_linfo(env, insn_idx, "; ");
5420 	return err;
5421 }
5422 
5423 static int check_flow_keys_access(struct bpf_verifier_env *env,
5424 				  struct bpf_reg_state *reg, argno_t argno,
5425 				  int off, int size)
5426 {
5427 	/* Only a constant offset is allowed here; fold it into off. */
5428 	if (!tnum_is_const(reg->var_off)) {
5429 		char tn_buf[48];
5430 
5431 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5432 		verbose(env, "%s invalid variable offset to flow keys: off=%d, var_off=%s\n",
5433 			reg_arg_name(env, argno), off, tn_buf);
5434 		return -EACCES;
5435 	}
5436 	off += reg->var_off.value;
5437 
5438 	if (size < 0 || off < 0 ||
5439 	    (u64)off + size > sizeof(struct bpf_flow_keys)) {
5440 		verbose(env, "invalid access to flow keys off=%d size=%d\n",
5441 			off, size);
5442 		return -EACCES;
5443 	}
5444 	return 0;
5445 }
5446 
5447 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
5448 			     struct bpf_reg_state *reg, argno_t argno, int off, int size,
5449 			     enum bpf_access_type t)
5450 {
5451 	struct bpf_insn_access_aux info = {};
5452 	bool valid;
5453 
5454 	if (reg_smin(reg) < 0) {
5455 		verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n",
5456 			reg_arg_name(env, argno));
5457 		return -EACCES;
5458 	}
5459 
5460 	switch (reg->type) {
5461 	case PTR_TO_SOCK_COMMON:
5462 		valid = bpf_sock_common_is_valid_access(off, size, t, &info);
5463 		break;
5464 	case PTR_TO_SOCKET:
5465 		valid = bpf_sock_is_valid_access(off, size, t, &info);
5466 		break;
5467 	case PTR_TO_TCP_SOCK:
5468 		valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
5469 		break;
5470 	case PTR_TO_XDP_SOCK:
5471 		valid = bpf_xdp_sock_is_valid_access(off, size, t, &info);
5472 		break;
5473 	default:
5474 		valid = false;
5475 	}
5476 
5477 	if (valid) {
5478 		env->insn_aux_data[insn_idx].ctx_field_size =
5479 			info.ctx_field_size;
5480 		return 0;
5481 	}
5482 
5483 	verbose(env, "%s invalid %s access off=%d size=%d\n",
5484 		reg_arg_name(env, argno), reg_type_str(env, reg->type), off, size);
5485 
5486 	return -EACCES;
5487 }
5488 
5489 static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
5490 {
5491 	return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
5492 }
5493 
5494 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
5495 {
5496 	const struct bpf_reg_state *reg = reg_state(env, regno);
5497 
5498 	return reg->type == PTR_TO_CTX;
5499 }
5500 
5501 static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
5502 {
5503 	const struct bpf_reg_state *reg = reg_state(env, regno);
5504 
5505 	return type_is_sk_pointer(reg->type);
5506 }
5507 
5508 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
5509 {
5510 	const struct bpf_reg_state *reg = reg_state(env, regno);
5511 
5512 	return type_is_pkt_pointer(reg->type);
5513 }
5514 
5515 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
5516 {
5517 	const struct bpf_reg_state *reg = reg_state(env, regno);
5518 
5519 	/* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */
5520 	return reg->type == PTR_TO_FLOW_KEYS;
5521 }
5522 
5523 static bool is_arena_reg(struct bpf_verifier_env *env, int regno)
5524 {
5525 	const struct bpf_reg_state *reg = reg_state(env, regno);
5526 
5527 	return reg->type == PTR_TO_ARENA;
5528 }
5529 
5530 static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno,
5531 			       struct bpf_insn *insn)
5532 {
5533 	const struct bpf_reg_state *reg = reg_state(env, regno);
5534 
5535 	/*
5536 	 * A BPF_LOAD_ACQ is not rewritten to a BPF_PROBE_MEM load by the
5537 	 * verifier, unlike a regular BPF_LDX. The JIT would emit a plain load
5538 	 * with no exception table entry, so a fault (e.g. NULL deref) crashes
5539 	 * the kernel instead of being handled. Reject the source pointer types
5540 	 * that would have needed that protection, the remaining ones stay
5541 	 * allowed.
5542 	 */
5543 	return insn->imm == BPF_LOAD_ACQ && bpf_may_fault_on_deref(reg->type);
5544 }
5545 
5546 /* Return false if @regno contains a pointer whose type isn't supported for
5547  * atomic instruction @insn.
5548  */
5549 static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno,
5550 			       struct bpf_insn *insn)
5551 {
5552 	if (is_ctx_reg(env, regno))
5553 		return false;
5554 	if (is_pkt_reg(env, regno))
5555 		return false;
5556 	if (is_flow_key_reg(env, regno))
5557 		return false;
5558 	if (is_sk_reg(env, regno))
5559 		return false;
5560 	if (is_arena_reg(env, regno))
5561 		return bpf_jit_supports_insn(insn, true);
5562 	if (is_load_acq_unsafe(env, regno, insn))
5563 		return false;
5564 	return true;
5565 }
5566 
5567 static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
5568 #ifdef CONFIG_NET
5569 	[PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
5570 	[PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
5571 	[PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
5572 #endif
5573 	[CONST_PTR_TO_MAP] = btf_bpf_map_id,
5574 };
5575 
5576 static bool is_trusted_reg_type(struct bpf_verifier_env *env,
5577 				const struct bpf_reg_state *reg,
5578 				enum bpf_reg_type type)
5579 {
5580 	/* A referenced register is always trusted. */
5581 	if (reg_is_referenced(env, reg))
5582 		return true;
5583 
5584 	/* Types listed in the reg2btf_ids are always trusted */
5585 	if (reg2btf_ids[base_type(type)] &&
5586 	    !bpf_type_has_unsafe_modifiers(type))
5587 		return true;
5588 
5589 	/* If a register is not referenced, it is trusted if it has the
5590 	 * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the
5591 	 * other type modifiers may be safe, but we elect to take an opt-in
5592 	 * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are
5593 	 * not.
5594 	 *
5595 	 * Eventually, we should make PTR_TRUSTED the single source of truth
5596 	 * for whether a register is trusted.
5597 	 */
5598 	return type_flag(type) & BPF_REG_TRUSTED_MODIFIERS &&
5599 	       !bpf_type_has_unsafe_modifiers(type);
5600 }
5601 
5602 static bool is_trusted_reg(struct bpf_verifier_env *env,
5603 			   const struct bpf_reg_state *reg)
5604 {
5605 	return is_trusted_reg_type(env, reg, reg->type);
5606 }
5607 
5608 static bool is_rcu_reg(const struct bpf_reg_state *reg)
5609 {
5610 	return reg->type & MEM_RCU;
5611 }
5612 
5613 static void clear_trusted_flags(enum bpf_type_flag *flag)
5614 {
5615 	*flag &= ~(BPF_REG_TRUSTED_MODIFIERS | MEM_RCU);
5616 }
5617 
5618 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
5619 				   const struct bpf_reg_state *reg,
5620 				   int off, int size, bool strict)
5621 {
5622 	struct tnum reg_off;
5623 	int ip_align;
5624 
5625 	/* Byte size accesses are always allowed. */
5626 	if (!strict || size == 1)
5627 		return 0;
5628 
5629 	/* For platforms that do not have a Kconfig enabling
5630 	 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of
5631 	 * NET_IP_ALIGN is universally set to '2'.  And on platforms
5632 	 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get
5633 	 * to this code only in strict mode where we want to emulate
5634 	 * the NET_IP_ALIGN==2 checking.  Therefore use an
5635 	 * unconditional IP align value of '2'.
5636 	 */
5637 	ip_align = 2;
5638 
5639 	reg_off = tnum_add(reg->var_off, tnum_const(ip_align + off));
5640 	if (!tnum_is_aligned(reg_off, size)) {
5641 		char tn_buf[48];
5642 
5643 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5644 		verbose(env,
5645 			"misaligned packet access off %d+%s+%d size %d\n",
5646 			ip_align, tn_buf, off, size);
5647 		return -EACCES;
5648 	}
5649 
5650 	return 0;
5651 }
5652 
5653 static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
5654 				       const struct bpf_reg_state *reg,
5655 				       const char *pointer_desc,
5656 				       int off, int size, bool strict)
5657 {
5658 	struct tnum reg_off;
5659 
5660 	/* Byte size accesses are always allowed. */
5661 	if (!strict || size == 1)
5662 		return 0;
5663 
5664 	reg_off = tnum_add(reg->var_off, tnum_const(off));
5665 	if (!tnum_is_aligned(reg_off, size)) {
5666 		char tn_buf[48];
5667 
5668 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5669 		verbose(env, "misaligned %saccess off %s+%d size %d\n",
5670 			pointer_desc, tn_buf, off, size);
5671 		return -EACCES;
5672 	}
5673 
5674 	return 0;
5675 }
5676 
5677 static int check_ptr_alignment(struct bpf_verifier_env *env,
5678 			       const struct bpf_reg_state *reg, int off,
5679 			       int size, bool strict_alignment_once)
5680 {
5681 	bool strict = env->strict_alignment || strict_alignment_once;
5682 	const char *pointer_desc = "";
5683 
5684 	switch (reg->type) {
5685 	case PTR_TO_PACKET:
5686 	case PTR_TO_PACKET_META:
5687 		/* Special case, because of NET_IP_ALIGN. Given metadata sits
5688 		 * right in front, treat it the very same way.
5689 		 */
5690 		return check_pkt_ptr_alignment(env, reg, off, size, strict);
5691 	case PTR_TO_FLOW_KEYS:
5692 		pointer_desc = "flow keys ";
5693 		break;
5694 	case PTR_TO_MAP_KEY:
5695 		pointer_desc = "key ";
5696 		break;
5697 	case PTR_TO_MAP_VALUE:
5698 		pointer_desc = "value ";
5699 		if (reg->map_ptr->map_type == BPF_MAP_TYPE_INSN_ARRAY)
5700 			strict = true;
5701 		break;
5702 	case PTR_TO_CTX:
5703 		pointer_desc = "context ";
5704 		break;
5705 	case PTR_TO_STACK:
5706 		pointer_desc = "stack ";
5707 		/* The stack spill tracking logic in check_stack_write_fixed_off()
5708 		 * and check_stack_read_fixed_off() relies on stack accesses being
5709 		 * aligned.
5710 		 */
5711 		strict = true;
5712 		break;
5713 	case PTR_TO_SOCKET:
5714 		pointer_desc = "sock ";
5715 		break;
5716 	case PTR_TO_SOCK_COMMON:
5717 		pointer_desc = "sock_common ";
5718 		break;
5719 	case PTR_TO_TCP_SOCK:
5720 		pointer_desc = "tcp_sock ";
5721 		break;
5722 	case PTR_TO_XDP_SOCK:
5723 		pointer_desc = "xdp_sock ";
5724 		break;
5725 	case PTR_TO_ARENA:
5726 		return 0;
5727 	default:
5728 		break;
5729 	}
5730 	return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
5731 					   strict);
5732 }
5733 
5734 static enum priv_stack_mode bpf_enable_priv_stack(struct bpf_prog *prog)
5735 {
5736 	if (!bpf_jit_supports_private_stack())
5737 		return NO_PRIV_STACK;
5738 
5739 	/* bpf_prog_check_recur() checks all prog types that use bpf trampoline
5740 	 * while kprobe/tp/perf_event/raw_tp don't use trampoline hence checked
5741 	 * explicitly.
5742 	 */
5743 	switch (prog->type) {
5744 	case BPF_PROG_TYPE_KPROBE:
5745 	case BPF_PROG_TYPE_TRACEPOINT:
5746 	case BPF_PROG_TYPE_PERF_EVENT:
5747 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
5748 		return PRIV_STACK_ADAPTIVE;
5749 	case BPF_PROG_TYPE_TRACING:
5750 	case BPF_PROG_TYPE_LSM:
5751 	case BPF_PROG_TYPE_STRUCT_OPS:
5752 		if (prog->aux->priv_stack_requested || bpf_prog_check_recur(prog))
5753 			return PRIV_STACK_ADAPTIVE;
5754 		fallthrough;
5755 	default:
5756 		break;
5757 	}
5758 
5759 	return NO_PRIV_STACK;
5760 }
5761 
5762 static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
5763 {
5764 	if (env->prog->jit_requested)
5765 		return round_up(stack_depth, 16);
5766 
5767 	/* round up to 32-bytes, since this is granularity
5768 	 * of interpreter stack size
5769 	 */
5770 	return round_up(max_t(u32, stack_depth, 1), 32);
5771 }
5772 
5773 /* temporary state used for call frame depth calculation */
5774 struct bpf_subprog_call_depth_info {
5775 	int ret_insn; /* caller instruction where we return to. */
5776 	int caller; /* caller subprogram idx */
5777 	int frame; /* # of consecutive static call stack frames on top of stack */
5778 	int callx_insn; /* callx instruction whose callees are being walked */
5779 	int callx_next; /* next callee of callx_insn to walk */
5780 	bool via_callx; /* the subprogram is entered via callx */
5781 };
5782 
5783 /* starting from main bpf function walk all instructions of the function
5784  * and recursively walk all callees that given function can call.
5785  * Ignore jump and exit insns.
5786  */
5787 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
5788 					 struct bpf_subprog_call_depth_info *dinfo,
5789 					 bool priv_stack_supported)
5790 {
5791 	struct bpf_subprog_info *subprog = env->subprog_info;
5792 	struct bpf_insn *insn = env->prog->insnsi;
5793 	int depth = 0, frame = 0, i, subprog_end, subprog_depth;
5794 	bool tail_call_reachable = false;
5795 	int total;
5796 	int tmp;
5797 
5798 	/* no caller idx */
5799 	dinfo[idx].caller = -1;
5800 	dinfo[idx].via_callx = false;
5801 
5802 	i = subprog[idx].start;
5803 	if (!priv_stack_supported)
5804 		subprog[idx].priv_stack_mode = NO_PRIV_STACK;
5805 process_func:
5806 	dinfo[idx].callx_insn = -1;
5807 	if (subprog[idx].has_ld_abs) {
5808 		for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) {
5809 			if (subprog[tmp].is_cb) {
5810 				verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n");
5811 				return -EINVAL;
5812 			}
5813 		}
5814 	}
5815 
5816 	/* protect against potential stack overflow that might happen when
5817 	 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack
5818 	 * depth for such case down to 256 so that the worst case scenario
5819 	 * would result in 8k stack size (32 which is tailcall limit * 256 =
5820 	 * 8k).
5821 	 *
5822 	 * To get the idea what might happen, see an example:
5823 	 * func1 -> sub rsp, 128
5824 	 *  subfunc1 -> sub rsp, 256
5825 	 *  tailcall1 -> add rsp, 256
5826 	 *   func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320)
5827 	 *   subfunc2 -> sub rsp, 64
5828 	 *   subfunc22 -> sub rsp, 128
5829 	 *   tailcall2 -> add rsp, 128
5830 	 *    func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416)
5831 	 *
5832 	 * tailcall will unwind the current stack frame but it will not get rid
5833 	 * of caller's stack as shown on the example above.
5834 	 */
5835 	if (idx && subprog[idx].has_tail_call && depth >= 256) {
5836 		verbose(env,
5837 			"tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n",
5838 			depth);
5839 		return -EACCES;
5840 	}
5841 
5842 	subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth);
5843 	if (IS_ENABLED(CONFIG_X86_64) && subprog[idx].stack_arg_cnt) {
5844 		/* x86-64 uses R9 for both private stack frame pointer and arg6. */
5845 		subprog[idx].priv_stack_mode = NO_PRIV_STACK;
5846 	} else if (priv_stack_supported) {
5847 		/* Request private stack support only if the subprog stack
5848 		 * depth is no less than BPF_PRIV_STACK_MIN_SIZE. This is to
5849 		 * avoid jit penalty if the stack usage is small.
5850 		 */
5851 		if (subprog[idx].priv_stack_mode == PRIV_STACK_UNKNOWN &&
5852 		    subprog_depth >= BPF_PRIV_STACK_MIN_SIZE)
5853 			subprog[idx].priv_stack_mode = PRIV_STACK_ADAPTIVE;
5854 	}
5855 
5856 	if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
5857 		if (subprog_depth > env->max_stack_depth)
5858 			env->max_stack_depth = subprog_depth;
5859 		if (subprog_depth > env->stack_limit) {
5860 			verbose(env, "stack size of subprog %d is %d. Too large\n",
5861 				idx, subprog_depth);
5862 			return -EACCES;
5863 		}
5864 	} else {
5865 		depth += subprog_depth;
5866 		if (depth > env->max_stack_depth)
5867 			env->max_stack_depth = depth;
5868 		if (depth > env->stack_limit) {
5869 			total = 0;
5870 			for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller)
5871 				total++;
5872 
5873 			verbose(env, "combined stack size of %d calls is %d. Too large\n",
5874 				total, depth);
5875 			return -EACCES;
5876 		}
5877 	}
5878 continue_func:
5879 	subprog_end = subprog[idx + 1].start;
5880 	for (; i < subprog_end; i++) {
5881 		int next_insn, sidx;
5882 
5883 		if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off &&
5884 		    bpf_is_throw_kfunc(insn + i)) {
5885 			bool err = false;
5886 
5887 			for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) {
5888 				if (subprog[tmp].is_cb) {
5889 					err = true;
5890 					break;
5891 				}
5892 			}
5893 			if (!err)
5894 				continue;
5895 			verbose(env,
5896 				"bpf_throw kfunc (insn %d) cannot be called from callback subprog %d\n",
5897 				i, idx);
5898 			return -EINVAL;
5899 		}
5900 
5901 		if (bpf_is_callx(insn + i) || bpf_calls_callback(env, i)) {
5902 			/*
5903 			 * Walk the callees and the callbacks recorded by
5904 			 * the main verification pass one by one, returning to
5905 			 * this insn after each.
5906 			 */
5907 			if (dinfo[idx].callx_insn != i) {
5908 				dinfo[idx].callx_insn = i;
5909 				dinfo[idx].callx_next = 0;
5910 			}
5911 			sidx = next_callx_callee(env, idx, dinfo[idx].callx_next);
5912 			if (sidx < 0)
5913 				continue;
5914 			dinfo[idx].callx_next = sidx + 1;
5915 			dinfo[idx].ret_insn = i;
5916 			next_insn = subprog[sidx].start;
5917 		} else if (bpf_pseudo_call(insn + i) || bpf_pseudo_func(insn + i)) {
5918 			/* find the callee */
5919 			next_insn = i + insn[i].imm + 1;
5920 			sidx = bpf_find_subprog(env, next_insn);
5921 			if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn))
5922 				return -EFAULT;
5923 			if (subprog[sidx].is_async_cb) {
5924 				/* async callbacks don't increase bpf prog stack size unless called directly */
5925 				if (!bpf_pseudo_call(insn + i))
5926 					continue;
5927 				if (subprog[sidx].is_exception_cb) {
5928 					verbose(env, "insn %d cannot call exception cb directly", i);
5929 					return -EINVAL;
5930 				}
5931 			}
5932 			/* remember insn to return to */
5933 			dinfo[idx].ret_insn = i + 1;
5934 		} else {
5935 			continue;
5936 		}
5937 
5938 		/*
5939 		 * sort_subprogs_topo() tolerates cycles in the call graph that
5940 		 * go through the address of a function being taken, since it
5941 		 * doesn't know what it is taken for. Such cycle is a recursion
5942 		 * unless it's an async callback, which are skipped above.
5943 		 * The main verification pass limits the depth of the recursion,
5944 		 * but it doesn't follow calls of global functions.
5945 		 */
5946 		for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) {
5947 			if (tmp != sidx)
5948 				continue;
5949 			verbose(env, "recursive call from %s() to %s()\n",
5950 				bpf_subprog_name(env, idx), bpf_subprog_name(env, sidx));
5951 			bpf_diag_program_structure(
5952 				env, i, "recursive subprogram call",
5953 				"Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.",
5954 				"This call or callback would make the subprogram call graph recursive. "
5955 				"The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis.");
5956 			return -EINVAL;
5957 		}
5958 
5959 		/* store caller info for after we return from callee */
5960 		dinfo[idx].frame = frame;
5961 
5962 		/* push caller idx into callee's dinfo */
5963 		dinfo[sidx].caller = idx;
5964 		dinfo[sidx].via_callx = bpf_is_callx(insn + i);
5965 
5966 		i = next_insn;
5967 
5968 		idx = sidx;
5969 		if (!priv_stack_supported)
5970 			subprog[idx].priv_stack_mode = NO_PRIV_STACK;
5971 
5972 		/* sync tail_call_reachable with callee state on entry */
5973 		tail_call_reachable = subprog[idx].has_tail_call;
5974 
5975 		frame = bpf_subprog_is_global(env, idx) ? 0 : frame + 1;
5976 		if (frame >= MAX_CALL_FRAMES) {
5977 			verbose(env, "the call stack of %d frames is too deep !\n",
5978 				frame);
5979 			return -E2BIG;
5980 		}
5981 		goto process_func;
5982 	}
5983 	/* if tail call got detected across bpf2bpf calls then mark each of the
5984 	 * currently present subprog frames as tail call reachable subprogs;
5985 	 * this info will be utilized by JIT so that we will be preserving the
5986 	 * tail call counter throughout bpf2bpf calls combined with tailcalls
5987 	 */
5988 	if (tail_call_reachable) {
5989 		for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) {
5990 			if (subprog[tmp].is_cb) {
5991 				verbose(env, "cannot tail call within callback\n");
5992 				return -EINVAL;
5993 			}
5994 			if (subprog[tmp].stack_arg_cnt) {
5995 				verbose(env, "tail_calls are not allowed in programs with stack args\n");
5996 				return -EINVAL;
5997 			}
5998 			/*
5999 			 * JITs pass tail call counter in a register that is
6000 			 * not available when the callee is called via callx.
6001 			 */
6002 			if (dinfo[tmp].via_callx) {
6003 				verbose(env, "tail_calls are not allowed in functions called via callx\n");
6004 				return -EINVAL;
6005 			}
6006 			subprog[tmp].tail_call_reachable = true;
6007 		}
6008 	} else if (!idx && subprog[0].has_tail_call && subprog[0].stack_arg_cnt) {
6009 		verbose(env, "tail_calls are not allowed in programs with stack args\n");
6010 		return -EINVAL;
6011 	}
6012 
6013 	if (subprog[0].tail_call_reachable)
6014 		env->prog->aux->tail_call_reachable = true;
6015 
6016 	/* end of for() loop means the last insn of the 'subprog'
6017 	 * was reached. Doesn't matter whether it was JA or EXIT
6018 	 */
6019 	if (frame == 0 && dinfo[idx].caller < 0)
6020 		return 0;
6021 	if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE)
6022 		depth -= round_up_stack_depth(env, subprog[idx].stack_depth);
6023 
6024 	/* pop caller idx from callee */
6025 	idx = dinfo[idx].caller;
6026 
6027 	/* retrieve caller state from its frame */
6028 	frame = dinfo[idx].frame;
6029 	i = dinfo[idx].ret_insn;
6030 
6031 	/* reset tail_call_reachable to the parent's actual state */
6032 	tail_call_reachable = subprog[idx].tail_call_reachable;
6033 
6034 	goto continue_func;
6035 }
6036 
6037 static int check_max_stack_depth(struct bpf_verifier_env *env)
6038 {
6039 	enum priv_stack_mode priv_stack_mode = PRIV_STACK_UNKNOWN;
6040 	struct bpf_subprog_call_depth_info *dinfo;
6041 	struct bpf_subprog_info *si = env->subprog_info;
6042 	bool priv_stack_supported;
6043 	int ret;
6044 
6045 	dinfo = kvzalloc_objs(*dinfo, env->subprog_cnt, GFP_KERNEL_ACCOUNT);
6046 	if (!dinfo)
6047 		return -ENOMEM;
6048 
6049 	for (int i = 0; i < env->subprog_cnt; i++) {
6050 		if (si[i].has_tail_call) {
6051 			priv_stack_mode = NO_PRIV_STACK;
6052 			break;
6053 		}
6054 	}
6055 
6056 	if (priv_stack_mode == PRIV_STACK_UNKNOWN)
6057 		priv_stack_mode = bpf_enable_priv_stack(env->prog);
6058 
6059 	/* All async_cb subprogs use normal kernel stack. If a particular
6060 	 * subprog appears in both main prog and async_cb subtree, that
6061 	 * subprog will use normal kernel stack to avoid potential nesting.
6062 	 * The reverse subprog traversal ensures when main prog subtree is
6063 	 * checked, the subprogs appearing in async_cb subtrees are already
6064 	 * marked as using normal kernel stack, so stack size checking can
6065 	 * be done properly.
6066 	 */
6067 	for (int i = env->subprog_cnt - 1; i >= 0; i--) {
6068 		if (!i || si[i].is_async_cb) {
6069 			priv_stack_supported = !i && priv_stack_mode == PRIV_STACK_ADAPTIVE;
6070 			ret = check_max_stack_depth_subprog(env, i, dinfo,
6071 					priv_stack_supported);
6072 			if (ret < 0) {
6073 				kvfree(dinfo);
6074 				return ret;
6075 			}
6076 		}
6077 	}
6078 
6079 	for (int i = 0; i < env->subprog_cnt; i++) {
6080 		if (si[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
6081 			env->prog->aux->jits_use_priv_stack = true;
6082 			break;
6083 		}
6084 	}
6085 
6086 	kvfree(dinfo);
6087 
6088 	return 0;
6089 }
6090 
6091 static int __check_buffer_access(struct bpf_verifier_env *env,
6092 				 const char *buf_info,
6093 				 const struct bpf_reg_state *reg,
6094 				 argno_t argno, int off, int size,
6095 				 u32 *access_end)
6096 {
6097 	s64 start;
6098 
6099 	if (!tnum_is_const(reg->var_off)) {
6100 		char tn_buf[48];
6101 
6102 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
6103 		verbose(env,
6104 			"%s invalid variable buffer offset: off=%d, var_off=%s\n",
6105 			reg_arg_name(env, argno), off, tn_buf);
6106 		return -EACCES;
6107 	}
6108 
6109 	start = (s64)reg->var_off.value + off;
6110 	if (start < 0) {
6111 		verbose(env,
6112 			"%s invalid negative %s buffer offset: off=%d, var_off=%lld\n",
6113 			reg_arg_name(env, argno), buf_info, off, (s64)reg->var_off.value);
6114 		return -EACCES;
6115 	}
6116 
6117 	*access_end = start + size;
6118 	return 0;
6119 }
6120 
6121 static int check_tp_buffer_access(struct bpf_verifier_env *env,
6122 				  const struct bpf_reg_state *reg,
6123 				  argno_t argno, int off, int size)
6124 {
6125 	u32 access_end;
6126 	int err;
6127 
6128 	err = __check_buffer_access(env, "tracepoint", reg, argno, off, size, &access_end);
6129 	if (err)
6130 		return err;
6131 
6132 	env->prog->aux->max_tp_access = max(access_end, env->prog->aux->max_tp_access);
6133 
6134 	return 0;
6135 }
6136 
6137 static int check_buffer_access(struct bpf_verifier_env *env,
6138 			       const struct bpf_reg_state *reg,
6139 			       argno_t argno, int off, int size,
6140 			       bool zero_size_allowed,
6141 			       u32 *max_access, bool write)
6142 {
6143 	const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr";
6144 	struct bpf_prog_aux *aux = env->prog->aux;
6145 	u32 access_end;
6146 	int err;
6147 
6148 	err = __check_buffer_access(env, buf_info, reg, argno, off, size, &access_end);
6149 	if (err)
6150 		return err;
6151 
6152 	*max_access = max(access_end, *max_access);
6153 	if (write && size) {
6154 		if (!aux->max_rdwr_write || access_end - size < aux->min_rdwr_write)
6155 			aux->min_rdwr_write = access_end - size;
6156 		aux->max_rdwr_write = max(access_end, aux->max_rdwr_write);
6157 	}
6158 
6159 	return 0;
6160 }
6161 
6162 /* BPF architecture zero extends alu32 ops into 64-bit registesr */
6163 static void zext_32_to_64(struct bpf_reg_state *reg)
6164 {
6165 	reg_step_check_unsigned(reg, 32);
6166 	reg->var_off = tnum_subreg(reg->var_off);
6167 	reg_set_urange64(reg, reg_u32_min(reg), reg_u32_max(reg));
6168 }
6169 
6170 /* truncate register to smaller size (in bytes)
6171  * must be called with size < BPF_REG_SIZE
6172  */
6173 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
6174 {
6175 	u64 mask = (1ULL << (size * 8)) - 1;
6176 
6177 	reg_step_check_unsigned(reg, size * 8);
6178 
6179 	/* clear high bits in bit representation */
6180 	reg->var_off = tnum_cast(reg->var_off, size);
6181 
6182 	/* fix arithmetic bounds */
6183 	if ((reg_umin(reg) & ~mask) == (reg_umax(reg) & ~mask))
6184 		reg_set_urange64(reg, reg_umin(reg) & mask, reg_umax(reg) & mask);
6185 	else
6186 		reg_set_urange64(reg, 0, mask);
6187 
6188 	/* If size is smaller than 32bit register the 32bit register
6189 	 * values are also truncated so we push 64-bit bounds into
6190 	 * 32-bit bounds. Above were truncated < 32-bits already.
6191 	 */
6192 	if (size < 4)
6193 		__mark_reg32_unbounded(reg);
6194 
6195 	reg_bounds_sync(reg);
6196 }
6197 
6198 static void set_sext64_default_val(struct bpf_reg_state *reg, int size)
6199 {
6200 	if (size == 1) {
6201 		reg_set_srange64(reg, S8_MIN, S8_MAX);
6202 		reg_set_srange32(reg, S8_MIN, S8_MAX);
6203 	} else if (size == 2) {
6204 		reg_set_srange64(reg, S16_MIN, S16_MAX);
6205 		reg_set_srange32(reg, S16_MIN, S16_MAX);
6206 	} else {
6207 		/* size == 4 */
6208 		reg_set_srange64(reg, S32_MIN, S32_MAX);
6209 		reg_set_srange32(reg, S32_MIN, S32_MAX);
6210 	}
6211 	reg->var_off = tnum_unknown;
6212 }
6213 
6214 static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size)
6215 {
6216 	s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval;
6217 	u64 top_smax_value, top_smin_value;
6218 	u64 num_bits = size * 8;
6219 	s64 limit = 1LL << (num_bits - 1);
6220 
6221 	/* If every value fits the signed N-bit range sign extension is a noop. */
6222 	if (reg_smin(reg) >= -limit && reg_smax(reg) < limit)
6223 		return;
6224 
6225 	/* See commen at reg_step_check_unsigned(). */
6226 	reg_step_reset(reg);
6227 
6228 	if (tnum_is_const(reg->var_off)) {
6229 		u64_cval = reg->var_off.value;
6230 		if (size == 1)
6231 			reg->var_off = tnum_const((s8)u64_cval);
6232 		else if (size == 2)
6233 			reg->var_off = tnum_const((s16)u64_cval);
6234 		else
6235 			/* size == 4 */
6236 			reg->var_off = tnum_const((s32)u64_cval);
6237 
6238 		u64_cval = reg->var_off.value;
6239 		reg->r64 = cnum64_from_urange(u64_cval, u64_cval);
6240 		reg->r32 = cnum32_from_urange((u32)u64_cval, (u32)u64_cval);
6241 		return;
6242 	}
6243 
6244 	top_smax_value = ((u64)reg_smax(reg) >> num_bits) << num_bits;
6245 	top_smin_value = ((u64)reg_smin(reg) >> num_bits) << num_bits;
6246 
6247 	if (top_smax_value != top_smin_value)
6248 		goto out;
6249 
6250 	/* find the s64_min and s64_min after sign extension */
6251 	if (size == 1) {
6252 		init_s64_max = (s8)reg_smax(reg);
6253 		init_s64_min = (s8)reg_smin(reg);
6254 	} else if (size == 2) {
6255 		init_s64_max = (s16)reg_smax(reg);
6256 		init_s64_min = (s16)reg_smin(reg);
6257 	} else {
6258 		init_s64_max = (s32)reg_smax(reg);
6259 		init_s64_min = (s32)reg_smin(reg);
6260 	}
6261 
6262 	s64_max = max(init_s64_max, init_s64_min);
6263 	s64_min = min(init_s64_max, init_s64_min);
6264 
6265 	/* both of s64_max/s64_min positive or negative */
6266 	if ((s64_max >= 0) == (s64_min >= 0)) {
6267 		reg_set_srange64(reg, s64_min, s64_max);
6268 		reg_set_srange32(reg, s64_min, s64_max);
6269 		reg->var_off = tnum_range(s64_min, s64_max);
6270 		return;
6271 	}
6272 
6273 out:
6274 	set_sext64_default_val(reg, size);
6275 }
6276 
6277 static void set_sext32_default_val(struct bpf_reg_state *reg, int size)
6278 {
6279 	if (size == 1)
6280 		reg_set_srange32(reg, S8_MIN, S8_MAX);
6281 	else
6282 		/* size == 2 */
6283 		reg_set_srange32(reg, S16_MIN, S16_MAX);
6284 	reg->var_off = tnum_subreg(tnum_unknown);
6285 }
6286 
6287 static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size)
6288 {
6289 	s32 init_s32_max, init_s32_min, s32_max, s32_min, u32_val;
6290 	u32 top_smax_value, top_smin_value;
6291 	u32 num_bits = size * 8;
6292 
6293 	/*
6294 	 * If value fits the non-negative half of the signed N-bit range,
6295 	 * sign extension is a noop.
6296 	 */
6297 	if (reg_umax(reg) < (1ULL << (num_bits - 1)))
6298 		return;
6299 
6300 	/* See comment at reg_step_check_unsigned(). */
6301 	reg_step_reset(reg);
6302 
6303 	if (tnum_is_const(reg->var_off)) {
6304 		u32_val = reg->var_off.value;
6305 		if (size == 1)
6306 			reg->var_off = tnum_const((s8)u32_val);
6307 		else
6308 			reg->var_off = tnum_const((s16)u32_val);
6309 
6310 		u32_val = reg->var_off.value;
6311 		reg_set_srange32(reg, u32_val, u32_val);
6312 		return;
6313 	}
6314 
6315 	top_smax_value = ((u32)reg_s32_max(reg) >> num_bits) << num_bits;
6316 	top_smin_value = ((u32)reg_s32_min(reg) >> num_bits) << num_bits;
6317 
6318 	if (top_smax_value != top_smin_value)
6319 		goto out;
6320 
6321 	/* find the s32_min and s32_min after sign extension */
6322 	if (size == 1) {
6323 		init_s32_max = (s8)reg_s32_max(reg);
6324 		init_s32_min = (s8)reg_s32_min(reg);
6325 	} else {
6326 		/* size == 2 */
6327 		init_s32_max = (s16)reg_s32_max(reg);
6328 		init_s32_min = (s16)reg_s32_min(reg);
6329 	}
6330 	s32_max = max(init_s32_max, init_s32_min);
6331 	s32_min = min(init_s32_max, init_s32_min);
6332 
6333 	if ((s32_min >= 0) == (s32_max >= 0)) {
6334 		reg_set_srange32(reg, s32_min, s32_max);
6335 		reg->var_off = tnum_subreg(tnum_range(s32_min, s32_max));
6336 		return;
6337 	}
6338 
6339 out:
6340 	set_sext32_default_val(reg, size);
6341 }
6342 
6343 bool bpf_map_is_rdonly(const struct bpf_map *map)
6344 {
6345 	/* A map is considered read-only if the following condition are true:
6346 	 *
6347 	 * 1) BPF program side cannot change any of the map content. The
6348 	 *    BPF_F_RDONLY_PROG flag is throughout the lifetime of a map
6349 	 *    and was set at map creation time.
6350 	 * 2) The map value(s) have been initialized from user space by a
6351 	 *    loader and then "frozen", such that no new map update/delete
6352 	 *    operations from syscall side are possible for the rest of
6353 	 *    the map's lifetime from that point onwards.
6354 	 * 3) Any parallel/pending map update/delete operations from syscall
6355 	 *    side have been completed. Only after that point, it's safe to
6356 	 *    assume that map value(s) are immutable.
6357 	 */
6358 	return (map->map_flags & BPF_F_RDONLY_PROG) &&
6359 	       READ_ONCE(map->frozen) &&
6360 	       !bpf_map_write_active(map);
6361 }
6362 
6363 int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
6364 			bool is_ldsx)
6365 {
6366 	void *ptr;
6367 	u64 addr;
6368 	int err;
6369 
6370 	if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY || map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
6371 		return -EINVAL;
6372 	err = map->ops->map_direct_value_addr(map, &addr, off);
6373 	if (err)
6374 		return err;
6375 	ptr = (void *)(long)addr + off;
6376 
6377 	switch (size) {
6378 	case sizeof(u8):
6379 		*val = is_ldsx ? (s64)*(s8 *)ptr : (u64)*(u8 *)ptr;
6380 		break;
6381 	case sizeof(u16):
6382 		*val = is_ldsx ? (s64)*(s16 *)ptr : (u64)*(u16 *)ptr;
6383 		break;
6384 	case sizeof(u32):
6385 		*val = is_ldsx ? (s64)*(s32 *)ptr : (u64)*(u32 *)ptr;
6386 		break;
6387 	case sizeof(u64):
6388 		*val = *(u64 *)ptr;
6389 		break;
6390 	default:
6391 		return -EINVAL;
6392 	}
6393 	return 0;
6394 }
6395 
6396 static int cmp_func_ptrs(const void *_a, const void *_b)
6397 {
6398 	const struct bpf_func_ptr *a = _a, *b = _b;
6399 
6400 	if (a->map != b->map)
6401 		return a->map < b->map ? -1 : 1;
6402 	if (a->map_off != b->map_off)
6403 		return a->map_off < b->map_off ? -1 : 1;
6404 	return 0;
6405 }
6406 
6407 /* Find the first pointer to a function at or after 'off' in the value of 'map' */
6408 static u32 func_ptr_lower_bound(struct bpf_verifier_env *env, const struct bpf_map *map, u64 off)
6409 {
6410 	u32 l = 0, r = env->func_ptr_cnt, m;
6411 	struct bpf_func_ptr *p;
6412 
6413 	while (l < r) {
6414 		m = l + (r - l) / 2;
6415 		p = &env->func_ptrs[m];
6416 		if (p->map < map || (p->map == map && p->map_off < off))
6417 			l = m + 1;
6418 		else
6419 			r = m;
6420 	}
6421 	return l;
6422 }
6423 
6424 /*
6425  * Return pointers to functions that overlap with 'size' bytes at offset 'off'
6426  * of the value of 'map' and their number in 'cnt'.
6427  */
6428 struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env,
6429 					     const struct bpf_map *map,
6430 					     u64 off, u64 size, u32 *cnt)
6431 {
6432 	u32 first, last;
6433 
6434 	*cnt = 0;
6435 	if (!env->func_ptr_cnt || !size)
6436 		return NULL;
6437 
6438 	/* a pointer that starts up to 7 bytes before 'off' overlaps too */
6439 	first = func_ptr_lower_bound(env, map, off >= sizeof(u64) ? off - sizeof(u64) + 1 : 0);
6440 	last = func_ptr_lower_bound(env, map, off + size);
6441 	if (first >= last)
6442 		return NULL;
6443 
6444 	*cnt = last - first;
6445 	return &env->func_ptrs[first];
6446 }
6447 
6448 /* Return all pointers to functions in the value of 'map' */
6449 struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env,
6450 				       const struct bpf_map *map, u32 *cnt)
6451 {
6452 	return bpf_map_range_func_ptrs(env, map, 0, (u64)map->value_size, cnt);
6453 }
6454 
6455 /* instructions [off, off + len) replaced the instruction at 'off' */
6456 void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len)
6457 {
6458 	struct bpf_func_ptr *p;
6459 	u32 i;
6460 
6461 	if (len <= 1)
6462 		return;
6463 
6464 	for (i = 0; i < env->func_ptr_cnt; i++) {
6465 		p = &env->func_ptrs[i];
6466 		if (p->xlated_off <= off || p->xlated_off == BPF_FUNC_PTR_DELETED)
6467 			continue;
6468 		p->xlated_off += len - 1;
6469 	}
6470 }
6471 
6472 /*
6473  * Instructions [off, off + len) are about to be removed. It's called before
6474  * the starts of subprogs are adjusted. A subprog is gone when all of its
6475  * instructions are. Otherwise, e.g. when its first instruction is a nop,
6476  * it starts where the removed instructions did.
6477  */
6478 void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len)
6479 {
6480 	struct bpf_func_ptr *p;
6481 	int subprog;
6482 	u32 i;
6483 
6484 	for (i = 0; i < env->func_ptr_cnt; i++) {
6485 		p = &env->func_ptrs[i];
6486 		if (p->xlated_off < off || p->xlated_off == BPF_FUNC_PTR_DELETED)
6487 			continue;
6488 		if (p->xlated_off >= off + len) {
6489 			p->xlated_off -= len;
6490 			continue;
6491 		}
6492 		subprog = bpf_find_subprog(env, p->xlated_off);
6493 		if (subprog > 0 && env->subprog_info[subprog + 1].start > off + len)
6494 			p->xlated_off = off;
6495 		else
6496 			p->xlated_off = BPF_FUNC_PTR_DELETED;
6497 	}
6498 }
6499 
6500 #define BTF_TYPE_SAFE_RCU(__type)  __PASTE(__type, __safe_rcu)
6501 #define BTF_TYPE_SAFE_RCU_OR_NULL(__type)  __PASTE(__type, __safe_rcu_or_null)
6502 #define BTF_TYPE_SAFE_TRUSTED(__type)  __PASTE(__type, __safe_trusted)
6503 #define BTF_TYPE_SAFE_TRUSTED_OR_NULL(__type)  __PASTE(__type, __safe_trusted_or_null)
6504 
6505 /*
6506  * Allow list few fields as RCU trusted or full trusted.
6507  * This logic doesn't allow mix tagging and will be removed once GCC supports
6508  * btf_type_tag.
6509  */
6510 
6511 /* RCU trusted: these fields are trusted in RCU CS and never NULL */
6512 BTF_TYPE_SAFE_RCU(struct task_struct) {
6513 	const cpumask_t *cpus_ptr;
6514 	struct css_set __rcu *cgroups;
6515 	struct task_struct __rcu *real_parent;
6516 	struct task_struct *group_leader;
6517 };
6518 
6519 BTF_TYPE_SAFE_RCU(struct cgroup) {
6520 	/* cgrp->kn is always accessible as documented in kernel/cgroup/cgroup.c */
6521 	struct kernfs_node *kn;
6522 };
6523 
6524 BTF_TYPE_SAFE_RCU(struct css_set) {
6525 	struct cgroup *dfl_cgrp;
6526 };
6527 
6528 BTF_TYPE_SAFE_RCU(struct cgroup_subsys_state) {
6529 	struct cgroup *cgroup;
6530 };
6531 
6532 /* RCU trusted: these fields are trusted in RCU CS and can be NULL */
6533 BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct) {
6534 	struct file __rcu *exe_file;
6535 #ifdef CONFIG_MEMCG
6536 	struct task_struct __rcu *owner;
6537 #endif
6538 };
6539 
6540 /* skb->sk, req->sk are not RCU protected, but we mark them as such
6541  * because bpf prog accessible sockets are SOCK_RCU_FREE.
6542  */
6543 BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff) {
6544 	struct sock *sk;
6545 };
6546 
6547 BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock) {
6548 	struct sock *sk;
6549 };
6550 
6551 /* full trusted: these fields are trusted even outside of RCU CS and never NULL */
6552 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta) {
6553 	struct seq_file *seq;
6554 };
6555 
6556 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task) {
6557 	struct bpf_iter_meta *meta;
6558 	struct task_struct *task;
6559 };
6560 
6561 BTF_TYPE_SAFE_TRUSTED(struct linux_binprm) {
6562 	struct file *file;
6563 };
6564 
6565 BTF_TYPE_SAFE_TRUSTED(struct file) {
6566 	struct inode *f_inode;
6567 };
6568 
6569 /*
6570  * The pointer fields in the sched_ext ops argument containers are pinned by the
6571  * callers for the duration of the ops calls and are never NULL.
6572  */
6573 BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args) {
6574 #ifdef CONFIG_EXT_GROUP_SCHED
6575 	struct cgroup *cgroup;
6576 #endif
6577 };
6578 
6579 BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args) {
6580 	struct task_struct *task;
6581 };
6582 
6583 BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args) {
6584 	struct sched_ext_ops *ops;
6585 };
6586 
6587 BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args) {
6588 	struct sched_ext_ops *ops;
6589 };
6590 
6591 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) {
6592 	struct inode *d_inode;
6593 };
6594 
6595 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm) {
6596 	struct mm_struct *mm;
6597 };
6598 
6599 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) {
6600 	struct sock *sk;
6601 };
6602 
6603 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct) {
6604 	struct mm_struct *vm_mm;
6605 	struct file *vm_file;
6606 };
6607 
6608 static bool type_is_rcu(struct bpf_verifier_env *env,
6609 			struct bpf_reg_state *reg,
6610 			const char *field_name, u32 btf_id)
6611 {
6612 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct task_struct));
6613 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup));
6614 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct css_set));
6615 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup_subsys_state));
6616 
6617 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu");
6618 }
6619 
6620 static bool type_is_rcu_or_null(struct bpf_verifier_env *env,
6621 				struct bpf_reg_state *reg,
6622 				const char *field_name, u32 btf_id)
6623 {
6624 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct));
6625 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff));
6626 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock));
6627 
6628 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null");
6629 }
6630 
6631 static bool type_is_trusted(struct bpf_verifier_env *env,
6632 			    struct bpf_reg_state *reg,
6633 			    const char *field_name, u32 btf_id)
6634 {
6635 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta));
6636 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task));
6637 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm));
6638 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file));
6639 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args));
6640 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args));
6641 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args));
6642 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args));
6643 
6644 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted");
6645 }
6646 
6647 static bool type_is_trusted_or_null(struct bpf_verifier_env *env,
6648 				    struct bpf_reg_state *reg,
6649 				    const char *field_name, u32 btf_id)
6650 {
6651 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket));
6652 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry));
6653 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm));
6654 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct));
6655 
6656 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id,
6657 					  "__safe_trusted_or_null");
6658 }
6659 
6660 /*
6661  * The member holds the kernel address of arena memory, which the rewritten
6662  * load turns into an arena pointer, see arena_member_fixup(). NULL needs no
6663  * tracking, PTR_TO_ARENA covers it.
6664  */
6665 static int check_arena_member_load(struct bpf_verifier_env *env, struct bpf_reg_state *regs,
6666 				   int value_regno, enum bpf_type_flag flag, const char *tname,
6667 				   const char *field_name)
6668 {
6669 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx];
6670 	struct bpf_insn *insn;
6671 
6672 	/* a helper reading the member as bytes needs no rewrite */
6673 	if (value_regno < 0)
6674 		return 0;
6675 	if (!env->prog->aux->arena) {
6676 		verbose(env,
6677 			"arena pointer member %s of struct %s requires a program with an associated arena\n",
6678 			field_name, tname);
6679 		return -EINVAL;
6680 	}
6681 	/* only a plain load or an acquire load is rewritten */
6682 	insn = &env->prog->insnsi[env->insn_idx];
6683 	if (BPF_CLASS(insn->code) != BPF_LDX &&
6684 	    !(insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && insn->imm == BPF_LOAD_ACQ)) {
6685 		verbose(env,
6686 			"arena pointer member %s of struct %s can only be read with a plain or an acquire load\n",
6687 			field_name, tname);
6688 		return -EACCES;
6689 	}
6690 
6691 	aux->arena_member = true;
6692 	if (flag & PTR_MAYBE_NULL)
6693 		aux->arena_member_nullable = true;
6694 	mark_reg_unknown(env, regs, value_regno);
6695 	regs[value_regno].type = PTR_TO_ARENA;
6696 	return 0;
6697 }
6698 
6699 static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
6700 				   struct bpf_reg_state *regs, struct bpf_reg_state *reg,
6701 				   argno_t argno, int off, int size,
6702 				   enum bpf_access_type atype,
6703 				   int value_regno)
6704 {
6705 	const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
6706 	const char *tname = btf_name_by_offset(reg->btf, t->name_off);
6707 	const char *field_name = NULL;
6708 	enum bpf_type_flag flag = 0;
6709 	u32 btf_id = 0;
6710 	s64 min_off;
6711 	int ret;
6712 
6713 	if (!env->allow_ptr_leaks) {
6714 		verbose(env,
6715 			"'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
6716 			tname);
6717 		return -EPERM;
6718 	}
6719 	if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) {
6720 		verbose(env,
6721 			"Cannot access kernel 'struct %s' from non-GPL compatible program\n",
6722 			tname);
6723 		return -EINVAL;
6724 	}
6725 
6726 	if (check_add_overflow(reg_smin(reg), off, &min_off)) {
6727 		verbose(env,
6728 			"%s is ptr_%s access, offset computation overflows: register's minimal offset is %lld, instruction offset is %d\n",
6729 			reg_arg_name(env, argno), tname, reg_smin(reg), off);
6730 		return -EACCES;
6731 	}
6732 
6733 	if (min_off < 0) {
6734 		verbose(env,
6735 			"%s is ptr_%s invalid negative access: off=%lld\n",
6736 			reg_arg_name(env, argno), tname, min_off);
6737 		return -EACCES;
6738 	}
6739 
6740 	if (reg->type & MEM_USER) {
6741 		verbose(env,
6742 			"%s is ptr_%s access user memory\n",
6743 			reg_arg_name(env, argno), tname);
6744 		return -EACCES;
6745 	}
6746 
6747 	if (reg->type & MEM_PERCPU) {
6748 		verbose(env,
6749 			"%s is ptr_%s access percpu memory\n",
6750 			reg_arg_name(env, argno), tname);
6751 		return -EACCES;
6752 	}
6753 
6754 	if (atype != BPF_READ && bpf_may_fault_on_deref(reg->type)) {
6755 		verbose(env, "only read is supported\n");
6756 		return -EACCES;
6757 	}
6758 
6759 	if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) {
6760 		if (!tnum_is_const(reg->var_off)) {
6761 			char tn_buf[48];
6762 
6763 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
6764 			verbose(env,
6765 				"%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n",
6766 				reg_arg_name(env, argno), tname, off, tn_buf);
6767 			return -EACCES;
6768 		}
6769 
6770 		off += reg->var_off.value;
6771 
6772 		if (!btf_is_kernel(reg->btf)) {
6773 			verifier_bug(env, "reg->btf must be kernel btf");
6774 			return -EFAULT;
6775 		}
6776 		ret = env->ops->btf_struct_access(&env->log, reg, off, size);
6777 		if (ret < 0)
6778 			verbose(env,
6779 				"%s cannot write into ptr_%s at off=%d size=%d\n",
6780 				reg_arg_name(env, argno), tname, off, size);
6781 	} else {
6782 		/* Writes are permitted with default btf_struct_access for
6783 		 * program allocated objects (which always have id > 0).
6784 		 */
6785 		if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) {
6786 			verbose(env, "only read is supported\n");
6787 			return -EACCES;
6788 		}
6789 
6790 		/*
6791 		 * A fault-prone allocated object may still be read through a
6792 		 * BPF_PROBE_MEM load after its lifetime protection ends. Writes
6793 		 * through such pointers were rejected above.
6794 		 */
6795 		if (type_is_alloc(reg->type) && !bpf_may_fault_on_deref(reg->type) &&
6796 		    !type_is_non_owning_ref(reg->type) &&
6797 		    !(reg->type & MEM_RCU) && !reg_is_referenced(env, reg)) {
6798 			verifier_bug(env, "allocated object must have a referenced id");
6799 			return -EFAULT;
6800 		}
6801 
6802 		ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name);
6803 	}
6804 
6805 	if (ret < 0)
6806 		return ret;
6807 
6808 	if (ret == PTR_TO_ARENA)
6809 		return check_arena_member_load(env, regs, value_regno, flag, tname, field_name);
6810 
6811 	if (ret != PTR_TO_BTF_ID) {
6812 		/* just mark; */
6813 
6814 	} else if (type_flag(reg->type) & PTR_UNTRUSTED) {
6815 		/* If this is an untrusted pointer, all pointers formed by walking it
6816 		 * also inherit the untrusted flag.
6817 		 */
6818 		flag = PTR_UNTRUSTED;
6819 
6820 	} else if (is_trusted_reg(env, reg) || is_rcu_reg(reg)) {
6821 		/* By default any pointer obtained from walking a trusted pointer is no
6822 		 * longer trusted, unless the field being accessed has explicitly been
6823 		 * marked as inheriting its parent's state of trust (either full or RCU).
6824 		 * For example:
6825 		 * 'cgroups' pointer is untrusted if task->cgroups dereference
6826 		 * happened in a sleepable program outside of bpf_rcu_read_lock()
6827 		 * section. In a non-sleepable program it's trusted while in RCU CS (aka MEM_RCU).
6828 		 * Note bpf_rcu_read_unlock() converts MEM_RCU pointers to PTR_UNTRUSTED.
6829 		 *
6830 		 * A regular RCU-protected pointer with __rcu tag can also be deemed
6831 		 * trusted if we are in an RCU CS. Such pointer can be NULL.
6832 		 */
6833 		if (type_is_trusted(env, reg, field_name, btf_id)) {
6834 			flag |= PTR_TRUSTED;
6835 		} else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) {
6836 			flag |= PTR_TRUSTED | PTR_MAYBE_NULL;
6837 		} else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) {
6838 			if (type_is_rcu(env, reg, field_name, btf_id)) {
6839 				/* ignore __rcu tag and mark it MEM_RCU */
6840 				flag |= MEM_RCU;
6841 			} else if (flag & MEM_RCU ||
6842 				   type_is_rcu_or_null(env, reg, field_name, btf_id)) {
6843 				/* __rcu tagged pointers can be NULL */
6844 				flag |= MEM_RCU | PTR_MAYBE_NULL;
6845 
6846 				/* We always trust them */
6847 				if (type_is_rcu_or_null(env, reg, field_name, btf_id) &&
6848 				    flag & PTR_UNTRUSTED)
6849 					flag &= ~PTR_UNTRUSTED;
6850 			} else if (flag & (MEM_PERCPU | MEM_USER)) {
6851 				/* keep as-is */
6852 			} else {
6853 				/* walking unknown pointers yields old deprecated PTR_TO_BTF_ID */
6854 				clear_trusted_flags(&flag);
6855 			}
6856 		} else {
6857 			/*
6858 			 * If not in RCU CS or MEM_RCU pointer can be NULL then
6859 			 * aggressively mark as untrusted otherwise such
6860 			 * pointers will be plain PTR_TO_BTF_ID without flags
6861 			 * and will be allowed to be passed into helpers for
6862 			 * compat reasons.
6863 			 */
6864 			flag = PTR_UNTRUSTED;
6865 		}
6866 	} else {
6867 		/* Old compat. Deprecated */
6868 		clear_trusted_flags(&flag);
6869 	}
6870 
6871 	/*
6872 	 * A __nullable member only matters where NULL would otherwise be
6873 	 * assumed away, which is anywhere but on a plain or an untrusted
6874 	 * pointer: those are probed on a dereference, which tolerates NULL, and
6875 	 * the BTF_TYPE_SAFE_*_OR_NULL lists never applied to them either. A
6876 	 * per-CPU pointer counts, per_cpu_ptr() does not check for NULL.
6877 	 */
6878 	if (bpf_may_fault_on_deref(PTR_TO_BTF_ID | (flag & ~PTR_MAYBE_NULL)))
6879 		flag &= ~PTR_MAYBE_NULL;
6880 
6881 	if (atype == BPF_READ && value_regno >= 0) {
6882 		u32 parent_id = reg_lifetime_id(env, reg);
6883 
6884 		ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);
6885 		if (ret < 0)
6886 			return ret;
6887 		if ((regs[value_regno].type & PTR_TRUSTED) &&
6888 		    !(regs[value_regno].type & (MEM_RCU | MEM_PERCPU | MEM_USER)))
6889 			regs[value_regno].parent_id = parent_id;
6890 	}
6891 
6892 	return 0;
6893 }
6894 
6895 static int check_ptr_to_map_access(struct bpf_verifier_env *env,
6896 				   struct bpf_reg_state *regs, struct bpf_reg_state *reg,
6897 				   argno_t argno, int off, int size,
6898 				   enum bpf_access_type atype,
6899 				   int value_regno)
6900 {
6901 	struct bpf_map *map = reg->map_ptr;
6902 	struct bpf_reg_state map_reg;
6903 	enum bpf_type_flag flag = 0;
6904 	const struct btf_type *t;
6905 	const char *tname;
6906 	u32 btf_id;
6907 	int ret;
6908 
6909 	if (!btf_vmlinux) {
6910 		verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");
6911 		return -ENOTSUPP;
6912 	}
6913 
6914 	if (!map->ops->map_btf_id || !*map->ops->map_btf_id) {
6915 		verbose(env, "map_ptr access not supported for map type %d\n",
6916 			map->map_type);
6917 		return -ENOTSUPP;
6918 	}
6919 
6920 	t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id);
6921 	tname = btf_name_by_offset(btf_vmlinux, t->name_off);
6922 
6923 	if (!env->allow_ptr_leaks) {
6924 		verbose(env,
6925 			"'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
6926 			tname);
6927 		return -EPERM;
6928 	}
6929 
6930 	if (off < 0) {
6931 		verbose(env, "%s is %s invalid negative access: off=%d\n",
6932 			reg_arg_name(env, argno), tname, off);
6933 		return -EACCES;
6934 	}
6935 
6936 	if (atype != BPF_READ) {
6937 		verbose(env, "only read from %s is supported\n", tname);
6938 		return -EACCES;
6939 	}
6940 
6941 	/* Simulate access to a PTR_TO_BTF_ID */
6942 	memset(&map_reg, 0, sizeof(map_reg));
6943 	ret = mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID,
6944 			      btf_vmlinux, *map->ops->map_btf_id, 0);
6945 	if (ret < 0)
6946 		return ret;
6947 	ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL);
6948 	if (ret < 0)
6949 		return ret;
6950 
6951 	if (value_regno >= 0) {
6952 		ret = mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag);
6953 		if (ret < 0)
6954 			return ret;
6955 	}
6956 
6957 	return 0;
6958 }
6959 
6960 /*
6961  * Check that the stack access at the given offset is within bounds. The
6962  * maximum valid offset is -1.
6963  *
6964  * The minimum valid offset is -env->stack_limit for writes, and
6965  * -state->allocated_stack for reads.
6966  */
6967 static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,
6968                                           s64 off,
6969                                           struct bpf_func_state *state,
6970                                           enum bpf_access_type t)
6971 {
6972 	int min_valid_off;
6973 
6974 	if (t == BPF_WRITE || env->allow_uninit_stack)
6975 		min_valid_off = -(int)env->stack_limit;
6976 	else
6977 		min_valid_off = -state->allocated_stack;
6978 
6979 	if (off < min_valid_off || off > -1)
6980 		return -EACCES;
6981 	return 0;
6982 }
6983 
6984 /* Check that the stack access at 'regno + off' falls within the maximum stack
6985  * bounds.
6986  *
6987  * 'off' includes `regno->offset`, but not its dynamic part (if any).
6988  */
6989 static int check_stack_access_within_bounds(
6990 		struct bpf_verifier_env *env, struct bpf_reg_state *reg,
6991 		argno_t argno, int off, int access_size,
6992 		enum bpf_access_type type)
6993 {
6994 	struct bpf_func_state *state = bpf_func(env, reg);
6995 	s64 min_off, max_off;
6996 	int err;
6997 	char *err_extra;
6998 
6999 	if (type == BPF_READ)
7000 		err_extra = " read from";
7001 	else
7002 		err_extra = " write to";
7003 
7004 	if (tnum_is_const(reg->var_off)) {
7005 		min_off = (s64)reg->var_off.value + off;
7006 		max_off = min_off + access_size;
7007 	} else {
7008 		if (reg_smax(reg) >= BPF_MAX_VAR_OFF ||
7009 		    reg_smin(reg) <= -BPF_MAX_VAR_OFF) {
7010 			verbose(env, "invalid unbounded variable-offset%s stack %s\n",
7011 				err_extra, reg_arg_name(env, argno));
7012 			return -EACCES;
7013 		}
7014 		min_off = reg_smin(reg) + off;
7015 		max_off = reg_smax(reg) + off + access_size;
7016 	}
7017 
7018 	err = check_stack_slot_within_bounds(env, min_off, state, type);
7019 	if (!err && max_off > 0)
7020 		err = -EINVAL; /* out of stack access into non-negative offsets */
7021 	if (!err && access_size < 0)
7022 		/* access_size should not be negative (or overflow an int); others checks
7023 		 * along the way should have prevented such an access.
7024 		 */
7025 		err = -EFAULT; /* invalid negative access size; integer overflow? */
7026 
7027 	if (err) {
7028 		if (tnum_is_const(reg->var_off)) {
7029 			verbose(env, "invalid%s stack %s off=%lld size=%d\n",
7030 				err_extra, reg_arg_name(env, argno), min_off, access_size);
7031 		} else {
7032 			char tn_buf[48];
7033 
7034 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
7035 			verbose(env, "invalid variable-offset%s stack %s var_off=%s off=%d size=%d\n",
7036 				err_extra, reg_arg_name(env, argno), tn_buf, off, access_size);
7037 		}
7038 		return err;
7039 	}
7040 
7041 	/*
7042 	 * Every stack access passes through here, including buffers passed to
7043 	 * helpers and kfuncs and accesses into a caller's frame, so check the
7044 	 * fastcall contract of the frame that owns the slots once for all of
7045 	 * them. Zero-sized accesses touch no stack and keep the rewrite enabled.
7046 	 */
7047 	if (access_size)
7048 		check_fastcall_stack_contract(env, state, env->insn_idx, min_off);
7049 
7050 	/* Note that there is no stack access with offset zero, so the needed stack
7051 	 * size is -min_off, not -min_off+1.
7052 	 */
7053 	return bpf_grow_stack_state(env, state, -min_off /* size */);
7054 }
7055 
7056 static bool get_func_retval_range(struct bpf_prog *prog,
7057 				  struct bpf_retval_range *range)
7058 {
7059 	if (prog->type == BPF_PROG_TYPE_LSM &&
7060 		prog->expected_attach_type == BPF_LSM_MAC &&
7061 		!bpf_lsm_get_retval_range(prog, range)) {
7062 		return true;
7063 	}
7064 	return false;
7065 }
7066 
7067 static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val)
7068 {
7069 	struct bpf_reg_state fake_reg;
7070 
7071 	if (!val)
7072 		return;
7073 
7074 	fake_reg.type = SCALAR_VALUE;
7075 	__mark_reg_known(&fake_reg, val);
7076 
7077 	scalar_step_add(dst_reg, dst_reg, &fake_reg);
7078 	scalar32_min_max_add(dst_reg, &fake_reg);
7079 	scalar_min_max_add(dst_reg, &fake_reg);
7080 	dst_reg->var_off = tnum_add(dst_reg->var_off, fake_reg.var_off);
7081 
7082 	reg_bounds_sync(dst_reg);
7083 }
7084 
7085 /* there is no support for callx in the interpreter */
7086 static int require_callx_jit(struct bpf_verifier_env *env)
7087 {
7088 	if (!env->prog->jit_requested) {
7089 		verbose(env, "JIT is required to use callx\n");
7090 		return -EOPNOTSUPP;
7091 	}
7092 	if (!bpf_jit_supports_callx()) {
7093 		verbose(env, "JIT doesn't support callx\n");
7094 		return -EOPNOTSUPP;
7095 	}
7096 	env->prog->jit_required = true;
7097 	return 0;
7098 }
7099 
7100 static void mark_reg_func_ptr(struct bpf_verifier_env *env, struct bpf_reg_state *regs,
7101 			      int regno, int subprog)
7102 {
7103 	mark_reg_known_zero(env, regs, regno);
7104 	regs[regno].type = PTR_TO_FUNC;
7105 	regs[regno].subprogno = subprog;
7106 }
7107 
7108 /* a read from a table of functions branches into that many states at most */
7109 #define BPF_MAX_FUNC_PTR_TARGETS 64
7110 /* and that many pointers at most are next to what is read at variable offset */
7111 #define BPF_MAX_FUNC_PTR_RANGE 4096
7112 
7113 /* Count up to 'limit' offsets in [min_off, max_off] that the read is possible at */
7114 static u32 count_read_offsets(struct tnum offs, u64 min_off, u64 max_off, u32 limit)
7115 {
7116 	u64 o = min_off, next;
7117 	u32 cnt = 0;
7118 
7119 	if (!tnum_in(offs, tnum_const(o))) {
7120 		next = tnum_step(offs, o);
7121 		if (next <= o)
7122 			return 0;
7123 		o = next;
7124 	}
7125 	while (o <= max_off && cnt < limit) {
7126 		cnt++;
7127 		next = tnum_step(offs, o);
7128 		if (next <= o)
7129 			break;
7130 		o = next;
7131 	}
7132 	return cnt;
7133 }
7134 
7135 /*
7136  * A read from a frozen read-only map that has pointers to functions, see
7137  * resolve_func_ptrs(). The bounds and var_off of the offset tell where the read
7138  * is possible at, e.g. var_off is the stride of the elements of an array.
7139  * A read of exactly one pointer yields PTR_TO_FUNC. When the offset is variable
7140  * and only pointers can be read, which is how an element of a table of
7141  * functions is loaded, the verification continues with each of them. Other
7142  * reads that overlap with a pointer are rejected, because their result is not
7143  * known until the program is jitted.
7144  *
7145  * Return -ENOENT if there are no pointers to functions in the bytes that are read.
7146  */
7147 static int check_func_ptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off,
7148 			       int size, int value_regno)
7149 {
7150 	u64 min_off = reg_umin(reg) + off, max_off = reg_umax(reg) + off;
7151 	struct tnum offs = tnum_add(reg->var_off, tnum_const(off));
7152 	struct bpf_reg_state *regs = cur_regs(env);
7153 	struct bpf_map *map = reg->map_ptr;
7154 	struct bpf_verifier_state *branch;
7155 	int subprog, targets[BPF_MAX_FUNC_PTR_TARGETS];
7156 	struct bpf_func_ptr *ptrs;
7157 	u32 i, cnt, n = 0;
7158 	u64 o, lo, hi;
7159 	int err;
7160 
7161 	ptrs = bpf_map_range_func_ptrs(env, map, min_off, max_off - min_off + size, &cnt);
7162 	if (!ptrs)
7163 		return -ENOENT;
7164 	if (cnt > BPF_MAX_FUNC_PTR_RANGE)
7165 		goto overlap;
7166 
7167 	for (i = 0; i < cnt; i++) {
7168 		/* the offsets at which the read overlaps with the pointer */
7169 		lo = ptrs[i].map_off >= size ? ptrs[i].map_off - size + 1 : 0;
7170 		lo = max(lo, min_off);
7171 		hi = min(ptrs[i].map_off + sizeof(u64) - 1, max_off);
7172 		for (o = lo; o <= hi; o++) {
7173 			if (!tnum_in(offs, tnum_const(o)))
7174 				continue;
7175 			if (o != ptrs[i].map_off || size != sizeof(u64) || value_regno < 0)
7176 				goto overlap;
7177 			if (n == BPF_MAX_FUNC_PTR_TARGETS) {
7178 				verbose(env, "read from map '%s' may yield more than %d pointers to functions\n",
7179 					map->name, BPF_MAX_FUNC_PTR_TARGETS);
7180 				return -E2BIG;
7181 			}
7182 			subprog = bpf_find_subprog(env, ptrs[i].xlated_off);
7183 			if (verifier_bug_if(subprog <= 0, env,
7184 					    "no function at insn %u for map '%s' offset %u",
7185 					    ptrs[i].xlated_off, map->name, ptrs[i].map_off))
7186 				return -EFAULT;
7187 			targets[n++] = subprog;
7188 		}
7189 	}
7190 	/* pointers are next to what is read, e.g. the data of an array of vtables */
7191 	if (!n)
7192 		return -ENOENT;
7193 	/* either every offset that the read is possible at is a pointer or none */
7194 	if (count_read_offsets(offs, min_off, max_off, n + 1) != n)
7195 		goto overlap;
7196 
7197 	/* the map has the offset of the function until the program is jitted */
7198 	err = require_callx_jit(env);
7199 	if (err)
7200 		return err;
7201 
7202 	for (i = 0; i < n - 1; i++) {
7203 		branch = push_stack(env, env->insn_idx + 1, env->insn_idx,
7204 				    env->cur_state->speculative);
7205 		if (IS_ERR(branch))
7206 			return PTR_ERR(branch);
7207 		mark_reg_func_ptr(env, branch->frame[branch->curframe]->regs, value_regno,
7208 				  targets[i]);
7209 	}
7210 	mark_reg_func_ptr(env, regs, value_regno, targets[n - 1]);
7211 	return 0;
7212 
7213 overlap:
7214 	verbose(env, "read of %d bytes at offset [%llu,%llu] of map '%s' overlaps with a pointer to a function\n",
7215 		size, min_off, max_off, map->name);
7216 	return -EACCES;
7217 }
7218 
7219 static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off,
7220 			      int bpf_size, int value_regno, bool is_ldsx)
7221 {
7222 	struct bpf_reg_state *regs = cur_regs(env);
7223 	int size = bpf_size_to_bytes(bpf_size);
7224 	struct bpf_map *map = reg->map_ptr;
7225 	int err;
7226 
7227 	switch (map->map_type) {
7228 	case BPF_MAP_TYPE_INSN_ARRAY:
7229 		if (bpf_size != BPF_DW) {
7230 			verbose(env, "Invalid read of %d bytes from insn_array\n", size);
7231 			return -EACCES;
7232 		}
7233 		regs[value_regno] = *reg;
7234 		add_scalar_to_reg(&regs[value_regno], off);
7235 		regs[value_regno].type = PTR_TO_INSN;
7236 		return 0;
7237 	case BPF_MAP_TYPE_PERCPU_ARRAY:
7238 		goto reg_unknown;
7239 	default:
7240 		break;
7241 	}
7242 
7243 	if (env->func_ptr_cnt) {
7244 		err = check_func_ptr_read(env, reg, off, size, value_regno);
7245 		if (err != -ENOENT)
7246 			return err;
7247 	}
7248 
7249 	/* If map is read-only, track its contents as scalars. */
7250 	if (tnum_is_const(reg->var_off) &&
7251 	    bpf_map_is_rdonly(map) &&
7252 	    map->ops->map_direct_value_addr) {
7253 		int map_off = off + reg->var_off.value;
7254 		u64 val = 0;
7255 
7256 		err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx);
7257 		if (err)
7258 			return err;
7259 
7260 		regs[value_regno].type = SCALAR_VALUE;
7261 		__mark_reg_known(&regs[value_regno], val);
7262 		return 0;
7263 	}
7264 
7265 reg_unknown:
7266 	mark_reg_unknown(env, regs, value_regno);
7267 	return 0;
7268 }
7269 
7270 /* check whether memory at (regno + off) is accessible for t = (read | write)
7271  * if t==write, value_regno is a register which value is stored into memory
7272  * if t==read, value_regno is a register which will receive the value from memory
7273  * if t==write && value_regno==-1, some unknown value is stored into memory
7274  * if t==read && value_regno==-1, don't care what we read from memory
7275  */
7276 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno,
7277 			    int off, int bpf_size, enum bpf_access_type t,
7278 			    int value_regno, bool strict_alignment_once, bool is_ldsx)
7279 {
7280 	struct bpf_reg_state *regs = cur_regs(env);
7281 	enum bpf_reg_type ptr_type = reg->type;
7282 	int size, err = 0;
7283 
7284 	size = bpf_size_to_bytes(bpf_size);
7285 	if (size < 0)
7286 		return size;
7287 
7288 	err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
7289 	if (err)
7290 		return err;
7291 
7292 	if (reg->type == PTR_TO_MAP_KEY) {
7293 		if (t == BPF_WRITE) {
7294 			verbose(env, "write to change key %s not allowed\n",
7295 				reg_arg_name(env, argno));
7296 			return -EACCES;
7297 		}
7298 
7299 		err = check_mem_region_access(env, reg, argno, off, size,
7300 					      reg->map_ptr->key_size, false);
7301 		if (err)
7302 			return err;
7303 		if (value_regno >= 0)
7304 			mark_reg_unknown(env, regs, value_regno);
7305 	} else if (reg->type == PTR_TO_MAP_VALUE) {
7306 		struct btf_field *kptr_field = NULL;
7307 
7308 		if (t == BPF_WRITE && value_regno >= 0 &&
7309 		    is_pointer_value(env, value_regno)) {
7310 			verbose(env, "R%d leaks addr into map\n", value_regno);
7311 			return -EACCES;
7312 		}
7313 		err = check_map_access_type(env, reg, off, size, t);
7314 		if (err)
7315 			return err;
7316 		err = check_map_access(env, reg, argno, off, size, false, ACCESS_DIRECT);
7317 		if (err)
7318 			return err;
7319 		if (tnum_is_const(reg->var_off))
7320 			kptr_field = btf_record_find(reg->map_ptr->record,
7321 						     off + reg->var_off.value, BPF_KPTR | BPF_UPTR);
7322 		if (kptr_field) {
7323 			err = check_map_kptr_access(env, value_regno, insn_idx, kptr_field);
7324 		} else if (t == BPF_READ && value_regno >= 0) {
7325 			err = check_map_mem_read(env, reg, off, bpf_size, value_regno, is_ldsx);
7326 		}
7327 	} else if (base_type(reg->type) == PTR_TO_MEM) {
7328 		bool rdonly_mem = type_is_rdonly_mem(reg->type);
7329 		bool rdonly_untrusted = rdonly_mem && (reg->type & PTR_UNTRUSTED);
7330 
7331 		if (type_may_be_null(reg->type)) {
7332 			verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno),
7333 				reg_type_str(env, reg->type));
7334 			bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno),
7335 					       reg_arg_name(env, argno), reg,
7336 						      BPF_DIAG_DEREF_NULLABLE_PTR, 0);
7337 			return -EACCES;
7338 		}
7339 
7340 		if (t == BPF_WRITE && rdonly_mem) {
7341 			verbose(env, "%s cannot write into %s\n",
7342 				reg_arg_name(env, argno), reg_type_str(env, reg->type));
7343 			return -EACCES;
7344 		}
7345 
7346 		if (t == BPF_WRITE && value_regno >= 0 &&
7347 		    is_pointer_value(env, value_regno)) {
7348 			verbose(env, "R%d leaks addr into mem\n", value_regno);
7349 			return -EACCES;
7350 		}
7351 
7352 		if (rdonly_untrusted && !env->allow_ptr_leaks) {
7353 			verbose(env, "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
7354 				reg_type_str(env, reg->type));
7355 			bpf_diag_policy(env, insn_idx, "read from untrusted read-only memory",
7356 					"the access requires CAP_PERFMON",
7357 					"Load the program with CAP_PERFMON, or avoid dereferencing untrusted pointers.");
7358 			return -EPERM;
7359 		}
7360 
7361 		/*
7362 		 * Accesses to untrusted PTR_TO_MEM are done through probe
7363 		 * instructions, hence no need to check bounds in that case.
7364 		 */
7365 		if (!rdonly_untrusted)
7366 			err = check_mem_region_access(env, reg, argno, off, size,
7367 						      reg->mem_size, false);
7368 		if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem))
7369 			mark_reg_unknown(env, regs, value_regno);
7370 	} else if (reg->type == PTR_TO_CTX) {
7371 		struct bpf_insn_access_aux info = {
7372 			.reg_type = SCALAR_VALUE,
7373 			.is_ldsx = is_ldsx,
7374 			.log = &env->log,
7375 		};
7376 		struct bpf_retval_range range;
7377 
7378 		if (t == BPF_WRITE && value_regno >= 0 &&
7379 		    is_pointer_value(env, value_regno)) {
7380 			verbose(env, "R%d leaks addr into ctx\n", value_regno);
7381 			return -EACCES;
7382 		}
7383 
7384 		err = check_ctx_access(env, insn_idx, reg, argno, off, size, t, &info);
7385 		if (!err && t == BPF_READ && value_regno >= 0) {
7386 			/* ctx access returns either a scalar, or a
7387 			 * PTR_TO_PACKET[_META,_END]. In the latter
7388 			 * case, we know the offset is zero.
7389 			 */
7390 			if (info.reg_type == SCALAR_VALUE) {
7391 				if (info.is_retval && get_func_retval_range(env->prog, &range)) {
7392 					mark_reg_unknown(env, regs, value_regno);
7393 					err = __mark_reg_s32_range(env, regs, value_regno,
7394 								   range.minval, range.maxval);
7395 					if (err)
7396 						return err;
7397 				} else {
7398 					mark_reg_unknown(env, regs, value_regno);
7399 				}
7400 			} else {
7401 				mark_reg_known_zero(env, regs,
7402 						    value_regno);
7403 				if (base_type(info.reg_type) == PTR_TO_BTF_ID) {
7404 					regs[value_regno].btf = info.btf;
7405 					regs[value_regno].btf_id = info.btf_id;
7406 					regs[value_regno].id = info.ref_id;
7407 				} else if (base_type(info.reg_type) == PTR_TO_MEM) {
7408 					regs[value_regno].mem_size = info.mem_size;
7409 				}
7410 				if (type_may_be_null(info.reg_type) && !regs[value_regno].id)
7411 					regs[value_regno].id = ++env->id_gen;
7412 			}
7413 			regs[value_regno].type = info.reg_type;
7414 		}
7415 
7416 	} else if (reg->type == PTR_TO_STACK) {
7417 		/* Basic bounds checks. */
7418 		err = check_stack_access_within_bounds(env, reg, argno, off, size, t);
7419 		if (err)
7420 			return err;
7421 
7422 		if (t == BPF_READ)
7423 			err = check_stack_read(env, reg, argno, off, size,
7424 					       value_regno);
7425 		else
7426 			err = check_stack_write(env, reg, off, size,
7427 						value_regno, insn_idx);
7428 	} else if (reg_is_pkt_pointer(reg)) {
7429 		if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
7430 			verbose(env, "cannot write into packet\n");
7431 			return -EACCES;
7432 		}
7433 		if (t == BPF_WRITE && value_regno >= 0 &&
7434 		    is_pointer_value(env, value_regno)) {
7435 			verbose(env, "R%d leaks addr into packet\n",
7436 				value_regno);
7437 			return -EACCES;
7438 		}
7439 		err = check_packet_access(env, reg, argno, off, size, false);
7440 		if (!err && t == BPF_READ && value_regno >= 0)
7441 			mark_reg_unknown(env, regs, value_regno);
7442 	} else if (reg->type == PTR_TO_FLOW_KEYS) {
7443 		if (t == BPF_WRITE && value_regno >= 0 &&
7444 		    is_pointer_value(env, value_regno)) {
7445 			verbose(env, "R%d leaks addr into flow keys\n",
7446 				value_regno);
7447 			return -EACCES;
7448 		}
7449 
7450 		err = check_flow_keys_access(env, reg, argno, off, size);
7451 		if (!err && t == BPF_READ && value_regno >= 0)
7452 			mark_reg_unknown(env, regs, value_regno);
7453 	} else if (type_is_sk_pointer(reg->type)) {
7454 		if (t == BPF_WRITE) {
7455 			verbose(env, "%s cannot write into %s\n",
7456 				reg_arg_name(env, argno), reg_type_str(env, reg->type));
7457 			return -EACCES;
7458 		}
7459 		err = check_sock_access(env, insn_idx, reg, argno, off, size, t);
7460 		if (!err && value_regno >= 0)
7461 			mark_reg_unknown(env, regs, value_regno);
7462 	} else if (reg->type == PTR_TO_TP_BUFFER) {
7463 		err = check_tp_buffer_access(env, reg, argno, off, size);
7464 		if (!err && t == BPF_READ && value_regno >= 0)
7465 			mark_reg_unknown(env, regs, value_regno);
7466 	} else if (base_type(reg->type) == PTR_TO_BTF_ID &&
7467 		   !type_may_be_null(reg->type)) {
7468 		err = check_ptr_to_btf_access(env, regs, reg, argno, off, size, t,
7469 					      value_regno);
7470 	} else if (reg->type == CONST_PTR_TO_MAP) {
7471 		err = check_ptr_to_map_access(env, regs, reg, argno, off, size, t,
7472 					      value_regno);
7473 	} else if (base_type(reg->type) == PTR_TO_BUF &&
7474 		   !type_may_be_null(reg->type)) {
7475 		bool rdonly_mem = type_is_rdonly_mem(reg->type);
7476 		u32 *max_access;
7477 
7478 		if (rdonly_mem) {
7479 			if (t == BPF_WRITE) {
7480 				verbose(env, "%s cannot write into %s\n",
7481 					reg_arg_name(env, argno), reg_type_str(env, reg->type));
7482 				return -EACCES;
7483 			}
7484 			max_access = &env->prog->aux->max_rdonly_access;
7485 		} else {
7486 			max_access = &env->prog->aux->max_rdwr_access;
7487 		}
7488 
7489 		err = check_buffer_access(env, reg, argno, off, size, false,
7490 					  max_access, t == BPF_WRITE);
7491 
7492 		if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ))
7493 			mark_reg_unknown(env, regs, value_regno);
7494 	} else if (reg->type == PTR_TO_ARENA) {
7495 		if (t == BPF_READ && value_regno >= 0)
7496 			mark_reg_unknown(env, regs, value_regno);
7497 	} else {
7498 		enum bpf_diag_invalid_deref_kind kind = BPF_DIAG_DEREF_INVALID_PTR;
7499 
7500 		verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno),
7501 			reg_type_str(env, reg->type));
7502 		if (reg->type == SCALAR_VALUE)
7503 			kind = BPF_DIAG_DEREF_SCALAR;
7504 		else if (type_may_be_null(reg->type))
7505 			kind = BPF_DIAG_DEREF_NULLABLE_PTR;
7506 		bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno),
7507 				       reg_arg_name(env, argno), reg, kind, 0);
7508 		return -EACCES;
7509 	}
7510 
7511 	if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
7512 	    regs[value_regno].type == SCALAR_VALUE) {
7513 		if (!is_ldsx) {
7514 			/* b/h/w load zero-extends, mark upper bits as known 0 */
7515 			coerce_reg_to_size(&regs[value_regno], size);
7516 		} else {
7517 			/*
7518 			 * Sign-extension can change the register value relative
7519 			 * to a scalar it is linked with by id (e.g. a zero-
7520 			 * extending fill of the same spilled stack slot), thus
7521 			 * drop the shared id in that case.
7522 			 */
7523 			bool no_sext = reg_umax(&regs[value_regno]) <
7524 					(1ULL << (size * BITS_PER_BYTE - 1));
7525 
7526 			coerce_reg_to_size_sx(&regs[value_regno], size);
7527 			if (!no_sext)
7528 				clear_scalar_id(&regs[value_regno]);
7529 		}
7530 	}
7531 
7532 	if (!err && bpf_is_mem_insn(&env->prog->insnsi[insn_idx]) && ptr_type != PTR_TO_STACK)
7533 		mark_non_stack_access(env, insn_idx);
7534 
7535 	return err;
7536 }
7537 
7538 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
7539 			     bool allow_trust_mismatch);
7540 
7541 static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn,
7542 			  bool strict_alignment_once, bool is_ldsx,
7543 			  bool allow_trust_mismatch, const char *ctx)
7544 {
7545 	struct bpf_verifier_state *vstate = env->cur_state;
7546 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
7547 	struct bpf_reg_state *regs = cur_regs(env);
7548 	enum bpf_reg_type src_reg_type;
7549 	int err;
7550 
7551 	/* Handle stack arg read */
7552 	if (is_stack_arg_ldx(insn)) {
7553 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
7554 		if (err)
7555 			return err;
7556 		return check_stack_arg_read(env, state, insn->off, insn->dst_reg);
7557 	}
7558 
7559 	/* check src operand */
7560 	err = check_reg_arg(env, insn->src_reg, SRC_OP);
7561 	if (err)
7562 		return err;
7563 
7564 	/* check dst operand */
7565 	err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
7566 	if (err)
7567 		return err;
7568 
7569 	src_reg_type = regs[insn->src_reg].type;
7570 
7571 	/*
7572 	 * check_stack_read_fixed_off() may refine the modification's origin to
7573 	 * the source stack slot.
7574 	 */
7575 	bpf_diag_mod_begin(env, &regs[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE);
7576 	err = check_mem_access(env, env->insn_idx, regs + insn->src_reg,
7577 			       argno_from_reg(insn->src_reg), insn->off,
7578 			       BPF_SIZE(insn->code), BPF_READ, insn->dst_reg,
7579 			       strict_alignment_once, is_ldsx);
7580 	/*
7581 	 * Paths through a kernel struct pointer and through other pointers
7582 	 * may share the insn, see save_aux_ptr_type(). Record a load that did
7583 	 * not produce an arena pointer from an __arena member, the only load
7584 	 * through a kernel struct pointer that does, next to the one
7585 	 * check_arena_member_load() records, so that arena_member_fixup() can
7586 	 * tell when the paths disagree.
7587 	 */
7588 	if (!err && !(base_type(src_reg_type) == PTR_TO_BTF_ID &&
7589 		      regs[insn->dst_reg].type == PTR_TO_ARENA))
7590 		env->insn_aux_data[env->insn_idx].non_arena_load = true;
7591 	err = err ?: save_aux_ptr_type(env, src_reg_type,
7592 				       allow_trust_mismatch);
7593 	err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], ctx);
7594 	if (!err)
7595 		bpf_diag_mod_end(env);
7596 
7597 	return err;
7598 }
7599 
7600 static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn,
7601 			   bool strict_alignment_once)
7602 {
7603 	struct bpf_verifier_state *vstate = env->cur_state;
7604 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
7605 	struct bpf_reg_state *regs = cur_regs(env);
7606 	enum bpf_reg_type dst_reg_type;
7607 	int err;
7608 
7609 	/* Handle stack arg write */
7610 	if (is_stack_arg_stx(insn)) {
7611 		err = check_reg_arg(env, insn->src_reg, SRC_OP);
7612 		if (err)
7613 			return err;
7614 		return check_stack_arg_write(env, state, insn->off, regs + insn->src_reg);
7615 	}
7616 
7617 	/* check src1 operand */
7618 	err = check_reg_arg(env, insn->src_reg, SRC_OP);
7619 	if (err)
7620 		return err;
7621 
7622 	/* check src2 operand */
7623 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
7624 	if (err)
7625 		return err;
7626 
7627 	dst_reg_type = regs[insn->dst_reg].type;
7628 
7629 	/* Check if (dst_reg + off) is writeable. */
7630 	err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg,
7631 			       argno_from_reg(insn->dst_reg), insn->off,
7632 			       BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg,
7633 			       strict_alignment_once, false);
7634 	err = err ?: save_aux_ptr_type(env, dst_reg_type, false);
7635 
7636 	return err;
7637 }
7638 
7639 static int check_atomic_rmw(struct bpf_verifier_env *env,
7640 			    struct bpf_insn *insn)
7641 {
7642 	struct bpf_reg_state *dst_reg;
7643 	int load_reg;
7644 	int err;
7645 
7646 	if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) {
7647 		verbose(env, "invalid atomic operand size\n");
7648 		return -EINVAL;
7649 	}
7650 
7651 	/* check src1 operand */
7652 	err = check_reg_arg(env, insn->src_reg, SRC_OP);
7653 	if (err)
7654 		return err;
7655 
7656 	/* check src2 operand */
7657 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
7658 	if (err)
7659 		return err;
7660 
7661 	if (insn->imm == BPF_CMPXCHG) {
7662 		/* Check comparison of R0 with memory location */
7663 		const u32 aux_reg = BPF_REG_0;
7664 
7665 		err = check_reg_arg(env, aux_reg, SRC_OP);
7666 		if (err)
7667 			return err;
7668 
7669 		if (is_pointer_value(env, aux_reg)) {
7670 			verbose(env, "R%d leaks addr into mem\n", aux_reg);
7671 			return -EACCES;
7672 		}
7673 	}
7674 
7675 	if (is_pointer_value(env, insn->src_reg)) {
7676 		verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
7677 		return -EACCES;
7678 	}
7679 
7680 	if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {
7681 		verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
7682 			insn->dst_reg,
7683 			reg_type_str(env, reg_state(env, insn->dst_reg)->type));
7684 		return -EACCES;
7685 	}
7686 
7687 	load_reg = bpf_atomic_load_reg(insn);
7688 	if (load_reg >= 0) {
7689 		/* check and record load of old value */
7690 		err = check_reg_arg(env, load_reg, DST_OP);
7691 		if (err)
7692 			return err;
7693 	}
7694 
7695 	dst_reg = cur_regs(env) + insn->dst_reg;
7696 
7697 	/* Check whether we can read the memory, with second call for fetch
7698 	 * case to simulate the register fill.
7699 	 */
7700 	err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off,
7701 			       BPF_SIZE(insn->code), BPF_READ, -1, true, false);
7702 	if (!err && load_reg >= 0) {
7703 		bpf_diag_mod_begin(env, cur_regs(env) + load_reg, NULL, BPF_DIAG_MOD_WRITE);
7704 		err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg),
7705 				       insn->off, BPF_SIZE(insn->code),
7706 				       BPF_READ, load_reg, true, false);
7707 		if (!err)
7708 			bpf_diag_mod_end(env);
7709 	}
7710 	if (err)
7711 		return err;
7712 
7713 	err = save_aux_ptr_type(env, dst_reg->type, false);
7714 	if (err)
7715 		return err;
7716 	/* Check whether we can write into the same memory. */
7717 	err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off,
7718 			       BPF_SIZE(insn->code), BPF_WRITE, -1, true, false);
7719 	if (err)
7720 		return err;
7721 	return 0;
7722 }
7723 
7724 static int check_atomic_load(struct bpf_verifier_env *env,
7725 			     struct bpf_insn *insn)
7726 {
7727 	int err;
7728 
7729 	err = check_reg_arg(env, insn->src_reg, SRC_OP);
7730 	if (err)
7731 		return err;
7732 
7733 	if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) {
7734 		verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n",
7735 			insn->src_reg,
7736 			reg_type_str(env, reg_state(env, insn->src_reg)->type));
7737 		return -EACCES;
7738 	}
7739 
7740 	return check_load_mem(env, insn, true, false, false, "atomic_load");
7741 }
7742 
7743 static int check_atomic_store(struct bpf_verifier_env *env,
7744 			      struct bpf_insn *insn)
7745 {
7746 	int err;
7747 
7748 	err = check_store_reg(env, insn, true);
7749 	if (err)
7750 		return err;
7751 
7752 	if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {
7753 		verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
7754 			insn->dst_reg,
7755 			reg_type_str(env, reg_state(env, insn->dst_reg)->type));
7756 		return -EACCES;
7757 	}
7758 
7759 	return 0;
7760 }
7761 
7762 static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn)
7763 {
7764 	switch (insn->imm) {
7765 	case BPF_ADD:
7766 	case BPF_ADD | BPF_FETCH:
7767 	case BPF_AND:
7768 	case BPF_AND | BPF_FETCH:
7769 	case BPF_OR:
7770 	case BPF_OR | BPF_FETCH:
7771 	case BPF_XOR:
7772 	case BPF_XOR | BPF_FETCH:
7773 	case BPF_XCHG:
7774 	case BPF_CMPXCHG:
7775 		return check_atomic_rmw(env, insn);
7776 	case BPF_LOAD_ACQ:
7777 		if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
7778 			verbose(env,
7779 				"64-bit load-acquires are only supported on 64-bit arches\n");
7780 			return -EOPNOTSUPP;
7781 		}
7782 		return check_atomic_load(env, insn);
7783 	case BPF_STORE_REL:
7784 		if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
7785 			verbose(env,
7786 				"64-bit store-releases are only supported on 64-bit arches\n");
7787 			return -EOPNOTSUPP;
7788 		}
7789 		return check_atomic_store(env, insn);
7790 	default:
7791 		verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n",
7792 			insn->imm);
7793 		return -EINVAL;
7794 	}
7795 }
7796 
7797 /* When register 'regno' is used to read the stack (either directly or through
7798  * a helper function) make sure that it's within stack boundary and, depending
7799  * on the access type and privileges, that all elements of the stack are
7800  * initialized.
7801  *
7802  * All registers that have been spilled on the stack in the slots within the
7803  * read offsets are marked as read.
7804  */
7805 static int check_stack_range_initialized(
7806 		struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off,
7807 		int access_size, bool zero_size_allowed,
7808 		enum bpf_access_type type, struct bpf_call_arg_meta *meta)
7809 {
7810 	struct bpf_func_state *state = bpf_func(env, reg);
7811 	int err, min_off, max_off, i, j, slot, spi;
7812 	/* Some accesses can write anything into the stack, others are
7813 	 * read-only.
7814 	 */
7815 	bool clobber = type == BPF_WRITE;
7816 	/*
7817 	 * Negative access_size signals global subprog arg check where
7818 	 * STACK_POISON slots are acceptable. static stack liveness
7819 	 * might have determined that subprog doesn't read them,
7820 	 * but BTF based global subprog validation isn't accurate enough.
7821 	 */
7822 	bool allow_poison = access_size < 0 || clobber;
7823 	u32 arg_slot = arg_slot_from_argno(argno);
7824 	bool uninit = clobber && meta && arg_slot < MAX_BPF_FUNC_ARGS &&
7825 		      (meta->arg_raw_mem.mask & BIT(arg_slot));
7826 	bool raw_mode = uninit && env->allow_uninit_stack &&
7827 			!(meta->arg_raw_mem.var_size_mask & BIT(arg_slot));
7828 
7829 	access_size = abs(access_size);
7830 
7831 	if (access_size == 0 && !zero_size_allowed) {
7832 		verbose(env, "invalid zero-sized read\n");
7833 		return -EACCES;
7834 	}
7835 
7836 	err = check_stack_access_within_bounds(env, reg, argno, off, access_size, type);
7837 	if (err)
7838 		return err;
7839 
7840 	if (tnum_is_const(reg->var_off)) {
7841 		min_off = max_off = reg->var_off.value + off;
7842 	} else {
7843 		/* Variable offset is prohibited for unprivileged mode for
7844 		 * simplicity since it requires corresponding support in
7845 		 * Spectre masking for stack ALU.
7846 		 * See also retrieve_ptr_limit().
7847 		 */
7848 		if (!env->bypass_spec_v1) {
7849 			char tn_buf[48];
7850 
7851 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
7852 			verbose(env, "%s variable offset stack access prohibited for !root, var_off=%s\n",
7853 				reg_arg_name(env, argno), tn_buf);
7854 			return -EACCES;
7855 		}
7856 		/*
7857 		 * The call may touch any byte in the possible range, but does not
7858 		 * definitely initialize all of it. Fall back to ordinary stack checks.
7859 		 */
7860 		raw_mode = false;
7861 
7862 		min_off = reg_smin(reg) + off;
7863 		max_off = reg_smax(reg) + off;
7864 	}
7865 
7866 	/* Unprivileged outputs retain each byte's initialization state. */
7867 	if (raw_mode) {
7868 		meta->arg_raw_mem.size[arg_slot] = access_size;
7869 		return 0;
7870 	}
7871 
7872 	for (i = min_off; i < max_off + access_size; i++) {
7873 		struct bpf_stack_state *ss;
7874 		u8 *stype;
7875 
7876 		slot = -i - 1;
7877 		spi = slot / BPF_REG_SIZE;
7878 		if (state->allocated_stack <= slot) {
7879 			verbose(env, "allocated_stack too small\n");
7880 			return -EFAULT;
7881 		}
7882 
7883 		ss = bpf_stack_slot(state, spi);
7884 		stype = &ss->slot_type[slot % BPF_REG_SIZE];
7885 		if (*stype == STACK_MISC)
7886 			goto mark;
7887 		if ((*stype == STACK_ZERO) ||
7888 		    (*stype == STACK_INVALID && (uninit || env->allow_uninit_stack))) {
7889 			if (clobber && (*stype != STACK_INVALID || env->allow_uninit_stack)) {
7890 				/* helper can write anything into the stack */
7891 				*stype = STACK_MISC;
7892 			}
7893 			goto mark;
7894 		}
7895 
7896 		/*
7897 		 * Only the bytes marked STACK_SPILL hold the spilled register.
7898 		 * The rest of a narrowly spilled slot keeps its previous type
7899 		 * and must be initialized on its own.
7900 		 */
7901 		if (*stype == STACK_SPILL &&
7902 		    (ss->spilled_ptr.type == SCALAR_VALUE ||
7903 		     env->allow_ptr_leaks)) {
7904 			if (clobber) {
7905 				__mark_reg_unknown(env, &ss->spilled_ptr);
7906 				for (j = 0; j < BPF_REG_SIZE; j++)
7907 					scrub_spilled_slot(&ss->slot_type[j]);
7908 			}
7909 			goto mark;
7910 		}
7911 
7912 		if (*stype == STACK_POISON) {
7913 			if (allow_poison) {
7914 				if (uninit && env->allow_uninit_stack)
7915 					*stype = STACK_MISC;
7916 				goto mark;
7917 			}
7918 			verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n",
7919 				reg_arg_name(env, argno), min_off, i - min_off, access_size);
7920 		} else if (tnum_is_const(reg->var_off)) {
7921 			verbose(env, "invalid read from stack %s off %d+%d size %d\n",
7922 				reg_arg_name(env, argno), min_off, i - min_off, access_size);
7923 		} else {
7924 			char tn_buf[48];
7925 
7926 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
7927 			verbose(env, "invalid read from stack %s var_off %s+%d size %d\n",
7928 				reg_arg_name(env, argno), tn_buf, i - min_off, access_size);
7929 		}
7930 		return -EACCES;
7931 mark:
7932 		;
7933 	}
7934 	return 0;
7935 }
7936 
7937 static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
7938 				   argno_t argno, int access_size,
7939 				   enum bpf_access_type access_type, bool zero_size_allowed,
7940 				   struct bpf_call_arg_meta *meta, bool *known_memory)
7941 {
7942 	struct bpf_reg_state *regs = cur_regs(env);
7943 	u32 *max_access;
7944 
7945 	if (known_memory)
7946 		*known_memory = true;
7947 
7948 	switch (base_type(reg->type)) {
7949 	case PTR_TO_PACKET:
7950 	case PTR_TO_PACKET_META:
7951 		if (!may_access_direct_pkt_data(env, meta, access_type)) {
7952 			verbose(env, "function access to the packet is not allowed\n");
7953 			return -EACCES;
7954 		}
7955 		return check_packet_access(env, reg, argno, 0, access_size,
7956 					   zero_size_allowed);
7957 	case PTR_TO_MAP_KEY:
7958 		if (access_type == BPF_WRITE) {
7959 			verbose(env, "%s cannot write into %s\n",
7960 				reg_arg_name(env, argno), reg_type_str(env, reg->type));
7961 			return -EACCES;
7962 		}
7963 		return check_mem_region_access(env, reg, argno, 0, access_size,
7964 					       reg->map_ptr->key_size, false);
7965 	case PTR_TO_MAP_VALUE:
7966 		if (check_map_access_type(env, reg, 0, access_size, access_type))
7967 			return -EACCES;
7968 		return check_map_access(env, reg, argno, 0, access_size,
7969 					zero_size_allowed, ACCESS_HELPER);
7970 	case PTR_TO_MEM:
7971 		if (type_is_rdonly_mem(reg->type)) {
7972 			if (access_type == BPF_WRITE) {
7973 				verbose(env, "%s cannot write into %s\n",
7974 					reg_arg_name(env, argno), reg_type_str(env, reg->type));
7975 				return -EACCES;
7976 			}
7977 		}
7978 		return check_mem_region_access(env, reg, argno, 0,
7979 					       access_size, reg->mem_size,
7980 					       zero_size_allowed);
7981 	case PTR_TO_BUF:
7982 		if (type_is_rdonly_mem(reg->type)) {
7983 			if (access_type == BPF_WRITE) {
7984 				verbose(env, "%s cannot write into %s\n",
7985 					reg_arg_name(env, argno), reg_type_str(env, reg->type));
7986 				return -EACCES;
7987 			}
7988 
7989 			max_access = &env->prog->aux->max_rdonly_access;
7990 		} else {
7991 			max_access = &env->prog->aux->max_rdwr_access;
7992 		}
7993 		return check_buffer_access(env, reg, argno, 0,
7994 					   access_size, zero_size_allowed,
7995 					   max_access, access_type == BPF_WRITE);
7996 	case PTR_TO_STACK:
7997 		return check_stack_range_initialized(
7998 				env, reg,
7999 				argno, 0, access_size,
8000 				zero_size_allowed, access_type, meta);
8001 	case PTR_TO_BTF_ID:
8002 		return check_ptr_to_btf_access(env, regs, reg, argno, 0,
8003 					       access_size, access_type, -1);
8004 	case PTR_TO_CTX:
8005 		/* Only permit reading or writing syscall context using helper calls. */
8006 		if (is_var_ctx_off_allowed(env->prog)) {
8007 			int err = check_mem_region_access(env, reg, argno, 0, access_size, U16_MAX,
8008 							  zero_size_allowed);
8009 			if (err)
8010 				return err;
8011 			if (env->prog->aux->max_ctx_offset < reg_umax(reg) + access_size)
8012 				env->prog->aux->max_ctx_offset = reg_umax(reg) + access_size;
8013 			return 0;
8014 		}
8015 		fallthrough;
8016 	default: /* scalar_value or invalid ptr */
8017 		/* Allow zero-byte read from NULL, regardless of pointer type */
8018 		if (zero_size_allowed && access_size == 0 &&
8019 		    bpf_register_is_null(reg))
8020 			return 0;
8021 		if (known_memory && base_type(reg->type) != PTR_TO_CTX)
8022 			*known_memory = false;
8023 
8024 		verbose(env, "%s type=%s ", reg_arg_name(env, argno),
8025 			reg_type_str(env, reg->type));
8026 		verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
8027 		return -EACCES;
8028 	}
8029 }
8030 
8031 enum bpf_mem_size_failure {
8032 	BPF_MEM_SIZE_FAIL_NONE,
8033 	BPF_MEM_SIZE_FAIL_MEMORY,
8034 	BPF_MEM_SIZE_FAIL_SIZE,
8035 };
8036 
8037 /* verify arguments to helpers or kfuncs consisting of a pointer and an access
8038  * size.
8039  *
8040  * @mem_reg contains the pointer, @size_reg contains the access size.
8041  */
8042 static int check_mem_size_reg(struct bpf_verifier_env *env,
8043 			      struct bpf_reg_state *mem_reg,
8044 			      struct bpf_reg_state *size_reg, argno_t mem_argno,
8045 			      argno_t size_argno, u32 access_type,
8046 			      bool zero_size_allowed,
8047 			      struct bpf_call_arg_meta *meta,
8048 			      enum bpf_mem_size_failure *failure)
8049 {
8050 	int err = 0;
8051 
8052 	if (failure)
8053 		*failure = BPF_MEM_SIZE_FAIL_NONE;
8054 
8055 	/* This is used to refine r0 return value bounds for helpers
8056 	 * that enforce this value as an upper bound on return values.
8057 	 * See do_refine_retval_range() for helpers that can refine
8058 	 * the return value. C type of helper is u32 so we pull register
8059 	 * bound from umax_value however, if negative verifier errors
8060 	 * out. Only upper bounds can be learned because retval is an
8061 	 * int type and negative retvals are allowed.
8062 	 */
8063 	meta->msize_max_value = reg_umax(size_reg);
8064 
8065 	/*
8066 	 * Check variable ranges byte by byte instead of using raw mode. Keep the
8067 	 * MEM_UNINIT annotation so invalid bytes are accepted without marking them
8068 	 * initialized when the caller cannot read uninitialized stack memory.
8069 	 */
8070 	if (!tnum_is_const(size_reg->var_off))
8071 		meta->arg_raw_mem.var_size_mask |= BIT(arg_slot_from_argno(mem_argno));
8072 
8073 	if (reg_smin(size_reg) < 0) {
8074 		verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n",
8075 			reg_arg_name(env, size_argno));
8076 		err = -EACCES;
8077 		goto size_error;
8078 	}
8079 
8080 	if (reg_umin(size_reg) == 0 && !zero_size_allowed) {
8081 		verbose(env, "%s invalid zero-sized read: u64=[%lld,%lld]\n",
8082 			reg_arg_name(env, size_argno), reg_umin(size_reg), reg_umax(size_reg));
8083 		err = -EACCES;
8084 		goto size_error;
8085 	}
8086 
8087 	if (reg_umax(size_reg) >= BPF_MAX_VAR_SIZ) {
8088 		verbose(env, "%s unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
8089 			reg_arg_name(env, size_argno));
8090 		err = -EACCES;
8091 		goto size_error;
8092 	}
8093 
8094 	if (access_type & BPF_READ)
8095 		err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg),
8096 					      BPF_READ, zero_size_allowed, meta, NULL);
8097 	if (!err && access_type & BPF_WRITE)
8098 		err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg),
8099 					      BPF_WRITE, zero_size_allowed, meta, NULL);
8100 	if (err && failure)
8101 		*failure = BPF_MEM_SIZE_FAIL_MEMORY;
8102 
8103 	if (!err)
8104 		err = mark_arg_precision(env, size_argno);
8105 
8106 	return err;
8107 
8108 size_error:
8109 	if (failure)
8110 		*failure = BPF_MEM_SIZE_FAIL_SIZE;
8111 	return err;
8112 }
8113 
8114 static bool is_subprog(const struct bpf_call_arg_meta *meta)
8115 {
8116 	return meta->btf && !meta->func_id;
8117 }
8118 
8119 static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
8120 			 argno_t argno, u32 mem_size, enum bpf_access_type access_type,
8121 			 struct bpf_call_arg_meta *meta, bool *known_memory)
8122 {
8123 	int size, err = 0;
8124 
8125 	if (bpf_register_is_null(reg))
8126 		return mark_arg_precision(env, argno);
8127 	if (known_memory)
8128 		*known_memory = true;
8129 
8130 	if (mem_size > S32_MAX) {
8131 		verbose(env, "%s memory size %u is too large\n",
8132 			reg_arg_name(env, argno), mem_size);
8133 		return -EACCES;
8134 	}
8135 
8136 	/*
8137 	 * Only a global subprog may read poisoned stack slots:
8138 	 * its static stack liveness proved the callee body skips them.
8139 	 */
8140 	size = (is_subprog(meta) &&
8141 		base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size;
8142 
8143 	if (access_type & BPF_READ)
8144 		err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta,
8145 					      known_memory);
8146 	if (!err && (access_type & BPF_WRITE))
8147 		err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta,
8148 					      known_memory);
8149 
8150 	return err;
8151 }
8152 
8153 static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
8154 					argno_t argno, struct ret_mem_desc *ret_mem)
8155 {
8156 	int regno = reg_from_argno(argno);
8157 	int err;
8158 
8159 	if (ret_mem->found) {
8160 		verifier_bug(env, "only one allocation size argument permitted");
8161 		return -EFAULT;
8162 	}
8163 
8164 	if (!tnum_is_const(reg->var_off)) {
8165 		verbose(env, "%s is not a const\n", reg_arg_name(env, argno));
8166 		return -EINVAL;
8167 	}
8168 
8169 	if (reg->var_off.value > U32_MAX) {
8170 		verbose(env, "%s allocation size exceeds u32 max\n", reg_arg_name(env, argno));
8171 		return -EINVAL;
8172 	}
8173 
8174 	if (regno >= 0)
8175 		err = mark_chain_precision(env, regno);
8176 	else
8177 		err = mark_stack_arg_precision(env, arg_idx_from_argno(argno));
8178 	if (err)
8179 		return err;
8180 
8181 	ret_mem->size = reg->var_off.value;
8182 	ret_mem->found = true;
8183 
8184 	return 0;
8185 }
8186 
8187 static int process_const_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
8188 			     argno_t argno, struct bpf_call_arg_meta *meta)
8189 {
8190 	int regno = reg_from_argno(argno);
8191 	int err;
8192 
8193 	if (meta->arg_constant.found) {
8194 		verifier_bug(env, "only one constant argument permitted");
8195 		return -EFAULT;
8196 	}
8197 
8198 	if (!tnum_is_const(reg->var_off)) {
8199 		verbose(env, "%s must be a known constant\n", reg_arg_name(env, argno));
8200 		return -EINVAL;
8201 	}
8202 
8203 	if (regno >= 0)
8204 		err = mark_chain_precision(env, regno);
8205 	else
8206 		err = mark_stack_arg_precision(env, arg_idx_from_argno(argno));
8207 	if (err < 0)
8208 		return err;
8209 
8210 	meta->arg_constant.found = true;
8211 	meta->arg_constant.value = reg->var_off.value;
8212 
8213 	return 0;
8214 }
8215 
8216 enum {
8217 	PROCESS_SPIN_LOCK = (1 << 0),
8218 	PROCESS_RES_LOCK  = (1 << 1),
8219 	PROCESS_LOCK_IRQ  = (1 << 2),
8220 };
8221 
8222 /* Implementation details:
8223  * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL.
8224  * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL.
8225  * Two bpf_map_lookups (even with the same key) will have different reg->id.
8226  * Two separate bpf_obj_new will also have different reg->id.
8227  * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier
8228  * clears reg->id after value_or_null->value transition, since the verifier only
8229  * cares about the range of access to valid map value pointer and doesn't care
8230  * about actual address of the map element.
8231  * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps
8232  * reg->id > 0 after value_or_null->value transition. By doing so
8233  * two bpf_map_lookups will be considered two different pointers that
8234  * point to different bpf_spin_locks. Likewise for pointers to allocated objects
8235  * returned from bpf_obj_new.
8236  * The verifier allows taking only one bpf_spin_lock at a time to avoid
8237  * dead-locks.
8238  * Since only one bpf_spin_lock is allowed the checks are simpler than
8239  * reg_is_refcounted() logic. The verifier needs to remember only
8240  * one spin_lock instead of array of acquired_refs.
8241  * env->cur_state->active_locks remembers which map value element or allocated
8242  * object got locked and clears it after bpf_spin_unlock.
8243  */
8244 static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int flags)
8245 {
8246 	bool is_lock = flags & PROCESS_SPIN_LOCK, is_res_lock = flags & PROCESS_RES_LOCK;
8247 	const char *lock_str = is_res_lock ? "bpf_res_spin" : "bpf_spin";
8248 	struct bpf_verifier_state *cur = env->cur_state;
8249 	struct bpf_reference_state *lock;
8250 	bool is_const = tnum_is_const(reg->var_off);
8251 	bool is_irq = flags & PROCESS_LOCK_IRQ;
8252 	u64 val = reg->var_off.value;
8253 	struct bpf_map *map = NULL;
8254 	struct btf *btf = NULL;
8255 	struct btf_record *rec;
8256 	u32 spin_lock_off;
8257 	int err;
8258 
8259 	if (!is_const) {
8260 		verbose(env,
8261 			"%s doesn't have constant offset. %s_lock has to be at the constant offset\n",
8262 			reg_arg_name(env, argno), lock_str);
8263 		return -EINVAL;
8264 	}
8265 	if (reg->type == PTR_TO_MAP_VALUE) {
8266 		map = reg->map_ptr;
8267 		if (!map->btf) {
8268 			verbose(env,
8269 				"map '%s' has to have BTF in order to use %s_lock\n",
8270 				map->name, lock_str);
8271 			return -EINVAL;
8272 		}
8273 	} else {
8274 		btf = reg->btf;
8275 	}
8276 
8277 	rec = reg_btf_record(reg);
8278 	if (!btf_record_has_field(rec, is_res_lock ? BPF_RES_SPIN_LOCK : BPF_SPIN_LOCK)) {
8279 		verbose(env, "%s '%s' has no valid %s_lock\n", map ? "map" : "local",
8280 			map ? map->name : "kptr", lock_str);
8281 		return -EINVAL;
8282 	}
8283 	spin_lock_off = is_res_lock ? rec->res_spin_lock_off : rec->spin_lock_off;
8284 	if (spin_lock_off != val) {
8285 		verbose(env, "off %lld doesn't point to 'struct %s_lock' that is at %d\n",
8286 			val, lock_str, spin_lock_off);
8287 		return -EINVAL;
8288 	}
8289 	if (is_lock) {
8290 		void *ptr;
8291 		int type;
8292 
8293 		if (map)
8294 			ptr = map;
8295 		else
8296 			ptr = btf;
8297 
8298 		if (!is_res_lock && cur->active_locks) {
8299 			lock = find_lock_state(cur, REF_TYPE_LOCK, 0, NULL);
8300 			if (lock) {
8301 				verbose(env,
8302 					"Locking two bpf_spin_locks are not allowed\n");
8303 				bpf_diag_lock(
8304 					env, env->insn_idx, "nested spin lock",
8305 					"This path already holds a bpf_spin_lock. The verifier allows only one regular BPF spin lock at a time.",
8306 					"Unlock the current bpf_spin_lock before taking another one.", lock);
8307 				return -EINVAL;
8308 			}
8309 		} else if (is_res_lock && cur->active_locks) {
8310 			lock = find_lock_state(cur, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ,
8311 					       reg->id, ptr);
8312 			if (lock) {
8313 				verbose(env, "Acquiring the same lock again, AA deadlock detected\n");
8314 				bpf_diag_lock(
8315 					env, env->insn_idx, "recursive resource spin lock",
8316 					"This path already holds the same resource spin lock. Taking it again would deadlock.",
8317 					"Avoid reacquiring the same resource spin lock before it is unlocked.", lock);
8318 				return -EINVAL;
8319 			}
8320 		}
8321 
8322 		if (is_res_lock && is_irq)
8323 			type = REF_TYPE_RES_LOCK_IRQ;
8324 		else if (is_res_lock)
8325 			type = REF_TYPE_RES_LOCK;
8326 		else
8327 			type = REF_TYPE_LOCK;
8328 		err = acquire_lock_state(env, env->insn_idx, type, reg->id, ptr);
8329 		if (err < 0) {
8330 			verbose(env, "Failed to acquire lock state\n");
8331 			return err;
8332 		}
8333 	} else {
8334 		void *ptr;
8335 		int type;
8336 
8337 		if (map)
8338 			ptr = map;
8339 		else
8340 			ptr = btf;
8341 
8342 		if (!cur->active_locks) {
8343 			verbose(env, "%s_unlock without taking a lock\n", lock_str);
8344 			bpf_diag_res(
8345 				env, env->insn_idx, "unlock without lock",
8346 				"This unlock operation has no matching active lock on the current path.",
8347 				"Take the matching lock before this unlock, or remove the unmatched unlock path.");
8348 			return -EINVAL;
8349 		}
8350 
8351 		if (is_res_lock && is_irq)
8352 			type = REF_TYPE_RES_LOCK_IRQ;
8353 		else if (is_res_lock)
8354 			type = REF_TYPE_RES_LOCK;
8355 		else
8356 			type = REF_TYPE_LOCK;
8357 
8358 		lock = find_lock_state(cur, type, reg->id, ptr);
8359 		if (!lock) {
8360 			verbose(env, "%s_unlock of different lock\n", lock_str);
8361 			lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id,
8362 					       cur->active_lock_ptr);
8363 			bpf_diag_lock(
8364 				env, env->insn_idx, "unlock of a different lock",
8365 				"This unlock does not match any active lock with the same tracked identity on the current path.",
8366 				"Unlock the same lock object that was most recently acquired.", lock);
8367 			return -EINVAL;
8368 		}
8369 		if (reg->id != cur->active_lock_id || ptr != cur->active_lock_ptr) {
8370 			verbose(env, "%s_unlock cannot be out of order\n", lock_str);
8371 			lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id,
8372 					       cur->active_lock_ptr);
8373 			bpf_diag_lock(
8374 				env, env->insn_idx, "unlock out of order",
8375 				"Locks must be released in last-in, first-out order, but this unlock does not match the currently active lock.",
8376 				"Release nested locks in the reverse order they were acquired.", lock);
8377 			return -EINVAL;
8378 		}
8379 		if (release_lock_state(env, type, reg->id, ptr)) {
8380 			verbose(env, "%s_unlock of different lock\n", lock_str);
8381 			bpf_diag_lock(
8382 				env, env->insn_idx, "unlock of a different lock",
8383 				"The verifier could not release a lock state matching this unlock operation.",
8384 				"Pass the same lock object and lock kind that were used for the matching lock operation.",
8385 				lock);
8386 			return -EINVAL;
8387 		}
8388 		/*
8389 		 * Invalidate non-owning refs before RCU demotion clears their
8390 		 * NON_OWN_REF flag.
8391 		 */
8392 		invalidate_non_owning_refs(env);
8393 
8394 		if (!in_rcu_cs(env))
8395 			invalidate_rcu_protected_refs(env);
8396 	}
8397 	return 0;
8398 }
8399 
8400 /* Check if @regno is a pointer to a specific field in a map value */
8401 static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
8402 				   enum btf_field_type field_type,
8403 				   struct bpf_map_desc *map_desc)
8404 {
8405 	bool is_const = tnum_is_const(reg->var_off);
8406 	struct bpf_map *map = reg->map_ptr;
8407 	u64 val = reg->var_off.value;
8408 	const char *struct_name = btf_field_type_name(field_type);
8409 	int field_off = -1;
8410 
8411 	if (!is_const) {
8412 		verbose(env,
8413 			"%s doesn't have constant offset. %s has to be at the constant offset\n",
8414 			reg_arg_name(env, argno), struct_name);
8415 		return -EINVAL;
8416 	}
8417 	if (!map->btf) {
8418 		verbose(env, "map '%s' has to have BTF in order to use %s\n", map->name,
8419 			struct_name);
8420 		return -EINVAL;
8421 	}
8422 	if (!btf_record_has_field(map->record, field_type)) {
8423 		verbose(env, "map '%s' has no valid %s\n", map->name, struct_name);
8424 		return -EINVAL;
8425 	}
8426 	switch (field_type) {
8427 	case BPF_TIMER:
8428 		field_off = map->record->timer_off;
8429 		break;
8430 	case BPF_TASK_WORK:
8431 		field_off = map->record->task_work_off;
8432 		break;
8433 	case BPF_RCU_HEAD:
8434 		field_off = map->record->rcu_head_off;
8435 		break;
8436 	case BPF_WORKQUEUE:
8437 		field_off = map->record->wq_off;
8438 		break;
8439 	default:
8440 		verifier_bug(env, "unsupported BTF field type: %s\n", struct_name);
8441 		return -EINVAL;
8442 	}
8443 	if (field_off != val) {
8444 		verbose(env, "off %lld doesn't point to 'struct %s' that is at %d\n",
8445 			val, struct_name, field_off);
8446 		return -EINVAL;
8447 	}
8448 	if (map_desc->ptr) {
8449 		verifier_bug(env, "Two map pointers in a %s helper", struct_name);
8450 		return -EFAULT;
8451 	}
8452 	map_desc->uid = reg->map_uid;
8453 	map_desc->ptr = map;
8454 	return 0;
8455 }
8456 
8457 static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
8458 			      struct bpf_map_desc *map)
8459 {
8460 	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
8461 		verbose(env, "bpf_timer cannot be used for PREEMPT_RT.\n");
8462 		return -EOPNOTSUPP;
8463 	}
8464 	return check_map_field_pointer(env, reg, argno, BPF_TIMER, map);
8465 }
8466 
8467 static int process_kptr_func(struct bpf_verifier_env *env, int regno,
8468 			     struct bpf_call_arg_meta *meta)
8469 {
8470 	struct bpf_reg_state *reg = reg_state(env, regno);
8471 	struct btf_field *kptr_field;
8472 	struct bpf_map *map_ptr;
8473 	struct btf_record *rec;
8474 	u32 kptr_off;
8475 
8476 	if (type_is_ptr_alloc_obj(reg->type)) {
8477 		rec = reg_btf_record(reg);
8478 	} else { /* PTR_TO_MAP_VALUE */
8479 		map_ptr = reg->map_ptr;
8480 		if (!map_ptr->btf) {
8481 			verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n",
8482 				map_ptr->name);
8483 			return -EINVAL;
8484 		}
8485 		rec = map_ptr->record;
8486 		meta->map.ptr = map_ptr;
8487 	}
8488 
8489 	if (!tnum_is_const(reg->var_off)) {
8490 		verbose(env,
8491 			"R%d doesn't have constant offset. kptr has to be at the constant offset\n",
8492 			regno);
8493 		return -EINVAL;
8494 	}
8495 
8496 	if (!btf_record_has_field(rec, BPF_KPTR)) {
8497 		verbose(env, "R%d has no valid kptr\n", regno);
8498 		return -EINVAL;
8499 	}
8500 
8501 	kptr_off = reg->var_off.value;
8502 	kptr_field = btf_record_find(rec, kptr_off, BPF_KPTR);
8503 	if (!kptr_field) {
8504 		verbose(env, "off=%d doesn't point to kptr\n", kptr_off);
8505 		return -EACCES;
8506 	}
8507 	if (kptr_field->type != BPF_KPTR_REF && kptr_field->type != BPF_KPTR_PERCPU) {
8508 		verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off);
8509 		return -EACCES;
8510 	}
8511 	meta->kptr_field = kptr_field;
8512 	return 0;
8513 }
8514 
8515 static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno,
8516 			      const char *call_name, const char *reason, const char *suggestion);
8517 __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx,
8518 						 argno_t argno, const char *call_name,
8519 						 const char *suggestion, const char *fmt, ...);
8520 
8521 /*
8522  * Validate dynptr arguments for helper, kfunc and subprog.
8523  *
8524  * @meta carries the dynptr and referenced-object state. The dynptr is populated
8525  * when the argument is tagged with MEM_UNINIT (i.e., the dynptr argument that
8526  * will be constructed) and consumed when the argument is expected to be an
8527  * initialized dynptr. The reference tracks the parent object (e.g., file or skb
8528  * in qdisc program) when constructing a dynptr.
8529  *
8530  * There are two register types representing a bpf_dynptr, one is PTR_TO_STACK
8531  * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR.
8532  *
8533  * In both cases we deal with the first 8 bytes, but need to mark the next 8
8534  * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of
8535  * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object.
8536  *
8537  * Mutability of bpf_dynptr is at two levels: the dynptr and the memory the
8538  * dynptr points to. At the first level, the verifier will make sure a
8539  * CONST_PTR_TO_DYNPTR cannot be reinitialized or destroyed. The mutability of
8540  * a dynptr's view (i.e., start and offset) is not tracked as there is not such
8541  * use case. The second level is tracked using the upper bit of bpf_dynptr->size
8542  * and checked dynamically during runtime.
8543  */
8544 static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
8545 			       argno_t argno, int insn_idx, enum bpf_arg_type arg_type,
8546 			       struct bpf_call_arg_meta *meta)
8547 {
8548 	int spi, err = 0;
8549 
8550 	/*  MEM_UNINIT - Points to memory that is an appropriate candidate for
8551 	 *		 constructing a mutable bpf_dynptr object.
8552 	 *
8553 	 *		 Currently, this is only possible with PTR_TO_STACK
8554 	 *		 pointing to a region of at least 16 bytes which doesn't
8555 	 *		 contain an existing bpf_dynptr.
8556 	 *
8557 	 *  OBJ_RELEASE - Points to a initialized bpf_dynptr that will be
8558 	 *		  destroyed.
8559 	 *
8560 	 *  None       - Points to a initialized dynptr that cannot be
8561 	 *		 reinitialized or destroyed. However, the view of the
8562 	 *		 dynptr and the memory it points to may be mutated.
8563 	 */
8564 	if (arg_type & MEM_UNINIT) {
8565 		int i;
8566 
8567 		if (!is_dynptr_reg_valid_uninit(env, reg)) {
8568 			verbose(env, "Dynptr has to be an uninitialized dynptr\n");
8569 			bpf_diag_res(
8570 				env, insn_idx, "dynptr is already initialized",
8571 				"This function constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.",
8572 				"Use a fresh stack dynptr slot, or release/destroy the existing dynptr before reusing the slot.");
8573 			return -EINVAL;
8574 		}
8575 
8576 		/* we write BPF_DW bits (8 bytes) at a time */
8577 		for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) {
8578 			err = check_mem_access(env, insn_idx, reg, argno,
8579 					       i, BPF_DW, BPF_WRITE, -1, false, false);
8580 			if (err)
8581 				return err;
8582 		}
8583 
8584 		err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx,
8585 					      &meta->ref_obj, &meta->dynptr);
8586 	} else /* OBJ_RELEASE and None case from above */ {
8587 		/* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */
8588 		if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) {
8589 			verbose(env, "CONST_PTR_TO_DYNPTR cannot be released\n");
8590 			bpf_diag_res(
8591 				env, insn_idx, "const dynptr release",
8592 				"This release operation was given a const dynptr. Const dynptr values are verifier-provided views and cannot be released by the program.",
8593 				"Release only mutable dynptrs that the program initialized or reserved.");
8594 			return -EINVAL;
8595 		}
8596 
8597 		if (!is_dynptr_reg_valid_init(env, reg)) {
8598 			verbose(env, "Expected an initialized dynptr as %s\n",
8599 				reg_arg_name(env, argno));
8600 			bpf_diag_res(
8601 				env, insn_idx, "uninitialized dynptr use",
8602 				"This operation requires an initialized dynptr, but the stack slot does not currently hold a valid dynptr on this path.",
8603 				"Initialize the dynptr on every path before this call, and avoid overwriting or releasing it before this use.");
8604 			return -EINVAL;
8605 		}
8606 
8607 		/* Fold modifiers (in this case, OBJ_RELEASE) when checking expected type */
8608 		if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) {
8609 			enum bpf_dynptr_type expected_type = arg_to_dynptr_type(arg_type);
8610 			enum bpf_dynptr_type actual_type = dynptr_reg_type(env, reg);
8611 
8612 			verbose(env, "Expected a dynptr of type %s as %s\n",
8613 				dynptr_type_str(expected_type), reg_arg_name(env, argno));
8614 			bpf_diag_call_arg_fmt(
8615 				env, insn_idx, argno, meta->func_name,
8616 				"Use a dynptr constructor that matches this operation, or call an operation that accepts the dynptr's current type.",
8617 				"the dynptr is initialized with backing object type %s, but this operation expects dynptr type %s",
8618 				dynptr_type_str(actual_type), dynptr_type_str(expected_type));
8619 			return -EINVAL;
8620 		}
8621 
8622 		if (reg->type != CONST_PTR_TO_DYNPTR) {
8623 			struct bpf_func_state *state = bpf_func(env, reg);
8624 
8625 			spi = dynptr_get_spi(env, reg);
8626 			if (spi < 0)
8627 				return spi;
8628 
8629 			mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS);
8630 
8631 			reg = &bpf_stack_slot(state, spi)->spilled_ptr;
8632 		}
8633 
8634 		meta->dynptr.type = reg->dynptr.type;
8635 		meta->dynptr.id = reg->id;
8636 		meta->dynptr.parent_id = reg->parent_id;
8637 	}
8638 	return err;
8639 }
8640 
8641 static bool is_iter_kfunc(struct bpf_call_arg_meta *meta)
8642 {
8643 	return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY);
8644 }
8645 
8646 static bool is_iter_new_kfunc(struct bpf_call_arg_meta *meta)
8647 {
8648 	return meta->kfunc_flags & KF_ITER_NEW;
8649 }
8650 
8651 static bool is_iter_destroy_kfunc(struct bpf_call_arg_meta *meta)
8652 {
8653 	return meta->kfunc_flags & KF_ITER_DESTROY;
8654 }
8655 
8656 static bool iter_next_borrows_struct(struct bpf_call_arg_meta *meta)
8657 {
8658 	const struct btf_type *t;
8659 
8660 	if (!(meta->kfunc_flags & KF_ITER_NEXT) ||
8661 	    (meta->kfunc_flags & (KF_ITER_RET_INDEPENDENT | KF_ACQUIRE)))
8662 		return false;
8663 
8664 	t = btf_type_skip_modifiers(meta->btf, meta->func_proto->type, NULL);
8665 	if (!t || !btf_type_is_ptr(t))
8666 		return false;
8667 	t = btf_type_skip_modifiers(meta->btf, t->type, NULL);
8668 	return t && __btf_type_is_struct(t);
8669 }
8670 
8671 static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx,
8672 			      const struct btf_param *arg)
8673 {
8674 	/* btf_check_iter_kfuncs() guarantees that first argument of any iter
8675 	 * kfunc is iter state pointer
8676 	 */
8677 	if (is_iter_kfunc(meta))
8678 		return arg_idx == 0;
8679 
8680 	/* iter passed as an argument to a generic kfunc */
8681 	return btf_param_match_suffix(meta->btf, arg, "__iter");
8682 }
8683 
8684 static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
8685 			    u32 arg, argno_t argno, int insn_idx,
8686 			    struct bpf_call_arg_meta *meta)
8687 {
8688 	struct bpf_func_state *state = bpf_func(env, reg);
8689 	const struct btf_type *t;
8690 	int spi, err, i, nr_slots, btf_id;
8691 
8692 	if (reg->type != PTR_TO_STACK) {
8693 		verbose(env, "%s expected pointer to an iterator on stack\n",
8694 			reg_arg_name(env, argno));
8695 		bpf_diag_call_arg_fmt(
8696 			env, insn_idx, argno, meta->func_name,
8697 			"Pass the address of a stack iterator object for iterator new, next, and destroy calls.",
8698 			"iterator state must live in verifier-tracked stack memory, but %s is %s",
8699 			reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type));
8700 		return -EINVAL;
8701 	}
8702 
8703 	/* For iter_{new,next,destroy} functions, btf_check_iter_kfuncs()
8704 	 * ensures struct convention, so we wouldn't need to do any BTF
8705 	 * validation here. But given iter state can be passed as a parameter
8706 	 * to any kfunc, if arg has "__iter" suffix, we need to be a bit more
8707 	 * conservative here.
8708 	 */
8709 	btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg);
8710 	if (btf_id < 0) {
8711 		verbose(env, "expected valid iter pointer as %s\n",
8712 			reg_arg_name(env, argno));
8713 		bpf_diag_call_arg(
8714 			env, insn_idx, argno, meta->func_name,
8715 			"the function expects a recognized iterator state pointer, but this argument does not match a valid iterator type",
8716 			"Pass the exact iterator state type expected by this function.");
8717 		return -EINVAL;
8718 	}
8719 	t = btf_type_by_id(meta->btf, btf_id);
8720 	nr_slots = t->size / BPF_REG_SIZE;
8721 
8722 	if (is_iter_new_kfunc(meta)) {
8723 		/* bpf_iter_<type>_new() expects pointer to uninit iter state */
8724 		if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) {
8725 			verbose(env, "expected uninitialized iter_%s as %s\n",
8726 				iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno));
8727 			bpf_diag_res(
8728 				env, insn_idx, "iterator is already initialized",
8729 				"Iterator creation requires an uninitialized iterator stack object, but this stack range already contains iterator state.",
8730 				"Use a fresh iterator stack slot, or destroy the existing iterator before reusing the slot.");
8731 			return -EINVAL;
8732 		}
8733 
8734 		for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) {
8735 			err = check_mem_access(env, insn_idx, reg, argno,
8736 					       i, BPF_DW, BPF_WRITE, -1, false, false);
8737 			if (err)
8738 				return err;
8739 		}
8740 
8741 		err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots);
8742 		if (err)
8743 			return err;
8744 	} else {
8745 		/* iter_next() or iter_destroy(), as well as any kfunc
8746 		 * accepting iter argument, expect initialized iter state
8747 		 */
8748 		err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots);
8749 		switch (err) {
8750 		case 0:
8751 			break;
8752 		case -EINVAL:
8753 			verbose(env, "expected an initialized iter_%s as %s\n",
8754 				iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno));
8755 			bpf_diag_res(
8756 				env, insn_idx, "uninitialized iterator use",
8757 				"This iterator operation requires an initialized iterator state object, but the stack range does not contain a live iterator on this path.",
8758 				"Call the matching iterator new kfunc on every path before calling next or destroy, and do not destroy the iterator before this use.");
8759 			return err;
8760 		case -EPROTO:
8761 			verbose(env, "expected an RCU CS when using %s\n", meta->func_name);
8762 			bpf_diag_ctx_required(
8763 				env, insn_idx, meta->func_name, BPF_DIAG_CONTEXT_RCU,
8764 				"Wrap iterator use in bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced.");
8765 			return err;
8766 		default:
8767 			return err;
8768 		}
8769 
8770 		spi = iter_get_spi(env, reg, nr_slots);
8771 		if (spi < 0)
8772 			return spi;
8773 
8774 		mark_stack_slots_scratched(env, spi, nr_slots);
8775 
8776 		/* remember meta->iter info for process_iter_next_call() */
8777 		meta->iter.spi = spi;
8778 		meta->iter.frameno = reg->frameno;
8779 		update_ref_obj(&meta->ref_obj, &bpf_stack_slot(state, spi)->spilled_ptr);
8780 
8781 		/* The iterator stays. What was borrowed from the previous element goes. */
8782 		if (iter_next_borrows_struct(meta)) {
8783 			err = invalidate_reference(env, meta->ref_obj.id);
8784 			if (err)
8785 				return err;
8786 		}
8787 
8788 		if (is_iter_destroy_kfunc(meta)) {
8789 			err = unmark_stack_slots_iter(env, reg, nr_slots);
8790 			if (err)
8791 				return err;
8792 		}
8793 	}
8794 
8795 	return 0;
8796 }
8797 
8798 /* Look for a previous loop entry at insn_idx: nearest parent state
8799  * stopped at insn_idx with callsites matching those in cur->frame.
8800  */
8801 static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env,
8802 						  struct bpf_verifier_state *cur,
8803 						  int insn_idx)
8804 {
8805 	struct bpf_verifier_state_list *sl;
8806 	struct bpf_verifier_state *st;
8807 	struct list_head *pos, *head;
8808 
8809 	/* Explored states are pushed in stack order, most recent states come first */
8810 	head = bpf_explored_state(env, insn_idx);
8811 	list_for_each(pos, head) {
8812 		sl = container_of(pos, struct bpf_verifier_state_list, node);
8813 		/* If st->branches != 0 state is a part of current DFS verification path,
8814 		 * hence cur & st for a loop.
8815 		 */
8816 		st = &sl->state;
8817 		if (st->insn_idx == insn_idx && st->branches && same_callsites(st, cur) &&
8818 		    st->dfs_depth < cur->dfs_depth)
8819 			return st;
8820 	}
8821 
8822 	return NULL;
8823 }
8824 
8825 /*
8826  * Check if scalar registers are exact for the purpose of not widening.
8827  * More lenient than regs_exact()
8828  */
8829 static bool scalars_exact_for_widen(const struct bpf_reg_state *rold,
8830 				    const struct bpf_reg_state *rcur)
8831 {
8832 	return !memcmp(rold, rcur, offsetof(struct bpf_reg_state, id));
8833 }
8834 
8835 static void maybe_widen_reg(struct bpf_verifier_env *env,
8836 			    struct bpf_reg_state *rold, struct bpf_reg_state *rcur)
8837 {
8838 	if (rold->type != SCALAR_VALUE)
8839 		return;
8840 	if (rold->type != rcur->type)
8841 		return;
8842 	if (rold->precise || rcur->precise || scalars_exact_for_widen(rold, rcur))
8843 		return;
8844 	__mark_reg_unknown(env, rcur);
8845 }
8846 
8847 static int widen_imprecise_scalars(struct bpf_verifier_env *env,
8848 				   struct bpf_verifier_state *old,
8849 				   struct bpf_verifier_state *cur)
8850 {
8851 	struct bpf_func_state *fold, *fcur;
8852 	int i, fr, num_slots;
8853 
8854 	for (fr = old->curframe; fr >= 0; fr--) {
8855 		fold = old->frame[fr];
8856 		fcur = cur->frame[fr];
8857 
8858 		for (i = 0; i < MAX_BPF_REG; i++)
8859 			maybe_widen_reg(env,
8860 					&fold->regs[i],
8861 					&fcur->regs[i]);
8862 
8863 		num_slots = min(bpf_stack_nr_slots(fold), bpf_stack_nr_slots(fcur));
8864 		for (i = 0; i < num_slots; i++) {
8865 			struct bpf_stack_state *old_ss = bpf_stack_slot(fold, i);
8866 			struct bpf_stack_state *cur_ss = bpf_stack_slot(fcur, i);
8867 
8868 			if (!bpf_is_spilled_reg(old_ss) || !bpf_is_spilled_reg(cur_ss))
8869 				continue;
8870 
8871 			maybe_widen_reg(env, &old_ss->spilled_ptr, &cur_ss->spilled_ptr);
8872 		}
8873 	}
8874 	return 0;
8875 }
8876 
8877 static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_st,
8878 						 struct bpf_call_arg_meta *meta)
8879 {
8880 	int iter_frameno = meta->iter.frameno;
8881 	int iter_spi = meta->iter.spi;
8882 
8883 	return &bpf_stack_slot(cur_st->frame[iter_frameno], iter_spi)->spilled_ptr;
8884 }
8885 
8886 /* process_iter_next_call() is called when verifier gets to iterator's next
8887  * "method" (e.g., bpf_iter_num_next() for numbers iterator) call. We'll refer
8888  * to it as just "iter_next()" in comments below.
8889  *
8890  * BPF verifier relies on a crucial contract for any iter_next()
8891  * implementation: it should *eventually* return NULL, and once that happens
8892  * it should keep returning NULL. That is, once iterator exhausts elements to
8893  * iterate, it should never reset or spuriously return new elements.
8894  *
8895  * With the assumption of such contract, process_iter_next_call() simulates
8896  * a fork in the verifier state to validate loop logic correctness and safety
8897  * without having to simulate infinite amount of iterations.
8898  *
8899  * In current state, we first assume that iter_next() returned NULL and
8900  * iterator state is set to DRAINED (BPF_ITER_STATE_DRAINED). In such
8901  * conditions we should not form an infinite loop and should eventually reach
8902  * exit.
8903  *
8904  * Besides that, we also fork current state and enqueue it for later
8905  * verification. In a forked state we keep iterator state as ACTIVE
8906  * (BPF_ITER_STATE_ACTIVE) and assume non-NULL return from iter_next(). We
8907  * also bump iteration depth to prevent erroneous infinite loop detection
8908  * later on (see iter_active_depths_differ() comment for details). In this
8909  * state we assume that we'll eventually loop back to another iter_next()
8910  * calls (it could be in exactly same location or in some other instruction,
8911  * it doesn't matter, we don't make any unnecessary assumptions about this,
8912  * everything revolves around iterator state in a stack slot, not which
8913  * instruction is calling iter_next()). When that happens, we either will come
8914  * to iter_next() with equivalent state and can conclude that next iteration
8915  * will proceed in exactly the same way as we just verified, so it's safe to
8916  * assume that loop converges. If not, we'll go on another iteration
8917  * simulation with a different input state, until all possible starting states
8918  * are validated or we reach maximum number of instructions limit.
8919  *
8920  * This way, we will either exhaustively discover all possible input states
8921  * that iterator loop can start with and eventually will converge, or we'll
8922  * effectively regress into bounded loop simulation logic and either reach
8923  * maximum number of instructions if loop is not provably convergent, or there
8924  * is some statically known limit on number of iterations (e.g., if there is
8925  * an explicit `if n > 100 then break;` statement somewhere in the loop).
8926  *
8927  * Iteration convergence logic in is_state_visited() relies on exact
8928  * states comparison, which ignores read and precision marks.
8929  * This is necessary because read and precision marks are not finalized
8930  * while in the loop. Exact comparison might preclude convergence for
8931  * simple programs like below:
8932  *
8933  *     i = 0;
8934  *     while(iter_next(&it))
8935  *       i++;
8936  *
8937  * At each iteration step i++ would produce a new distinct state and
8938  * eventually instruction processing limit would be reached.
8939  *
8940  * To avoid such behavior speculatively forget (widen) range for
8941  * imprecise scalar registers, if those registers were not precise at the
8942  * end of the previous iteration and do not match exactly.
8943  *
8944  * This is a conservative heuristic that allows to verify wide range of programs,
8945  * however it precludes verification of programs that conjure an
8946  * imprecise value on the first loop iteration and use it as precise on a second.
8947  * For example, the following safe program would fail to verify:
8948  *
8949  *     struct bpf_num_iter it;
8950  *     int arr[10];
8951  *     int i = 0, a = 0;
8952  *     bpf_iter_num_new(&it, 0, 10);
8953  *     while (bpf_iter_num_next(&it)) {
8954  *       if (a == 0) {
8955  *         a = 1;
8956  *         i = 7; // Because i changed verifier would forget
8957  *                // it's range on second loop entry.
8958  *       } else {
8959  *         arr[i] = 42; // This would fail to verify.
8960  *       }
8961  *     }
8962  *     bpf_iter_num_destroy(&it);
8963  */
8964 static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx,
8965 				  struct bpf_call_arg_meta *meta)
8966 {
8967 	struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;
8968 	struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr;
8969 	struct bpf_reg_state *cur_iter, *queued_iter;
8970 
8971 	BTF_TYPE_EMIT(struct bpf_iter);
8972 
8973 	cur_iter = get_iter_from_state(cur_st, meta);
8974 
8975 	if (cur_iter->iter.state != BPF_ITER_STATE_ACTIVE &&
8976 	    cur_iter->iter.state != BPF_ITER_STATE_DRAINED) {
8977 		verifier_bug(env, "unexpected iterator state %d (%s)",
8978 			     cur_iter->iter.state, iter_state_str(cur_iter->iter.state));
8979 		return -EFAULT;
8980 	}
8981 
8982 	if (cur_iter->iter.state == BPF_ITER_STATE_ACTIVE) {
8983 		/* Because iter_next() call is a checkpoint is_state_visitied()
8984 		 * should guarantee parent state with same call sites and insn_idx.
8985 		 */
8986 		if (!cur_st->parent || cur_st->parent->insn_idx != insn_idx ||
8987 		    !same_callsites(cur_st->parent, cur_st)) {
8988 			verifier_bug(env, "bad parent state for iter next call");
8989 			return -EFAULT;
8990 		}
8991 		/* Note cur_st->parent in the call below, it is necessary to skip
8992 		 * checkpoint created for cur_st by is_state_visited()
8993 		 * right at this instruction.
8994 		 */
8995 		prev_st = find_prev_entry(env, cur_st->parent, insn_idx);
8996 		/* branch out active iter state */
8997 		queued_st = push_stack(env, insn_idx + 1, insn_idx, false);
8998 		if (IS_ERR(queued_st))
8999 			return PTR_ERR(queued_st);
9000 
9001 		queued_iter = get_iter_from_state(queued_st, meta);
9002 		queued_iter->iter.state = BPF_ITER_STATE_ACTIVE;
9003 		queued_iter->iter.depth++;
9004 		if (prev_st)
9005 			widen_imprecise_scalars(env, prev_st, queued_st);
9006 
9007 		queued_fr = queued_st->frame[queued_st->curframe];
9008 		mark_ptr_not_null_reg(&queued_fr->regs[BPF_REG_0]);
9009 	}
9010 
9011 	/* switch to DRAINED state, but keep the depth unchanged */
9012 	/* mark current iter state as drained and assume returned NULL */
9013 	cur_iter->iter.state = BPF_ITER_STATE_DRAINED;
9014 	__mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]);
9015 
9016 	return 0;
9017 }
9018 
9019 static bool arg_type_is_mem_size(enum bpf_arg_type type)
9020 {
9021 	return type == ARG_MEM_SIZE || type == ARG_MEM_SIZE_OR_ZERO;
9022 }
9023 
9024 static bool arg_type_is_raw_mem(enum bpf_arg_type type)
9025 {
9026 	/*
9027 	 * A map value output buffer (e.g. bpf_map_pop_elem) is also a raw
9028 	 * (uninitialized) memory argument, and like ARG_PTR_TO_MEM it may be
9029 	 * passed as a PTR_TO_STACK that reaches check_stack_range_initialized().
9030 	 */
9031 	return (base_type(type) == ARG_PTR_TO_MEM ||
9032 		base_type(type) == ARG_PTR_TO_MAP_VALUE) &&
9033 	       type & MEM_UNINIT;
9034 }
9035 
9036 static bool arg_type_is_release(enum bpf_arg_type type)
9037 {
9038 	return type & OBJ_RELEASE;
9039 }
9040 
9041 static bool arg_type_is_dynptr(enum bpf_arg_type type)
9042 {
9043 	return base_type(type) == ARG_PTR_TO_DYNPTR;
9044 }
9045 
9046 /*
9047  * An argument that only ever takes a scalar, so a zero register passed to it
9048  * is a value rather than a NULL pointer.
9049  */
9050 static bool arg_type_is_scalar(enum bpf_arg_type type)
9051 {
9052 	switch (base_type(type)) {
9053 	case ARG_SCALAR:
9054 	case ARG_CONST_SCALAR:
9055 	case ARG_MEM_SIZE:
9056 	case ARG_MEM_SIZE_OR_ZERO:
9057 	case ARG_CONST_MEM_SIZE:
9058 	case ARG_CONST_ALLOC_SIZE_OR_ZERO:
9059 		return true;
9060 	default:
9061 		return false;
9062 	}
9063 }
9064 
9065 /*
9066  * A helper has no BTF and a nonzero function ID. A kfunc has both, while a
9067  * BPF subprogram has BTF and a zero function ID.
9068  */
9069 static bool is_helper(const struct bpf_call_arg_meta *meta)
9070 {
9071 	return !meta->btf && meta->func_id;
9072 }
9073 
9074 static bool is_helper_call(const struct bpf_call_arg_meta *meta, enum bpf_func_id func_id)
9075 {
9076 	return is_helper(meta) && meta->func_id == func_id;
9077 }
9078 
9079 static bool is_kfunc(const struct bpf_call_arg_meta *meta)
9080 {
9081 	return meta->btf && meta->func_id;
9082 }
9083 
9084 static bool is_kfunc_call(const struct bpf_call_arg_meta *meta, u32 btf_id)
9085 {
9086 	return is_kfunc(meta) && meta->func_id == btf_id;
9087 }
9088 
9089 static int resolve_map_arg_type(struct bpf_verifier_env *env,
9090 				const struct bpf_call_arg_meta *meta,
9091 				enum bpf_arg_type *arg_type)
9092 {
9093 	if (!meta->map.ptr) {
9094 		/* kernel subsystem misconfigured verifier */
9095 		verifier_bug(env, "invalid map_ptr to access map->type");
9096 		return -EFAULT;
9097 	}
9098 
9099 	switch (meta->map.ptr->map_type) {
9100 	case BPF_MAP_TYPE_SOCKMAP:
9101 	case BPF_MAP_TYPE_SOCKHASH:
9102 		if (*arg_type == ARG_PTR_TO_MAP_VALUE) {
9103 			*arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON;
9104 		} else {
9105 			verbose(env, "invalid arg_type for sockmap/sockhash\n");
9106 			return -EINVAL;
9107 		}
9108 		break;
9109 	case BPF_MAP_TYPE_BLOOM_FILTER:
9110 		if (is_helper_call(meta, BPF_FUNC_map_peek_elem))
9111 			*arg_type = ARG_PTR_TO_MAP_VALUE;
9112 		break;
9113 	default:
9114 		break;
9115 	}
9116 	return 0;
9117 }
9118 
9119 static int resolve_func_arg_type(struct bpf_verifier_env *env,
9120 				 struct bpf_reg_state *reg, u32 arg, argno_t argno,
9121 				 struct bpf_call_arg_meta *meta,
9122 				 enum bpf_arg_type *arg_type, u32 *arg_size);
9123 static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
9124 				     argno_t argno, enum bpf_arg_type arg_type,
9125 				     const struct btf *arg_btf, u32 arg_btf_id,
9126 				     struct bpf_call_arg_meta *meta, int insn_idx);
9127 static bool is_kfunc_arg_nonown_allowed(const struct btf *btf,
9128 					const struct btf_param *arg);
9129 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf,
9130 					  const struct btf_param *arg,
9131 					  const char *name);
9132 static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta);
9133 static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta);
9134 static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta);
9135 static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta);
9136 static bool is_bpf_obj_drop_kfunc(u32 func_id);
9137 static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id);
9138 static bool is_bpf_rbtree_add_kfunc(u32 func_id);
9139 static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta);
9140 static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env);
9141 static int process_irq_flag(struct bpf_verifier_env *env,
9142 			    struct bpf_reg_state *reg, argno_t argno,
9143 			    struct bpf_call_arg_meta *meta);
9144 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,
9145 					   struct bpf_reg_state *reg,
9146 					   argno_t argno,
9147 					   struct bpf_call_arg_meta *meta);
9148 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,
9149 					     struct bpf_reg_state *reg,
9150 					     argno_t argno,
9151 					     struct bpf_call_arg_meta *meta);
9152 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
9153 					   struct bpf_reg_state *reg,
9154 					   argno_t argno,
9155 					   struct bpf_call_arg_meta *meta);
9156 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,
9157 					     struct bpf_reg_state *reg,
9158 					     argno_t argno,
9159 					     struct bpf_call_arg_meta *meta);
9160 static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env);
9161 
9162 struct bpf_reg_types {
9163 	const enum bpf_reg_type types[10];
9164 	u32 *btf_id;
9165 };
9166 
9167 static const struct bpf_reg_types sock_types = {
9168 	.types = {
9169 		PTR_TO_SOCK_COMMON,
9170 		PTR_TO_SOCKET,
9171 		PTR_TO_TCP_SOCK,
9172 		PTR_TO_XDP_SOCK,
9173 	},
9174 };
9175 
9176 #ifdef CONFIG_NET
9177 static const struct bpf_reg_types btf_id_sock_common_types = {
9178 	.types = {
9179 		PTR_TO_SOCK_COMMON,
9180 		PTR_TO_SOCKET,
9181 		PTR_TO_TCP_SOCK,
9182 		PTR_TO_XDP_SOCK,
9183 		PTR_TO_BTF_ID,
9184 		PTR_TO_BTF_ID | PTR_TRUSTED,
9185 	},
9186 	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
9187 };
9188 #endif
9189 
9190 static const struct bpf_reg_types mem_types = {
9191 	.types = {
9192 		PTR_TO_STACK,
9193 		PTR_TO_PACKET,
9194 		PTR_TO_PACKET_META,
9195 		PTR_TO_MAP_KEY,
9196 		PTR_TO_MAP_VALUE,
9197 		PTR_TO_MEM,
9198 		PTR_TO_MEM | MEM_RINGBUF,
9199 		PTR_TO_BUF,
9200 		PTR_TO_BTF_ID | PTR_TRUSTED,
9201 		PTR_TO_CTX,
9202 	},
9203 };
9204 
9205 static const struct bpf_reg_types map_value_or_alloc_obj_types = {
9206 	.types = {
9207 		PTR_TO_MAP_VALUE,
9208 		PTR_TO_BTF_ID | MEM_ALLOC,
9209 	}
9210 };
9211 
9212 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } };
9213 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } };
9214 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } };
9215 static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } };
9216 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
9217 static const struct bpf_reg_types btf_ptr_types = {
9218 	.types = {
9219 		PTR_TO_BTF_ID,
9220 		PTR_TO_BTF_ID | PTR_TRUSTED,
9221 		PTR_TO_BTF_ID | MEM_RCU,
9222 	},
9223 };
9224 static const struct bpf_reg_types percpu_btf_ptr_types = {
9225 	.types = {
9226 		PTR_TO_BTF_ID | MEM_PERCPU,
9227 		PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU,
9228 		PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED,
9229 	}
9230 };
9231 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } };
9232 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } };
9233 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } };
9234 static const struct bpf_reg_types map_value_types = { .types = { PTR_TO_MAP_VALUE } };
9235 static const struct bpf_reg_types arena_types = {
9236 	.types = {
9237 		PTR_TO_ARENA,
9238 		SCALAR_VALUE,
9239 	}
9240 };
9241 static const struct bpf_reg_types ctx_out_types = {
9242 	.types = { PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED },
9243 };
9244 
9245 static const struct bpf_reg_types alloc_obj_drop_types = {
9246 	.types = {
9247 		PTR_TO_BTF_ID | MEM_ALLOC,
9248 		PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU,
9249 	}
9250 };
9251 
9252 static const struct bpf_reg_types alloc_obj_types = {
9253 	.types = {
9254 		PTR_TO_BTF_ID | MEM_ALLOC,
9255 		PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU,
9256 		PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF,
9257 		PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU,
9258 	}
9259 };
9260 
9261 static const struct bpf_reg_types kptr_xchg_dest_types = {
9262 	.types = {
9263 		PTR_TO_MAP_VALUE,
9264 		PTR_TO_BTF_ID | MEM_ALLOC,
9265 		PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF,
9266 		PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU,
9267 	}
9268 };
9269 static const struct bpf_reg_types dynptr_types = {
9270 	.types = {
9271 		PTR_TO_STACK,
9272 		CONST_PTR_TO_DYNPTR,
9273 	}
9274 };
9275 
9276 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
9277 	[ARG_PTR_TO_MAP_KEY]		= &mem_types,
9278 	[ARG_PTR_TO_MAP_VALUE]		= &mem_types,
9279 	[ARG_MEM_SIZE]			= &scalar_types,
9280 	[ARG_MEM_SIZE_OR_ZERO]		= &scalar_types,
9281 	[ARG_CONST_ALLOC_SIZE_OR_ZERO]	= &scalar_types,
9282 	[ARG_SCALAR]			= &scalar_types,
9283 	[ARG_CONST_MAP_PTR]		= &const_map_ptr_types,
9284 	[ARG_PTR_TO_CTX]		= &context_types,
9285 	[ARG_PTR_TO_SOCK_COMMON]	= &sock_types,
9286 #ifdef CONFIG_NET
9287 	[ARG_PTR_TO_BTF_ID_SOCK_COMMON]	= &btf_id_sock_common_types,
9288 #endif
9289 	[ARG_PTR_TO_SOCKET]		= &fullsock_types,
9290 	[ARG_PTR_TO_BTF_ID]		= &btf_ptr_types,
9291 	[ARG_PTR_TO_SPIN_LOCK]		= &map_value_or_alloc_obj_types,
9292 	[ARG_PTR_TO_MEM]		= &mem_types,
9293 	[ARG_PTR_TO_RINGBUF_MEM]	= &ringbuf_mem_types,
9294 	[ARG_PTR_TO_PERCPU_BTF_ID]	= &percpu_btf_ptr_types,
9295 	[ARG_PTR_TO_FUNC]		= &func_ptr_types,
9296 	[ARG_PTR_TO_STACK]		= &stack_ptr_types,
9297 	[ARG_PTR_TO_CONST_STR]		= &const_str_ptr_types,
9298 	[ARG_PTR_TO_TIMER]		= &map_value_types,
9299 	[ARG_KPTR_XCHG_DEST]		= &kptr_xchg_dest_types,
9300 	[ARG_PTR_TO_DYNPTR]		= &dynptr_types,
9301 	[ARG_CONST_SCALAR]		= &scalar_types,
9302 	[ARG_CONST_MEM_SIZE]		= &scalar_types,
9303 	[ARG_PTR_TO_ALLOC_BTF_ID]	= &alloc_obj_drop_types,
9304 	[ARG_PTR_TO_REFCOUNTED_KPTR]	= &alloc_obj_types,
9305 	[ARG_PTR_TO_ITER]		= &stack_ptr_types,
9306 	[ARG_PTR_TO_LIST_HEAD]		= &map_value_or_alloc_obj_types,
9307 	[ARG_PTR_TO_LIST_NODE]		= &alloc_obj_types,
9308 	[ARG_PTR_TO_RB_ROOT]		= &map_value_or_alloc_obj_types,
9309 	[ARG_PTR_TO_RB_NODE]		= &alloc_obj_types,
9310 	[ARG_PTR_TO_RES_SPIN_LOCK]	= &map_value_or_alloc_obj_types,
9311 	[ARG_PTR_TO_WORKQUEUE]		= &map_value_types,
9312 	[ARG_PTR_TO_TASK_WORK]		= &map_value_types,
9313 	[ARG_PTR_TO_RCU_HEAD]		= &map_value_types,
9314 	[ARG_PTR_TO_IRQ_FLAG]		= &stack_ptr_types,
9315 	[ARG_PTR_TO_ARENA]		= &arena_types,
9316 	[ARG_PTR_TO_CTX_OUT]		= &ctx_out_types,
9317 };
9318 
9319 static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno,
9320 			      const char *call_name, const char *reason,
9321 			      const char *suggestion)
9322 {
9323 	int arg = arg_from_argno(argno);
9324 	int regno = reg_from_argno(argno);
9325 	int stack_slot = -1;
9326 
9327 	if (arg < 0 && regno >= BPF_REG_1 && regno <= BPF_REG_5)
9328 		arg = regno;
9329 	if (arg > MAX_BPF_FUNC_REG_ARGS)
9330 		stack_slot = arg - MAX_BPF_FUNC_REG_ARGS - 1;
9331 
9332 	bpf_diag_call_type(env, insn_idx, arg, regno, stack_slot,
9333 			   call_name && *call_name ? call_name : "call",
9334 			   reg_arg_name(env, argno), reason, suggestion);
9335 }
9336 
9337 static const char *bpf_diag_arg_name(struct bpf_verifier_env *env, argno_t argno)
9338 {
9339 	return bpf_diag_fmt(env, "%s", reg_arg_name(env, argno));
9340 }
9341 
9342 __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx,
9343 						 argno_t argno, const char *call_name,
9344 						 const char *suggestion, const char *fmt, ...)
9345 {
9346 	const char *reason;
9347 	va_list args;
9348 
9349 	va_start(args, fmt);
9350 	reason = bpf_diag_vfmt(env, fmt, args);
9351 	va_end(args);
9352 
9353 	bpf_diag_call_arg(env, insn_idx, argno, call_name, reason, suggestion);
9354 }
9355 
9356 static int check_func_arg_nullability(struct bpf_verifier_env *env,
9357 				      struct bpf_reg_state *reg, u32 arg, argno_t argno,
9358 				      enum bpf_arg_type arg_type,
9359 				      struct bpf_call_arg_meta *meta, int insn_idx)
9360 {
9361 	const char *expected_type = "pointer";
9362 
9363 	if (arg_type_is_scalar(arg_type) || type_may_be_null(arg_type) ||
9364 	    (!bpf_register_is_null(reg) && !type_may_be_null(reg->type)))
9365 		return 0;
9366 
9367 	if (meta->btf) {
9368 		u32 arg_btf_id;
9369 
9370 		arg_btf_id = btf_params(meta->func_proto)[arg].type;
9371 		expected_type = bpf_diag_fmt(env, "value of type %s",
9372 					     bpf_diag_fmt_btf_type(env, meta->btf, arg_btf_id));
9373 	}
9374 
9375 	verbose(env, "Possibly NULL pointer passed to trusted %s\n",
9376 		reg_arg_name(env, argno));
9377 	bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
9378 			      "Add a NULL check and make the call only on the non-NULL path.",
9379 			      "the pointer may be NULL, but this call requires a non-NULL %s",
9380 			      expected_type);
9381 	return -EACCES;
9382 }
9383 
9384 static int check_func_arg_release(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
9385 				  u32 arg, argno_t argno, enum bpf_arg_type arg_type,
9386 				  struct bpf_call_arg_meta *meta, int insn_idx)
9387 {
9388 	const char *expected_type = "pointer";
9389 
9390 	if (!arg_type_is_release(arg_type))
9391 		return 0;
9392 
9393 	if (arg_type_is_dynptr(arg_type) || reg_is_referenced(env, reg) ||
9394 	    bpf_register_is_null(reg))
9395 		return 0;
9396 
9397 	verbose(env, "release function %s expects referenced PTR_TO_BTF_ID passed to %s\n",
9398 		meta->func_name, reg_arg_name(env, argno));
9399 
9400 	if (is_kfunc(meta)) {
9401 		const struct btf_param *btf_arg;
9402 		const struct btf_type *t;
9403 		u32 ref_id;
9404 
9405 		btf_arg = &btf_params(meta->func_proto)[arg];
9406 		ref_id = btf_arg->type;
9407 		t = btf_type_skip_modifiers(meta->btf, btf_arg->type, NULL);
9408 		if (btf_type_is_ptr(t))
9409 			btf_type_skip_modifiers(meta->btf, t->type, &ref_id);
9410 		expected_type = bpf_diag_fmt(env, "value of type %s",
9411 					     bpf_diag_fmt_btf_type(env, meta->btf, ref_id));
9412 	}
9413 
9414 	bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
9415 			      bpf_diag_fmt(env, "Pass the resource-owning %s returned by the matching acquire call, or avoid the release function after ownership has already been transferred or released.",
9416 					   expected_type),
9417 			      "release functions require a value that owns a live resource returned by a matching acquire function");
9418 	return -EINVAL;
9419 }
9420 
9421 static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env,
9422 					       const enum bpf_reg_type *types, int count)
9423 {
9424 	size_t len = 0, size = 1;
9425 	char *buf;
9426 	int i;
9427 
9428 	for (i = 0; i < count; i++)
9429 		size += strlen(reg_type_str(env, types[i])) + (i ? 2 : 0);
9430 
9431 	buf = bpf_diag_fmt_buf(env, size);
9432 	if (!buf)
9433 		return "";
9434 
9435 	for (i = 0; i < count; i++)
9436 		len += scnprintf(buf + len, size - len, "%s%s", i ? ", " : "",
9437 				 reg_type_str(env, types[i]));
9438 	return buf;
9439 }
9440 
9441 static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
9442 			  enum bpf_arg_type arg_type, struct bpf_call_arg_meta *meta)
9443 {
9444 	enum bpf_reg_type expected, type = reg->type;
9445 	const struct bpf_reg_types *compatible;
9446 	const char *actual, *accepted;
9447 	int i, j;
9448 
9449 	compatible = compatible_reg_types[base_type(arg_type)];
9450 	if (!compatible) {
9451 		verifier_bug(env, "unsupported arg type %d", arg_type);
9452 		return -EFAULT;
9453 	}
9454 	if (is_kfunc(meta) && base_type(arg_type) == ARG_PTR_TO_BTF_ID &&
9455 	    (base_type(type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(type)]))
9456 		goto found;
9457 
9458 	/* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY,
9459 	 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY
9460 	 *
9461 	 * Same for MAYBE_NULL:
9462 	 *
9463 	 * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL,
9464 	 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL
9465 	 *
9466 	 * ARG_PTR_TO_MEM is compatible with PTR_TO_MEM that is tagged with a dynptr type.
9467 	 *
9468 	 * Therefore we fold these flags depending on the arg_type before comparison.
9469 	 */
9470 	if (arg_type & MEM_RDONLY)
9471 		type &= ~MEM_RDONLY;
9472 	if (arg_type & PTR_MAYBE_NULL)
9473 		type &= ~PTR_MAYBE_NULL;
9474 	if (base_type(arg_type) == ARG_PTR_TO_MEM)
9475 		type &= ~DYNPTR_TYPE_FLAG_MASK;
9476 	/* Allow allocated memory for BTF-defined ARG_PTR_TO_MEM but not helpers. */
9477 	if (!is_helper(meta) && base_type(arg_type) == ARG_PTR_TO_MEM &&
9478 	    type_is_ptr_alloc_obj(type))
9479 		type = PTR_TO_MEM;
9480 
9481 	/* Local kptr types are allowed as the source argument of bpf_kptr_xchg */
9482 	if (is_helper_call(meta, BPF_FUNC_kptr_xchg) && type_is_alloc(type) &&
9483 	    reg_from_argno(argno) == BPF_REG_2) {
9484 		type &= ~MEM_ALLOC;
9485 		type &= ~MEM_PERCPU;
9486 	}
9487 
9488 	for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
9489 		expected = compatible->types[i];
9490 		if (expected == NOT_INIT)
9491 			break;
9492 
9493 		if (type == expected)
9494 			goto found;
9495 	}
9496 
9497 	verbose(env, "%s type=%s expected=", reg_arg_name(env, argno), reg_type_str(env, reg->type));
9498 	for (j = 0; j + 1 < i; j++)
9499 		verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
9500 	verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
9501 	actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type));
9502 	accepted = bpf_diag_expected_reg_types(env, compatible->types, i);
9503 	bpf_diag_call_arg_fmt(env, env->insn_idx, argno, meta->func_name,
9504 			      bpf_diag_fmt(env, "Pass %s.", bpf_diag_arg_type_plain(arg_type)),
9505 			      "it has type %s, but this argument accepts %s",
9506 			      actual, accepted);
9507 	return -EACCES;
9508 
9509 found:
9510 	return 0;
9511 }
9512 
9513 static struct btf_field *
9514 reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields)
9515 {
9516 	struct btf_field *field;
9517 	struct btf_record *rec;
9518 
9519 	rec = reg_btf_record(reg);
9520 	if (!rec)
9521 		return NULL;
9522 
9523 	field = btf_record_find(rec, off, fields);
9524 	if (!field)
9525 		return NULL;
9526 
9527 	return field;
9528 }
9529 
9530 static int check_func_arg_reg_off(struct bpf_verifier_env *env,
9531 				  const struct bpf_reg_state *reg, argno_t argno,
9532 				  enum bpf_arg_type arg_type)
9533 {
9534 	u32 type = reg->type;
9535 
9536 	/* When referenced register is passed to release function, its fixed
9537 	 * offset must be 0.
9538 	 *
9539 	 * We will check arg_type_is_release reg has id when storing
9540 	 * meta->release_regno.
9541 	 */
9542 	if (arg_type_is_release(arg_type)) {
9543 		/* ARG_PTR_TO_DYNPTR with OBJ_RELEASE is a bit special, as it
9544 		 * may not directly point to the object being released, but to
9545 		 * dynptr pointing to such object, which might be at some offset
9546 		 * on the stack. In that case, we simply to fallback to the
9547 		 * default handling.
9548 		 */
9549 		if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK)
9550 			return 0;
9551 
9552 		/* Doing check_ptr_off_reg check for the offset will catch this
9553 		 * because fixed_off_ok is false, but checking here allows us
9554 		 * to give the user a better error message.
9555 		 */
9556 		if (!tnum_is_const(reg->var_off) || reg->var_off.value != 0) {
9557 			verbose(env, "%s must have zero offset when passed to release func or trusted arg to kfunc\n",
9558 				reg_arg_name(env, argno));
9559 			return -EINVAL;
9560 		}
9561 	}
9562 
9563 	switch (type) {
9564 	/* Pointer types where both fixed and variable offset is explicitly allowed: */
9565 	case PTR_TO_STACK:
9566 	case PTR_TO_PACKET:
9567 	case PTR_TO_PACKET_META:
9568 	case PTR_TO_MAP_KEY:
9569 	case PTR_TO_MAP_VALUE:
9570 	case PTR_TO_MEM:
9571 	case PTR_TO_MEM | MEM_RDONLY:
9572 	case PTR_TO_MEM | MEM_RINGBUF:
9573 	case PTR_TO_BUF:
9574 	case PTR_TO_BUF | MEM_RDONLY:
9575 	case PTR_TO_ARENA:
9576 	case SCALAR_VALUE:
9577 		return 0;
9578 	/* All the rest must be rejected, except PTR_TO_BTF_ID which allows
9579 	 * fixed offset.
9580 	 */
9581 	case PTR_TO_BTF_ID:
9582 	case PTR_TO_BTF_ID | MEM_ALLOC:
9583 	case PTR_TO_BTF_ID | PTR_TRUSTED:
9584 	case PTR_TO_BTF_ID | MEM_RCU:
9585 	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF:
9586 	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU:
9587 		/* When referenced PTR_TO_BTF_ID is passed to release function,
9588 		 * its fixed offset must be 0. bpf_refcount_acquire() returns the
9589 		 * pointer it was given while incrementing the refcount at the
9590 		 * refcount field offset, so it needs a zero offset too. In the
9591 		 * other cases, fixed offset can be non-zero. var_off always must
9592 		 * be 0 for PTR_TO_BTF_ID, hence we still need to do checks
9593 		 * instead of returning.
9594 		 */
9595 		return __check_ptr_off_reg(env, reg, argno,
9596 					   base_type(arg_type) != ARG_PTR_TO_REFCOUNTED_KPTR);
9597 	case PTR_TO_CTX:
9598 		/*
9599 		 * Allow fixed and variable offsets for syscall context, but
9600 		 * only when the argument is passed as memory, not ctx,
9601 		 * otherwise we may get modified ctx in tail called programs and
9602 		 * global subprogs (that may act as extension prog hooks).
9603 		 */
9604 		if (base_type(arg_type) != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog))
9605 			return 0;
9606 		fallthrough;
9607 	default:
9608 		return __check_ptr_off_reg(env, reg, argno, false);
9609 	}
9610 }
9611 
9612 static int check_arg_const_str(struct bpf_verifier_env *env,
9613 			       struct bpf_reg_state *reg, argno_t argno)
9614 {
9615 	struct bpf_map *map = reg->map_ptr;
9616 	int err;
9617 	int map_off;
9618 	u64 map_addr;
9619 	char *str_ptr;
9620 	u32 cnt;
9621 
9622 	if (reg->type != PTR_TO_MAP_VALUE)
9623 		return -EINVAL;
9624 
9625 	if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) {
9626 		verbose(env, "%s points to insn_array map which cannot be used as const string\n",
9627 			reg_arg_name(env, argno));
9628 		return -EACCES;
9629 	}
9630 
9631 	if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
9632 		verbose(env, "%s points to percpu_array map which cannot be used as const string\n",
9633 			reg_arg_name(env, argno));
9634 		return -EACCES;
9635 	}
9636 
9637 	if (!bpf_map_is_rdonly(map)) {
9638 		verbose(env, "%s does not point to a readonly map'\n", reg_arg_name(env, argno));
9639 		return -EACCES;
9640 	}
9641 
9642 	if (!tnum_is_const(reg->var_off)) {
9643 		verbose(env, "%s is not a constant address'\n", reg_arg_name(env, argno));
9644 		return -EACCES;
9645 	}
9646 
9647 	if (!map->ops->map_direct_value_addr) {
9648 		verbose(env, "no direct value access support for this map type\n");
9649 		return -EACCES;
9650 	}
9651 
9652 	err = check_map_access(env, reg, argno, 0,
9653 			       map->value_size - reg->var_off.value, false,
9654 			       ACCESS_HELPER);
9655 	if (err)
9656 		return err;
9657 
9658 	map_off = reg->var_off.value;
9659 	err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
9660 	if (err) {
9661 		verbose(env, "direct value access on string failed\n");
9662 		return err;
9663 	}
9664 
9665 	str_ptr = (char *)(long)(map_addr);
9666 	if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) {
9667 		verbose(env, "string is not zero-terminated\n");
9668 		return -EINVAL;
9669 	}
9670 	/* the bytes of a pointer to a function are not known until the program is jitted */
9671 	if (bpf_map_range_func_ptrs(env, map, map_off, strlen(str_ptr + map_off) + 1, &cnt)) {
9672 		verbose(env, "string overlaps with a pointer to a function\n");
9673 		return -EACCES;
9674 	}
9675 	return 0;
9676 }
9677 
9678 /* Returns constant key value in `value` if possible, else negative error */
9679 static int get_constant_map_key(struct bpf_verifier_env *env,
9680 				struct bpf_reg_state *key,
9681 				u32 key_size,
9682 				s64 *value)
9683 {
9684 	struct bpf_func_state *state = bpf_func(env, key);
9685 	struct bpf_stack_state *ss;
9686 	struct bpf_reg_state *reg;
9687 	int slot, spi, off;
9688 	int spill_size = 0;
9689 	int zero_size = 0;
9690 	int stack_off;
9691 	int i, err;
9692 	u8 *stype;
9693 
9694 	if (!env->bpf_capable)
9695 		return -EOPNOTSUPP;
9696 	if (key->type != PTR_TO_STACK)
9697 		return -EOPNOTSUPP;
9698 	if (!tnum_is_const(key->var_off))
9699 		return -EOPNOTSUPP;
9700 
9701 	stack_off = key->var_off.value;
9702 	slot = -stack_off - 1;
9703 	spi = slot / BPF_REG_SIZE;
9704 	off = slot % BPF_REG_SIZE;
9705 	ss = bpf_stack_slot(state, spi);
9706 	stype = ss->slot_type;
9707 
9708 	/* First handle precisely tracked STACK_ZERO */
9709 	for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--)
9710 		zero_size++;
9711 	if (zero_size >= key_size) {
9712 		*value = 0;
9713 		return 0;
9714 	}
9715 
9716 	/* Check that stack contains a scalar spill of expected size */
9717 	if (!bpf_is_spilled_scalar_reg(ss))
9718 		return -EOPNOTSUPP;
9719 	for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--)
9720 		spill_size++;
9721 	if (spill_size != key_size)
9722 		return -EOPNOTSUPP;
9723 
9724 	reg = &ss->spilled_ptr;
9725 	if (!tnum_is_const(reg->var_off))
9726 		/* Stack value not statically known */
9727 		return -EOPNOTSUPP;
9728 
9729 	/* We are relying on a constant value. So mark as precise
9730 	 * to prevent pruning on it.
9731 	 */
9732 	bpf_bt_set_frame_slot(&env->bt, key->frameno, spi);
9733 	err = mark_chain_precision_batch(env, env->cur_state);
9734 	if (err < 0)
9735 		return err;
9736 
9737 	*value = reg->var_off.value;
9738 	return 0;
9739 }
9740 
9741 static bool can_elide_value_nullness(const struct bpf_map *map);
9742 
9743 static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
9744 			       argno_t argno, struct bpf_call_arg_meta *meta)
9745 {
9746 	/* Use map_uid (which is unique id of inner map) to reject:
9747 	 * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
9748 	 * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
9749 	 * if (inner_map1 && inner_map2) {
9750 	 *     timer = bpf_map_lookup_elem(inner_map1);
9751 	 *     if (timer)
9752 	 *         // mismatch would have been allowed
9753 	 *         bpf_timer_init(timer, inner_map2);
9754 	 * }
9755 	 *
9756 	 * Comparing map_ptr is enough to distinguish normal and outer maps.
9757 	 */
9758 	if (meta->map.ptr &&
9759 	    (meta->map.ptr != reg->map_ptr || meta->map.uid != reg->map_uid)) {
9760 		argno_t obj_argno = argno_from_reg(reg_from_argno(argno) - 1);
9761 		struct btf_record *rec = meta->map.ptr->record;
9762 		const char *obj_name = "workqueue";
9763 
9764 		if (rec->timer_off >= 0)
9765 			obj_name = "timer";
9766 		else if (rec->task_work_off >= 0)
9767 			obj_name = "bpf_task_work";
9768 		else if (rec->rcu_head_off >= 0)
9769 			obj_name = "bpf_rcu_head";
9770 
9771 		verbose(env, "%s pointer in %s map_uid=%d ",
9772 			obj_name, reg_arg_name(env, obj_argno), meta->map.uid);
9773 		verbose(env, "doesn't match map pointer in %s map_uid=%d\n",
9774 			reg_arg_name(env, argno), reg->map_uid);
9775 		return -EINVAL;
9776 	}
9777 
9778 	meta->map.ptr = reg->map_ptr;
9779 	meta->map.uid = reg->map_uid;
9780 	return 0;
9781 }
9782 
9783 /*
9784  * MEM_WRITE alone denotes an input/output buffer. MEM_UNINIT marks an output
9785  * whose incoming contents are not read.
9786  */
9787 static enum bpf_access_type func_arg_access_type(enum bpf_arg_type arg_type)
9788 {
9789 	if (!(arg_type & MEM_WRITE))
9790 		return BPF_READ;
9791 	if (arg_type & MEM_UNINIT)
9792 		return BPF_WRITE;
9793 	return BPF_READ | BPF_WRITE;
9794 }
9795 
9796 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 prev_slot,
9797 			  struct bpf_call_arg_meta *meta,
9798 			  int insn_idx)
9799 {
9800 	const struct btf_param *btf_arg = is_kfunc(meta) ?
9801 					  &btf_params(meta->func_proto)[arg] : NULL;
9802 	const struct bpf_func_proto *fn = meta->fn;
9803 	struct bpf_func_state *caller = cur_func(env);
9804 	struct bpf_reg_state *regs = cur_regs(env);
9805 	argno_t argno = argno_from_arg(slot + 1);
9806 	struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, slot);
9807 	enum bpf_arg_type arg_type = fn->arg_type[arg];
9808 	int regno = reg_from_argno(argno);
9809 	u32 arg_size = arg_type & MEM_FIXED_SIZE ? fn->arg_size[arg] : 0;
9810 	u32 key_size;
9811 	int err = 0;
9812 
9813 	if (arg_type == ARG_PTR_TO_PROG_AUX) {
9814 		cur_aux(env)->arg_prog = regno;
9815 		return 0;
9816 	}
9817 
9818 	if (arg_type == ARG_IGNORE)
9819 		return 0;
9820 
9821 	if (regno >= 0) {
9822 		err = check_reg_arg(env, regno, SRC_OP);
9823 		if (err)
9824 			return err;
9825 	}
9826 
9827 	/* Preserve the legacy helper behavior for privileged pointer leaks. */
9828 	if (arg_type == ARG_ANYTHING) {
9829 		if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
9830 			verbose(env, "%s leaks addr into helper function\n",
9831 				reg_arg_name(env, argno));
9832 			return -EACCES;
9833 		}
9834 		return 0;
9835 	}
9836 
9837 	err = resolve_func_arg_type(env, reg, arg, argno, meta, &arg_type, &arg_size);
9838 	if (err)
9839 		return err;
9840 
9841 	/*
9842 	 * A slice of a packet is packet data under another name. A helper which
9843 	 * may reallocate the packet must not go on reading from the old one.
9844 	 */
9845 	if (is_helper(meta) && reg_is_dynptr_slice_pkt(reg) &&
9846 	    bpf_helper_changes_pkt_data(meta->func_id)) {
9847 		verbose(env, "function access to the packet is not allowed\n");
9848 		return -EACCES;
9849 	}
9850 
9851 	if (arg_type_is_raw_mem(arg_type))
9852 		meta->arg_raw_mem.mask |= BIT(slot);
9853 
9854 	if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) {
9855 		err = mark_arg_precision(env, argno);
9856 		if (err)
9857 			return err;
9858 		return 0;
9859 	}
9860 
9861 	err = check_func_arg_nullability(env, reg, arg, argno, arg_type, meta, insn_idx);
9862 	if (err)
9863 		return err;
9864 
9865 	err = check_reg_type(env, reg, argno, arg_type, meta);
9866 	if (err)
9867 		return err;
9868 
9869 	err = check_func_arg_reg_off(env, reg, argno, arg_type);
9870 	if (err)
9871 		return err;
9872 
9873 	err = check_func_arg_release(env, reg, arg, argno, arg_type, meta, insn_idx);
9874 	if (err)
9875 		return err;
9876 
9877 	if (reg_is_referenced(env, reg))
9878 		update_ref_obj(&meta->ref_obj, reg);
9879 
9880 	switch (base_type(arg_type)) {
9881 	case ARG_CONST_SCALAR:
9882 		err = process_const_arg(env, reg, argno, meta);
9883 		if (err < 0) {
9884 			if (err == -EINVAL)
9885 				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
9886 						      "Pass a compile-time constant or a value the verifier can prove is constant at this call.",
9887 						      "the function requires this scalar argument to be a verifier-known constant, but %s is variable on this path",
9888 						      reg_arg_name(env, argno));
9889 			return err;
9890 		}
9891 		break;
9892 	case ARG_SCALAR:
9893 		break;
9894 	case ARG_CONST_MAP_PTR:
9895 		/* bpf_map_xxx(map_ptr) call: remember that map_ptr */
9896 		err = process_map_ptr_arg(env, reg, argno, meta);
9897 		if (err)
9898 			return err;
9899 		break;
9900 	case ARG_PTR_TO_MAP_KEY:
9901 		/* bpf_map_xxx(..., map_ptr, ..., key) call:
9902 		 * check that [key, key + map->key_size) are within
9903 		 * stack limits and initialized
9904 		 */
9905 		if (!meta->map.ptr) {
9906 			/* in function declaration map_ptr must come before
9907 			 * map_key, so that it's verified and known before
9908 			 * we have to check map_key here. Otherwise it means
9909 			 * that kernel subsystem misconfigured verifier
9910 			 */
9911 			verifier_bug(env, "invalid map_ptr to access map->key");
9912 			return -EFAULT;
9913 		}
9914 		key_size = meta->map.ptr->key_size;
9915 		err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, meta,
9916 					      NULL);
9917 		if (err)
9918 			return err;
9919 		if (can_elide_value_nullness(meta->map.ptr)) {
9920 			err = get_constant_map_key(env, reg, key_size, &meta->const_map_key);
9921 			if (err < 0) {
9922 				meta->const_map_key = -1;
9923 				if (err == -EOPNOTSUPP)
9924 					err = 0;
9925 				else
9926 					return err;
9927 			}
9928 		}
9929 		break;
9930 	case ARG_PTR_TO_MAP_VALUE:
9931 		if (type_may_be_null(arg_type) && bpf_register_is_null(reg))
9932 			return 0;
9933 
9934 		/* bpf_map_xxx(..., map_ptr, ..., value) call:
9935 		 * check [value, value + map->value_size) validity
9936 		 */
9937 		if (!meta->map.ptr) {
9938 			/* kernel subsystem misconfigured verifier */
9939 			verifier_bug(env, "invalid map_ptr to access map->value");
9940 			return -EFAULT;
9941 		}
9942 
9943 		err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size,
9944 					      arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ,
9945 					      false, meta, NULL);
9946 		break;
9947 	case ARG_PTR_TO_BTF_ID:
9948 	case ARG_PTR_TO_BTF_ID_SOCK_COMMON:
9949 	{
9950 		const u32 *arg_btf_id = fn->arg_btf_id[arg];
9951 		const struct btf *arg_btf = is_kfunc(meta) ? meta->btf : btf_vmlinux;
9952 		enum bpf_reg_type reg_type;
9953 
9954 		if (!meta->btf) {
9955 			const struct bpf_reg_types *compatible;
9956 
9957 			if (base_type(reg->type) != PTR_TO_BTF_ID)
9958 				break;
9959 
9960 			if (is_helper_call(meta, BPF_FUNC_kptr_xchg))
9961 				return map_kptr_match_type(env, meta->kptr_field, reg, regno) ?
9962 				       -EACCES : 0;
9963 
9964 			if (!arg_btf_id) {
9965 				compatible = compatible_reg_types[base_type(arg_type)];
9966 				if (!compatible->btf_id) {
9967 					verifier_bug(env, "missing arg compatible BTF ID");
9968 					return -EFAULT;
9969 				}
9970 				arg_btf_id = compatible->btf_id;
9971 			}
9972 			if (arg_btf_id == BPF_PTR_POISON) {
9973 				verbose(env, "verifier internal error:");
9974 				verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n",
9975 					reg_arg_name(env, argno));
9976 				return -EACCES;
9977 			}
9978 		}
9979 
9980 		reg_type = reg->type & ~(arg_type & PTR_MAYBE_NULL);
9981 		if ((arg_type & PTR_TRUSTED) &&
9982 		    (!is_trusted_reg_type(env, reg, reg_type) ||
9983 		     bpf_type_has_unsafe_modifiers(reg_type))) {
9984 			if (!(arg_type & MEM_RCU)) {
9985 				const char *actual_type, *arg_name, *expected_type;
9986 
9987 				expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id);
9988 				verbose(env, "%s must be referenced or trusted\n",
9989 					reg_arg_name(env, argno));
9990 				arg_name = reg_arg_name(env, argno);
9991 				actual_type = bpf_diag_reg_type_plain(env, reg->type);
9992 				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
9993 						      "Pass a pointer acquired from a verifier-tracked source, or call this function only inside the required protection if it accepts RCU pointers.",
9994 						      "the function requires a trusted or resource-owning pointer to %s, but %s is %s",
9995 						      expected_type, arg_name, actual_type);
9996 				return -EINVAL;
9997 			}
9998 			if (!is_rcu_reg(reg)) {
9999 				const char *actual_type, *arg_name, *expected_type;
10000 
10001 				expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id);
10002 				verbose(env, "%s must be a rcu pointer\n",
10003 					reg_arg_name(env, argno));
10004 				arg_name = reg_arg_name(env, argno);
10005 				actual_type = bpf_diag_reg_type_plain(env, reg->type);
10006 				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
10007 						      "Use this function with a pointer that is valid in an RCU read lock region.",
10008 						      "the function requires an RCU-protected pointer to %s, but %s is %s",
10009 						      expected_type, arg_name, actual_type);
10010 				return -EINVAL;
10011 			}
10012 		}
10013 
10014 		err = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, arg_btf,
10015 						*arg_btf_id, meta, insn_idx);
10016 		if (err < 0)
10017 			return err;
10018 		break;
10019 	}
10020 	case ARG_PTR_TO_PERCPU_BTF_ID:
10021 		if (!reg->btf_id) {
10022 			verbose(env, "Helper has invalid btf_id in %s\n", reg_arg_name(env, argno));
10023 			return -EACCES;
10024 		}
10025 		meta->ret_btf = reg->btf;
10026 		meta->ret_btf_id = reg->btf_id;
10027 		break;
10028 	case ARG_PTR_TO_SPIN_LOCK:
10029 		if (in_rbtree_lock_required_cb(env)) {
10030 			verbose(env, "can't spin_{lock,unlock} in rbtree cb\n");
10031 			return -EACCES;
10032 		}
10033 		if (is_helper_call(meta, BPF_FUNC_spin_lock)) {
10034 			err = process_spin_lock(env, reg, argno, PROCESS_SPIN_LOCK);
10035 			if (err)
10036 				return err;
10037 		} else if (is_helper_call(meta, BPF_FUNC_spin_unlock)) {
10038 			err = process_spin_lock(env, reg, argno, 0);
10039 			if (err)
10040 				return err;
10041 		} else {
10042 			verifier_bug(env, "spin lock arg on unexpected helper");
10043 			return -EFAULT;
10044 		}
10045 		break;
10046 	case ARG_PTR_TO_TIMER:
10047 		err = process_timer_func(env, reg, argno, &meta->map);
10048 		if (err)
10049 			return err;
10050 		break;
10051 	case ARG_PTR_TO_CTX:
10052 		if (is_bpf_cast_to_kern_ctx_kfunc(meta)) {
10053 			err = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog));
10054 			if (err < 0)
10055 				return -EINVAL;
10056 			meta->ret_btf_id = err;
10057 		}
10058 		break;
10059 	case ARG_PTR_TO_ARENA:
10060 		break;
10061 	case ARG_PTR_TO_ALLOC_BTF_ID:
10062 		if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) {
10063 			if (!is_bpf_obj_drop_kfunc(meta->func_id)) {
10064 				verbose(env, "%s expected for bpf_obj_drop()\n",
10065 					reg_arg_name(env, argno));
10066 				return -EINVAL;
10067 			}
10068 		} else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) {
10069 			if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) {
10070 				verbose(env, "%s expected for bpf_percpu_obj_drop()\n",
10071 					reg_arg_name(env, argno));
10072 				return -EINVAL;
10073 			}
10074 		}
10075 		if (!reg_is_referenced(env, reg)) {
10076 			verbose(env, "allocated object must be referenced\n");
10077 			bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
10078 					      "Pass the owned object pointer before it is released or transferred.",
10079 					      "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource",
10080 					      reg_arg_name(env, argno));
10081 			return -EINVAL;
10082 		}
10083 		if (meta->btf == btf_vmlinux) {
10084 			meta->arg_btf = reg->btf;
10085 			meta->arg_btf_id = reg->btf_id;
10086 		}
10087 		break;
10088 	case ARG_PTR_TO_FUNC:
10089 		meta->subprogno = reg->subprogno;
10090 		break;
10091 	case ARG_PTR_TO_MEM:
10092 	{
10093 		enum bpf_access_type access_type;
10094 		bool known_memory;
10095 
10096 		/* The access to this pointer is only checked when we hit the
10097 		 * next is_mem_size argument below.
10098 		 */
10099 		if (!(arg_type & MEM_FIXED_SIZE))
10100 			break;
10101 
10102 		access_type = func_arg_access_type(arg_type);
10103 
10104 		err = check_mem_reg(env, reg, argno, arg_size, access_type, meta, &known_memory);
10105 		if (err < 0) {
10106 			if (known_memory)
10107 				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
10108 					"Pass memory with at least the required number of accessible bytes and suitable read or write access.",
10109 					"the function expects %u bytes of memory, but the verifier cannot prove that %s provides a range of that size with the required read or write access",
10110 					arg_size,
10111 					bpf_diag_reg_type_plain(env, reg->type));
10112 			else
10113 				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
10114 					"Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.",
10115 					"the function expects %u bytes of memory, but it is %s and not verifier-known memory",
10116 					arg_size,
10117 					bpf_diag_reg_type_plain(env, reg->type));
10118 			return err;
10119 		}
10120 		/*
10121 		 * PTR_TO_PACKET gets passed as PTR_TO_MEM, preventing us from adjusting
10122 		 * bounds tracking information.
10123 		 */
10124 		if (is_subprog(meta) && meta->subprog_may_change_pkt &&
10125 		    (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))) {
10126 			verbose(env,
10127 				"cannot pass packet pointer %s to %s(): function may change packet data\n",
10128 				reg_arg_name(env, argno), meta->func_name);
10129 			return -EINVAL;
10130 		}
10131 		if (arg_type & MEM_ALIGNED)
10132 			err = check_ptr_alignment(env, reg, 0, arg_size, true);
10133 		break;
10134 	}
10135 	case ARG_CONST_MEM_SIZE:
10136 		err = process_const_arg(env, reg, argno, meta);
10137 		if (err < 0) {
10138 			if (err == -EINVAL)
10139 				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
10140 						      "Pass a compile-time constant or a value the verifier can prove is constant at this call.",
10141 						      "the function requires this memory size to be a verifier-known constant, but %s is variable on this path",
10142 						      reg_arg_name(env, argno));
10143 			return err;
10144 		}
10145 		fallthrough;
10146 	case ARG_MEM_SIZE:
10147 	case ARG_MEM_SIZE_OR_ZERO:
10148 	{
10149 		struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, prev_slot);
10150 		argno_t buff_argno = argno_from_arg(prev_slot + 1);
10151 		enum bpf_mem_size_failure failure;
10152 		const char *buff_arg, *size_arg;
10153 		bool zero_size_allowed;
10154 		u32 access_type;
10155 
10156 		if (meta->btf && bpf_register_is_null(buff_reg))
10157 			break;
10158 
10159 		access_type = func_arg_access_type(fn->arg_type[arg - 1]);
10160 
10161 		zero_size_allowed = meta->btf || base_type(arg_type) == ARG_MEM_SIZE_OR_ZERO;
10162 
10163 		err = check_mem_size_reg(env, buff_reg, reg, buff_argno, argno,
10164 					 access_type, zero_size_allowed, meta, &failure);
10165 		if (!err)
10166 			break;
10167 
10168 		buff_arg = bpf_diag_arg_name(env, buff_argno);
10169 		size_arg = bpf_diag_arg_name(env, argno);
10170 		verbose(env, "%s and ", reg_arg_name(env, buff_argno));
10171 		verbose(env, "%s memory, len pair leads to invalid memory access\n",
10172 			reg_arg_name(env, argno));
10173 		if (failure == BPF_MEM_SIZE_FAIL_MEMORY) {
10174 			bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, meta->func_name,
10175 					      "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.",
10176 					      "it is the memory pointer in a memory/length pair with %s, but %s does not provide a verifier-accessible range of the requested length",
10177 					      size_arg, buff_arg);
10178 		} else if (failure == BPF_MEM_SIZE_FAIL_SIZE) {
10179 			if (reg_smin(reg) < 0)
10180 				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
10181 					"Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.",
10182 					"the memory size in %s may be negative because its signed minimum is %lld",
10183 					size_arg, reg_smin(reg));
10184 			else if (!zero_size_allowed && reg_umin(reg) == 0)
10185 				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
10186 					"Ensure the memory size is non-zero before this call.",
10187 					"the memory size in %s may be zero, but the function requires a non-zero size",
10188 					size_arg);
10189 			else
10190 				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
10191 					"Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.",
10192 					"the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes",
10193 					size_arg, reg_umax(reg), BPF_MAX_VAR_SIZ);
10194 		}
10195 		break;
10196 	}
10197 	case ARG_PTR_TO_DYNPTR: {
10198 		if (is_bpf_dynptr_clone_kfunc(meta) &&
10199 		    (arg_type & MEM_UNINIT)) {
10200 			enum bpf_dynptr_type parent_type = meta->dynptr.type;
10201 
10202 			if (parent_type == BPF_DYNPTR_TYPE_INVALID) {
10203 				verifier_bug(env, "no dynptr type for parent of clone");
10204 				return -EFAULT;
10205 			}
10206 
10207 			arg_type |= (unsigned int)get_dynptr_type_flag(parent_type);
10208 		}
10209 
10210 		err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, meta);
10211 		if (err)
10212 			return err;
10213 		if (!arg && is_kfunc_dynptr_write(meta))
10214 			meta->dynptr_write_type = meta->dynptr.type;
10215 		break;
10216 	}
10217 	case ARG_PTR_TO_ITER:
10218 		if (is_bpf_iter_css_task_new_kfunc(meta) &&
10219 		    !check_css_task_iter_allowlist(env)) {
10220 			verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n");
10221 			return -EINVAL;
10222 		}
10223 		err = process_iter_arg(env, reg, arg, argno, insn_idx, meta);
10224 		if (err < 0)
10225 			return err;
10226 		break;
10227 	case ARG_PTR_TO_LIST_HEAD:
10228 		if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) &&
10229 		    !reg_is_referenced(env, reg)) {
10230 			verbose(env, "allocated object must be referenced\n");
10231 			return -EINVAL;
10232 		}
10233 		err = process_kf_arg_ptr_to_list_head(env, reg, argno, meta);
10234 		if (err < 0)
10235 			return err;
10236 		break;
10237 	case ARG_PTR_TO_RB_ROOT:
10238 		if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) &&
10239 		    !reg_is_referenced(env, reg)) {
10240 			verbose(env, "allocated object must be referenced\n");
10241 			return -EINVAL;
10242 		}
10243 		err = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta);
10244 		if (err < 0)
10245 			return err;
10246 		break;
10247 	case ARG_PTR_TO_LIST_NODE:
10248 		if (!(is_kfunc_arg_nonown_allowed(meta->btf, btf_arg) &&
10249 		      type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg))) {
10250 			if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
10251 				verbose(env, "%s expected pointer to allocated object\n",
10252 					reg_arg_name(env, argno));
10253 				return -EINVAL;
10254 			}
10255 			if (!reg_is_referenced(env, reg)) {
10256 				verbose(env, "allocated object must be referenced\n");
10257 				return -EINVAL;
10258 			}
10259 		}
10260 		err = process_kf_arg_ptr_to_list_node(env, reg, argno, meta);
10261 		if (err < 0)
10262 			return err;
10263 		break;
10264 	case ARG_PTR_TO_RB_NODE:
10265 		if (is_bpf_rbtree_add_kfunc(meta->func_id)) {
10266 			if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
10267 				verbose(env, "%s expected pointer to allocated object\n",
10268 					reg_arg_name(env, argno));
10269 				return -EINVAL;
10270 			}
10271 			if (!reg_is_referenced(env, reg)) {
10272 				verbose(env, "allocated object must be referenced\n");
10273 				return -EINVAL;
10274 			}
10275 		} else {
10276 			if (!type_is_non_owning_ref(reg->type) &&
10277 			    !reg_is_referenced(env, reg)) {
10278 				verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n",
10279 					meta->func_name);
10280 				return -EINVAL;
10281 			}
10282 			if (in_rbtree_lock_required_cb(env)) {
10283 				verbose(env, "%s not allowed in rbtree cb\n", meta->func_name);
10284 				return -EINVAL;
10285 			}
10286 		}
10287 		err = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta);
10288 		if (err < 0)
10289 			return err;
10290 		break;
10291 	case ARG_CONST_ALLOC_SIZE_OR_ZERO:
10292 		if (meta->btf && is_kfunc_arg_scalar_with_name(meta->btf, btf_arg,
10293 							       "rdonly_buf_size"))
10294 			meta->r0_rdonly = true;
10295 		err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem);
10296 		if (err < 0) {
10297 			if (meta->btf && err == -EINVAL)
10298 				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
10299 						      "Pass a verifier-known constant size for this function's buffer argument.",
10300 						      "the function uses this argument as a return-buffer size, but %s is invalid or variable on this path",
10301 						      reg_arg_name(env, argno));
10302 			return err;
10303 		}
10304 		break;
10305 	case ARG_PTR_TO_REFCOUNTED_KPTR:
10306 	{
10307 		struct btf_record *rec;
10308 
10309 		if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg))
10310 			meta->arg_owning_ref = true;
10311 
10312 		rec = reg_btf_record(reg);
10313 		if (!rec) {
10314 			verifier_bug(env, "Couldn't find btf_record");
10315 			return -EFAULT;
10316 		}
10317 
10318 		if (rec->refcount_off < 0) {
10319 			verbose(env, "%s doesn't point to a type with bpf_refcount field\n",
10320 				reg_arg_name(env, argno));
10321 			return -EINVAL;
10322 		}
10323 
10324 		meta->arg_btf = reg->btf;
10325 		meta->arg_btf_id = reg->btf_id;
10326 		break;
10327 	}
10328 	case ARG_PTR_TO_CONST_STR:
10329 	{
10330 		err = check_arg_const_str(env, reg, argno);
10331 		if (err)
10332 			return err;
10333 		break;
10334 	}
10335 	case ARG_PTR_TO_WORKQUEUE:
10336 		err = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map);
10337 		if (err < 0)
10338 			return err;
10339 		break;
10340 	case ARG_PTR_TO_TASK_WORK:
10341 		err = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map);
10342 		if (err < 0)
10343 			return err;
10344 		break;
10345 	case ARG_PTR_TO_RCU_HEAD:
10346 		err = check_map_field_pointer(env, reg, argno, BPF_RCU_HEAD, &meta->map);
10347 		if (err < 0)
10348 			return err;
10349 		break;
10350 	case ARG_PTR_TO_IRQ_FLAG:
10351 		err = process_irq_flag(env, reg, argno, meta);
10352 		if (err < 0)
10353 			return err;
10354 		break;
10355 	case ARG_PTR_TO_CTX_OUT:
10356 		if (reg->mem_size != arg_size) {
10357 			verbose(env, "%s expected %u bytes of ctx-provided memory, got %u\n",
10358 				reg_arg_name(env, argno), arg_size, reg->mem_size);
10359 			return -EINVAL;
10360 		}
10361 		break;
10362 	case ARG_PTR_TO_RES_SPIN_LOCK:
10363 	{
10364 		int flags;
10365 
10366 		if (in_rbtree_lock_required_cb(env)) {
10367 			verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n");
10368 			return -EACCES;
10369 		}
10370 
10371 		flags = get_bpf_res_spin_lock_kfunc_flags(meta);
10372 		if (!flags)
10373 			return -EFAULT;
10374 		err = process_spin_lock(env, reg, argno, flags);
10375 		if (err < 0)
10376 			return err;
10377 		break;
10378 	}
10379 	case ARG_KPTR_XCHG_DEST:
10380 		err = process_kptr_func(env, regno, meta);
10381 		if (err)
10382 			return err;
10383 		break;
10384 	}
10385 
10386 	return err;
10387 }
10388 
10389 /*
10390  * The slots a parameter takes, from its BTF type. This has to agree with
10391  * btf_func_model_arg_slots(), which answers the same from the size the
10392  * func model recorded, or the verifier would check an argument at a slot
10393  * the JIT does not place it at.
10394  */
10395 static u32 btf_arg_slots(const struct btf_type *t)
10396 {
10397 	if (btf_type_is_int(t) || btf_type_is_struct(t))
10398 		return (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE;
10399 	return 1;
10400 }
10401 
10402 static u32 btf_proto_slots(const struct btf *btf, const struct btf_type *func_proto)
10403 {
10404 	const struct btf_param *args = btf_params(func_proto);
10405 	u32 i, nargs = btf_type_vlen(func_proto), slots_used = 0;
10406 
10407 	for (i = 0; i < nargs; i++)
10408 		slots_used += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL));
10409 
10410 	return slots_used;
10411 }
10412 
10413 /* The argument slot @slot holds an eightbyte of a by-value argument. */
10414 static int check_arg_extra_slot(struct bpf_verifier_env *env, struct bpf_func_state *caller,
10415 				u32 slot, struct bpf_call_arg_meta *meta)
10416 {
10417 	struct bpf_reg_state *reg = get_func_arg_reg(caller, cur_regs(env), slot);
10418 	argno_t argno = argno_from_arg(slot + 1);
10419 	int regno = reg_from_argno(argno);
10420 	int err;
10421 
10422 	if (regno >= 0) {
10423 		err = check_reg_arg(env, regno, SRC_OP);
10424 		if (err)
10425 			return err;
10426 	}
10427 
10428 	return check_reg_type(env, reg, argno, ARG_SCALAR, meta);
10429 }
10430 
10431 static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
10432 			   int insn_idx)
10433 {
10434 	u32 slot = 0, prev_slot = 0, proto_slots, nslots;
10435 	struct bpf_func_state *caller = cur_func(env);
10436 	const struct btf_param *args = NULL;
10437 	u32 arg, nargs = MAX_BPF_FUNC_REG_ARGS;
10438 	int err;
10439 
10440 	if (meta->btf) {
10441 		args = btf_params(meta->func_proto);
10442 		nargs = btf_type_vlen(meta->func_proto);
10443 	}
10444 
10445 	/*
10446 	 * A by-value argument of more than one eightbyte takes a slot per
10447 	 * eightbyte, so the slots a call occupies are no longer its parameter
10448 	 * count. Only a proto whose parameters take a slot each can name the
10449 	 * argument a stack slot belongs to.
10450 	 */
10451 	proto_slots = meta->btf ? btf_proto_slots(meta->btf, meta->func_proto) : nargs;
10452 
10453 	if (proto_slots > MAX_BPF_FUNC_REG_ARGS) {
10454 		err = check_outgoing_stack_args(env, caller, proto_slots, meta->func_name,
10455 						meta->btf, proto_slots == nargs ? args : NULL);
10456 		if (err)
10457 			return err;
10458 	}
10459 
10460 	for (arg = 0; arg < nargs; arg++, prev_slot = slot, slot += nslots) {
10461 		const struct btf_type *t;
10462 		u32 k;
10463 
10464 		nslots = 1;
10465 		if (args) {
10466 			t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL);
10467 			nslots = btf_arg_slots(t);
10468 		}
10469 
10470 		if (meta->fn->arg_type[arg] == ARG_UNUSED)
10471 			break;
10472 		err = check_func_arg(env, arg, slot, prev_slot, meta, insn_idx);
10473 		if (err)
10474 			return err;
10475 
10476 		/*
10477 		 * check_func_arg() took the first slot. A parameter of more
10478 		 * than one eightbyte is always a scalar, so every slot it
10479 		 * takes beyond the first holds one too.
10480 		 */
10481 		for (k = 1; k < nslots; k++) {
10482 			err = check_arg_extra_slot(env, caller, slot + k, meta);
10483 			if (err)
10484 				return err;
10485 		}
10486 	}
10487 
10488 	return 0;
10489 }
10490 
10491 static int mark_raw_stack(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
10492 			  int insn_idx)
10493 {
10494 	struct bpf_func_state *caller = cur_func(env);
10495 	u32 slot;
10496 	int i, err;
10497 
10498 	/*
10499 	 * Validate every argument before initializing outputs: an input argument
10500 	 * may alias an output buffer. Use the normal stack-write checks to discard
10501 	 * stale spills and preserve the rules for special stack objects.
10502 	 */
10503 	for (slot = 0; slot < MAX_BPF_FUNC_ARGS; slot++) {
10504 		struct bpf_reg_state *reg;
10505 
10506 		if (!meta->arg_raw_mem.size[slot])
10507 			continue;
10508 		reg = get_func_arg_reg(caller, cur_regs(env), slot);
10509 
10510 		/*
10511 		 * check_stack_range_initialized() checked the range, its offset is a constant.
10512 		 * Not check_stack_write(): unknown bytes in its buffer leave a dynptr slice valid.
10513 		 */
10514 		for (i = 0; i < meta->arg_raw_mem.size[slot]; i++) {
10515 			err = check_stack_write_fixed_off(env, bpf_func(env, reg),
10516 							  reg->var_off.value + i, 1, -1, insn_idx);
10517 			if (err)
10518 				return err;
10519 		}
10520 	}
10521 
10522 	return 0;
10523 }
10524 
10525 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
10526 {
10527 	enum bpf_attach_type eatype = env->prog->expected_attach_type;
10528 	enum bpf_prog_type type = resolve_prog_type(env->prog);
10529 	const char *afname = env->prog->aux->attach_func_name;
10530 
10531 	if (func_id != BPF_FUNC_map_update_elem &&
10532 	    func_id != BPF_FUNC_map_delete_elem)
10533 		return false;
10534 
10535 	/* It's not possible to get access to a locked struct sock in these
10536 	 * contexts, so updating is safe.
10537 	 */
10538 	switch (type) {
10539 	case BPF_PROG_TYPE_TRACING:
10540 		if (eatype == BPF_TRACE_ITER &&
10541 		    !strcmp(afname + strlen(BPF_ITER_FUNC_PREFIX), "sockmap"))
10542 			return true;
10543 		break;
10544 	case BPF_PROG_TYPE_SOCK_OPS:
10545 		/* map_update allowed only via dedicated helpers with event type checks */
10546 		if (func_id == BPF_FUNC_map_delete_elem)
10547 			return true;
10548 		break;
10549 	case BPF_PROG_TYPE_SK_REUSEPORT:
10550 	case BPF_PROG_TYPE_SK_LOOKUP:
10551 		return true;
10552 	default:
10553 		break;
10554 	}
10555 
10556 	verbose(env, "cannot update sockmap in this context\n");
10557 	return false;
10558 }
10559 
10560 bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env)
10561 {
10562 	return env->prog->jit_requested &&
10563 	       bpf_jit_supports_subprog_tailcalls();
10564 }
10565 
10566 /*
10567  * Is rcu_read_lock_held() true at run time? Unlike for in_rcu_cs(), disabled
10568  * preemption or interrupts will not do.
10569  */
10570 static bool has_rcu_read_lock(struct bpf_verifier_env *env)
10571 {
10572 	struct bpf_verifier_state *state = env->cur_state;
10573 
10574 	if (state->active_rcu_locks || in_rcu_callback(env))
10575 		return true;
10576 	if (in_sleepable(env))
10577 		return false;
10578 	return !env->no_rcu_read_lock || state->frame[0]->in_async_callback_fn;
10579 }
10580 
10581 /* May the map be used outside of an RCU read side critical section? */
10582 static bool map_needs_no_rcu(const struct bpf_map *map)
10583 {
10584 	switch (map->map_type) {
10585 	case BPF_MAP_TYPE_HASH:
10586 	case BPF_MAP_TYPE_RHASH:
10587 	case BPF_MAP_TYPE_LRU_HASH:
10588 	case BPF_MAP_TYPE_ARRAY:
10589 	case BPF_MAP_TYPE_PERCPU_HASH:
10590 	case BPF_MAP_TYPE_PERCPU_ARRAY:
10591 	case BPF_MAP_TYPE_LRU_PERCPU_HASH:
10592 	case BPF_MAP_TYPE_LPM_TRIE:
10593 	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
10594 	case BPF_MAP_TYPE_HASH_OF_MAPS:
10595 	case BPF_MAP_TYPE_RINGBUF:
10596 	case BPF_MAP_TYPE_USER_RINGBUF:
10597 	case BPF_MAP_TYPE_INODE_STORAGE:
10598 	case BPF_MAP_TYPE_SK_STORAGE:
10599 	case BPF_MAP_TYPE_TASK_STORAGE:
10600 	case BPF_MAP_TYPE_CGRP_STORAGE:
10601 	case BPF_MAP_TYPE_QUEUE:
10602 	case BPF_MAP_TYPE_STACK:
10603 	case BPF_MAP_TYPE_ARENA:
10604 	case BPF_MAP_TYPE_INSN_ARRAY:
10605 	case BPF_MAP_TYPE_PROG_ARRAY:
10606 	case BPF_MAP_TYPE_BLOOM_FILTER:
10607 		return true;
10608 	default:
10609 		return false;
10610 	}
10611 }
10612 
10613 static int check_map_func_compatibility(struct bpf_verifier_env *env,
10614 					struct bpf_map *map, int func_id)
10615 {
10616 	if (!map)
10617 		return 0;
10618 
10619 	/*
10620 	 * check_map_prog_compatibility() asks the program.  That misses the
10621 	 * sleepable callback of another program, and an inner map.  They may
10622 	 * use any map under the RCU read lock.  Disabled preemption or
10623 	 * interrupts will not do: the maps ask for rcu_read_lock_held().
10624 	 */
10625 	if (!has_rcu_read_lock(env) && !map_needs_no_rcu(map)) {
10626 		verbose(env, "map '%s' needs bpf_rcu_read_lock() in sleepable context\n",
10627 			map->name);
10628 		return -EINVAL;
10629 	}
10630 
10631 	/* We need a two way check, first is from map perspective ... */
10632 	switch (map->map_type) {
10633 	case BPF_MAP_TYPE_PROG_ARRAY:
10634 		if (func_id != BPF_FUNC_tail_call)
10635 			goto error;
10636 		break;
10637 	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
10638 		if (func_id != BPF_FUNC_perf_event_read &&
10639 		    func_id != BPF_FUNC_perf_event_output &&
10640 		    func_id != BPF_FUNC_skb_output &&
10641 		    func_id != BPF_FUNC_perf_event_read_value &&
10642 		    func_id != BPF_FUNC_xdp_output)
10643 			goto error;
10644 		break;
10645 	case BPF_MAP_TYPE_RINGBUF:
10646 		if (func_id != BPF_FUNC_ringbuf_output &&
10647 		    func_id != BPF_FUNC_ringbuf_reserve &&
10648 		    func_id != BPF_FUNC_ringbuf_query &&
10649 		    func_id != BPF_FUNC_ringbuf_reserve_dynptr &&
10650 		    func_id != BPF_FUNC_ringbuf_submit_dynptr &&
10651 		    func_id != BPF_FUNC_ringbuf_discard_dynptr)
10652 			goto error;
10653 		break;
10654 	case BPF_MAP_TYPE_USER_RINGBUF:
10655 		if (func_id != BPF_FUNC_user_ringbuf_drain)
10656 			goto error;
10657 		break;
10658 	case BPF_MAP_TYPE_STACK_TRACE:
10659 		if (func_id != BPF_FUNC_get_stackid)
10660 			goto error;
10661 		break;
10662 	case BPF_MAP_TYPE_CGROUP_ARRAY:
10663 		if (func_id != BPF_FUNC_skb_under_cgroup &&
10664 		    func_id != BPF_FUNC_current_task_under_cgroup)
10665 			goto error;
10666 		break;
10667 	case BPF_MAP_TYPE_CGROUP_STORAGE:
10668 	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
10669 		if (func_id != BPF_FUNC_get_local_storage)
10670 			goto error;
10671 		break;
10672 	case BPF_MAP_TYPE_DEVMAP:
10673 	case BPF_MAP_TYPE_DEVMAP_HASH:
10674 		if (func_id != BPF_FUNC_redirect_map &&
10675 		    func_id != BPF_FUNC_map_lookup_elem)
10676 			goto error;
10677 		break;
10678 	/* Restrict bpf side of cpumap and xskmap, open when use-cases
10679 	 * appear.
10680 	 */
10681 	case BPF_MAP_TYPE_CPUMAP:
10682 		if (func_id != BPF_FUNC_redirect_map)
10683 			goto error;
10684 		break;
10685 	case BPF_MAP_TYPE_XSKMAP:
10686 		if (func_id != BPF_FUNC_redirect_map &&
10687 		    func_id != BPF_FUNC_map_lookup_elem)
10688 			goto error;
10689 		break;
10690 	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
10691 	case BPF_MAP_TYPE_HASH_OF_MAPS:
10692 		if (func_id != BPF_FUNC_map_lookup_elem)
10693 			goto error;
10694 		break;
10695 	case BPF_MAP_TYPE_SOCKMAP:
10696 		if (func_id != BPF_FUNC_sk_redirect_map &&
10697 		    func_id != BPF_FUNC_sock_map_update &&
10698 		    func_id != BPF_FUNC_msg_redirect_map &&
10699 		    func_id != BPF_FUNC_sk_select_reuseport &&
10700 		    func_id != BPF_FUNC_map_lookup_elem &&
10701 		    !may_update_sockmap(env, func_id))
10702 			goto error;
10703 		break;
10704 	case BPF_MAP_TYPE_SOCKHASH:
10705 		if (func_id != BPF_FUNC_sk_redirect_hash &&
10706 		    func_id != BPF_FUNC_sock_hash_update &&
10707 		    func_id != BPF_FUNC_msg_redirect_hash &&
10708 		    func_id != BPF_FUNC_sk_select_reuseport &&
10709 		    func_id != BPF_FUNC_map_lookup_elem &&
10710 		    !may_update_sockmap(env, func_id))
10711 			goto error;
10712 		break;
10713 	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
10714 		if (func_id != BPF_FUNC_sk_select_reuseport)
10715 			goto error;
10716 		break;
10717 	case BPF_MAP_TYPE_QUEUE:
10718 	case BPF_MAP_TYPE_STACK:
10719 		if (func_id != BPF_FUNC_map_peek_elem &&
10720 		    func_id != BPF_FUNC_map_pop_elem &&
10721 		    func_id != BPF_FUNC_map_push_elem)
10722 			goto error;
10723 		break;
10724 	case BPF_MAP_TYPE_SK_STORAGE:
10725 		if (func_id != BPF_FUNC_sk_storage_get &&
10726 		    func_id != BPF_FUNC_sk_storage_delete &&
10727 		    func_id != BPF_FUNC_kptr_xchg)
10728 			goto error;
10729 		break;
10730 	case BPF_MAP_TYPE_INODE_STORAGE:
10731 		if (func_id != BPF_FUNC_inode_storage_get &&
10732 		    func_id != BPF_FUNC_inode_storage_delete &&
10733 		    func_id != BPF_FUNC_kptr_xchg)
10734 			goto error;
10735 		break;
10736 	case BPF_MAP_TYPE_TASK_STORAGE:
10737 		if (func_id != BPF_FUNC_task_storage_get &&
10738 		    func_id != BPF_FUNC_task_storage_delete &&
10739 		    func_id != BPF_FUNC_kptr_xchg)
10740 			goto error;
10741 		break;
10742 	case BPF_MAP_TYPE_CGRP_STORAGE:
10743 		if (func_id != BPF_FUNC_cgrp_storage_get &&
10744 		    func_id != BPF_FUNC_cgrp_storage_delete &&
10745 		    func_id != BPF_FUNC_kptr_xchg)
10746 			goto error;
10747 		break;
10748 	case BPF_MAP_TYPE_BLOOM_FILTER:
10749 		if (func_id != BPF_FUNC_map_peek_elem &&
10750 		    func_id != BPF_FUNC_map_push_elem)
10751 			goto error;
10752 		break;
10753 	case BPF_MAP_TYPE_INSN_ARRAY:
10754 		goto error;
10755 	default:
10756 		break;
10757 	}
10758 
10759 	/* ... and second from the function itself. */
10760 	switch (func_id) {
10761 	case BPF_FUNC_tail_call:
10762 		if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
10763 			goto error;
10764 		if (env->subprog_cnt > 1 && !bpf_allow_tail_call_in_subprogs(env)) {
10765 			verbose(env, "mixing of tail_calls and bpf-to-bpf calls is not supported\n");
10766 			return -EINVAL;
10767 		}
10768 		break;
10769 	case BPF_FUNC_perf_event_read:
10770 	case BPF_FUNC_perf_event_output:
10771 	case BPF_FUNC_perf_event_read_value:
10772 	case BPF_FUNC_skb_output:
10773 	case BPF_FUNC_xdp_output:
10774 		if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
10775 			goto error;
10776 		break;
10777 	case BPF_FUNC_ringbuf_output:
10778 	case BPF_FUNC_ringbuf_reserve:
10779 	case BPF_FUNC_ringbuf_query:
10780 	case BPF_FUNC_ringbuf_reserve_dynptr:
10781 	case BPF_FUNC_ringbuf_submit_dynptr:
10782 	case BPF_FUNC_ringbuf_discard_dynptr:
10783 		if (map->map_type != BPF_MAP_TYPE_RINGBUF)
10784 			goto error;
10785 		break;
10786 	case BPF_FUNC_user_ringbuf_drain:
10787 		if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF)
10788 			goto error;
10789 		break;
10790 	case BPF_FUNC_get_stackid:
10791 		if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
10792 			goto error;
10793 		break;
10794 	case BPF_FUNC_current_task_under_cgroup:
10795 	case BPF_FUNC_skb_under_cgroup:
10796 		if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
10797 			goto error;
10798 		break;
10799 	case BPF_FUNC_redirect_map:
10800 		if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
10801 		    map->map_type != BPF_MAP_TYPE_DEVMAP_HASH &&
10802 		    map->map_type != BPF_MAP_TYPE_CPUMAP &&
10803 		    map->map_type != BPF_MAP_TYPE_XSKMAP)
10804 			goto error;
10805 		break;
10806 	case BPF_FUNC_sk_redirect_map:
10807 	case BPF_FUNC_msg_redirect_map:
10808 	case BPF_FUNC_sock_map_update:
10809 		if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
10810 			goto error;
10811 		break;
10812 	case BPF_FUNC_sk_redirect_hash:
10813 	case BPF_FUNC_msg_redirect_hash:
10814 	case BPF_FUNC_sock_hash_update:
10815 		if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
10816 			goto error;
10817 		break;
10818 	case BPF_FUNC_get_local_storage:
10819 		if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
10820 		    map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
10821 			goto error;
10822 		break;
10823 	case BPF_FUNC_sk_select_reuseport:
10824 		if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY &&
10825 		    map->map_type != BPF_MAP_TYPE_SOCKMAP &&
10826 		    map->map_type != BPF_MAP_TYPE_SOCKHASH)
10827 			goto error;
10828 		break;
10829 	case BPF_FUNC_map_pop_elem:
10830 		if (map->map_type != BPF_MAP_TYPE_QUEUE &&
10831 		    map->map_type != BPF_MAP_TYPE_STACK)
10832 			goto error;
10833 		break;
10834 	case BPF_FUNC_map_peek_elem:
10835 	case BPF_FUNC_map_push_elem:
10836 		if (map->map_type != BPF_MAP_TYPE_QUEUE &&
10837 		    map->map_type != BPF_MAP_TYPE_STACK &&
10838 		    map->map_type != BPF_MAP_TYPE_BLOOM_FILTER)
10839 			goto error;
10840 		break;
10841 	case BPF_FUNC_map_lookup_percpu_elem:
10842 		if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
10843 		    map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
10844 		    map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH)
10845 			goto error;
10846 		break;
10847 	case BPF_FUNC_sk_storage_get:
10848 	case BPF_FUNC_sk_storage_delete:
10849 		if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
10850 			goto error;
10851 		break;
10852 	case BPF_FUNC_inode_storage_get:
10853 	case BPF_FUNC_inode_storage_delete:
10854 		if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
10855 			goto error;
10856 		break;
10857 	case BPF_FUNC_task_storage_get:
10858 	case BPF_FUNC_task_storage_delete:
10859 		if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
10860 			goto error;
10861 		break;
10862 	case BPF_FUNC_cgrp_storage_get:
10863 	case BPF_FUNC_cgrp_storage_delete:
10864 		if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE)
10865 			goto error;
10866 		break;
10867 	default:
10868 		break;
10869 	}
10870 
10871 	return 0;
10872 error:
10873 	verbose(env, "cannot pass map_type %d into func %s#%d\n",
10874 		map->map_type, func_id_name(func_id), func_id);
10875 	return -EINVAL;
10876 }
10877 
10878 static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg)
10879 {
10880 	bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE;
10881 	bool has_size = fn->arg_size[arg] != 0;
10882 	bool is_next_size = false;
10883 
10884 	if (arg + 1 < ARRAY_SIZE(fn->arg_type))
10885 		is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]);
10886 
10887 	if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM)
10888 		return is_next_size;
10889 
10890 	return has_size == is_next_size || is_next_size == is_fixed;
10891 }
10892 
10893 static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
10894 {
10895 	/* bpf_xxx(..., buf, len) call will access 'len'
10896 	 * bytes from memory 'buf'. Both arg types need
10897 	 * to be paired, so make sure there's no buggy
10898 	 * helper function specification.
10899 	 */
10900 	if (arg_type_is_mem_size(fn->arg1_type) ||
10901 	    check_args_pair_invalid(fn, 0) ||
10902 	    check_args_pair_invalid(fn, 1) ||
10903 	    check_args_pair_invalid(fn, 2) ||
10904 	    check_args_pair_invalid(fn, 3) ||
10905 	    check_args_pair_invalid(fn, 4))
10906 		return false;
10907 
10908 	return true;
10909 }
10910 
10911 static bool check_btf_id_ok(const struct bpf_func_proto *fn)
10912 {
10913 	int i;
10914 
10915 	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
10916 		if (fn->arg_type[i] == ARG_UNUSED)
10917 			break;
10918 		if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID)
10919 			return !!fn->arg_btf_id[i];
10920 		if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK)
10921 			return fn->arg_btf_id[i] == BPF_PTR_POISON;
10922 		if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] &&
10923 		    /* arg_btf_id and arg_size are in a union. */
10924 		    (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM ||
10925 		     !(fn->arg_type[i] & MEM_FIXED_SIZE)))
10926 			return false;
10927 	}
10928 
10929 	return true;
10930 }
10931 
10932 static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn)
10933 {
10934 	int i;
10935 
10936 	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
10937 		enum bpf_arg_type arg_type = fn->arg_type[i];
10938 
10939 		if (arg_type == ARG_UNUSED)
10940 			break;
10941 		if (base_type(arg_type) != ARG_PTR_TO_MEM)
10942 			continue;
10943 		if (!(arg_type & (MEM_WRITE | MEM_RDONLY)))
10944 			return false;
10945 	}
10946 
10947 	return true;
10948 }
10949 
10950 static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta)
10951 {
10952 	int i;
10953 
10954 	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
10955 		enum bpf_arg_type arg_type = fn->arg_type[i];
10956 
10957 		if (arg_type == ARG_UNUSED)
10958 			break;
10959 		if (arg_type_is_release(arg_type)) {
10960 			if (meta->release_regno)
10961 				return false;
10962 			meta->release_regno = i + 1;
10963 		}
10964 	}
10965 
10966 	return true;
10967 }
10968 
10969 static bool check_arg_prog_aux(struct bpf_verifier_env *env,
10970 			       const struct bpf_func_proto *proto)
10971 {
10972 	bool seen = false;
10973 	argno_t argno;
10974 	u32 i;
10975 
10976 	for (i = 0; i < ARRAY_SIZE(proto->arg_type); i++) {
10977 		if (proto->arg_type[i] == ARG_UNUSED)
10978 			break;
10979 		if (proto->arg_type[i] != ARG_PTR_TO_PROG_AUX)
10980 			continue;
10981 
10982 		if (seen) {
10983 			verifier_bug(env, "Only 1 prog->aux argument supported");
10984 			return false;
10985 		}
10986 
10987 		argno = argno_from_arg(i + 1);
10988 		if (reg_from_argno(argno) < 0) {
10989 			verbose(env, "%s prog->aux cannot be a stack argument\n",
10990 				reg_arg_name(env, argno));
10991 			return false;
10992 		}
10993 
10994 		seen = true;
10995 	}
10996 
10997 	return true;
10998 }
10999 
11000 static int check_func_proto(struct bpf_verifier_env *env, const struct bpf_func_proto *fn,
11001 			    struct bpf_call_arg_meta *meta)
11002 {
11003 	return check_arg_prog_aux(env, fn) &&
11004 	       check_arg_pair_ok(fn) &&
11005 	       check_mem_arg_rw_flag_ok(fn) &&
11006 	       check_proto_release_reg(fn, meta) &&
11007 	       check_btf_id_ok(fn) ? 0 : -EINVAL;
11008 }
11009 
11010 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
11011  * are now invalid, so turn them into unknown SCALAR_VALUE.
11012  *
11013  * This also applies to dynptr slices belonging to skb and xdp dynptrs,
11014  * since these slices point to packet data.
11015  */
11016 static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
11017 {
11018 	struct bpf_func_state *state;
11019 	struct bpf_reg_state *reg;
11020 
11021 	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
11022 		if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) {
11023 			bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_PKT_DATA_CHANGE);
11024 			mark_reg_invalid(env, reg);
11025 		}
11026 	}));
11027 }
11028 
11029 static bool holds_pkt_pointers(struct bpf_verifier_env *env)
11030 {
11031 	struct bpf_func_state *state;
11032 	struct bpf_reg_state *reg;
11033 	bool ret = false;
11034 
11035 	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
11036 		if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))
11037 			ret = true;
11038 	}));
11039 
11040 	return ret;
11041 }
11042 
11043 enum {
11044 	AT_PKT_END = -1,
11045 	BEYOND_PKT_END = -2,
11046 };
11047 
11048 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open)
11049 {
11050 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
11051 	struct bpf_reg_state *reg = &state->regs[regn];
11052 
11053 	if (reg->type != PTR_TO_PACKET)
11054 		/* PTR_TO_PACKET_META is not supported yet */
11055 		return;
11056 
11057 	/* The 'reg' is pkt > pkt_end or pkt >= pkt_end.
11058 	 * How far beyond pkt_end it goes is unknown.
11059 	 * if (!range_open) it's the case of pkt >= pkt_end
11060 	 * if (range_open) it's the case of pkt > pkt_end
11061 	 * hence this pointer is at least 1 byte bigger than pkt_end
11062 	 */
11063 	if (range_open)
11064 		reg->range = BEYOND_PKT_END;
11065 	else
11066 		reg->range = AT_PKT_END;
11067 }
11068 
11069 static int __release_reference_nomark(struct bpf_verifier_state *state, int id)
11070 {
11071 	int i;
11072 
11073 	for (i = 0; i < state->acquired_refs; i++) {
11074 		if (state->refs[i].type != REF_TYPE_PTR)
11075 			continue;
11076 		if (state->refs[i].id == id) {
11077 			release_reference_state(state, i);
11078 			return 0;
11079 		}
11080 	}
11081 	return -EINVAL;
11082 }
11083 
11084 static int release_reference_nomark(struct bpf_verifier_env *env, int id)
11085 {
11086 	int err;
11087 
11088 	err = __release_reference_nomark(env->cur_state, id);
11089 	if (!err)
11090 		bpf_diag_record_ref_release(env, env->insn_idx, id);
11091 	return err;
11092 }
11093 
11094 static int idstack_push(struct bpf_idmap *idmap, u32 id)
11095 {
11096 	int i;
11097 
11098 	if (!id)
11099 		return 0;
11100 
11101 	for (i = 0; i < idmap->cnt; i++)
11102 		if (idmap->map[i].old == id)
11103 			return 0;
11104 
11105 	if (WARN_ON_ONCE(idmap->cnt >= BPF_ID_MAP_SIZE))
11106 		return -EFAULT;
11107 
11108 	idmap->map[idmap->cnt++].old = id;
11109 	return 0;
11110 }
11111 
11112 static int idstack_pop(struct bpf_idmap *idmap)
11113 {
11114 	if (!idmap->cnt)
11115 		return 0;
11116 
11117 	return idmap->map[--idmap->cnt].old;
11118 }
11119 
11120 /* Invalidate id and objects derived from it iteratively in a DFS manner */
11121 static int invalidate_reference(struct bpf_verifier_env *env, int id)
11122 {
11123 	u32 mask = (1 << STACK_SPILL) | (1 << STACK_DYNPTR);
11124 	struct bpf_verifier_state *vstate = env->cur_state;
11125 	struct bpf_idmap *idstack = &env->idmap_scratch;
11126 	struct bpf_stack_state *stack;
11127 	struct bpf_func_state *state;
11128 	struct bpf_reg_state *reg;
11129 	int i, err;
11130 
11131 	idstack->cnt = 0;
11132 	err = idstack_push(idstack, id);
11133 	if (err)
11134 		return err;
11135 
11136 	while ((id = idstack_pop(idstack))) {
11137 		/*
11138 		 * Child references are inaccessible after parent is released,
11139 		 * any child references that exist at this point are a leak.
11140 		 */
11141 		for (i = 0; i < vstate->acquired_refs; i++) {
11142 			if (vstate->refs[i].type != REF_TYPE_PTR)
11143 				continue;
11144 			if (vstate->refs[i].parent_id != id)
11145 				continue;
11146 			verbose(env, "Leaking reference id=%d alloc_insn=%d. Release it first.\n",
11147 				vstate->refs[i].id, vstate->refs[i].insn_idx);
11148 			return -EINVAL;
11149 		}
11150 
11151 		bpf_for_each_reg_in_vstate_mask(vstate, state, reg, stack, mask, ({
11152 			if (reg->id != id && reg->parent_id != id)
11153 				continue;
11154 
11155 			/* Free objects derived from the current object */
11156 			if (reg->parent_id == id) {
11157 				err = idstack_push(idstack, reg->id);
11158 				if (err)
11159 					return err;
11160 			}
11161 
11162 			/*
11163 			 * A dynptr occupies two stack slots that invalidate_dynptr()
11164 			 * clears together. Record both scrubs before invalidating it.
11165 			 */
11166 			if (stack && stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) {
11167 				struct bpf_stack_state *dyn_stack = stack;
11168 
11169 				if (reg->dynptr.first_slot)
11170 					dyn_stack--;
11171 				bpf_diag_record_scrub(env, &dyn_stack[0].spilled_ptr,
11172 						      BPF_DIAG_MOD_REF_RELEASE);
11173 				bpf_diag_record_scrub(env, &dyn_stack[1].spilled_ptr,
11174 						      BPF_DIAG_MOD_REF_RELEASE);
11175 				invalidate_dynptr(env, dyn_stack);
11176 				continue;
11177 			}
11178 			bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_REF_RELEASE);
11179 			if (!stack || stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL)
11180 				mark_reg_invalid(env, reg);
11181 		}));
11182 	}
11183 
11184 	return 0;
11185 }
11186 
11187 static int release_reference(struct bpf_verifier_env *env, int id)
11188 {
11189 	if (find_reference_state(env->cur_state, id))
11190 		WARN_ON_ONCE(release_reference_nomark(env, id));
11191 
11192 	return invalidate_reference(env, id);
11193 }
11194 
11195 static void invalidate_non_owning_refs(struct bpf_verifier_env *env)
11196 {
11197 	struct bpf_func_state *unused;
11198 	struct bpf_reg_state *reg;
11199 
11200 	bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
11201 		if (type_is_non_owning_ref(reg->type)) {
11202 			bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_NON_OWN_REF);
11203 			mark_reg_invalid(env, reg);
11204 		}
11205 	}));
11206 }
11207 
11208 static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env)
11209 {
11210 	struct bpf_stack_state *stack;
11211 	struct bpf_func_state *state;
11212 	struct bpf_reg_state *reg;
11213 	u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);
11214 
11215 	bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({
11216 		if (reg->type & MEM_RCU) {
11217 			bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE);
11218 			reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL | NON_OWN_REF);
11219 			reg->type |= PTR_UNTRUSTED;
11220 			bpf_diag_mod_end(env);
11221 		}
11222 	}));
11223 }
11224 
11225 static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id)
11226 {
11227 	struct bpf_func_state *state;
11228 	struct bpf_reg_state *reg;
11229 	int err;
11230 
11231 	err = release_reference_nomark(env, id);
11232 	if (err)
11233 		return err;
11234 
11235 	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
11236 		if (reg->id != id)
11237 			continue;
11238 		if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) {
11239 			bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE);
11240 			if (!type_may_be_null(reg->type))
11241 				reg->id = 0;
11242 			reg->type &= ~MEM_ALLOC;
11243 			reg->type |= MEM_RCU;
11244 			bpf_diag_mod_end(env);
11245 		}
11246 	}));
11247 
11248 	return err;
11249 }
11250 
11251 static void clear_caller_saved_regs(struct bpf_verifier_env *env,
11252 				    struct bpf_reg_state *regs)
11253 {
11254 	int i;
11255 
11256 	bpf_diag_record_caller_saved(env, regs);
11257 
11258 	/* after the call registers r0 - r5 were scratched */
11259 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
11260 		bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);
11261 		__check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK);
11262 	}
11263 }
11264 
11265 static void invalidate_outgoing_stack_args(struct bpf_verifier_env *env,
11266 					   struct bpf_func_state *state)
11267 {
11268 	int i, nslots = state->out_stack_arg_cnt;
11269 
11270 	for (i = 0; i < nslots; i++) {
11271 		bpf_diag_record_scrub(env, &state->stack_arg_regs[i], BPF_DIAG_MOD_CALLER_SAVED);
11272 		bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]);
11273 	}
11274 }
11275 
11276 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
11277 				   struct bpf_func_state *caller,
11278 				   struct bpf_func_state *callee,
11279 				   int insn_idx);
11280 
11281 static int set_callee_state(struct bpf_verifier_env *env,
11282 			    struct bpf_func_state *caller,
11283 			    struct bpf_func_state *callee, int insn_idx);
11284 
11285 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite,
11286 			    set_callee_state_fn set_callee_state_cb,
11287 			    struct bpf_verifier_state *state)
11288 {
11289 	struct bpf_func_state *caller, *callee;
11290 	int err;
11291 
11292 	if (state->curframe + 1 >= MAX_CALL_FRAMES) {
11293 		verbose(env, "the call stack of %d frames is too deep\n",
11294 			state->curframe + 2);
11295 		return -E2BIG;
11296 	}
11297 
11298 	if (state->frame[state->curframe + 1]) {
11299 		verifier_bug(env, "Frame %d already allocated", state->curframe + 1);
11300 		return -EFAULT;
11301 	}
11302 
11303 	caller = state->frame[state->curframe];
11304 	callee = kzalloc_obj(*callee, GFP_KERNEL_ACCOUNT);
11305 	if (!callee)
11306 		return -ENOMEM;
11307 	state->frame[state->curframe + 1] = callee;
11308 
11309 	/* callee cannot access r0, r6 - r9 for reading and has to write
11310 	 * into its own stack before reading from it.
11311 	 * callee can read/write into caller's stack
11312 	 */
11313 	init_func_state(env, callee,
11314 			/* remember the callsite, it will be used by bpf_exit */
11315 			callsite,
11316 			state->curframe + 1 /* frameno within this callchain */,
11317 			subprog /* subprog number within this prog */);
11318 	err = set_callee_state_cb(env, caller, callee, callsite);
11319 	if (err)
11320 		goto err_out;
11321 
11322 	/* only increment it after check_reg_arg() finished */
11323 	state->curframe++;
11324 
11325 	return 0;
11326 
11327 err_out:
11328 	free_func_state(callee);
11329 	state->frame[state->curframe + 1] = NULL;
11330 	return err;
11331 }
11332 
11333 static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const struct btf *btf,
11334 				  const struct btf_type *func_proto, struct bpf_func_proto *proto)
11335 {
11336 	const struct btf_param *args = btf_params(func_proto);
11337 	u32 arg, slot = 0;
11338 
11339 	memset(proto, 0, sizeof(*proto));
11340 	for (arg = 0; arg < btf_type_vlen(func_proto); arg++) {
11341 		enum bpf_arg_type arg_type = sub->args[slot].arg_type;
11342 		const struct btf_type *t;
11343 
11344 		if (arg_type & PTR_UNTRUSTED) {
11345 			/*
11346 			 * An __arg_untrusted argument accepts any caller value. The
11347 			 * callee treats it as read-only and uses probe-read instructions
11348 			 * to protect against invalid memory access.
11349 			 */
11350 			arg_type = ARG_IGNORE;
11351 		} else if (base_type(arg_type) == ARG_PTR_TO_ARENA) {
11352 			arg_type |= PTR_MAYBE_NULL;
11353 		} else if (base_type(arg_type) == ARG_PTR_TO_MEM) {
11354 			proto->arg_size[arg] = sub->args[slot].mem_size;
11355 			arg_type |= MEM_FIXED_SIZE | MEM_WRITE;
11356 		} else if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) {
11357 			proto->arg_btf_id[arg] = &sub->args[slot].btf_id;
11358 		}
11359 		proto->arg_type[arg] = arg_type;
11360 		t = btf_type_skip_modifiers(btf, args[arg].type, NULL);
11361 		slot += btf_arg_slots(t);
11362 	}
11363 }
11364 
11365 static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct btf *btf)
11366 {
11367 	struct bpf_subprog_info *sub = subprog_info(env, subprog);
11368 	struct bpf_func_state *caller = cur_func(env);
11369 	const struct btf_type *func, *func_proto;
11370 	struct bpf_call_arg_meta meta;
11371 	struct bpf_func_proto *fn;
11372 	int ret, err;
11373 	u32 i;
11374 
11375 	memset(&meta, 0, sizeof(meta));
11376 	meta.btf = btf;
11377 	meta.subprog_may_change_pkt = sub->changes_pkt_data;
11378 	meta.func_name = bpf_subprog_name(env, subprog);
11379 
11380 	ret = btf_prepare_func_args(env, subprog);
11381 	if (ret) {
11382 		if (bpf_in_stack_arg_cnt(sub) > 0) {
11383 			err = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt,
11384 							bpf_subprog_name(env, subprog),
11385 							NULL, NULL);
11386 			if (err)
11387 				return err;
11388 		}
11389 		return ret;
11390 	}
11391 
11392 	func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id);
11393 	func_proto = btf_type_by_id(btf, func->type);
11394 
11395 	fn = &env->bpf_subprog_scratch;
11396 	gen_subprog_arg_proto(sub, btf, func_proto, fn);
11397 	meta.fn = fn;
11398 	meta.func_proto = func_proto;
11399 
11400 	/*
11401 	 * Check if all register arguments declared in BTF are initialized.
11402 	 * If they aren't the BTF is unreliable.
11403 	 */
11404 	if (!bpf_subprog_is_global(env, subprog)) {
11405 		for (i = 0; i < min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++)
11406 			if (caller->regs[BPF_REG_1 + i].type == NOT_INIT)
11407 				return -EINVAL;
11408 	}
11409 	return check_func_args(env, &meta, env->insn_idx);
11410 }
11411 
11412 /*
11413  * Check that call-site argument states match a subprog's BTF signature.
11414  *
11415  * Returns:
11416  * EFAULT - there is a verifier bug. Abort verification.
11417  * Other errors - there is a type mismatch or BTF is not available.
11418  * 0 - BTF matches with what bpf_reg_state expects.
11419  */
11420 static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog)
11421 {
11422 	struct bpf_prog *prog = env->prog;
11423 	struct btf *btf = prog->aux->btf;
11424 	u32 btf_id;
11425 	int err;
11426 
11427 	if (!prog->aux->func_info)
11428 		return -EINVAL;
11429 
11430 	btf_id = prog->aux->func_info[subprog].type_id;
11431 	if (!btf_id)
11432 		return -EFAULT;
11433 
11434 	if (prog->aux->func_info_aux[subprog].unreliable)
11435 		return -EINVAL;
11436 
11437 	err = btf_check_func_arg_match(env, subprog, btf);
11438 	/* Compiler optimizations can remove arguments from static functions
11439 	 * or mismatched type can be passed into a global function.
11440 	 * In such cases mark the function as unreliable from BTF point of view.
11441 	 */
11442 	if (err)
11443 		prog->aux->func_info_aux[subprog].unreliable = true;
11444 	return err;
11445 }
11446 
11447 static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
11448 			      int insn_idx, int subprog,
11449 			      set_callee_state_fn set_callee_state_cb)
11450 {
11451 	struct bpf_verifier_state *state = env->cur_state, *callback_state;
11452 	struct bpf_func_state *caller, *callee;
11453 	int err;
11454 
11455 	caller = state->frame[state->curframe];
11456 	err = btf_check_subprog_call(env, subprog);
11457 	if (err == -EFAULT)
11458 		return err;
11459 	if (bpf_in_stack_arg_cnt(&env->subprog_info[subprog])) {
11460 		verbose(env, "callback subprog cannot have stack args\n");
11461 		return -EINVAL;
11462 	}
11463 
11464 	/* set_callee_state is used for direct subprog calls, but we are
11465 	 * interested in validating only BPF helpers that can call subprogs as
11466 	 * callbacks
11467 	 */
11468 	env->subprog_info[subprog].is_cb = true;
11469 	if (bpf_pseudo_kfunc_call(insn) &&
11470 	    !is_callback_calling_kfunc(insn->imm)) {
11471 		verifier_bug(env, "kfunc %s#%d not marked as callback-calling",
11472 			     func_id_name(insn->imm), insn->imm);
11473 		return -EFAULT;
11474 	} else if (!bpf_pseudo_kfunc_call(insn) &&
11475 		   !is_callback_calling_function(insn->imm)) { /* helper */
11476 		verifier_bug(env, "helper %s#%d not marked as callback-calling",
11477 			     func_id_name(insn->imm), insn->imm);
11478 		return -EFAULT;
11479 	}
11480 
11481 	if (bpf_is_async_callback_calling_insn(insn)) {
11482 		struct bpf_verifier_state *async_cb;
11483 
11484 		/* there is no real recursion here. timer and workqueue callbacks are async */
11485 		env->subprog_info[subprog].is_async_cb = true;
11486 		async_cb = push_async_cb(env, env->subprog_info[subprog].start,
11487 					 insn_idx, subprog,
11488 					 is_async_cb_sleepable(env, insn));
11489 		if (IS_ERR(async_cb))
11490 			return PTR_ERR(async_cb);
11491 		if (is_async_cb_sleepable(env, insn))
11492 			mark_sleepable_async_cb(env);
11493 		callee = async_cb->frame[0];
11494 		callee->async_entry_cnt = state->frame[0]->async_entry_cnt + 1;
11495 
11496 		/* Convert bpf_timer_set_callback() args into timer callback args */
11497 		err = set_callee_state_cb(env, caller, callee, insn_idx);
11498 		if (err)
11499 			return err;
11500 
11501 		return 0;
11502 	}
11503 
11504 	/*
11505 	 * The address of the callback might be taken in another function or
11506 	 * read from a map, see check_func_ptr_read(). Like with callx it's
11507 	 * known here what is called.
11508 	 */
11509 	err = record_callx_edge(env, caller->subprogno, subprog);
11510 	if (err)
11511 		return err;
11512 
11513 	if (!bpf_pseudo_kfunc_call(insn) &&
11514 	    (insn->imm == BPF_FUNC_loop || insn->imm == BPF_FUNC_for_each_map_elem))
11515 		env->subprog_info[subprog].is_loop_cb = true;
11516 	else
11517 		env->subprog_info[subprog].is_called = true;
11518 
11519 	/* for callback functions enqueue entry to callback and
11520 	 * proceed with next instruction within current frame.
11521 	 */
11522 	callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);
11523 	if (IS_ERR(callback_state))
11524 		return PTR_ERR(callback_state);
11525 
11526 	err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,
11527 			       callback_state);
11528 	if (err)
11529 		return err;
11530 
11531 	callback_state->callback_unroll_depth++;
11532 	callback_state->frame[callback_state->curframe - 1]->callback_depth++;
11533 	caller->callback_depth = 0;
11534 	return 0;
11535 }
11536 
11537 static int process_bpf_exit_full(struct bpf_verifier_env *env,
11538 				 bool *do_print_state, bool exception_exit);
11539 
11540 /*
11541  * Call of a static subprog. The callee is verified in the context of
11542  * the caller, hence set up a new frame and continue from the first
11543  * instruction of the callee.
11544  */
11545 static int check_static_func_call(struct bpf_verifier_env *env, int subprog,
11546 				  int *insn_idx)
11547 {
11548 	struct bpf_verifier_state *state = env->cur_state;
11549 	struct bpf_subprog_info *caller_info;
11550 	u16 callee_incoming, stack_arg_cnt;
11551 	struct bpf_func_state *caller;
11552 	int err;
11553 
11554 	caller = state->frame[state->curframe];
11555 	env->subprog_info[subprog].is_called = true;
11556 
11557 	/*
11558 	 * Track caller's total stack arg count (incoming + max outgoing).
11559 	 * This is needed so the JIT knows how much stack arg space to allocate.
11560 	 */
11561 	caller_info = &env->subprog_info[caller->subprogno];
11562 	callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]);
11563 	stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming;
11564 	if (stack_arg_cnt > caller_info->stack_arg_cnt)
11565 		caller_info->stack_arg_cnt = stack_arg_cnt;
11566 
11567 	/*
11568 	 * For regular function entry setup new frame and continue
11569 	 * from that frame.
11570 	 */
11571 	err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);
11572 	if (err)
11573 		return err;
11574 
11575 	bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED);
11576 	clear_caller_saved_regs(env, caller->regs);
11577 
11578 	/* and go analyze first insn of the callee */
11579 	*insn_idx = env->subprog_info[subprog].start - 1;
11580 
11581 	if (env->log.level & BPF_LOG_LEVEL) {
11582 		verbose(env, "caller:\n");
11583 		print_verifier_state(env, state, caller->frameno, true);
11584 		verbose(env, "callee:\n");
11585 		print_verifier_state(env, state, state->curframe, true);
11586 	}
11587 
11588 	return 0;
11589 }
11590 
11591 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
11592 			   int *insn_idx)
11593 {
11594 	struct bpf_verifier_state *state = env->cur_state;
11595 	struct bpf_func_state *caller;
11596 	int err, subprog, target_insn;
11597 	u32 i, nregs;
11598 
11599 	target_insn = *insn_idx + insn->imm + 1;
11600 	subprog = bpf_find_subprog(env, target_insn);
11601 	if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program",
11602 			    target_insn))
11603 		return -EFAULT;
11604 
11605 	caller = state->frame[state->curframe];
11606 	err = btf_check_subprog_call(env, subprog);
11607 	if (err == -EFAULT)
11608 		return err;
11609 	if (bpf_subprog_is_global(env, subprog)) {
11610 		struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog);
11611 		const char *sub_name = bpf_subprog_name(env, subprog);
11612 		const char *operation;
11613 		bool returns_void;
11614 
11615 		if (env->cur_state->active_locks) {
11616 			verbose(env, "global function calls are not allowed while holding a lock,\n"
11617 				     "use static function instead\n");
11618 			operation = bpf_diag_fmt(env, "global function %s()", sub_name);
11619 			bpf_diag_ctx_active(env, *insn_idx, operation, BPF_DIAG_CONTEXT_LOCK,
11620 					    "Release the lock before calling the global function, or use a static function instead.");
11621 			return -EINVAL;
11622 		}
11623 
11624 		if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) {
11625 			const char *suggestion;
11626 
11627 			verbose(env, "sleepable global function %s() called in %s\n",
11628 				sub_name, non_sleepable_context_description(env));
11629 			if (in_sleepable(env))
11630 				suggestion = "Move the call outside the critical section, or use a non-sleepable function.";
11631 			else
11632 				suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable function.";
11633 			operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name);
11634 			bpf_diag_ctx_forbidden(env, *insn_idx, operation, suggestion);
11635 			return -EINVAL;
11636 		}
11637 
11638 		if (err) {
11639 			verbose(env, "Caller passes invalid args into func#%d ('%s')\n",
11640 				subprog, sub_name);
11641 			return err;
11642 		}
11643 
11644 		if (env->log.level & BPF_LOG_LEVEL)
11645 			verbose(env, "Func#%d ('%s') is global and assumed valid.\n",
11646 				subprog, sub_name);
11647 		if (in_sleepable_context(env))
11648 			sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] = true;
11649 		else if (has_rcu_read_lock(env))
11650 			sub_aux->called[BPF_FUNC_CTX_NON_SLEEPABLE] = true;
11651 		else
11652 			sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK] = true;
11653 		returns_void = subprog_returns_void(env, subprog);
11654 		if (env->subprog_info[subprog].changes_pkt_data)
11655 			clear_all_pkt_pointers(env);
11656 		if (returns_void)
11657 			bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED);
11658 		else
11659 			bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE);
11660 		clear_caller_saved_regs(env, caller->regs);
11661 		invalidate_outgoing_stack_args(env, cur_func(env));
11662 		/*
11663 		 * What the callers of a global function hold is not known
11664 		 * while it is verified.
11665 		 */
11666 		if (!env->subprog_info[subprog].changes_pkt_data &&
11667 		    (state->frame[0]->subprogno || holds_pkt_pointers(env)))
11668 			sub_aux->called_with_pkt_ptrs = true;
11669 
11670 		/*
11671 		 * A non-void global function returns a 64-bit SCALAR_VALUE in
11672 		 * R0, or a >8 byte SCALAR_VALUE in the R0:R2 register pair.
11673 		 */
11674 		if (!returns_void) {
11675 			nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1;
11676 			mark_reg_unknown(env, caller->regs, ret_regs[0]);
11677 			bpf_diag_mod_end(env);
11678 			for (i = 1; i < nregs; i++) {
11679 				bpf_diag_mod_begin(env, &caller->regs[ret_regs[i]], NULL,
11680 						   BPF_DIAG_MOD_WRITE);
11681 				mark_reg_unknown(env, caller->regs, ret_regs[i]);
11682 				bpf_diag_mod_end(env);
11683 			}
11684 		}
11685 
11686 		if (env->subprog_info[subprog].can_unwind) {
11687 			struct bpf_verifier_state *branch;
11688 
11689 			if (!bpf_is_unwind_goto(insn + 1)) {
11690 				verbose(env, "no unwind_goto after the call of %s()\n", sub_name);
11691 				return -EINVAL;
11692 			}
11693 			branch = push_stack(env, *insn_idx + 1, *insn_idx, false);
11694 			if (IS_ERR(branch))
11695 				return PTR_ERR(branch);
11696 			branch->unwound = true;
11697 		}
11698 
11699 		if (env->subprog_info[subprog].might_throw) {
11700 			struct bpf_verifier_state *branch;
11701 
11702 			branch = push_stack(env, *insn_idx + 1, *insn_idx, false);
11703 			if (IS_ERR(branch)) {
11704 				verbose(env, "failed to push state for global subprog exception path\n");
11705 				return PTR_ERR(branch);
11706 			}
11707 			return process_bpf_exit_full(env, NULL, true);
11708 		}
11709 
11710 		/* continue with next insn after call */
11711 		return 0;
11712 	}
11713 
11714 	return check_static_func_call(env, subprog, insn_idx);
11715 }
11716 
11717 /*
11718  * callx dst_reg: call a bpf subprog whose address is in dst_reg.
11719  *
11720  * The address of a subprog is either loaded into a register by ld_imm64 with
11721  * src_reg == BPF_PSEUDO_FUNC, or it is read from a frozen read-only map, see
11722  * resolve_func_ptrs(). Both are possible for static subprogs only. Hence all
11723  * possible callees of callx are discovered by add_subprogs() and are reachable
11724  * in the control flow graph before the main verification pass begins.
11725  * PTR_TO_FUNC register identifies the callee, so from here on callx is verified
11726  * as a direct call of that static subprog.
11727  */
11728 static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn,
11729 			    int *insn_idx)
11730 {
11731 	struct bpf_func_state *caller = cur_func(env);
11732 	struct bpf_reg_state *reg;
11733 	const char *reason;
11734 	int err, subprog;
11735 
11736 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
11737 	if (err)
11738 		return err;
11739 
11740 	reg = reg_state(env, insn->dst_reg);
11741 	if (reg->type != PTR_TO_FUNC) {
11742 		verbose(env, "R%d has type %s, expected func\n", insn->dst_reg,
11743 			reg_type_str(env, reg->type));
11744 		reason = bpf_diag_fmt(
11745 			env, "R%d holds %s, but callx can only call through the address of a static BPF function.",
11746 			insn->dst_reg, bpf_diag_reg_type_plain(env, reg->type));
11747 		bpf_diag_register_type(
11748 			env, *insn_idx, insn->dst_reg, "indirect call through a non-function pointer", reason,
11749 			"Load the address of a static BPF function into the register before callx.");
11750 		return -EACCES;
11751 	}
11752 
11753 	/*
11754 	 * Arithmetic on PTR_TO_FUNC is allowed, but only unmodified address
11755 	 * of a subprog can be called.
11756 	 */
11757 	err = check_ptr_off_reg(env, reg, insn->dst_reg);
11758 	if (err)
11759 		return err;
11760 
11761 	/* the callx is valid, but only a JIT can execute it */
11762 	err = require_callx_jit(env);
11763 	if (err)
11764 		return err;
11765 
11766 	/* PTR_TO_FUNC is a pointer to a static subprog */
11767 	subprog = reg->subprogno;
11768 	err = btf_check_subprog_call(env, subprog);
11769 	if (err == -EFAULT)
11770 		return err;
11771 
11772 	err = record_callx_edge(env, caller->subprogno, subprog);
11773 	if (err)
11774 		return err;
11775 
11776 	return check_static_func_call(env, subprog, insn_idx);
11777 }
11778 
11779 int map_set_for_each_callback_args(struct bpf_verifier_env *env,
11780 				   struct bpf_func_state *caller,
11781 				   struct bpf_func_state *callee)
11782 {
11783 	/* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn,
11784 	 *      void *callback_ctx, u64 flags);
11785 	 * callback_fn(struct bpf_map *map, void *key, void *value,
11786 	 *      void *callback_ctx);
11787 	 */
11788 	callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
11789 
11790 	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
11791 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
11792 	callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr;
11793 	callee->regs[BPF_REG_2].map_uid = caller->regs[BPF_REG_1].map_uid;
11794 
11795 	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
11796 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
11797 	callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;
11798 	callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid;
11799 	callee->regs[BPF_REG_3].id = ++env->id_gen;
11800 
11801 	/* pointer to stack or null */
11802 	callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
11803 
11804 	/* unused */
11805 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
11806 	return 0;
11807 }
11808 
11809 static int set_callee_state(struct bpf_verifier_env *env,
11810 			    struct bpf_func_state *caller,
11811 			    struct bpf_func_state *callee, int insn_idx)
11812 {
11813 	int i;
11814 
11815 	/* copy r1 - r5 args that callee can access.  The copy includes parent
11816 	 * pointers, which connects us up to the liveness chain
11817 	 */
11818 	for (i = BPF_REG_1; i <= BPF_REG_5; i++)
11819 		callee->regs[i] = caller->regs[i];
11820 	return 0;
11821 }
11822 
11823 static int set_map_elem_callback_state(struct bpf_verifier_env *env,
11824 				       struct bpf_func_state *caller,
11825 				       struct bpf_func_state *callee,
11826 				       int insn_idx)
11827 {
11828 	struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];
11829 	struct bpf_map *map;
11830 	int err;
11831 
11832 	/* valid map_ptr and poison value does not matter */
11833 	map = insn_aux->map_ptr_state.map_ptr;
11834 	if (!map->ops->map_set_for_each_callback_args ||
11835 	    !map->ops->map_for_each_callback) {
11836 		verbose(env, "callback function not allowed for map\n");
11837 		return -ENOTSUPP;
11838 	}
11839 
11840 	err = map->ops->map_set_for_each_callback_args(env, caller, callee);
11841 	if (err)
11842 		return err;
11843 
11844 	switch (map->map_type) {
11845 	case BPF_MAP_TYPE_HASH:
11846 	case BPF_MAP_TYPE_PERCPU_HASH:
11847 	case BPF_MAP_TYPE_LRU_HASH:
11848 	case BPF_MAP_TYPE_LRU_PERCPU_HASH:
11849 	case BPF_MAP_TYPE_RHASH:
11850 		/* Hash tables run the callback under rcu_read_lock(). */
11851 		callee->in_rcu_callback_fn = true;
11852 		break;
11853 	default:
11854 		break;
11855 	}
11856 
11857 	callee->in_callback_fn = true;
11858 	callee->callback_ret_range = retval_range(0, 1);
11859 	return 0;
11860 }
11861 
11862 static int set_loop_callback_state(struct bpf_verifier_env *env,
11863 				   struct bpf_func_state *caller,
11864 				   struct bpf_func_state *callee,
11865 				   int insn_idx)
11866 {
11867 	/* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx,
11868 	 *	    u64 flags);
11869 	 * callback_fn(u64 index, void *callback_ctx);
11870 	 */
11871 	callee->regs[BPF_REG_1].type = SCALAR_VALUE;
11872 	callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
11873 
11874 	/* unused */
11875 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
11876 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
11877 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
11878 
11879 	callee->in_callback_fn = true;
11880 	callee->callback_ret_range = retval_range(0, 1);
11881 	return 0;
11882 }
11883 
11884 static int set_timer_callback_state(struct bpf_verifier_env *env,
11885 				    struct bpf_func_state *caller,
11886 				    struct bpf_func_state *callee,
11887 				    int insn_idx)
11888 {
11889 	struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr;
11890 	u32 map_uid = caller->regs[BPF_REG_1].map_uid;
11891 
11892 	/* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn);
11893 	 * callback_fn(struct bpf_map *map, void *key, void *value);
11894 	 */
11895 	callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
11896 	__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
11897 	callee->regs[BPF_REG_1].map_ptr = map_ptr;
11898 	callee->regs[BPF_REG_1].map_uid = map_uid;
11899 
11900 	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
11901 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
11902 	callee->regs[BPF_REG_2].map_ptr = map_ptr;
11903 	callee->regs[BPF_REG_2].map_uid = map_uid;
11904 
11905 	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
11906 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
11907 	callee->regs[BPF_REG_3].map_ptr = map_ptr;
11908 	callee->regs[BPF_REG_3].map_uid = map_uid;
11909 	callee->regs[BPF_REG_3].id = ++env->id_gen;
11910 
11911 	/* unused */
11912 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
11913 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
11914 	callee->in_async_callback_fn = true;
11915 	callee->callback_ret_range = retval_range(0, 0);
11916 	return 0;
11917 }
11918 
11919 static int set_find_vma_callback_state(struct bpf_verifier_env *env,
11920 				       struct bpf_func_state *caller,
11921 				       struct bpf_func_state *callee,
11922 				       int insn_idx)
11923 {
11924 	/* bpf_find_vma(struct task_struct *task, u64 addr,
11925 	 *               void *callback_fn, void *callback_ctx, u64 flags)
11926 	 * (callback_fn)(struct task_struct *task,
11927 	 *               struct vm_area_struct *vma, void *callback_ctx);
11928 	 */
11929 	callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
11930 
11931 	callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID;
11932 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
11933 	callee->regs[BPF_REG_2].btf =  btf_vmlinux;
11934 	callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA];
11935 
11936 	/* pointer to stack or null */
11937 	callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4];
11938 
11939 	/* unused */
11940 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
11941 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
11942 	callee->in_callback_fn = true;
11943 	/* bpf_find_vma() runs the callback under mmap_lock and RCU. */
11944 	callee->in_rcu_callback_fn = true;
11945 	callee->callback_ret_range = retval_range(0, 1);
11946 	return 0;
11947 }
11948 
11949 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env,
11950 					   struct bpf_func_state *caller,
11951 					   struct bpf_func_state *callee,
11952 					   int insn_idx)
11953 {
11954 	/* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void
11955 	 *			  callback_ctx, u64 flags);
11956 	 * callback_fn(const struct bpf_dynptr_t* dynptr, void *callback_ctx);
11957 	 */
11958 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_0]);
11959 	mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL);
11960 	callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
11961 
11962 	/* unused */
11963 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
11964 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
11965 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
11966 
11967 	callee->in_callback_fn = true;
11968 	callee->callback_ret_range = retval_range(0, 1);
11969 	return 0;
11970 }
11971 
11972 static int set_rbtree_add_callback_state(struct bpf_verifier_env *env,
11973 					 struct bpf_func_state *caller,
11974 					 struct bpf_func_state *callee,
11975 					 int insn_idx)
11976 {
11977 	/* void bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
11978 	 *                     bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b));
11979 	 *
11980 	 * 'struct bpf_rb_node *node' arg to bpf_rbtree_add_impl is the same PTR_TO_BTF_ID w/ offset
11981 	 * that 'less' callback args will be receiving. However, 'node' arg was release_reference'd
11982 	 * by this point, so look at 'root'
11983 	 */
11984 	struct btf_field *field;
11985 
11986 	field = reg_find_field_offset(&caller->regs[BPF_REG_1],
11987 				      caller->regs[BPF_REG_1].var_off.value,
11988 				      BPF_RB_ROOT);
11989 	if (!field || !field->graph_root.value_btf_id)
11990 		return -EFAULT;
11991 
11992 	mark_reg_graph_node(callee->regs, BPF_REG_1, &field->graph_root);
11993 	ref_set_non_owning(env, &callee->regs[BPF_REG_1]);
11994 	mark_reg_graph_node(callee->regs, BPF_REG_2, &field->graph_root);
11995 	ref_set_non_owning(env, &callee->regs[BPF_REG_2]);
11996 
11997 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
11998 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
11999 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
12000 	callee->in_callback_fn = true;
12001 	callee->callback_ret_range = retval_range(0, 1);
12002 	return 0;
12003 }
12004 
12005 static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env,
12006 						 struct bpf_func_state *caller,
12007 						 struct bpf_func_state *callee,
12008 						 int insn_idx)
12009 {
12010 	struct bpf_map *map_ptr = caller->regs[BPF_REG_3].map_ptr;
12011 	u32 map_uid = caller->regs[BPF_REG_3].map_uid;
12012 
12013 	/*
12014 	 * callback_fn(struct bpf_map *map, void *key, void *value);
12015 	 */
12016 	callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
12017 	__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
12018 	callee->regs[BPF_REG_1].map_ptr = map_ptr;
12019 	callee->regs[BPF_REG_1].map_uid = map_uid;
12020 
12021 	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
12022 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
12023 	callee->regs[BPF_REG_2].map_ptr = map_ptr;
12024 	callee->regs[BPF_REG_2].map_uid = map_uid;
12025 
12026 	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
12027 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
12028 	callee->regs[BPF_REG_3].map_ptr = map_ptr;
12029 	callee->regs[BPF_REG_3].map_uid = map_uid;
12030 	callee->regs[BPF_REG_3].id = ++env->id_gen;
12031 
12032 	/* unused */
12033 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
12034 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
12035 	callee->in_async_callback_fn = true;
12036 	callee->callback_ret_range = retval_range(S32_MIN, S32_MAX);
12037 	return 0;
12038 }
12039 
12040 static int set_rcu_callback_state(struct bpf_verifier_env *env,
12041 				  struct bpf_func_state *caller,
12042 				  struct bpf_func_state *callee,
12043 				  int insn_idx)
12044 {
12045 	struct bpf_map *map_ptr = caller->regs[BPF_REG_2].map_ptr;
12046 	u32 map_uid = caller->regs[BPF_REG_2].map_uid;
12047 
12048 	/*
12049 	 * callback_fn(struct bpf_map *map, void *key, void *value);
12050 	 */
12051 	callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
12052 	__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
12053 	callee->regs[BPF_REG_1].map_ptr = map_ptr;
12054 	callee->regs[BPF_REG_1].map_uid = map_uid;
12055 
12056 	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
12057 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
12058 	callee->regs[BPF_REG_2].map_ptr = map_ptr;
12059 	callee->regs[BPF_REG_2].map_uid = map_uid;
12060 
12061 	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
12062 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
12063 	callee->regs[BPF_REG_3].map_ptr = map_ptr;
12064 	callee->regs[BPF_REG_3].map_uid = map_uid;
12065 
12066 	/* unused */
12067 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
12068 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
12069 	callee->in_async_callback_fn = true;
12070 	callee->callback_ret_range = retval_range(S32_MIN, S32_MAX);
12071 	return 0;
12072 }
12073 
12074 static bool is_rbtree_lock_required_kfunc(u32 btf_id);
12075 
12076 static void account_processed_insn(struct bpf_verifier_env *env)
12077 {
12078 	struct bpf_func_state *frame = cur_func(env);
12079 
12080 	env->insn_processed++;
12081 	frame->insns_subtotal++;
12082 	env->subprog_info[frame->subprogno].insns_self++;
12083 }
12084 
12085 static void account_processed_insns(struct bpf_verifier_env *env,
12086 				    struct bpf_func_state *callee,
12087 				    struct bpf_func_state *caller)
12088 {
12089 	u32 insns;
12090 
12091 	if (!callee)
12092 		return;
12093 
12094 	insns = callee->insns_subtotal;
12095 
12096 	env->subprog_info[callee->subprogno].insns_total += insns;
12097 	if (caller)
12098 		caller->insns_subtotal += insns;
12099 	callee->insns_subtotal = 0;
12100 }
12101 
12102 static void account_current_path(struct bpf_verifier_env *env)
12103 {
12104 	struct bpf_verifier_state *state = env->cur_state;
12105 	int frame;
12106 
12107 	for (frame = state->curframe; frame >= 0; frame--)
12108 		account_processed_insns(env, state->frame[frame],
12109 					frame ? state->frame[frame - 1] : NULL);
12110 }
12111 
12112 /*
12113  * Are we currently verifying the callback for an rbtree kfunc that must
12114  * be called with a lock held, or one of that callback's subprogs? If so,
12115  * no need to complain about an unreleased lock.
12116  */
12117 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
12118 {
12119 	struct bpf_verifier_state *state = env->cur_state;
12120 	struct bpf_insn *insn = env->prog->insnsi;
12121 	struct bpf_func_state *callee;
12122 	int kfunc_btf_id;
12123 	u32 frame;
12124 
12125 	for (frame = state->curframe; frame; frame--) {
12126 		callee = state->frame[frame];
12127 		if (!callee->in_callback_fn)
12128 			continue;
12129 
12130 		kfunc_btf_id = insn[callee->callsite].imm;
12131 		if (is_rbtree_lock_required_kfunc(kfunc_btf_id))
12132 			return true;
12133 	}
12134 
12135 	return false;
12136 }
12137 
12138 static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg)
12139 {
12140 	if (range.return_32bit)
12141 		return range.minval <= reg_s32_min(reg) && reg_s32_max(reg) <= range.maxval;
12142 	else
12143 		return range.minval <= reg_smin(reg) && reg_smax(reg) <= range.maxval;
12144 }
12145 
12146 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
12147 {
12148 	struct bpf_verifier_state *state = env->cur_state, *prev_st;
12149 	struct bpf_func_state *caller, *callee;
12150 	struct bpf_reg_state *r0;
12151 	bool in_callback_fn;
12152 	u32 i, nregs;
12153 	int err;
12154 
12155 	callee = state->frame[state->curframe];
12156 	r0 = &callee->regs[BPF_REG_0];
12157 	nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1;
12158 	for (i = 0; i < nregs; i++) {
12159 		if (callee->regs[ret_regs[i]].type != PTR_TO_STACK)
12160 			continue;
12161 		/* technically it's ok to return caller's stack pointer
12162 		 * (or caller's caller's pointer) back to the caller,
12163 		 * since these pointers are valid. Only current stack
12164 		 * pointer will be invalid as soon as function exits,
12165 		 * but let's be conservative
12166 		 */
12167 		verbose(env, "cannot return stack pointer to the caller\n");
12168 		return -EINVAL;
12169 	}
12170 
12171 	caller = state->frame[state->curframe - 1];
12172 	if (bpf_is_unwind_exit(&env->prog->insnsi[*insn_idx])) {
12173 		/* the helper or the kfunc that called it doesn't expect that */
12174 		if (callee->in_callback_fn) {
12175 			verbose(env, "callback cannot unwind\n");
12176 			return -EINVAL;
12177 		}
12178 		if (!bpf_is_unwind_goto(&env->prog->insnsi[callee->callsite + 1])) {
12179 			verbose(env, "unwind_exit, but no unwind_goto after the call at insn %d\n",
12180 				callee->callsite);
12181 			return -EINVAL;
12182 		}
12183 		state->unwound = true;
12184 	}
12185 	if (callee->in_callback_fn) {
12186 		if (r0->type != SCALAR_VALUE) {
12187 			verbose(env, "R0 not a scalar value\n");
12188 			return -EACCES;
12189 		}
12190 
12191 		/* we are going to rely on register's precise value */
12192 		err = mark_chain_precision(env, BPF_REG_0);
12193 		if (err)
12194 			return err;
12195 
12196 		/* enforce R0 return value range, and bpf_callback_t returns 64bit */
12197 		if (!retval_range_within(callee->callback_ret_range, r0)) {
12198 			verbose_invalid_scalar(env, r0, callee->callback_ret_range,
12199 					       "At callback return", "R0");
12200 			return -EINVAL;
12201 		}
12202 		if (!bpf_calls_callback(env, callee->callsite)) {
12203 			verifier_bug(env, "in callback at %d, callsite %d !calls_callback",
12204 				     *insn_idx, callee->callsite);
12205 			return -EFAULT;
12206 		}
12207 	} else {
12208 		/*
12209 		 * return to the caller whatever the callee had in the
12210 		 * return register(s)
12211 		 */
12212 		for (i = 0; i < nregs; i++) {
12213 			u32 regno = ret_regs[i];
12214 
12215 			bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno],
12216 					   BPF_DIAG_MOD_WRITE);
12217 			caller->regs[regno] = callee->regs[regno];
12218 			bpf_diag_mod_end(env);
12219 		}
12220 	}
12221 
12222 	/* Invalidate callee-local dynptrs and their slices before the frame goes away. */
12223 	err = destroy_dynptrs_in_stack_slots(env, callee, 0,
12224 					     callee->allocated_stack / BPF_REG_SIZE - 1);
12225 	if (err)
12226 		return err;
12227 	invalidate_stack_slices(env, callee->frameno, INT_MIN, 0);
12228 
12229 	/* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
12230 	 * there function call logic would reschedule callback visit. If iteration
12231 	 * converges is_state_visited() would prune that visit eventually.
12232 	 */
12233 	in_callback_fn = callee->in_callback_fn;
12234 	if (in_callback_fn)
12235 		*insn_idx = callee->callsite;
12236 	else
12237 		*insn_idx = callee->callsite + 1;
12238 
12239 	if (env->log.level & BPF_LOG_LEVEL) {
12240 		verbose(env, "returning from callee:\n");
12241 		print_verifier_state(env, state, callee->frameno, true);
12242 		verbose(env, "to caller at %d:\n", *insn_idx);
12243 		print_verifier_state(env, state, caller->frameno, true);
12244 	}
12245 	account_processed_insns(env, callee, caller);
12246 	/* clear everything in the callee. In case of exceptional exits using
12247 	 * bpf_throw, this will be done by copy_verifier_state for extra frames. */
12248 	free_func_state(callee);
12249 	state->frame[state->curframe--] = NULL;
12250 	invalidate_outgoing_stack_args(env, caller);
12251 
12252 	/* for callbacks widen imprecise scalars to make programs like below verify:
12253 	 *
12254 	 *   struct ctx { int i; }
12255 	 *   void cb(int idx, struct ctx *ctx) { ctx->i++; ... }
12256 	 *   ...
12257 	 *   struct ctx = { .i = 0; }
12258 	 *   bpf_loop(100, cb, &ctx, 0);
12259 	 *
12260 	 * This is similar to what is done in process_iter_next_call() for open
12261 	 * coded iterators.
12262 	 */
12263 	prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL;
12264 	if (prev_st) {
12265 		err = widen_imprecise_scalars(env, prev_st, state);
12266 		if (err)
12267 			return err;
12268 	}
12269 	return 0;
12270 }
12271 
12272 static int do_refine_retval_range(struct bpf_verifier_env *env,
12273 				  struct bpf_reg_state *regs, int ret_type,
12274 				  int func_id,
12275 				  struct bpf_call_arg_meta *meta)
12276 {
12277 	struct bpf_retval_range range;
12278 	struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];
12279 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
12280 
12281 	if (ret_type != RET_INTEGER)
12282 		return 0;
12283 
12284 	switch (func_id) {
12285 	case BPF_FUNC_get_stack:
12286 	case BPF_FUNC_get_task_stack:
12287 	case BPF_FUNC_probe_read_str:
12288 	case BPF_FUNC_probe_read_kernel_str:
12289 	case BPF_FUNC_probe_read_user_str:
12290 		reg_set_srange64(ret_reg, -MAX_ERRNO, meta->msize_max_value);
12291 		reg_set_srange32(ret_reg, -MAX_ERRNO, meta->msize_max_value);
12292 		reg_bounds_sync(ret_reg);
12293 		break;
12294 	case BPF_FUNC_get_smp_processor_id:
12295 		reg_set_urange64(ret_reg, 0, nr_cpu_ids - 1);
12296 		reg_set_urange32(ret_reg, 0, nr_cpu_ids - 1);
12297 		reg_bounds_sync(ret_reg);
12298 		break;
12299 	case BPF_FUNC_get_retval:
12300 		/*
12301 		 * bpf_get_retval may see arbitrary value passed by bpf_prog_run_array_cg for
12302 		 * CGROUP_GETSOCKOPT type.
12303 		 */
12304 		if (prog_type == BPF_PROG_TYPE_CGROUP_SOCKOPT &&
12305 		    env->prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT)
12306 			break;
12307 
12308 		if (prog_type == BPF_PROG_TYPE_LSM &&
12309 		    env->prog->expected_attach_type == BPF_LSM_CGROUP) {
12310 			if (!env->prog->aux->attach_func_proto->type)
12311 				break;
12312 			bpf_lsm_get_retval_range(env->prog, &range);
12313 		} else {
12314 			range.minval = -MAX_ERRNO;
12315 			range.maxval = 0;
12316 		}
12317 
12318 		reg_set_srange64(ret_reg, range.minval, range.maxval);
12319 		reg_set_srange32(ret_reg, range.minval, range.maxval);
12320 		reg_bounds_sync(ret_reg);
12321 		break;
12322 	}
12323 
12324 	return reg_bounds_sanity_check(env, ret_reg, "retval");
12325 }
12326 
12327 static int
12328 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
12329 		int func_id, int insn_idx)
12330 {
12331 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
12332 	struct bpf_map *map = meta->map.ptr;
12333 
12334 	if (func_id != BPF_FUNC_tail_call &&
12335 	    func_id != BPF_FUNC_map_lookup_elem &&
12336 	    func_id != BPF_FUNC_map_update_elem &&
12337 	    func_id != BPF_FUNC_map_delete_elem &&
12338 	    func_id != BPF_FUNC_map_push_elem &&
12339 	    func_id != BPF_FUNC_map_pop_elem &&
12340 	    func_id != BPF_FUNC_map_peek_elem &&
12341 	    func_id != BPF_FUNC_for_each_map_elem &&
12342 	    func_id != BPF_FUNC_redirect_map &&
12343 	    func_id != BPF_FUNC_map_lookup_percpu_elem)
12344 		return 0;
12345 
12346 	if (map == NULL) {
12347 		verifier_bug(env, "expected map for helper call");
12348 		return -EFAULT;
12349 	}
12350 
12351 	/* In case of read-only, some additional restrictions
12352 	 * need to be applied in order to prevent altering the
12353 	 * state of the map from program side.
12354 	 */
12355 	if ((map->map_flags & BPF_F_RDONLY_PROG) &&
12356 	    (func_id == BPF_FUNC_map_delete_elem ||
12357 	     func_id == BPF_FUNC_map_update_elem ||
12358 	     func_id == BPF_FUNC_map_push_elem ||
12359 	     func_id == BPF_FUNC_map_pop_elem)) {
12360 		verbose(env, "write into map forbidden\n");
12361 		return -EACCES;
12362 	}
12363 
12364 	if (!aux->map_ptr_state.map_ptr)
12365 		bpf_map_ptr_store(aux, meta->map.ptr,
12366 				  !meta->map.ptr->bypass_spec_v1, false);
12367 	else if (aux->map_ptr_state.map_ptr != meta->map.ptr)
12368 		bpf_map_ptr_store(aux, meta->map.ptr,
12369 				  !meta->map.ptr->bypass_spec_v1, true);
12370 	return 0;
12371 }
12372 
12373 static int
12374 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
12375 		int func_id, int insn_idx)
12376 {
12377 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
12378 	struct bpf_reg_state *reg;
12379 	struct bpf_map *map = meta->map.ptr;
12380 	u64 val, max;
12381 	int err;
12382 
12383 	if (func_id != BPF_FUNC_tail_call)
12384 		return 0;
12385 	if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) {
12386 		verbose(env, "expected prog array map for tail call");
12387 		return -EINVAL;
12388 	}
12389 
12390 	reg = reg_state(env, BPF_REG_3);
12391 	val = reg->var_off.value;
12392 	max = map->max_entries;
12393 
12394 	if (!(is_reg_const(reg, false) && val < max)) {
12395 		bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
12396 		return 0;
12397 	}
12398 
12399 	err = mark_chain_precision(env, BPF_REG_3);
12400 	if (err)
12401 		return err;
12402 	if (bpf_map_key_unseen(aux))
12403 		bpf_map_key_store(aux, val);
12404 	else if (!bpf_map_key_poisoned(aux) &&
12405 		  bpf_map_key_immediate(aux) != val)
12406 		bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
12407 	return 0;
12408 }
12409 
12410 static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit)
12411 {
12412 	struct bpf_verifier_state *state = env->cur_state;
12413 	enum bpf_prog_type type = resolve_prog_type(env->prog);
12414 	struct bpf_reg_state *reg = reg_state(env, BPF_REG_0);
12415 	bool refs_lingering = false;
12416 	int i;
12417 
12418 	if (!exception_exit && cur_func(env)->frameno)
12419 		return 0;
12420 
12421 	for (i = 0; i < state->acquired_refs; i++) {
12422 		if (state->refs[i].type != REF_TYPE_PTR)
12423 			continue;
12424 		/* Allow struct_ops programs to return a referenced kptr back to
12425 		 * kernel. Type checks are performed later in check_return_code.
12426 		 */
12427 		if (type == BPF_PROG_TYPE_STRUCT_OPS && !exception_exit &&
12428 		    reg->id == state->refs[i].id)
12429 			continue;
12430 		verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
12431 			state->refs[i].id, state->refs[i].insn_idx);
12432 		bpf_diag_leak(env, state->refs[i].id, state->refs[i].insn_idx, env->insn_idx);
12433 		refs_lingering = true;
12434 	}
12435 	return refs_lingering ? -EINVAL : 0;
12436 }
12437 
12438 static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit, bool check_lock, const char *prefix)
12439 {
12440 	int err;
12441 
12442 	if (check_lock && env->cur_state->active_locks) {
12443 		verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix);
12444 		bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_LOCK,
12445 				    "Release the BPF spin lock before this operation on every path.");
12446 		return -EINVAL;
12447 	}
12448 
12449 	err = check_reference_leak(env, exception_exit);
12450 	if (err) {
12451 		verbose(env, "%s would lead to reference leak\n", prefix);
12452 		return err;
12453 	}
12454 
12455 	if (check_lock && env->cur_state->active_irq_id) {
12456 		verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix);
12457 		bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_IRQ,
12458 				    "Restore the saved IRQ state before this operation on every path.");
12459 		return -EINVAL;
12460 	}
12461 
12462 	if (check_lock && env->cur_state->active_rcu_locks) {
12463 		verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix);
12464 		bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_RCU,
12465 				    "Call bpf_rcu_read_unlock() before this operation on every path.");
12466 		return -EINVAL;
12467 	}
12468 
12469 	if (check_lock && env->cur_state->active_preempt_locks) {
12470 		verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix);
12471 		bpf_diag_ctx_active(
12472 			env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_PREEMPT,
12473 			"Call bpf_preempt_enable() before this operation on every path.");
12474 		return -EINVAL;
12475 	}
12476 
12477 	return 0;
12478 }
12479 
12480 static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
12481 				   struct bpf_reg_state *regs)
12482 {
12483 	struct bpf_reg_state *fmt_reg = &regs[BPF_REG_3];
12484 	struct bpf_reg_state *data_len_reg = &regs[BPF_REG_5];
12485 	struct bpf_map *fmt_map = fmt_reg->map_ptr;
12486 	struct bpf_bprintf_data data = {};
12487 	int err, fmt_map_off, num_args;
12488 	u64 fmt_addr;
12489 	char *fmt;
12490 
12491 	/* data must be an array of u64 */
12492 	if (data_len_reg->var_off.value % 8)
12493 		return -EINVAL;
12494 	num_args = data_len_reg->var_off.value / 8;
12495 
12496 	/* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const
12497 	 * and map_direct_value_addr is set.
12498 	 */
12499 	fmt_map_off = fmt_reg->var_off.value;
12500 	err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr,
12501 						  fmt_map_off);
12502 	if (err) {
12503 		verbose(env, "failed to retrieve map value address\n");
12504 		return -EFAULT;
12505 	}
12506 	fmt = (char *)(long)fmt_addr + fmt_map_off;
12507 
12508 	/* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we
12509 	 * can focus on validating the format specifiers.
12510 	 */
12511 	err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, num_args, &data);
12512 	if (err < 0)
12513 		verbose(env, "Invalid format string\n");
12514 
12515 	return err;
12516 }
12517 
12518 static int check_get_func_ip(struct bpf_verifier_env *env)
12519 {
12520 	enum bpf_prog_type type = resolve_prog_type(env->prog);
12521 	int func_id = BPF_FUNC_get_func_ip;
12522 
12523 	if (type == BPF_PROG_TYPE_TRACING) {
12524 		if (!bpf_prog_has_trampoline(env->prog)) {
12525 			verbose(env, "func %s#%d supported only for fentry/fexit/fsession/fmod_ret programs\n",
12526 				func_id_name(func_id), func_id);
12527 			return -ENOTSUPP;
12528 		}
12529 		return 0;
12530 	} else if (type == BPF_PROG_TYPE_KPROBE) {
12531 		return 0;
12532 	}
12533 
12534 	verbose(env, "func %s#%d not supported for program type %d\n",
12535 		func_id_name(func_id), func_id, type);
12536 	return -ENOTSUPP;
12537 }
12538 
12539 static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env)
12540 {
12541 	return &env->insn_aux_data[env->insn_idx];
12542 }
12543 
12544 /* Returns 1 if R4 is a known zero, 0 if it is not, a negative errno on error. */
12545 static int loop_flag_is_zero(struct bpf_verifier_env *env)
12546 {
12547 	struct bpf_reg_state *reg = reg_state(env, BPF_REG_4);
12548 	int err;
12549 
12550 	if (!bpf_register_is_null(reg))
12551 		return 0;
12552 
12553 	err = mark_chain_precision(env, BPF_REG_4);
12554 	if (err)
12555 		return err;
12556 	return 1;
12557 }
12558 
12559 static int update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)
12560 {
12561 	struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;
12562 	int flag_is_zero;
12563 
12564 	if (!state->initialized) {
12565 		flag_is_zero = loop_flag_is_zero(env);
12566 		if (flag_is_zero < 0)
12567 			return flag_is_zero;
12568 		state->initialized = 1;
12569 		state->fit_for_inline = flag_is_zero;
12570 		state->callback_subprogno = subprogno;
12571 		return 0;
12572 	}
12573 
12574 	if (!state->fit_for_inline)
12575 		return 0;
12576 
12577 	flag_is_zero = loop_flag_is_zero(env);
12578 	if (flag_is_zero < 0)
12579 		return flag_is_zero;
12580 	state->fit_for_inline = (flag_is_zero &&
12581 				 state->callback_subprogno == subprogno);
12582 	return 0;
12583 }
12584 
12585 /* Returns whether or not the given map can potentially elide
12586  * lookup return value nullness check. This is possible if the key
12587  * is statically known.
12588  */
12589 static bool can_elide_value_nullness(const struct bpf_map *map)
12590 {
12591 	if (map->map_flags & BPF_F_INNER_MAP)
12592 		return false;
12593 
12594 	switch (map->map_type) {
12595 	case BPF_MAP_TYPE_ARRAY:
12596 	case BPF_MAP_TYPE_PERCPU_ARRAY:
12597 		return true;
12598 	default:
12599 		return false;
12600 	}
12601 }
12602 
12603 int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id,
12604 			 const struct bpf_func_proto **ptr)
12605 {
12606 	if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID)
12607 		return -ERANGE;
12608 
12609 	if (!env->ops->get_func_proto)
12610 		return -EINVAL;
12611 
12612 	*ptr = env->ops->get_func_proto(func_id, env->prog);
12613 	return *ptr && (*ptr)->func ? 0 : -EINVAL;
12614 }
12615 
12616 /* Is any frame on the current path a callback that runs under RCU? */
12617 static bool in_rcu_callback(struct bpf_verifier_env *env)
12618 {
12619 	struct bpf_verifier_state *state = env->cur_state;
12620 	int i;
12621 
12622 	for (i = 0; i <= state->curframe; i++)
12623 		if (state->frame[i]->in_rcu_callback_fn)
12624 			return true;
12625 	return false;
12626 }
12627 
12628 /* Check if we're in a sleepable context. */
12629 static inline bool in_sleepable_context(struct bpf_verifier_env *env)
12630 {
12631 	return !in_rcu_cs(env) && !in_rcu_callback(env);
12632 }
12633 
12634 static const char *non_sleepable_context_description(struct bpf_verifier_env *env)
12635 {
12636 	if (in_sleepable(env)) {
12637 		if (env->cur_state->active_rcu_locks)
12638 			return "rcu_read_lock region";
12639 		if (env->cur_state->active_preempt_locks)
12640 			return "non-preemptible region";
12641 		if (env->cur_state->active_irq_id)
12642 			return "IRQ-disabled region";
12643 		if (env->cur_state->active_locks)
12644 			return "lock region";
12645 		return "callback running under rcu_read_lock";
12646 	}
12647 	return "non-sleepable prog";
12648 }
12649 
12650 static int release_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
12651 		       bool convert_rcu, bool release_dynptr)
12652 {
12653 	int err = -EINVAL;
12654 
12655 	if (bpf_register_is_null(reg))
12656 		return 0;
12657 
12658 	if (release_dynptr)
12659 		err = unmark_stack_slots_dynptr(env, reg);
12660 	else if (convert_rcu)
12661 		err = ref_convert_alloc_rcu_protected(env, reg->id);
12662 	else if (reg_is_referenced(env, reg))
12663 		err = release_reference(env, reg->id);
12664 
12665 	return err;
12666 }
12667 
12668 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
12669 			     int *insn_idx_p)
12670 {
12671 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
12672 	bool returns_cpu_specific_alloc_ptr = false;
12673 	const struct bpf_func_proto *fn = NULL;
12674 	enum bpf_return_type ret_type;
12675 	enum bpf_type_flag ret_flag;
12676 	struct bpf_reg_state *regs;
12677 	struct bpf_call_arg_meta meta;
12678 	const char *operation;
12679 	int insn_idx = *insn_idx_p;
12680 	bool changes_data;
12681 	int i, err, func_id;
12682 
12683 	/* find function prototype */
12684 	func_id = insn->imm;
12685 	err = bpf_get_helper_proto(env, insn->imm, &fn);
12686 	if (err == -ERANGE) {
12687 		verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id);
12688 		return -EINVAL;
12689 	}
12690 
12691 	if (err) {
12692 		verbose(env, "program of this type cannot use helper %s#%d\n",
12693 			func_id_name(func_id), func_id);
12694 		operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id);
12695 		bpf_diag_policy(
12696 			env, insn_idx, operation, "this program type does not allow the helper",
12697 			"Use a helper allowed for this program type, or move the logic to a compatible program type.");
12698 		return err;
12699 	}
12700 
12701 	/* eBPF programs must be GPL compatible to use GPL-ed functions */
12702 	if (!env->prog->gpl_compatible && fn->gpl_only) {
12703 		verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
12704 		operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id);
12705 		bpf_diag_policy(
12706 			env, insn_idx, operation,
12707 			"this helper is restricted to GPL-compatible programs",
12708 			"Use a GPL-compatible license, or replace the helper with one that is available to non-GPL programs.");
12709 		return -EINVAL;
12710 	}
12711 
12712 	if (fn->allowed && !fn->allowed(env->prog)) {
12713 		verbose(env, "helper call is not allowed in probe\n");
12714 		operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id);
12715 		bpf_diag_policy(
12716 			env, insn_idx, operation,
12717 			"the helper-specific policy callback rejected this program",
12718 			"Use the helper only from an allowed attach point or program configuration.");
12719 		return -EINVAL;
12720 	}
12721 
12722 	/* With LD_ABS/IND some JITs save/restore skb from r1. */
12723 	changes_data = bpf_helper_changes_pkt_data(func_id);
12724 	if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
12725 		verifier_bug(env, "func %s#%d: r1 != ctx", func_id_name(func_id), func_id);
12726 		return -EFAULT;
12727 	}
12728 
12729 	memset(&meta, 0, sizeof(meta));
12730 
12731 	err = check_func_proto(env, fn, &meta);
12732 	if (err) {
12733 		verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id);
12734 		return err;
12735 	}
12736 
12737 	if (fn->might_sleep && !in_sleepable_context(env)) {
12738 		const char *suggestion;
12739 
12740 		verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id,
12741 			non_sleepable_context_description(env));
12742 		if (in_sleepable(env))
12743 			suggestion = "Move the helper call outside the critical section, or use a non-sleepable helper.";
12744 		else
12745 			suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable helper.";
12746 		operation = bpf_diag_fmt(env, "sleepable helper %s#%d",
12747 					 func_id_name(func_id), func_id);
12748 		bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion);
12749 		return -EINVAL;
12750 	}
12751 
12752 	/* Track non-sleepable context for helpers. */
12753 	if (!in_sleepable_context(env))
12754 		env->insn_aux_data[insn_idx].non_sleepable = true;
12755 
12756 	meta.func_id = func_id;
12757 	meta.func_name = func_id_name(func_id);
12758 	meta.fn = fn;
12759 	err = check_func_args(env, &meta, insn_idx);
12760 	if (err)
12761 		return err;
12762 
12763 	err = record_func_map(env, &meta, func_id, insn_idx);
12764 	if (err)
12765 		return err;
12766 
12767 	err = record_func_key(env, &meta, func_id, insn_idx);
12768 	if (err)
12769 		return err;
12770 
12771 	regs = cur_regs(env);
12772 
12773 	err = mark_raw_stack(env, &meta, insn_idx);
12774 	if (err)
12775 		return err;
12776 
12777 	if (meta.release_regno) {
12778 		struct bpf_reg_state *reg = &regs[meta.release_regno];
12779 		bool convert_rcu = (func_id == BPF_FUNC_kptr_xchg) && in_rcu_cs(env) &&
12780 				   (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU);
12781 
12782 		err = release_reg(env, reg, convert_rcu, !!meta.dynptr.id);
12783 		if (err)
12784 			return err;
12785 	}
12786 
12787 	switch (func_id) {
12788 	case BPF_FUNC_tail_call:
12789 		err = check_resource_leak(env, false, true, "tail_call");
12790 		if (err)
12791 			return err;
12792 		break;
12793 	case BPF_FUNC_get_local_storage:
12794 		/* check that flags argument in get_local_storage(map, flags) is 0,
12795 		 * this is required because get_local_storage() can't return an error.
12796 		 */
12797 		if (!bpf_register_is_null(&regs[BPF_REG_2])) {
12798 			verbose(env, "get_local_storage() doesn't support non-zero flags\n");
12799 			return -EINVAL;
12800 		}
12801 		err = mark_chain_precision(env, BPF_REG_2);
12802 		if (err)
12803 			return err;
12804 		break;
12805 	case BPF_FUNC_for_each_map_elem:
12806 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
12807 					 set_map_elem_callback_state);
12808 		break;
12809 	case BPF_FUNC_timer_set_callback:
12810 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
12811 					 set_timer_callback_state);
12812 		break;
12813 	case BPF_FUNC_find_vma:
12814 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
12815 					 set_find_vma_callback_state);
12816 		break;
12817 	case BPF_FUNC_snprintf:
12818 		err = check_bpf_snprintf_call(env, regs);
12819 		break;
12820 	case BPF_FUNC_loop:
12821 		err = update_loop_inline_state(env, meta.subprogno);
12822 		if (err)
12823 			return err;
12824 		/* Verifier relies on R1 value to determine if bpf_loop() iteration
12825 		 * is finished, thus mark it precise.
12826 		 */
12827 		err = mark_chain_precision(env, BPF_REG_1);
12828 		if (err)
12829 			return err;
12830 		if (cur_func(env)->callback_depth < reg_umax(&regs[BPF_REG_1])) {
12831 			err = push_callback_call(env, insn, insn_idx, meta.subprogno,
12832 						 set_loop_callback_state);
12833 		} else {
12834 			cur_func(env)->callback_depth = 0;
12835 			if (env->log.level & BPF_LOG_LEVEL2)
12836 				verbose(env, "frame%d bpf_loop iteration limit reached\n",
12837 					env->cur_state->curframe);
12838 		}
12839 		break;
12840 	case BPF_FUNC_dynptr_from_mem:
12841 		if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) {
12842 			verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n",
12843 				reg_type_str(env, regs[BPF_REG_1].type));
12844 			return -EACCES;
12845 		}
12846 		break;
12847 	case BPF_FUNC_set_retval:
12848 	{
12849 		struct bpf_retval_range range = {
12850 			.minval = -MAX_ERRNO,
12851 			.maxval = 0,
12852 			.return_32bit = true
12853 		};
12854 		struct bpf_reg_state *r1 = &regs[BPF_REG_1];
12855 
12856 		if (r1->type != SCALAR_VALUE) {
12857 			verbose(env, "R1 is not a scalar\n");
12858 			return -EINVAL;
12859 		}
12860 
12861 		/* CGROUP_GETSOCKOPT is allowed to return arbitrary value */
12862 		if (prog_type == BPF_PROG_TYPE_CGROUP_SOCKOPT &&
12863 		    env->prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT)
12864 			break;
12865 
12866 		if (prog_type == BPF_PROG_TYPE_LSM &&
12867 		    env->prog->expected_attach_type == BPF_LSM_CGROUP) {
12868 			if (!env->prog->aux->attach_func_proto->type) {
12869 				/* Make sure programs that attach to void
12870 				 * hooks don't try to modify return value.
12871 				 */
12872 				verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
12873 				return -EINVAL;
12874 			}
12875 			bpf_lsm_get_retval_range(env->prog, &range);
12876 		}
12877 
12878 		err = mark_chain_precision(env, BPF_REG_1);
12879 		if (err)
12880 			return err;
12881 
12882 		if (!retval_range_within(range, r1)) {
12883 			verbose_invalid_scalar(env, r1, range, "At bpf_set_retval", "R1");
12884 			return -EINVAL;
12885 		}
12886 
12887 		break;
12888 	}
12889 	case BPF_FUNC_dynptr_write:
12890 	{
12891 		enum bpf_dynptr_type dynptr_type = meta.dynptr.type;
12892 
12893 		if (dynptr_type == BPF_DYNPTR_TYPE_INVALID)
12894 			return -EFAULT;
12895 
12896 		if (dynptr_write_changes_pkt_data(dynptr_type))
12897 			/* this will trigger clear_all_pkt_pointers(), which will
12898 			 * invalidate all dynptr slices associated with the skb
12899 			 */
12900 			changes_data = true;
12901 
12902 		break;
12903 	}
12904 	case BPF_FUNC_per_cpu_ptr:
12905 	case BPF_FUNC_this_cpu_ptr:
12906 	{
12907 		struct bpf_reg_state *reg = &regs[BPF_REG_1];
12908 		const struct btf_type *type;
12909 
12910 		if (reg->type & MEM_RCU) {
12911 			type = btf_type_by_id(reg->btf, reg->btf_id);
12912 			if (!type || !btf_type_is_struct(type)) {
12913 				verbose(env, "Helper has invalid btf/btf_id in R1\n");
12914 				return -EFAULT;
12915 			}
12916 			returns_cpu_specific_alloc_ptr = true;
12917 			env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true;
12918 		}
12919 		break;
12920 	}
12921 	case BPF_FUNC_user_ringbuf_drain:
12922 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
12923 					 set_user_ringbuf_callback_state);
12924 		break;
12925 	}
12926 
12927 	if (err)
12928 		return err;
12929 
12930 	/* reset caller saved regs */
12931 	bpf_diag_record_caller_saved(env, regs);
12932 	bpf_diag_mod_begin(env, &regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE);
12933 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
12934 		bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);
12935 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
12936 	}
12937 	invalidate_outgoing_stack_args(env, cur_func(env));
12938 
12939 	/* update return register (already marked as written above) */
12940 	ret_type = fn->ret_type;
12941 	ret_flag = type_flag(ret_type);
12942 
12943 	switch (base_type(ret_type)) {
12944 	case RET_INTEGER:
12945 		/* sets type to SCALAR_VALUE */
12946 		mark_reg_unknown(env, regs, BPF_REG_0);
12947 		break;
12948 	case RET_VOID:
12949 		regs[BPF_REG_0].type = NOT_INIT;
12950 		break;
12951 	case RET_PTR_TO_MAP_VALUE:
12952 		/* There is no offset yet applied, variable or fixed */
12953 		mark_reg_known_zero(env, regs, BPF_REG_0);
12954 		/* remember map_ptr, so that check_map_access()
12955 		 * can check 'value_size' boundary of memory access
12956 		 * to map element returned from bpf_map_lookup_elem()
12957 		 */
12958 		if (meta.map.ptr == NULL) {
12959 			verifier_bug(env, "unexpected null map_ptr");
12960 			return -EFAULT;
12961 		}
12962 
12963 		if (func_id == BPF_FUNC_map_lookup_elem &&
12964 		    can_elide_value_nullness(meta.map.ptr) &&
12965 		    meta.const_map_key >= 0 &&
12966 		    meta.const_map_key < meta.map.ptr->max_entries)
12967 			ret_flag &= ~PTR_MAYBE_NULL;
12968 
12969 		regs[BPF_REG_0].map_ptr = meta.map.ptr;
12970 		regs[BPF_REG_0].map_uid = meta.map.uid;
12971 		regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag;
12972 		if (type_may_be_null(ret_flag) ||
12973 		    btf_record_has_field(meta.map.ptr->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) {
12974 			regs[BPF_REG_0].id = ++env->id_gen;
12975 		}
12976 		/* requires regs[BPF_REG_0].id to be set because of the map-in-map case */
12977 		refine_map_lookup_value(&regs[BPF_REG_0]);
12978 		break;
12979 	case RET_PTR_TO_SOCKET:
12980 		mark_reg_known_zero(env, regs, BPF_REG_0);
12981 		regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag;
12982 		break;
12983 	case RET_PTR_TO_SOCK_COMMON:
12984 		mark_reg_known_zero(env, regs, BPF_REG_0);
12985 		regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag;
12986 		break;
12987 	case RET_PTR_TO_TCP_SOCK:
12988 		mark_reg_known_zero(env, regs, BPF_REG_0);
12989 		regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag;
12990 		break;
12991 	case RET_PTR_TO_MEM:
12992 		mark_reg_known_zero(env, regs, BPF_REG_0);
12993 		regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
12994 		regs[BPF_REG_0].mem_size = meta.ret_mem.size;
12995 		break;
12996 	case RET_PTR_TO_MEM_OR_BTF_ID:
12997 	{
12998 		const struct btf_type *t;
12999 
13000 		mark_reg_known_zero(env, regs, BPF_REG_0);
13001 		t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL);
13002 		if (!btf_type_is_struct(t)) {
13003 			u32 tsize;
13004 			const struct btf_type *ret;
13005 			const char *tname;
13006 
13007 			/* resolve the type size of ksym. */
13008 			ret = btf_resolve_size(meta.ret_btf, t, &tsize);
13009 			if (IS_ERR(ret)) {
13010 				tname = btf_name_by_offset(meta.ret_btf, t->name_off);
13011 				verbose(env, "unable to resolve the size of type '%s': %ld\n",
13012 					tname, PTR_ERR(ret));
13013 				return -EINVAL;
13014 			}
13015 			regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
13016 			regs[BPF_REG_0].mem_size = tsize;
13017 		} else {
13018 			if (returns_cpu_specific_alloc_ptr) {
13019 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU;
13020 			} else {
13021 				/* MEM_RDONLY may be carried from ret_flag, but it
13022 				 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise
13023 				 * it will confuse the check of PTR_TO_BTF_ID in
13024 				 * check_mem_access().
13025 				 */
13026 				ret_flag &= ~MEM_RDONLY;
13027 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
13028 			}
13029 
13030 			regs[BPF_REG_0].btf = meta.ret_btf;
13031 			regs[BPF_REG_0].btf_id = meta.ret_btf_id;
13032 		}
13033 		break;
13034 	}
13035 	case RET_PTR_TO_BTF_ID:
13036 	{
13037 		struct btf *ret_btf;
13038 		int ret_btf_id;
13039 
13040 		mark_reg_known_zero(env, regs, BPF_REG_0);
13041 		regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
13042 		if (func_id == BPF_FUNC_kptr_xchg) {
13043 			ret_btf = meta.kptr_field->kptr.btf;
13044 			ret_btf_id = meta.kptr_field->kptr.btf_id;
13045 			if (!btf_is_kernel(ret_btf)) {
13046 				regs[BPF_REG_0].type |= MEM_ALLOC;
13047 				if (meta.kptr_field->type == BPF_KPTR_PERCPU)
13048 					regs[BPF_REG_0].type |= MEM_PERCPU;
13049 			}
13050 		} else {
13051 			if (fn->ret_btf_id == BPF_PTR_POISON) {
13052 				verifier_bug(env, "func %s has non-overwritten BPF_PTR_POISON return type",
13053 					     func_id_name(func_id));
13054 				return -EFAULT;
13055 			}
13056 			ret_btf = btf_vmlinux;
13057 			ret_btf_id = *fn->ret_btf_id;
13058 		}
13059 		if (ret_btf_id == 0) {
13060 			verbose(env, "invalid return type %u of func %s#%d\n",
13061 				base_type(ret_type), func_id_name(func_id),
13062 				func_id);
13063 			return -EINVAL;
13064 		}
13065 		regs[BPF_REG_0].btf = ret_btf;
13066 		regs[BPF_REG_0].btf_id = ret_btf_id;
13067 		break;
13068 	}
13069 	default:
13070 		verbose(env, "unknown return type %u of func %s#%d\n",
13071 			base_type(ret_type), func_id_name(func_id), func_id);
13072 		return -EINVAL;
13073 	}
13074 
13075 	if (type_may_be_null(regs[BPF_REG_0].type) && !regs[BPF_REG_0].id)
13076 		regs[BPF_REG_0].id = ++env->id_gen;
13077 
13078 	if (is_ptr_cast_function(func_id) &&
13079 	    find_reference_state(env->cur_state, meta.ref_obj.id)) {
13080 		struct bpf_verifier_state *branch;
13081 		struct bpf_reg_state *r0;
13082 
13083 		err = validate_ref_obj(env, &meta.ref_obj);
13084 		if (err)
13085 			return err;
13086 
13087 		bpf_diag_mod_end(env);
13088 
13089 		/*
13090 		 * In order for a release of any of the original or cast pointers
13091 		 * to invalidate all other pointers, reuse the same reference id for
13092 		 * the cast result.
13093 		 * This reference id can't be used for nullness propagation,
13094 		 * as cast might return NULL for a non-NULL input.
13095 		 * Hence, explore the NULL case as a separate branch.
13096 		 */
13097 		branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);
13098 		if (IS_ERR(branch))
13099 			return PTR_ERR(branch);
13100 
13101 		r0 = &branch->frame[branch->curframe]->regs[BPF_REG_0];
13102 		__mark_reg_known_zero(r0);
13103 		r0->type = SCALAR_VALUE;
13104 
13105 		bpf_diag_mod_begin(env, &regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE);
13106 		regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL;
13107 		regs[BPF_REG_0].id = meta.ref_obj.id;
13108 	} else if (is_acquire_function(func_id, meta.map.ptr)) {
13109 		int id = acquire_reference(env, insn_idx, 0);
13110 
13111 		if (id < 0)
13112 			return id;
13113 
13114 		regs[BPF_REG_0].id = id;
13115 	}
13116 
13117 	if (func_id == BPF_FUNC_dynptr_data)
13118 		regs[BPF_REG_0].parent_id = meta.dynptr.id;
13119 
13120 	err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta);
13121 	if (err)
13122 		return err;
13123 
13124 	bpf_diag_mod_end(env);
13125 
13126 	err = check_map_func_compatibility(env, meta.map.ptr, func_id);
13127 	if (err)
13128 		return err;
13129 
13130 	if ((func_id == BPF_FUNC_get_stack ||
13131 	     func_id == BPF_FUNC_get_task_stack) &&
13132 	    !env->prog->has_callchain_buf) {
13133 		const char *err_str;
13134 
13135 #ifdef CONFIG_PERF_EVENTS
13136 		err = get_callchain_buffers(sysctl_perf_event_max_stack);
13137 		err_str = "cannot get callchain buffer for func %s#%d\n";
13138 #else
13139 		err = -ENOTSUPP;
13140 		err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
13141 #endif
13142 		if (err) {
13143 			verbose(env, err_str, func_id_name(func_id), func_id);
13144 			return err;
13145 		}
13146 
13147 		env->prog->has_callchain_buf = true;
13148 	}
13149 
13150 	if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack)
13151 		env->prog->call_get_stack = true;
13152 
13153 	if (func_id == BPF_FUNC_get_func_ip) {
13154 		if (check_get_func_ip(env))
13155 			return -ENOTSUPP;
13156 		env->prog->call_get_func_ip = true;
13157 	}
13158 
13159 	if (func_id == BPF_FUNC_tail_call) {
13160 		if (env->cur_state->curframe) {
13161 			struct bpf_verifier_state *branch;
13162 
13163 			/*
13164 			 * A taken tail call is modeled as a return from the current
13165 			 * frame. A callback frame cannot be left that way because
13166 			 * prepare_func_exit() would apply its return contract to the
13167 			 * unknown R0 synthesized below. Stack-depth validation rejects
13168 			 * this construct anyway.
13169 			 */
13170 			if (cur_func(env)->in_callback_fn) {
13171 				verbose(env, "cannot tail call within callback\n");
13172 				return -EINVAL;
13173 			}
13174 			mark_reg_scratched(env, BPF_REG_0);
13175 			branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);
13176 			if (IS_ERR(branch))
13177 				return PTR_ERR(branch);
13178 			clear_all_pkt_pointers(env);
13179 			mark_reg_unknown(env, regs, BPF_REG_0);
13180 			err = prepare_func_exit(env, &env->insn_idx);
13181 			if (err)
13182 				return err;
13183 			env->insn_idx--;
13184 		} else {
13185 			changes_data = false;
13186 		}
13187 	}
13188 
13189 	if (changes_data)
13190 		clear_all_pkt_pointers(env);
13191 	return 0;
13192 }
13193 
13194 /*
13195  * Mark the register(s) holding a @size byte kfunc return value as unknown
13196  * scalars and return how many registers that took. Both halves of a register
13197  * pair are treated the same way.
13198  *
13199  * R0 is written in the diagnostic modification scope the caller opened before
13200  * the caller-saved scrub, since R0 gets no scrub record of its own and that
13201  * scope is what carries its pre-call value into the history. A scope tracks a
13202  * single register and does not nest, so close it before recording the rest of
13203  * the return value. A value that fits in R0 leaves the caller's scope open for
13204  * the caller to close, so a later write to R0 still lands in it.
13205  */
13206 static u32 mark_kfunc_ret_regs(struct bpf_verifier_env *env,
13207 			       struct bpf_reg_state *regs, u32 size)
13208 {
13209 	u32 i, nregs = ret_regs_cnt(size);
13210 
13211 	mark_reg_unknown(env, regs, ret_regs[0]);
13212 	if (nregs == 1)
13213 		return nregs;
13214 
13215 	bpf_diag_mod_end(env);
13216 	for (i = 1; i < nregs; i++) {
13217 		bpf_diag_mod_begin(env, &regs[ret_regs[i]], NULL, BPF_DIAG_MOD_WRITE);
13218 		mark_reg_unknown(env, regs, ret_regs[i]);
13219 		bpf_diag_mod_end(env);
13220 	}
13221 
13222 	return nregs;
13223 }
13224 
13225 static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta)
13226 {
13227 	return meta->kfunc_flags & KF_ACQUIRE;
13228 }
13229 
13230 static bool is_kfunc_release(struct bpf_call_arg_meta *meta)
13231 {
13232 	return meta->kfunc_flags & KF_RELEASE;
13233 }
13234 
13235 static bool is_kfunc_destructive(struct bpf_call_arg_meta *meta)
13236 {
13237 	return meta->kfunc_flags & KF_DESTRUCTIVE;
13238 }
13239 
13240 static bool is_kfunc_perfmon(struct bpf_call_arg_meta *meta)
13241 {
13242 	return meta->kfunc_flags & KF_PERFMON;
13243 }
13244 
13245 static bool is_kfunc_rcu(struct bpf_call_arg_meta *meta)
13246 {
13247 	return meta->kfunc_flags & KF_RCU;
13248 }
13249 
13250 static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta)
13251 {
13252 	return meta->kfunc_flags & KF_RCU_PROTECTED;
13253 }
13254 
13255 static bool is_kfunc_arg_mem_size(const struct btf *btf,
13256 				  const struct btf_param *arg)
13257 {
13258 	const struct btf_type *t;
13259 
13260 	t = btf_type_skip_modifiers(btf, arg->type, NULL);
13261 	if (!btf_type_is_scalar(t))
13262 		return false;
13263 
13264 	return btf_param_match_suffix(btf, arg, "__sz");
13265 }
13266 
13267 static bool is_kfunc_arg_const_mem_size(const struct btf *btf,
13268 					const struct btf_param *arg)
13269 {
13270 	const struct btf_type *t;
13271 
13272 	t = btf_type_skip_modifiers(btf, arg->type, NULL);
13273 	if (!btf_type_is_scalar(t))
13274 		return false;
13275 
13276 	return btf_param_match_suffix(btf, arg, "__szk");
13277 }
13278 
13279 static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg)
13280 {
13281 	return btf_param_match_suffix(btf, arg, "__k");
13282 }
13283 
13284 static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg)
13285 {
13286 	return btf_param_match_suffix(btf, arg, "__ign");
13287 }
13288 
13289 static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg)
13290 {
13291 	return btf_param_match_suffix(btf, arg, "__map");
13292 }
13293 
13294 static bool is_kfunc_arg_const_map(const struct btf *btf, const struct btf_param *arg)
13295 {
13296 	return btf_param_match_suffix(btf, arg, "__const_map");
13297 }
13298 
13299 static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg)
13300 {
13301 	return btf_param_match_suffix(btf, arg, "__alloc");
13302 }
13303 
13304 static bool is_kfunc_arg_uninit(const struct btf *btf, const struct btf_param *arg)
13305 {
13306 	return btf_param_match_suffix(btf, arg, "__uninit");
13307 }
13308 
13309 static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf_param *arg)
13310 {
13311 	return btf_param_match_suffix(btf, arg, "__refcounted_kptr");
13312 }
13313 
13314 static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param *arg)
13315 {
13316 	return btf_param_match_suffix(btf, arg, "__nullable") ||
13317 	       btf_param_match_suffix(btf, arg, "__arena");
13318 }
13319 
13320 static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, const struct btf_param *arg)
13321 {
13322 	return btf_param_match_suffix(btf, arg, "__nonown_allowed");
13323 }
13324 
13325 static bool is_kfunc_arg_const_str(const struct btf *btf, const struct btf_param *arg)
13326 {
13327 	return btf_param_match_suffix(btf, arg, "__str");
13328 }
13329 
13330 static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param *arg)
13331 {
13332 	return btf_param_match_suffix(btf, arg, "__irq_flag");
13333 }
13334 
13335 static bool is_kfunc_arg_ctx_out(const struct btf *btf, const struct btf_param *arg)
13336 {
13337 	return btf_param_match_suffix(btf, arg, "__ctx_out");
13338 }
13339 
13340 static bool is_kfunc_arg_arena(const struct btf *btf, const struct btf_param *arg)
13341 {
13342 	return btf_param_match_suffix(btf, arg, "__arena__nullable") ||
13343 	       btf_param_match_suffix(btf, arg, "__arena");
13344 }
13345 
13346 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf,
13347 					  const struct btf_param *arg,
13348 					  const char *name)
13349 {
13350 	int len, target_len = strlen(name);
13351 	const char *param_name;
13352 
13353 	param_name = btf_name_by_offset(btf, arg->name_off);
13354 	if (str_is_empty(param_name))
13355 		return false;
13356 	len = strlen(param_name);
13357 	if (len != target_len)
13358 		return false;
13359 	if (strcmp(param_name, name))
13360 		return false;
13361 
13362 	return true;
13363 }
13364 
13365 enum {
13366 	KF_ARG_DYNPTR_ID,
13367 	KF_ARG_LIST_HEAD_ID,
13368 	KF_ARG_LIST_NODE_ID,
13369 	KF_ARG_RB_ROOT_ID,
13370 	KF_ARG_RB_NODE_ID,
13371 	KF_ARG_WORKQUEUE_ID,
13372 	KF_ARG_RES_SPIN_LOCK_ID,
13373 	KF_ARG_TASK_WORK_ID,
13374 	KF_ARG_PROG_AUX_ID,
13375 	KF_ARG_TIMER_ID,
13376 	KF_ARG_RCU_HEAD_ID
13377 };
13378 
13379 BTF_ID_LIST(kf_arg_btf_ids)
13380 BTF_ID(struct, bpf_dynptr)
13381 BTF_ID(struct, bpf_list_head)
13382 BTF_ID(struct, bpf_list_node)
13383 BTF_ID(struct, bpf_rb_root)
13384 BTF_ID(struct, bpf_rb_node)
13385 BTF_ID(struct, bpf_wq)
13386 BTF_ID(struct, bpf_res_spin_lock)
13387 BTF_ID(struct, bpf_task_work)
13388 BTF_ID(struct, bpf_prog_aux)
13389 BTF_ID(struct, bpf_timer)
13390 BTF_ID(struct, bpf_rcu_head)
13391 
13392 static bool __is_kfunc_ptr_arg_type(const struct btf *btf,
13393 				    const struct btf_param *arg, int type)
13394 {
13395 	const struct btf_type *t;
13396 	u32 res_id;
13397 
13398 	t = btf_type_skip_modifiers(btf, arg->type, NULL);
13399 	if (!t)
13400 		return false;
13401 	if (!btf_type_is_ptr(t))
13402 		return false;
13403 	t = btf_type_skip_modifiers(btf, t->type, &res_id);
13404 	if (!t)
13405 		return false;
13406 	return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]);
13407 }
13408 
13409 static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg)
13410 {
13411 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID);
13412 }
13413 
13414 static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg)
13415 {
13416 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID);
13417 }
13418 
13419 static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg)
13420 {
13421 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID);
13422 }
13423 
13424 static bool is_kfunc_arg_rbtree_root(const struct btf *btf, const struct btf_param *arg)
13425 {
13426 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_ROOT_ID);
13427 }
13428 
13429 static bool is_kfunc_arg_rbtree_node(const struct btf *btf, const struct btf_param *arg)
13430 {
13431 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_NODE_ID);
13432 }
13433 
13434 static bool is_kfunc_arg_timer(const struct btf *btf, const struct btf_param *arg)
13435 {
13436 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TIMER_ID);
13437 }
13438 
13439 static bool is_kfunc_arg_wq(const struct btf *btf, const struct btf_param *arg)
13440 {
13441 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_WORKQUEUE_ID);
13442 }
13443 
13444 static bool is_kfunc_arg_task_work(const struct btf *btf, const struct btf_param *arg)
13445 {
13446 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID);
13447 }
13448 
13449 static bool is_kfunc_arg_rcu_head(const struct btf *btf, const struct btf_param *arg)
13450 {
13451 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RCU_HEAD_ID);
13452 }
13453 
13454 static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_param *arg)
13455 {
13456 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID);
13457 }
13458 
13459 static bool is_rbtree_node_type(const struct btf_type *t)
13460 {
13461 	return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_RB_NODE_ID]);
13462 }
13463 
13464 static bool is_list_node_type(const struct btf_type *t)
13465 {
13466 	return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_LIST_NODE_ID]);
13467 }
13468 
13469 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf,
13470 				  const struct btf_param *arg)
13471 {
13472 	const struct btf_type *t;
13473 
13474 	t = btf_type_resolve_func_ptr(btf, arg->type, NULL);
13475 	if (!t)
13476 		return false;
13477 
13478 	return true;
13479 }
13480 
13481 static bool is_kfunc_arg_prog_aux(const struct btf *btf, const struct btf_param *arg)
13482 {
13483 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_PROG_AUX_ID);
13484 }
13485 
13486 /*
13487  * A kfunc with KF_IMPLICIT_ARGS has two prototypes in BTF:
13488  *   - the _impl prototype with full arg list (meta->func_proto)
13489  *   - the BPF API prototype w/o implicit args (func->type in BTF)
13490  * To determine whether an argument is implicit, we compare its position
13491  * against the number of arguments in the prototype w/o implicit args.
13492  */
13493 static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_idx)
13494 {
13495 	const struct btf_type *func, *func_proto;
13496 	u32 argn;
13497 
13498 	if (!(meta->kfunc_flags & KF_IMPLICIT_ARGS))
13499 		return false;
13500 
13501 	func = btf_type_by_id(meta->btf, meta->func_id);
13502 	func_proto = btf_type_by_id(meta->btf, func->type);
13503 	argn = btf_type_vlen(func_proto);
13504 
13505 	return argn <= arg_idx;
13506 }
13507 
13508 #define BTF_MEMBER_MAX_DEPTH	4
13509 #define BTF_MEMBER_PATH_LEN	64
13510 
13511 struct btf_member_path {
13512 	const struct btf_member *member[BTF_MEMBER_MAX_DEPTH];
13513 	bool in_array[BTF_MEMBER_MAX_DEPTH];
13514 	const struct btf_type *bad_type;
13515 	int depth;
13516 	bool too_deep;
13517 };
13518 
13519 static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t,
13520 				    u32 member_kinds)
13521 {
13522 	if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t))
13523 		return true;
13524 	if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t))
13525 		return true;
13526 	return false;
13527 }
13528 
13529 /*
13530  * Returns true if every member of struct @t is of a kind listed in
13531  * @member_kinds, 4 levels of nesting allowed. An array member counts as its
13532  * element type.
13533  */
13534 static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf,
13535 				   const struct btf_type *t, u32 member_kinds, int rec,
13536 				   struct btf_member_path *path)
13537 {
13538 	const struct btf_type *member_type;
13539 	const struct btf_member *member;
13540 	bool in_array;
13541 	u32 i;
13542 
13543 	if (!btf_type_is_struct(t))
13544 		return false;
13545 
13546 	for_each_member(i, t, member) {
13547 		const struct btf_array *array;
13548 
13549 		in_array = false;
13550 		member_type = btf_type_skip_modifiers(btf, member->type, NULL);
13551 		/*
13552 		 * Every element of an array is laid out in the value being
13553 		 * returned, so an array counts as its element type however many
13554 		 * dimensions deep that is.
13555 		 */
13556 		while (btf_type_is_array(member_type)) {
13557 			array = btf_array(member_type);
13558 			if (!array->nelems) {
13559 				member_type = NULL;
13560 				goto bad_member;
13561 			}
13562 			member_type = btf_type_skip_modifiers(btf, array->type, NULL);
13563 			in_array = true;
13564 		}
13565 		if (btf_type_is_struct(member_type)) {
13566 			if (rec >= BTF_MEMBER_MAX_DEPTH - 1) {
13567 				verbose(env, "max struct nesting depth exceeded\n");
13568 				if (path)
13569 					path->too_deep = true;
13570 				return false;
13571 			}
13572 			if (!btf_struct_member_walk(env, btf, member_type, member_kinds,
13573 						    rec + 1, path))
13574 				goto bad_path;
13575 			continue;
13576 		}
13577 		if (!btf_member_kind_allowed(btf, member_type, member_kinds))
13578 			goto bad_member;
13579 	}
13580 	return true;
13581 
13582 bad_member:
13583 	if (path) {
13584 		path->depth = rec + 1;
13585 		path->bad_type = member_type;
13586 	}
13587 bad_path:
13588 	if (path && path->depth) {
13589 		path->member[rec] = member;
13590 		path->in_array[rec] = in_array;
13591 	}
13592 	return false;
13593 }
13594 
13595 bool btf_struct_is_composed_of(struct bpf_verifier_env *env,
13596 			       const struct btf *btf,
13597 			       const struct btf_type *t, u32 member_kinds)
13598 {
13599 	return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL);
13600 }
13601 
13602 static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env,
13603 				      const struct btf *btf,
13604 				      const struct btf_type *t)
13605 {
13606 	return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR);
13607 }
13608 
13609 static int resolve_func_arg_type(struct bpf_verifier_env *env,
13610 				 struct bpf_reg_state *reg, u32 arg, argno_t argno,
13611 				 struct bpf_call_arg_meta *meta,
13612 				 enum bpf_arg_type *arg_type, u32 *arg_size)
13613 {
13614 	const struct btf_param *args;
13615 	const struct btf_type *ref_t, *resolve_ret;
13616 	const struct btf *btf;
13617 	const char *ref_tname;
13618 	u32 ref_id;
13619 
13620 	if (base_type(*arg_type) == ARG_PTR_TO_MAP_VALUE)
13621 		return resolve_map_arg_type(env, meta, arg_type);
13622 
13623 	if (base_type(*arg_type) != ARG_PTR_TO_BTF_ID)
13624 		return 0;
13625 
13626 	/*
13627 	 * A kfunc that releases an object, or that takes a reference to one,
13628 	 * needs the object itself: memory of the program cannot stand in for it.
13629 	 */
13630 	if (!is_kfunc(meta) || arg_type_is_release(*arg_type) ||
13631 	    is_kfunc_acquire(meta) ||
13632 	    base_type(reg->type) == PTR_TO_BTF_ID ||
13633 	    reg2btf_ids[base_type(reg->type)])
13634 		return 0;
13635 
13636 	args = btf_params(meta->func_proto);
13637 	ref_id = *meta->fn->arg_btf_id[arg];
13638 	btf = is_kfunc_arg_map(meta->btf, &args[arg]) ? btf_vmlinux : meta->btf;
13639 	ref_t = btf_type_skip_modifiers(btf, ref_id, &ref_id);
13640 	ref_tname = btf_name_by_offset(btf, ref_t->name_off);
13641 
13642 	/*
13643 	 * A kfunc that takes a projection type (ie. xdp_md) casts it to the
13644 	 * underlying type. So memory of the program cannot stand in for it,
13645 	 * even if the projection type holds only scalars. A struct without
13646 	 * members (ie. iou_ctx) is a handle for an object of another type.
13647 	 */
13648 	if (btf_is_projection_type(ref_tname) ||
13649 	    !btf_type_is_scalar_struct(env, btf, ref_t) ||
13650 	    !btf_type_vlen(ref_t))
13651 		return 0;
13652 
13653 	resolve_ret = btf_resolve_size(btf, ref_t, arg_size);
13654 	if (IS_ERR(resolve_ret)) {
13655 		verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n",
13656 			reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname,
13657 			PTR_ERR(resolve_ret));
13658 		return -EINVAL;
13659 	}
13660 	*arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE |
13661 		    (*arg_type & (PTR_MAYBE_NULL | MEM_UNINIT));
13662 
13663 	return 0;
13664 }
13665 
13666 static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path,
13667 				char *buf, size_t buf_sz)
13668 {
13669 	size_t len = 0;
13670 	int i;
13671 
13672 	buf[0] = '\0';
13673 	for (i = 0; i < path->depth; i++) {
13674 		const char *name = btf_name_by_offset(btf, path->member[i]->name_off);
13675 
13676 		/* An anonymous struct or union has no name to spell. */
13677 		if (!name || !name[0])
13678 			continue;
13679 		len += scnprintf(buf + len, buf_sz - len, "%s%s", len ? "." : "", name);
13680 		if (path->in_array[i])
13681 			len += scnprintf(buf + len, buf_sz - len, "[]");
13682 	}
13683 }
13684 
13685 enum special_kfunc_type {
13686 	KF_bpf_obj_new_impl,
13687 	KF_bpf_obj_new,
13688 	KF_bpf_obj_drop_impl,
13689 	KF_bpf_obj_drop,
13690 	KF_bpf_refcount_acquire_impl,
13691 	KF_bpf_refcount_acquire,
13692 	KF_bpf_list_push_front_impl,
13693 	KF_bpf_list_push_front,
13694 	KF_bpf_list_push_back_impl,
13695 	KF_bpf_list_push_back,
13696 	KF_bpf_list_add,
13697 	KF_bpf_list_pop_front,
13698 	KF_bpf_list_pop_back,
13699 	KF_bpf_list_del,
13700 	KF_bpf_list_front,
13701 	KF_bpf_list_back,
13702 	KF_bpf_list_is_first,
13703 	KF_bpf_list_is_last,
13704 	KF_bpf_list_empty,
13705 	KF_bpf_cast_to_kern_ctx,
13706 	KF_bpf_rdonly_cast,
13707 	KF_bpf_rcu_read_lock,
13708 	KF_bpf_rcu_read_unlock,
13709 	KF_bpf_rbtree_remove,
13710 	KF_bpf_rbtree_add_impl,
13711 	KF_bpf_rbtree_add,
13712 	KF_bpf_rbtree_first,
13713 	KF_bpf_rbtree_root,
13714 	KF_bpf_rbtree_left,
13715 	KF_bpf_rbtree_right,
13716 	KF_bpf_dynptr_from_skb,
13717 	KF_bpf_dynptr_from_xdp,
13718 	KF_bpf_dynptr_from_skb_meta,
13719 	KF_bpf_xdp_pull_data,
13720 	KF_bpf_dynptr_slice,
13721 	KF_bpf_dynptr_slice_rdwr,
13722 	KF_bpf_dynptr_clone,
13723 	KF_bpf_percpu_obj_new_impl,
13724 	KF_bpf_percpu_obj_new,
13725 	KF_bpf_percpu_obj_drop_impl,
13726 	KF_bpf_percpu_obj_drop,
13727 	KF_bpf_throw,
13728 	KF_bpf_wq_set_callback,
13729 	KF_bpf_preempt_disable,
13730 	KF_bpf_preempt_enable,
13731 	KF_bpf_iter_css_task_new,
13732 	KF_bpf_session_cookie,
13733 	KF_bpf_get_kmem_cache,
13734 	KF_bpf_local_irq_save,
13735 	KF_bpf_local_irq_restore,
13736 	KF_bpf_iter_num_new,
13737 	KF_bpf_iter_num_next,
13738 	KF_bpf_iter_num_destroy,
13739 	KF_bpf_set_dentry_xattr,
13740 	KF_bpf_remove_dentry_xattr,
13741 	KF_bpf_res_spin_lock,
13742 	KF_bpf_res_spin_unlock,
13743 	KF_bpf_res_spin_lock_irqsave,
13744 	KF_bpf_res_spin_unlock_irqrestore,
13745 	KF_bpf_dynptr_from_file,
13746 	KF_bpf_dynptr_file_discard,
13747 	KF___bpf_trap,
13748 	KF_bpf_task_work_schedule_signal,
13749 	KF_bpf_task_work_schedule_resume,
13750 	KF_bpf_call_rcu,
13751 	KF_bpf_call_rcu_tasks_trace,
13752 	KF_bpf_arena_alloc_pages,
13753 	KF_bpf_arena_free_pages,
13754 	KF_bpf_session_is_return,
13755 };
13756 
13757 BTF_ID_LIST(special_kfunc_list)
13758 BTF_ID(func, bpf_obj_new_impl)
13759 BTF_ID(func, bpf_obj_new)
13760 BTF_ID(func, bpf_obj_drop_impl)
13761 BTF_ID(func, bpf_obj_drop)
13762 BTF_ID(func, bpf_refcount_acquire_impl)
13763 BTF_ID(func, bpf_refcount_acquire)
13764 BTF_ID(func, bpf_list_push_front_impl)
13765 BTF_ID(func, bpf_list_push_front)
13766 BTF_ID(func, bpf_list_push_back_impl)
13767 BTF_ID(func, bpf_list_push_back)
13768 BTF_ID(func, bpf_list_add)
13769 BTF_ID(func, bpf_list_pop_front)
13770 BTF_ID(func, bpf_list_pop_back)
13771 BTF_ID(func, bpf_list_del)
13772 BTF_ID(func, bpf_list_front)
13773 BTF_ID(func, bpf_list_back)
13774 BTF_ID(func, bpf_list_is_first)
13775 BTF_ID(func, bpf_list_is_last)
13776 BTF_ID(func, bpf_list_empty)
13777 BTF_ID(func, bpf_cast_to_kern_ctx)
13778 BTF_ID(func, bpf_rdonly_cast)
13779 BTF_ID(func, bpf_rcu_read_lock)
13780 BTF_ID(func, bpf_rcu_read_unlock)
13781 BTF_ID(func, bpf_rbtree_remove)
13782 BTF_ID(func, bpf_rbtree_add_impl)
13783 BTF_ID(func, bpf_rbtree_add)
13784 BTF_ID(func, bpf_rbtree_first)
13785 BTF_ID(func, bpf_rbtree_root)
13786 BTF_ID(func, bpf_rbtree_left)
13787 BTF_ID(func, bpf_rbtree_right)
13788 #ifdef CONFIG_NET
13789 BTF_ID(func, bpf_dynptr_from_skb)
13790 BTF_ID(func, bpf_dynptr_from_xdp)
13791 BTF_ID(func, bpf_dynptr_from_skb_meta)
13792 BTF_ID(func, bpf_xdp_pull_data)
13793 #else
13794 BTF_ID_UNUSED
13795 BTF_ID_UNUSED
13796 BTF_ID_UNUSED
13797 BTF_ID_UNUSED
13798 #endif
13799 BTF_ID(func, bpf_dynptr_slice)
13800 BTF_ID(func, bpf_dynptr_slice_rdwr)
13801 BTF_ID(func, bpf_dynptr_clone)
13802 BTF_ID(func, bpf_percpu_obj_new_impl)
13803 BTF_ID(func, bpf_percpu_obj_new)
13804 BTF_ID(func, bpf_percpu_obj_drop_impl)
13805 BTF_ID(func, bpf_percpu_obj_drop)
13806 BTF_ID(func, bpf_throw)
13807 BTF_ID(func, bpf_wq_set_callback)
13808 BTF_ID(func, bpf_preempt_disable)
13809 BTF_ID(func, bpf_preempt_enable)
13810 #ifdef CONFIG_CGROUPS
13811 BTF_ID(func, bpf_iter_css_task_new)
13812 #else
13813 BTF_ID_UNUSED
13814 #endif
13815 #ifdef CONFIG_BPF_EVENTS
13816 BTF_ID(func, bpf_session_cookie)
13817 #else
13818 BTF_ID_UNUSED
13819 #endif
13820 BTF_ID(func, bpf_get_kmem_cache)
13821 BTF_ID(func, bpf_local_irq_save)
13822 BTF_ID(func, bpf_local_irq_restore)
13823 BTF_ID(func, bpf_iter_num_new)
13824 BTF_ID(func, bpf_iter_num_next)
13825 BTF_ID(func, bpf_iter_num_destroy)
13826 #ifdef CONFIG_BPF_LSM
13827 BTF_ID(func, bpf_set_dentry_xattr)
13828 BTF_ID(func, bpf_remove_dentry_xattr)
13829 #else
13830 BTF_ID_UNUSED
13831 BTF_ID_UNUSED
13832 #endif
13833 BTF_ID(func, bpf_res_spin_lock)
13834 BTF_ID(func, bpf_res_spin_unlock)
13835 BTF_ID(func, bpf_res_spin_lock_irqsave)
13836 BTF_ID(func, bpf_res_spin_unlock_irqrestore)
13837 BTF_ID(func, bpf_dynptr_from_file)
13838 BTF_ID(func, bpf_dynptr_file_discard)
13839 BTF_ID(func, __bpf_trap)
13840 BTF_ID(func, bpf_task_work_schedule_signal)
13841 BTF_ID(func, bpf_task_work_schedule_resume)
13842 BTF_ID(func, bpf_call_rcu)
13843 BTF_ID(func, bpf_call_rcu_tasks_trace)
13844 #if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
13845 BTF_ID(func, bpf_arena_alloc_pages)
13846 BTF_ID(func, bpf_arena_free_pages)
13847 #else
13848 BTF_ID_UNUSED
13849 BTF_ID_UNUSED
13850 #endif
13851 #ifdef CONFIG_BPF_EVENTS
13852 BTF_ID(func, bpf_session_is_return)
13853 #else
13854 BTF_ID_UNUSED
13855 #endif
13856 
13857 static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta)
13858 {
13859 	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]);
13860 }
13861 
13862 static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta)
13863 {
13864 	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]);
13865 }
13866 
13867 static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta)
13868 {
13869 	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_iter_css_task_new]);
13870 }
13871 
13872 static bool is_bpf_obj_new_kfunc(u32 func_id)
13873 {
13874 	return func_id == special_kfunc_list[KF_bpf_obj_new] ||
13875 	       func_id == special_kfunc_list[KF_bpf_obj_new_impl];
13876 }
13877 
13878 static bool is_bpf_percpu_obj_new_kfunc(u32 func_id)
13879 {
13880 	return func_id == special_kfunc_list[KF_bpf_percpu_obj_new] ||
13881 	       func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl];
13882 }
13883 
13884 static bool is_bpf_obj_drop_kfunc(u32 func_id)
13885 {
13886 	return func_id == special_kfunc_list[KF_bpf_obj_drop] ||
13887 	       func_id == special_kfunc_list[KF_bpf_obj_drop_impl];
13888 }
13889 
13890 static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id)
13891 {
13892 	return func_id == special_kfunc_list[KF_bpf_percpu_obj_drop] ||
13893 	       func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl];
13894 }
13895 
13896 static bool is_bpf_refcount_acquire_kfunc(u32 func_id)
13897 {
13898 	return func_id == special_kfunc_list[KF_bpf_refcount_acquire] ||
13899 	       func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl];
13900 }
13901 
13902 static bool is_bpf_list_push_kfunc(u32 func_id)
13903 {
13904 	return func_id == special_kfunc_list[KF_bpf_list_push_front] ||
13905 	       func_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
13906 	       func_id == special_kfunc_list[KF_bpf_list_push_back] ||
13907 	       func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
13908 	       func_id == special_kfunc_list[KF_bpf_list_add];
13909 }
13910 
13911 static bool is_bpf_rbtree_add_kfunc(u32 func_id)
13912 {
13913 	return func_id == special_kfunc_list[KF_bpf_rbtree_add] ||
13914 	       func_id == special_kfunc_list[KF_bpf_rbtree_add_impl];
13915 }
13916 
13917 static bool is_call_rcu_kfunc(u32 func_id)
13918 {
13919 	return func_id == special_kfunc_list[KF_bpf_call_rcu] ||
13920 	       func_id == special_kfunc_list[KF_bpf_call_rcu_tasks_trace];
13921 }
13922 
13923 static bool is_task_work_add_kfunc(u32 func_id)
13924 {
13925 	return func_id == special_kfunc_list[KF_bpf_task_work_schedule_signal] ||
13926 	       func_id == special_kfunc_list[KF_bpf_task_work_schedule_resume];
13927 }
13928 
13929 static bool is_kfunc_ret_null(struct bpf_call_arg_meta *meta)
13930 {
13931 	if (is_bpf_refcount_acquire_kfunc(meta->func_id) && meta->arg_owning_ref)
13932 		return false;
13933 
13934 	return meta->kfunc_flags & KF_RET_NULL;
13935 }
13936 
13937 static bool is_kfunc_bpf_rcu_read_lock(struct bpf_call_arg_meta *meta)
13938 {
13939 	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_lock]);
13940 }
13941 
13942 static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_call_arg_meta *meta)
13943 {
13944 	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_unlock]);
13945 }
13946 
13947 static bool is_kfunc_bpf_preempt_disable(struct bpf_call_arg_meta *meta)
13948 {
13949 	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_disable]);
13950 }
13951 
13952 static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta)
13953 {
13954 	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_enable]);
13955 }
13956 
13957 /* These write to what their first argument, a dynptr, points to */
13958 BTF_SET_START(dynptr_write_kfuncs)
13959 BTF_ID(func, bpf_dynptr_copy)
13960 BTF_ID(func, bpf_dynptr_memset)
13961 #ifdef CONFIG_BPF_EVENTS
13962 BTF_ID(func, bpf_probe_read_user_dynptr)
13963 BTF_ID(func, bpf_probe_read_kernel_dynptr)
13964 BTF_ID(func, bpf_probe_read_user_str_dynptr)
13965 BTF_ID(func, bpf_probe_read_kernel_str_dynptr)
13966 BTF_ID(func, bpf_copy_from_user_dynptr)
13967 BTF_ID(func, bpf_copy_from_user_str_dynptr)
13968 BTF_ID(func, bpf_copy_from_user_task_dynptr)
13969 BTF_ID(func, bpf_copy_from_user_task_str_dynptr)
13970 #endif
13971 BTF_SET_END(dynptr_write_kfuncs)
13972 
13973 static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta)
13974 {
13975 	return meta->btf == btf_vmlinux &&
13976 	       btf_id_set_contains(&dynptr_write_kfuncs, meta->func_id);
13977 }
13978 
13979 /*
13980  * A write through the dynptr of an skb may reallocate its head. Before the
13981  * arguments are checked nobody knows what kind of dynptr it is.
13982  */
13983 bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta)
13984 {
13985 	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_xdp_pull_data]) ||
13986 	       is_kfunc_dynptr_write(meta);
13987 }
13988 
13989 static u32 kfunc_abi_slots(const struct btf_func_model *fm)
13990 {
13991 	const struct bpf_jit_arg_abi *abi = bpf_jit_arg_abi();
13992 	u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS];
13993 	u32 i, nslots, slots = 0;
13994 
13995 	for (i = 0; i < fm->nr_args; i++)
13996 		slots += btf_func_model_arg_slots(fm, i);
13997 
13998 	if (!abi)
13999 		return slots;
14000 
14001 	nslots = bpf_jit_place_args(abi, fm, pos_of_slot);
14002 	for (i = 0; i < nslots; i++)
14003 		if (pos_of_slot[i] + 1 > slots)
14004 			slots = pos_of_slot[i] + 1;
14005 
14006 	return slots;
14007 }
14008 
14009 static u32 __btf_func_arg_align(const struct btf *btf, const struct btf_type *t, int rec)
14010 {
14011 	const struct btf_member *member;
14012 	const struct btf_type *mt;
14013 	u32 align, i;
14014 
14015 	while (btf_type_is_array(t))
14016 		t = btf_type_skip_modifiers(btf, btf_array(t)->type, NULL);
14017 
14018 	if (btf_type_is_int(t))
14019 		return t->size > BPF_REG_SIZE ? t->size : BPF_REG_SIZE;
14020 	if (!btf_type_is_struct(t))
14021 		return BPF_REG_SIZE;
14022 	if (rec >= BTF_MEMBER_MAX_DEPTH)
14023 		return 0;
14024 
14025 	for_each_member(i, t, member) {
14026 		mt = btf_type_skip_modifiers(btf, member->type, NULL);
14027 		align = __btf_func_arg_align(btf, mt, rec + 1);
14028 		if (!align)
14029 			return 0;
14030 		if (align > BPF_REG_SIZE)
14031 			return 2 * BPF_REG_SIZE;
14032 	}
14033 	return BPF_REG_SIZE;
14034 }
14035 
14036 u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t)
14037 {
14038 	return __btf_func_arg_align(btf, t, 0);
14039 }
14040 
14041 static int
14042 get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
14043 		   const struct btf_param *args, int arg, int nargs, u32 slot,
14044 		   struct bpf_func_proto *proto)
14045 {
14046 	const struct btf_type *t, *ref_t = NULL, *resolve_ret;
14047 	const u32 *ref_id_ptr = NULL;
14048 	argno_t argno = argno_from_arg(slot + 1);
14049 	const char *ref_tname = NULL;
14050 	u32 ref_id, type_size;
14051 	int arg_type;
14052 
14053 	proto->arg_btf_id[arg] = NULL;
14054 
14055 	if (is_kfunc_arg_prog_aux(meta->btf, &args[arg]))
14056 		return ARG_PTR_TO_PROG_AUX;
14057 
14058 	if (is_kfunc_arg_ignore(meta->btf, &args[arg]) || is_kfunc_arg_implicit(meta, arg))
14059 		return ARG_IGNORE;
14060 
14061 	t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL);
14062 
14063 	/* Scalar arguments are classified from their BTF suffix/name alone. */
14064 	if (btf_type_is_scalar(t)) {
14065 		if (is_kfunc_arg_constant(meta->btf, &args[arg]))
14066 			return ARG_CONST_SCALAR;
14067 		if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg]))
14068 			return ARG_CONST_MEM_SIZE;
14069 		if (is_kfunc_arg_mem_size(meta->btf, &args[arg]))
14070 			return ARG_MEM_SIZE;
14071 		if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") ||
14072 		    is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size"))
14073 			return ARG_CONST_ALLOC_SIZE_OR_ZERO;
14074 		return ARG_SCALAR;
14075 	}
14076 
14077 	if (btf_type_is_struct(t)) {
14078 		if (!t->size || t->size > 2 * BPF_REG_SIZE) {
14079 			verbose(env,
14080 				"%s type %s has size %u, only 1 to %d bytes "
14081 				"can be passed by value\n",
14082 				reg_arg_name(env, argno), btf_type_str(t), t->size,
14083 				2 * BPF_REG_SIZE);
14084 			return -EINVAL;
14085 		}
14086 		if (!btf_type_is_scalar_struct(env, meta->btf, t)) {
14087 			verbose(env, "%s type %s is not composed of scalars\n",
14088 				reg_arg_name(env, argno), btf_type_str(t));
14089 			return -EINVAL;
14090 		}
14091 		return ARG_SCALAR;
14092 	}
14093 
14094 	if (!btf_type_is_ptr(t)) {
14095 		verbose(env, "Unrecognized %s type %s\n",
14096 			reg_arg_name(env, argno), btf_type_str(t));
14097 		return -EINVAL;
14098 	}
14099 	/* Keep a pointer to the BTF field containing the resolved referent ID. */
14100 	ref_id_ptr = &t->type;
14101 	ref_t = btf_type_skip_modifiers(meta->btf, *ref_id_ptr, &ref_id);
14102 	while (*ref_id_ptr != ref_id)
14103 		ref_id_ptr = &btf_type_by_id(meta->btf, *ref_id_ptr)->type;
14104 	ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off);
14105 
14106 	/* In this function, we verify the kfunc's BTF as per the argument type,
14107 	 * leaving the rest of the verification with respect to the register
14108 	 * type to our caller. When a set of conditions hold in the BTF type of
14109 	 * arguments, we resolve it to a known bpf_arg_type.
14110 	 */
14111 	if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]) ||
14112 	    is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_is_return]) ||
14113 	    is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_cookie]))
14114 		arg_type = ARG_PTR_TO_CTX;
14115 	else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg))
14116 		arg_type = ARG_PTR_TO_CTX;
14117 	else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg]))
14118 		arg_type = ARG_PTR_TO_ALLOC_BTF_ID;
14119 	else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg]))
14120 		arg_type = ARG_PTR_TO_REFCOUNTED_KPTR;
14121 	else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) {
14122 		arg_type = ARG_PTR_TO_DYNPTR;
14123 
14124 		if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb]))
14125 			arg_type |= DYNPTR_TYPE_SKB;
14126 		else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_xdp]))
14127 			arg_type |= DYNPTR_TYPE_XDP;
14128 		else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb_meta]))
14129 			arg_type |= DYNPTR_TYPE_SKB_META;
14130 		else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_file]) ||
14131 			 is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_file_discard]))
14132 			/* OBJ_RELEASE for the latter comes from KF_RELEASE below */
14133 			arg_type |= DYNPTR_TYPE_FILE;
14134 	} else if (is_kfunc_arg_iter(meta, arg, &args[arg]))
14135 		arg_type = ARG_PTR_TO_ITER;
14136 	else if (is_kfunc_arg_list_head(meta->btf, &args[arg]))
14137 		arg_type = ARG_PTR_TO_LIST_HEAD;
14138 	else if (is_kfunc_arg_list_node(meta->btf, &args[arg]))
14139 		arg_type = ARG_PTR_TO_LIST_NODE;
14140 	else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg]))
14141 		arg_type = ARG_PTR_TO_RB_ROOT;
14142 	else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg]))
14143 		arg_type = ARG_PTR_TO_RB_NODE;
14144 	else if (is_kfunc_arg_const_str(meta->btf, &args[arg]))
14145 		arg_type = ARG_PTR_TO_CONST_STR;
14146 	else if (is_kfunc_arg_const_map(meta->btf, &args[arg]))
14147 		arg_type = ARG_CONST_MAP_PTR;
14148 	else if (is_kfunc_arg_map(meta->btf, &args[arg]))
14149 		arg_type = ARG_PTR_TO_BTF_ID;
14150 	else if (is_kfunc_arg_wq(meta->btf, &args[arg]))
14151 		arg_type = ARG_PTR_TO_WORKQUEUE;
14152 	else if (is_kfunc_arg_timer(meta->btf, &args[arg]))
14153 		arg_type = ARG_PTR_TO_TIMER;
14154 	else if (is_kfunc_arg_task_work(meta->btf, &args[arg]))
14155 		arg_type = ARG_PTR_TO_TASK_WORK;
14156 	else if (is_kfunc_arg_rcu_head(meta->btf, &args[arg]))
14157 		arg_type = ARG_PTR_TO_RCU_HEAD;
14158 	else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg]))
14159 		arg_type = ARG_PTR_TO_IRQ_FLAG;
14160 	else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg]))
14161 		arg_type = ARG_PTR_TO_RES_SPIN_LOCK;
14162 	else if (is_kfunc_arg_ctx_out(meta->btf, &args[arg])) {
14163 		if (!btf_type_is_scalar(ref_t)) {
14164 			verbose(env, "%s __ctx_out argument must point to a scalar\n",
14165 				reg_arg_name(env, argno));
14166 			return -EINVAL;
14167 		}
14168 		resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size);
14169 		if (IS_ERR(resolve_ret)) {
14170 			verbose(env,
14171 				"%s reference type('%s %s') size cannot be determined: %ld\n",
14172 				reg_arg_name(env, argno), btf_type_str(ref_t),
14173 				ref_tname, PTR_ERR(resolve_ret));
14174 			return -EINVAL;
14175 		}
14176 		proto->arg_size[arg] = type_size;
14177 		arg_type = ARG_PTR_TO_CTX_OUT | MEM_FIXED_SIZE;
14178 	} else if (is_kfunc_arg_callback(env, meta->btf, &args[arg]))
14179 		arg_type = ARG_PTR_TO_FUNC;
14180 	else if (is_kfunc_arg_arena(meta->btf, &args[arg])) {
14181 		if (!bpf_jit_supports_arena_args()) {
14182 			verbose(env, "JIT does not support kfunc %s() with arena pointer arguments\n",
14183 				meta->func_name);
14184 			return -ENOTSUPP;
14185 		}
14186 		if (!env->prog->aux->arena) {
14187 			verbose(env,
14188 				"%s arena pointer requires a program with an associated arena\n",
14189 				reg_arg_name(env, argno));
14190 			return -EINVAL;
14191 		}
14192 		if (reg_from_argno(argno) < 0) {
14193 			verbose(env, "%s arena pointer cannot be a stack argument\n",
14194 				reg_arg_name(env, argno));
14195 			return -EINVAL;
14196 		}
14197 		/*
14198 		 * Both suffixes accept a constant zero. The function model determines
14199 		 * whether the JIT rebases it to the arena base or preserves NULL.
14200 		 * The common nullable path below records that verifier property.
14201 		 */
14202 		arg_type = ARG_PTR_TO_ARENA;
14203 	} else if (arg + 1 < nargs &&
14204 		 (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) ||
14205 		  is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) {
14206 		if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) &&
14207 		    !btf_type_is_scalar_struct(env, meta->btf, ref_t)) {
14208 			verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n",
14209 				reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname);
14210 			return -EINVAL;
14211 		}
14212 		arg_type = ARG_PTR_TO_MEM | MEM_WRITE;
14213 	} else if (btf_type_is_struct(ref_t))
14214 		/* A pointer to a struct without a size argument is classified as ARG_PTR_TO_BTF_ID */
14215 		arg_type = ARG_PTR_TO_BTF_ID;
14216 	else {
14217 		/*
14218 		 * Otherwise this is a fixed-size memory buffer supported by
14219 		 * check_helper_mem_access(): a pointer to a scalar or a struct of
14220 		 * scalars. The access size is derived from the pointed-to BTF type.
14221 		 */
14222 		if (!btf_type_is_scalar(ref_t) &&
14223 		    !btf_type_is_scalar_struct(env, meta->btf, ref_t)) {
14224 			verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n",
14225 				reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname);
14226 			return -EINVAL;
14227 		}
14228 		resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size);
14229 		if (IS_ERR(resolve_ret)) {
14230 			verbose(env,
14231 				"%s reference type('%s %s') size cannot be determined: %ld\n",
14232 				reg_arg_name(env, argno), btf_type_str(ref_t),
14233 				ref_tname, PTR_ERR(resolve_ret));
14234 			return -EINVAL;
14235 		}
14236 		proto->arg_size[arg] = type_size;
14237 		arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE;
14238 	}
14239 
14240 	if (is_kfunc_arg_uninit(meta->btf, &args[arg]))
14241 		arg_type |= MEM_UNINIT;
14242 
14243 	if (is_kfunc_arg_nullable(meta->btf, &args[arg]))
14244 		arg_type |= PTR_MAYBE_NULL;
14245 
14246 	/*
14247 	 * Only the first argument of a KF_RELEASE kfunc releases anything, and
14248 	 * bpf_fetch_kfunc_arg_meta() only ever records BPF_REG_1 for it.
14249 	 */
14250 	if (is_kfunc_release(meta) && arg == 0)
14251 		arg_type |= OBJ_RELEASE;
14252 
14253 	if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) {
14254 		if (is_kfunc_arg_map(meta->btf, &args[arg]))
14255 			proto->arg_btf_id[arg] = reg2btf_ids[CONST_PTR_TO_MAP];
14256 		else
14257 			proto->arg_btf_id[arg] = ref_id_ptr;
14258 
14259 		arg_type |= PTR_TRUSTED;
14260 
14261 		/*
14262 		 * A KF_RCU kfunc accepts an RCU-protected pointer where it would
14263 		 * otherwise demand a referenced or trusted one. Other argument kinds
14264 		 * have their own provenance requirements and must not inherit MEM_RCU.
14265 		 */
14266 		if (is_kfunc_rcu(meta))
14267 			arg_type |= MEM_RCU;
14268 	}
14269 
14270 	return arg_type;
14271 }
14272 
14273 static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
14274 			       const struct btf_func_model *fm, struct bpf_func_proto *proto)
14275 {
14276 	const struct bpf_jit_arg_abi *abi;
14277 	const struct btf *btf = meta->btf;
14278 	const struct btf_param *args;
14279 	const struct btf_type *t;
14280 	u32 i, nargs, slots_used;
14281 	int arg_type;
14282 
14283 	args = (const struct btf_param *)(meta->func_proto + 1);
14284 	nargs = btf_type_vlen(meta->func_proto);
14285 	if (nargs > MAX_BPF_FUNC_ARGS) {
14286 		verbose(env, "Function %s has %d > %d args\n", meta->func_name,
14287 			nargs, MAX_BPF_FUNC_ARGS);
14288 		return -EINVAL;
14289 	}
14290 
14291 	for (i = 0, slots_used = 0; i < nargs; i++) {
14292 		u32 nslots = btf_func_model_arg_slots(fm, i);
14293 
14294 		if (nslots > 1) {
14295 			t = btf_type_skip_modifiers(btf, args[i].type, NULL);
14296 			if (!btf_func_arg_align(btf, t)) {
14297 				verbose(env,
14298 					"Function %s arg#%d type %s nests structs more than "
14299 					"%d levels deep\n",
14300 					meta->func_name, i, btf_type_str(t),
14301 					BTF_MEMBER_MAX_DEPTH);
14302 				return -EINVAL;
14303 			}
14304 		}
14305 		slots_used += nslots;
14306 	}
14307 
14308 	if (slots_used > MAX_BPF_FUNC_ARGS) {
14309 		verbose(env, "Function %s needs %d > %d argument slots\n", meta->func_name,
14310 			slots_used, MAX_BPF_FUNC_ARGS);
14311 		return -EINVAL;
14312 	}
14313 
14314 	abi = bpf_jit_arg_abi();
14315 
14316 	for (i = 0, slots_used = 0; i < nargs; i++) {
14317 		u32 nslots = btf_func_model_arg_slots(fm, i);
14318 
14319 		if (!abi && nslots > 1) {
14320 			t = btf_type_skip_modifiers(btf, args[i].type, NULL);
14321 			verbose(env,
14322 				"Function %s arg#%d type %s cannot be passed at "
14323 				"argument slot %d on this architecture\n",
14324 				meta->func_name, i, btf_type_str(t), slots_used);
14325 			return -EINVAL;
14326 		}
14327 		slots_used += nslots;
14328 
14329 		arg_type = get_kfunc_arg_type(env, meta, args, i, nargs,
14330 					      slots_used - nslots, proto);
14331 		if (arg_type < 0)
14332 			return arg_type;
14333 
14334 		proto->arg_type[i] = arg_type;
14335 	}
14336 
14337 	if (slots_used > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) {
14338 		verbose(env, "JIT does not support kfunc %s() with %d argument slots\n",
14339 			meta->func_name, slots_used);
14340 		return -ENOTSUPP;
14341 	}
14342 
14343 	return check_arg_prog_aux(env, proto) ? 0 : -EINVAL;
14344 }
14345 
14346 static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
14347 				     argno_t argno, enum bpf_arg_type arg_type,
14348 				     const struct btf *arg_btf, u32 arg_btf_id,
14349 				     struct bpf_call_arg_meta *meta, int insn_idx)
14350 {
14351 	bool taking_projection, struct_same, strict_type_match = false;
14352 	const struct btf_type *arg_t, *reg_t;
14353 	const char *arg_tname, *reg_tname;
14354 	const struct btf *reg_btf;
14355 	u32 reg_btf_id;
14356 
14357 	if (base_type(reg->type) == PTR_TO_BTF_ID) {
14358 		reg_btf = reg->btf;
14359 		reg_btf_id = reg->btf_id;
14360 	} else {
14361 		reg_btf = btf_vmlinux;
14362 		reg_btf_id = *reg2btf_ids[base_type(reg->type)];
14363 	}
14364 
14365 	/*
14366 	 * Enforce strict type matching for arguments that release a reference,
14367 	 * or are no-cast aliases. We do _not_ enforce strict matching by default,
14368 	 * as we want to enable BPF programs to pass types that are bitwise
14369 	 * equivalent without forcing them to explicitly cast with something
14370 	 * like bpf_cast_to_kern_ctx().
14371 	 *
14372 	 * For example, say we had a type like the following:
14373 	 *
14374 	 * struct bpf_cpumask {
14375 	 *	cpumask_t cpumask;
14376 	 *	refcount_t usage;
14377 	 * };
14378 	 *
14379 	 * Note that as specified in <linux/cpumask.h>, cpumask_t is typedef'ed
14380 	 * to a struct cpumask, so it would be safe to pass a struct
14381 	 * bpf_cpumask * to a kfunc expecting a struct cpumask *.
14382 	 *
14383 	 * The philosophy here is similar to how we allow scalars of different
14384 	 * types to be passed to kfuncs as long as the size is the same. The
14385 	 * only difference here is that we're simply allowing
14386 	 * btf_struct_ids_match() to walk the struct at the 0th offset, and
14387 	 * resolve types.
14388 	 */
14389 	if ((arg_type_is_release(arg_type) && !is_helper_call(meta, BPF_FUNC_sk_release)) ||
14390 	    (is_kfunc(meta) && btf_type_ids_nocast_alias(&env->log, reg_btf, reg_btf_id,
14391 						    arg_btf, arg_btf_id)))
14392 		strict_type_match = true;
14393 
14394 	arg_t = btf_type_skip_modifiers(arg_btf, arg_btf_id, &arg_btf_id);
14395 	arg_tname = btf_name_by_offset(arg_btf, arg_t->name_off);
14396 	reg_t = btf_type_skip_modifiers(reg_btf, reg_btf_id, &reg_btf_id);
14397 	reg_tname = btf_name_by_offset(reg_btf, reg_t->name_off);
14398 
14399 	struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_btf_id,
14400 					  reg->var_off.value, arg_btf, arg_btf_id,
14401 					  strict_type_match, !type_is_alloc(reg->type));
14402 
14403 	/* If kfunc is accepting a projection type (ie. __sk_buff), it cannot
14404 	 * actually use it -- it must cast to the underlying type. So we allow
14405 	 * caller to pass in the underlying type. Only kernel BTF has that type:
14406 	 * a type of the same name in the BTF of the program is program memory.
14407 	 * And only a pointer to the start of the object points to that type.
14408 	 */
14409 	taking_projection = is_kfunc(meta) && btf_is_kernel(reg_btf) &&
14410 			    !reg->var_off.value &&
14411 			    btf_is_projection_of(arg_tname, reg_tname);
14412 	if (!taking_projection && !struct_same) {
14413 		verbose(env, "%s %s expected pointer to %s %s but %s has a pointer to %s %s\n",
14414 			meta->func_name, reg_arg_name(env, argno),
14415 			btf_type_str(arg_t), arg_tname,
14416 			reg_arg_name(env, argno), btf_type_str(reg_t), reg_tname);
14417 		return -EINVAL;
14418 	}
14419 	return 0;
14420 }
14421 
14422 static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
14423 			     struct bpf_call_arg_meta *meta)
14424 {
14425 	int err, spi, kfunc_class = IRQ_NATIVE_KFUNC;
14426 	bool irq_save;
14427 
14428 	if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_save]) ||
14429 	    is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) {
14430 		irq_save = true;
14431 		if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]))
14432 			kfunc_class = IRQ_LOCK_KFUNC;
14433 	} else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_restore]) ||
14434 		   is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) {
14435 		irq_save = false;
14436 		if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]))
14437 			kfunc_class = IRQ_LOCK_KFUNC;
14438 	} else {
14439 		verifier_bug(env, "unknown irq flags kfunc");
14440 		return -EFAULT;
14441 	}
14442 
14443 	if (irq_save) {
14444 		if (!is_irq_flag_reg_valid_uninit(env, reg)) {
14445 			verbose(env, "expected uninitialized irq flag as %s\n",
14446 				reg_arg_name(env, argno));
14447 			bpf_diag_res(env, env->insn_idx, "IRQ flag is already initialized",
14448 				     "Saving IRQ state requires an uninitialized stack slot for "
14449 				     "the IRQ flag, but this slot already contains tracked IRQ "
14450 				     "flag state.",
14451 				     "Use a fresh stack slot for this save operation, or restore "
14452 				     "the existing IRQ flag before reusing the slot.");
14453 			return -EINVAL;
14454 		}
14455 
14456 		err = check_mem_access(env, env->insn_idx, reg, argno, 0, BPF_DW,
14457 				       BPF_WRITE, -1, false, false);
14458 		if (err)
14459 			return err;
14460 
14461 		err = mark_stack_slot_irq_flag(env, meta, reg, env->insn_idx, kfunc_class);
14462 		if (err)
14463 			return err;
14464 	} else {
14465 		err = is_irq_flag_reg_valid_init(env, reg);
14466 		if (err) {
14467 			verbose(env, "expected an initialized irq flag as %s\n",
14468 				reg_arg_name(env, argno));
14469 			bpf_diag_res(env, env->insn_idx, "uninitialized IRQ flag restore",
14470 				     "Restoring IRQ state requires a stack slot that was "
14471 				     "initialized by a matching IRQ save operation on this path.",
14472 				     "Pass the same stack slot that was previously initialized by "
14473 				     "the matching IRQ save kfunc.");
14474 			return err;
14475 		}
14476 
14477 		spi = irq_flag_get_spi(env, reg);
14478 		if (spi < 0)
14479 			return spi;
14480 
14481 		mark_stack_slots_scratched(env, spi, 1);
14482 
14483 		err = unmark_stack_slot_irq_flag(env, reg, kfunc_class);
14484 		if (err)
14485 			return err;
14486 
14487 		if (!in_rcu_cs(env))
14488 			invalidate_rcu_protected_refs(env);
14489 	}
14490 	return 0;
14491 }
14492 
14493 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
14494 {
14495 	struct btf_record *rec = reg_btf_record(reg);
14496 
14497 	if (!env->cur_state->active_locks) {
14498 		verifier_bug(env, "%s w/o active lock", __func__);
14499 		return -EFAULT;
14500 	}
14501 
14502 	if (type_flag(reg->type) & NON_OWN_REF) {
14503 		verifier_bug(env, "NON_OWN_REF already set");
14504 		return -EFAULT;
14505 	}
14506 
14507 	reg->type |= NON_OWN_REF;
14508 	if (rec->refcount_off >= 0)
14509 		reg->type |= MEM_RCU;
14510 
14511 	return 0;
14512 }
14513 
14514 static void ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 id)
14515 {
14516 	struct bpf_func_state *unused;
14517 	struct bpf_reg_state *reg;
14518 	int err;
14519 
14520 	err = release_reference_nomark(env, id);
14521 	WARN_ON_ONCE(err);
14522 
14523 	bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
14524 		if (reg->id == id) {
14525 			reg->id = 0;
14526 			ref_set_non_owning(env, reg);
14527 		}
14528 	}));
14529 
14530 	return;
14531 }
14532 
14533 /* Implementation details:
14534  *
14535  * Each register points to some region of memory, which we define as an
14536  * allocation. Each allocation may embed a bpf_spin_lock which protects any
14537  * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same
14538  * allocation. The lock and the data it protects are colocated in the same
14539  * memory region.
14540  *
14541  * Hence, everytime a register holds a pointer value pointing to such
14542  * allocation, the verifier preserves a unique reg->id for it.
14543  *
14544  * The verifier remembers the lock 'ptr' and the lock 'id' whenever
14545  * bpf_spin_lock is called.
14546  *
14547  * To enable this, lock state in the verifier captures two values:
14548  *	active_lock.ptr = Register's type specific pointer
14549  *	active_lock.id  = A unique ID for each register pointer value
14550  *
14551  * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two
14552  * supported register types.
14553  *
14554  * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of
14555  * allocated objects is the reg->btf pointer.
14556  *
14557  * The active_lock.id is non-unique for maps supporting direct_value_addr, as we
14558  * can establish the provenance of the map value statically for each distinct
14559  * lookup into such maps. They always contain a single map value hence unique
14560  * IDs for each pseudo load pessimizes the algorithm and rejects valid programs.
14561  *
14562  * So, in case of global variables, they use array maps with max_entries = 1,
14563  * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point
14564  * into the same map value as max_entries is 1, as described above).
14565  *
14566  * In case of inner map lookups, the inner map pointer has same map_ptr as the
14567  * outer map pointer (in verifier context), but each lookup into an inner map
14568  * assigns a fresh reg->id to the lookup, so while lookups into distinct inner
14569  * maps from the same outer map share the same map_ptr as active_lock.ptr, they
14570  * will get different reg->id assigned to each lookup, hence different
14571  * active_lock.id.
14572  *
14573  * In case of allocated objects, active_lock.ptr is the reg->btf, and the
14574  * reg->id is a unique ID preserved after the NULL pointer check on the pointer
14575  * returned from bpf_obj_new. Each allocation receives a new reg->id.
14576  */
14577 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
14578 {
14579 	struct bpf_reference_state *s;
14580 	void *ptr;
14581 	u32 id;
14582 
14583 	switch ((int)reg->type) {
14584 	case PTR_TO_MAP_VALUE:
14585 		ptr = reg->map_ptr;
14586 		break;
14587 	case PTR_TO_BTF_ID | MEM_ALLOC:
14588 		ptr = reg->btf;
14589 		break;
14590 	default:
14591 		verifier_bug(env, "unknown reg type for lock check");
14592 		return -EFAULT;
14593 	}
14594 	id = reg->id;
14595 
14596 	if (!env->cur_state->active_locks)
14597 		return -EINVAL;
14598 	s = find_lock_state(env->cur_state, REF_TYPE_LOCK_MASK, id, ptr);
14599 	if (!s) {
14600 		verbose(env, "held lock and object are not in the same allocation\n");
14601 		return -EINVAL;
14602 	}
14603 	return 0;
14604 }
14605 
14606 static bool is_bpf_list_api_kfunc(u32 btf_id)
14607 {
14608 	return is_bpf_list_push_kfunc(btf_id) ||
14609 	       btf_id == special_kfunc_list[KF_bpf_list_pop_front] ||
14610 	       btf_id == special_kfunc_list[KF_bpf_list_pop_back] ||
14611 	       btf_id == special_kfunc_list[KF_bpf_list_del] ||
14612 	       btf_id == special_kfunc_list[KF_bpf_list_front] ||
14613 	       btf_id == special_kfunc_list[KF_bpf_list_back] ||
14614 	       btf_id == special_kfunc_list[KF_bpf_list_is_first] ||
14615 	       btf_id == special_kfunc_list[KF_bpf_list_is_last] ||
14616 	       btf_id == special_kfunc_list[KF_bpf_list_empty];
14617 }
14618 
14619 static bool is_bpf_rbtree_api_kfunc(u32 btf_id)
14620 {
14621 	return is_bpf_rbtree_add_kfunc(btf_id) ||
14622 	       btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
14623 	       btf_id == special_kfunc_list[KF_bpf_rbtree_first] ||
14624 	       btf_id == special_kfunc_list[KF_bpf_rbtree_root] ||
14625 	       btf_id == special_kfunc_list[KF_bpf_rbtree_left] ||
14626 	       btf_id == special_kfunc_list[KF_bpf_rbtree_right];
14627 }
14628 
14629 static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta)
14630 {
14631 	int flags = PROCESS_RES_LOCK;
14632 
14633 	if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock]) ||
14634 	    is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]))
14635 		flags |= PROCESS_SPIN_LOCK;
14636 	else if (!is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock]) &&
14637 		 !is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]))
14638 		return 0;
14639 	if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) ||
14640 	    is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]))
14641 		flags |= PROCESS_LOCK_IRQ;
14642 	return flags;
14643 }
14644 
14645 static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset)
14646 {
14647 	struct bpf_kfunc_meta kfunc;
14648 	int err;
14649 
14650 	err = fetch_kfunc_meta(env, func_id, offset, &kfunc);
14651 	if (err || !kfunc.flags)
14652 		return false;
14653 
14654 	return *kfunc.flags & KF_SPINLOCK_SAFE;
14655 }
14656 
14657 /*
14658  * True if insn calls a helper/kfunc that requires one of its arguments to
14659  * be a stack pointer with a constant offset.
14660  */
14661 bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx)
14662 {
14663 	const struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
14664 	const struct bpf_func_proto *fn;
14665 	struct bpf_kfunc_desc *desc;
14666 	u32 *flags, btf_id;
14667 	int i;
14668 
14669 	if (bpf_helper_call(insn)) {
14670 		if (bpf_get_helper_proto(env, insn->imm, &fn) < 0)
14671 			return false;
14672 	} else if (bpf_pseudo_kfunc_call(insn)) {
14673 		desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
14674 		if (!desc)
14675 			return false;
14676 		fn = &desc->proto;
14677 	} else {
14678 		return false;
14679 	}
14680 
14681 	/* Both initialized and uninitialized stack dynptrs need a fixed offset. */
14682 	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++)
14683 		if (arg_type_is_dynptr(fn->arg_type[i]))
14684 			return true;
14685 
14686 	/* vmlinux kfuncs only */
14687 	if (!bpf_pseudo_kfunc_call(insn) || insn->off != 0)
14688 		return false;
14689 	btf_id = insn->imm;
14690 
14691 	flags = btf_kfunc_flags(btf_vmlinux, btf_id, env->prog);
14692 	if (flags && (*flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY)))
14693 		return true;
14694 
14695 	if (btf_id == special_kfunc_list[KF_bpf_res_spin_lock] ||
14696 	    btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] ||
14697 	    btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] ||
14698 	    btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore] ||
14699 	    btf_id == special_kfunc_list[KF_bpf_local_irq_save] ||
14700 	    btf_id == special_kfunc_list[KF_bpf_local_irq_restore])
14701 		return true;
14702 
14703 	return false;
14704 }
14705 
14706 static bool is_sync_callback_calling_kfunc(u32 btf_id)
14707 {
14708 	return is_bpf_rbtree_add_kfunc(btf_id);
14709 }
14710 
14711 static bool is_async_callback_calling_kfunc(u32 btf_id)
14712 {
14713 	return is_bpf_wq_set_callback_kfunc(btf_id) ||
14714 	       is_task_work_add_kfunc(btf_id) ||
14715 	       is_call_rcu_kfunc(btf_id);
14716 }
14717 
14718 bool bpf_is_throw_kfunc(struct bpf_insn *insn)
14719 {
14720 	return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
14721 	       insn->imm == special_kfunc_list[KF_bpf_throw];
14722 }
14723 
14724 static bool is_bpf_wq_set_callback_kfunc(u32 btf_id)
14725 {
14726 	return btf_id == special_kfunc_list[KF_bpf_wq_set_callback];
14727 }
14728 
14729 static bool is_callback_calling_kfunc(u32 btf_id)
14730 {
14731 	return is_sync_callback_calling_kfunc(btf_id) ||
14732 	       is_async_callback_calling_kfunc(btf_id);
14733 }
14734 
14735 static bool is_rbtree_lock_required_kfunc(u32 btf_id)
14736 {
14737 	return is_bpf_rbtree_api_kfunc(btf_id);
14738 }
14739 
14740 static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env,
14741 					  enum btf_field_type head_field_type,
14742 					  u32 kfunc_btf_id)
14743 {
14744 	bool ret;
14745 
14746 	switch (head_field_type) {
14747 	case BPF_LIST_HEAD:
14748 		ret = is_bpf_list_api_kfunc(kfunc_btf_id);
14749 		break;
14750 	case BPF_RB_ROOT:
14751 		ret = is_bpf_rbtree_api_kfunc(kfunc_btf_id);
14752 		break;
14753 	default:
14754 		verbose(env, "verifier internal error: unexpected graph root argument type %s\n",
14755 			btf_field_type_name(head_field_type));
14756 		return false;
14757 	}
14758 
14759 	if (!ret)
14760 		verbose(env, "verifier internal error: %s head arg for unknown kfunc\n",
14761 			btf_field_type_name(head_field_type));
14762 	return ret;
14763 }
14764 
14765 static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env,
14766 					  enum btf_field_type node_field_type,
14767 					  u32 kfunc_btf_id)
14768 {
14769 	bool ret;
14770 
14771 	switch (node_field_type) {
14772 	case BPF_LIST_NODE:
14773 		ret = is_bpf_list_push_kfunc(kfunc_btf_id) ||
14774 		      kfunc_btf_id == special_kfunc_list[KF_bpf_list_del] ||
14775 		      kfunc_btf_id == special_kfunc_list[KF_bpf_list_is_first] ||
14776 		      kfunc_btf_id == special_kfunc_list[KF_bpf_list_is_last];
14777 		break;
14778 	case BPF_RB_NODE:
14779 		ret = (is_bpf_rbtree_add_kfunc(kfunc_btf_id) ||
14780 		       kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
14781 		       kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_left] ||
14782 		       kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_right]);
14783 		break;
14784 	default:
14785 		verbose(env, "verifier internal error: unexpected graph node argument type %s\n",
14786 			btf_field_type_name(node_field_type));
14787 		return false;
14788 	}
14789 
14790 	if (!ret)
14791 		verbose(env, "verifier internal error: %s node arg for unknown kfunc\n",
14792 			btf_field_type_name(node_field_type));
14793 	return ret;
14794 }
14795 
14796 static int
14797 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,
14798 				   struct bpf_reg_state *reg, argno_t argno,
14799 				   struct bpf_call_arg_meta *meta,
14800 				   enum btf_field_type head_field_type,
14801 				   struct btf_field **head_field)
14802 {
14803 	const char *head_type_name;
14804 	struct btf_field *field;
14805 	struct btf_record *rec;
14806 	u32 head_off;
14807 
14808 	if (meta->btf != btf_vmlinux) {
14809 		verifier_bug(env, "unexpected btf mismatch in kfunc call");
14810 		return -EFAULT;
14811 	}
14812 
14813 	if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id))
14814 		return -EFAULT;
14815 
14816 	head_type_name = btf_field_type_name(head_field_type);
14817 	if (!tnum_is_const(reg->var_off)) {
14818 		verbose(env,
14819 			"%s doesn't have constant offset. %s has to be at the constant offset\n",
14820 			reg_arg_name(env, argno), head_type_name);
14821 		return -EINVAL;
14822 	}
14823 
14824 	rec = reg_btf_record(reg);
14825 	head_off = reg->var_off.value;
14826 	field = btf_record_find(rec, head_off, head_field_type);
14827 	if (!field) {
14828 		verbose(env, "%s not found at offset=%u\n", head_type_name, head_off);
14829 		return -EINVAL;
14830 	}
14831 
14832 	/* All functions require bpf_list_head to be protected using a bpf_spin_lock */
14833 	if (check_reg_allocation_locked(env, reg)) {
14834 		verbose(env, "bpf_spin_lock at off=%d must be held for %s\n",
14835 			rec->spin_lock_off, head_type_name);
14836 		return -EINVAL;
14837 	}
14838 
14839 	if (*head_field) {
14840 		verifier_bug(env, "repeating %s arg", head_type_name);
14841 		return -EFAULT;
14842 	}
14843 	*head_field = field;
14844 	return 0;
14845 }
14846 
14847 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,
14848 					   struct bpf_reg_state *reg, argno_t argno,
14849 					   struct bpf_call_arg_meta *meta)
14850 {
14851 	return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_LIST_HEAD,
14852 							  &meta->arg_list_head.field);
14853 }
14854 
14855 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,
14856 					     struct bpf_reg_state *reg, argno_t argno,
14857 					     struct bpf_call_arg_meta *meta)
14858 {
14859 	return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_RB_ROOT,
14860 							  &meta->arg_rbtree_root.field);
14861 }
14862 
14863 static int
14864 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
14865 				   struct bpf_reg_state *reg, argno_t argno,
14866 				   struct bpf_call_arg_meta *meta,
14867 				   enum btf_field_type head_field_type,
14868 				   enum btf_field_type node_field_type,
14869 				   struct btf_field **node_field)
14870 {
14871 	const char *node_type_name;
14872 	const struct btf_type *et, *t;
14873 	struct btf_field *field;
14874 	u32 node_off;
14875 
14876 	if (meta->btf != btf_vmlinux) {
14877 		verifier_bug(env, "unexpected btf mismatch in kfunc call");
14878 		return -EFAULT;
14879 	}
14880 
14881 	if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id))
14882 		return -EFAULT;
14883 
14884 	node_type_name = btf_field_type_name(node_field_type);
14885 	if (!tnum_is_const(reg->var_off)) {
14886 		verbose(env,
14887 			"%s doesn't have constant offset. %s has to be at the constant offset\n",
14888 			reg_arg_name(env, argno), node_type_name);
14889 		return -EINVAL;
14890 	}
14891 
14892 	node_off = reg->var_off.value;
14893 	field = reg_find_field_offset(reg, node_off, node_field_type);
14894 	if (!field) {
14895 		verbose(env, "%s not found at offset=%u\n", node_type_name, node_off);
14896 		return -EINVAL;
14897 	}
14898 
14899 	field = *node_field;
14900 
14901 	et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id);
14902 	t = btf_type_by_id(reg->btf, reg->btf_id);
14903 	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf,
14904 				  field->graph_root.value_btf_id, true,
14905 				  !type_is_alloc(reg->type))) {
14906 		verbose(env, "operation on %s expects arg#1 %s at offset=%d "
14907 			"in struct %s, but arg is at offset=%d in struct %s\n",
14908 			btf_field_type_name(head_field_type),
14909 			btf_field_type_name(node_field_type),
14910 			field->graph_root.node_offset,
14911 			btf_name_by_offset(field->graph_root.btf, et->name_off),
14912 			node_off, btf_name_by_offset(reg->btf, t->name_off));
14913 		return -EINVAL;
14914 	}
14915 	meta->arg_btf = reg->btf;
14916 	meta->arg_btf_id = reg->btf_id;
14917 
14918 	if (node_off != field->graph_root.node_offset) {
14919 		verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n",
14920 			node_off, btf_field_type_name(node_field_type),
14921 			field->graph_root.node_offset,
14922 			btf_name_by_offset(field->graph_root.btf, et->name_off));
14923 		return -EINVAL;
14924 	}
14925 
14926 	return 0;
14927 }
14928 
14929 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
14930 					   struct bpf_reg_state *reg, argno_t argno,
14931 					   struct bpf_call_arg_meta *meta)
14932 {
14933 	return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,
14934 						  BPF_LIST_HEAD, BPF_LIST_NODE,
14935 						  &meta->arg_list_head.field);
14936 }
14937 
14938 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,
14939 					     struct bpf_reg_state *reg, argno_t argno,
14940 					     struct bpf_call_arg_meta *meta)
14941 {
14942 	return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,
14943 						  BPF_RB_ROOT, BPF_RB_NODE,
14944 						  &meta->arg_rbtree_root.field);
14945 }
14946 
14947 /*
14948  * css_task iter allowlist is needed to avoid dead locking on css_set_lock.
14949  * LSM hooks and iters (both sleepable and non-sleepable) are safe.
14950  * Any sleepable progs are also safe since bpf_check_attach_target() enforce
14951  * them can only be attached to some specific hook points.
14952  */
14953 static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env)
14954 {
14955 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
14956 
14957 	switch (prog_type) {
14958 	case BPF_PROG_TYPE_LSM:
14959 		return true;
14960 	case BPF_PROG_TYPE_TRACING:
14961 		if (env->prog->expected_attach_type == BPF_TRACE_ITER)
14962 			return true;
14963 		fallthrough;
14964 	default:
14965 		return in_sleepable(env);
14966 	}
14967 }
14968 
14969 int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env,
14970 			     s32 func_id,
14971 			     s16 offset,
14972 			     struct bpf_call_arg_meta *meta)
14973 {
14974 	struct bpf_kfunc_meta kfunc;
14975 	int err;
14976 
14977 	memset(meta, 0, sizeof(*meta));
14978 
14979 	err = fetch_kfunc_meta(env, func_id, offset, &kfunc);
14980 	if (err)
14981 		return err;
14982 
14983 	meta->btf = kfunc.btf;
14984 	meta->func_id = kfunc.id;
14985 	meta->func_proto = kfunc.proto;
14986 	meta->func_name = kfunc.name;
14987 
14988 	if (!kfunc.flags || !btf_kfunc_is_allowed(kfunc.btf, kfunc.id, env->prog))
14989 		return -EACCES;
14990 
14991 	meta->kfunc_flags = *kfunc.flags;
14992 
14993 	/* Only support release referenced argument passed by register */
14994 	if (is_kfunc_release(meta))
14995 		meta->release_regno = BPF_REG_1;
14996 
14997 	return 0;
14998 }
14999 
15000 /*
15001  * Describe a helper's stack access through argument @arg (0-based, R1-R5).
15002  * must_write means the verifier destroys prior state throughout size bytes.
15003  * In other words, must_write follows the verifier's model of the
15004  * function's behaviour, which is safe to use for is_state_visited() pruning.
15005  */
15006 struct arg_access_info
15007 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn,
15008 			      int arg, int insn_idx)
15009 {
15010 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
15011 	struct arg_access_info info = {
15012 		.size = U32_MAX,
15013 		.may_read = true,
15014 		.may_write = true,
15015 	};
15016 	const struct bpf_func_proto *fn;
15017 	enum bpf_access_type access_type;
15018 	enum bpf_arg_type at;
15019 	bool exact_size = true;
15020 	u64 size = U32_MAX;
15021 
15022 	if (bpf_get_helper_proto(env, insn->imm, &fn) < 0)
15023 		return info;
15024 
15025 	at = fn->arg_type[arg];
15026 	access_type = func_arg_access_type(at);
15027 
15028 	switch (base_type(at)) {
15029 	case ARG_PTR_TO_MAP_KEY:
15030 	case ARG_PTR_TO_MAP_VALUE: {
15031 		bool is_key = base_type(at) == ARG_PTR_TO_MAP_KEY;
15032 		u64 val;
15033 		int i, map_reg;
15034 
15035 		for (i = 0; i < arg; i++) {
15036 			if (base_type(fn->arg_type[i]) == ARG_CONST_MAP_PTR)
15037 				break;
15038 		}
15039 		if (i >= arg)
15040 			goto scan_all_maps;
15041 
15042 		map_reg = BPF_REG_1 + i;
15043 
15044 		if (!(aux->const_reg_map_mask & BIT(map_reg)))
15045 			goto scan_all_maps;
15046 
15047 		i = aux->const_reg_vals[map_reg];
15048 		if (i < env->used_map_cnt) {
15049 			/* Bloom-filter peek reads the value buffer. */
15050 			if (!is_key && insn->imm == BPF_FUNC_map_peek_elem &&
15051 			    env->used_maps[i]->map_type == BPF_MAP_TYPE_BLOOM_FILTER)
15052 				access_type = BPF_READ;
15053 			size = is_key ? env->used_maps[i]->key_size
15054 				      : env->used_maps[i]->value_size;
15055 			goto out;
15056 		}
15057 scan_all_maps:
15058 		/*
15059 		 * Map pointer is not known at this call site (e.g. different
15060 		 * maps on merged paths).  Conservatively return the largest
15061 		 * key_size or value_size across all maps used by the program.
15062 		 * This is only an upper bound, so it cannot establish a definite
15063 		 * write. Map-dependent argument types can also turn an output
15064 		 * into an input, so conservatively retain a read dependency.
15065 		 */
15066 		exact_size = false;
15067 		access_type |= BPF_READ;
15068 		val = 0;
15069 		for (i = 0; i < env->used_map_cnt; i++) {
15070 			struct bpf_map *map = env->used_maps[i];
15071 			u32 sz = is_key ? map->key_size : map->value_size;
15072 
15073 			if (sz > val)
15074 				val = sz;
15075 			if (map->inner_map_meta) {
15076 				sz = is_key ? map->inner_map_meta->key_size
15077 					    : map->inner_map_meta->value_size;
15078 				if (sz > val)
15079 					val = sz;
15080 			}
15081 		}
15082 		if (!val)
15083 			return info;
15084 		size = val;
15085 		goto out;
15086 	}
15087 	case ARG_PTR_TO_MEM:
15088 		if (at & MEM_FIXED_SIZE) {
15089 			size = fn->arg_size[arg];
15090 			goto out;
15091 		}
15092 		if (arg + 1 < ARRAY_SIZE(fn->arg_type) &&
15093 		    arg_type_is_mem_size(fn->arg_type[arg + 1])) {
15094 			int size_reg = BPF_REG_1 + arg + 1;
15095 
15096 			if (aux->const_reg_mask & BIT(size_reg)) {
15097 				size = aux->const_reg_vals[size_reg];
15098 				goto out;
15099 			}
15100 		}
15101 		/* Preserve access directions even when the extent is unknown. */
15102 		goto out;
15103 	case ARG_PTR_TO_DYNPTR:
15104 		size = BPF_DYNPTR_SIZE;
15105 		break;
15106 	case ARG_PTR_TO_STACK:
15107 		/*
15108 		 * Only used by bpf_calls_callback() helpers. The helper itself
15109 		 * doesn't access stack. The callback subprog does and it's
15110 		 * analyzed separately.
15111 		 */
15112 		return (struct arg_access_info) {};
15113 	default:
15114 		return info;
15115 	}
15116 out:
15117 	info.size = min_t(u64, size, U32_MAX);
15118 	info.may_read = !!(access_type & BPF_READ);
15119 	info.may_write = !!(access_type & BPF_WRITE);
15120 	info.must_write = info.may_write && exact_size && info.size != U32_MAX;
15121 	/* Generic unprivileged outputs retain prior initialization state. */
15122 	if (!env->allow_uninit_stack && arg_type_is_raw_mem(at)) {
15123 		info.may_read = true;
15124 		info.must_write = false;
15125 	}
15126 	return info;
15127 }
15128 
15129 /*
15130  * Describe a kfunc's stack access through argument slot @arg (0-based).
15131  * As for helpers, must_write describes destruction of prior verifier state,
15132  * rather than a guarantee that the callee writes every byte at runtime.
15133  */
15134 struct arg_access_info
15135 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn,
15136 			     int arg, int insn_idx)
15137 {
15138 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
15139 	struct arg_access_info info = {
15140 		.size = U32_MAX,
15141 		.may_read = true,
15142 		.may_write = true,
15143 	};
15144 	struct bpf_call_arg_meta meta;
15145 	const struct btf_param *args;
15146 	const struct btf_type *t, *ref_t;
15147 	const struct btf *btf;
15148 	u32 i, slot, nargs, type_size;
15149 	u64 size;
15150 
15151 	if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0)
15152 		return info;
15153 
15154 	btf = meta.btf;
15155 	args = btf_params(meta.func_proto);
15156 	nargs = btf_type_vlen(meta.func_proto);
15157 
15158 	/*
15159 	 * @arg is an argument slot and a 16-byte parameter takes two of them,
15160 	 * so walk the parameters to find the one that starts at this slot. A
15161 	 * slot holding the upper eightbyte of such a parameter belongs to no
15162 	 * pointer, and neither does a slot past the last parameter.
15163 	 */
15164 	for (i = 0, slot = 0; i < nargs && slot < arg; i++)
15165 		slot += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL));
15166 	if (i >= nargs || slot != arg)
15167 		return (struct arg_access_info) {};
15168 
15169 	t = btf_type_skip_modifiers(btf, args[i].type, NULL);
15170 	if (!btf_type_is_ptr(t))
15171 		return (struct arg_access_info) {};
15172 
15173 	/* dynptr: fixed 16-byte on-stack representation */
15174 	if (is_kfunc_arg_dynptr(btf, &args[i])) {
15175 		size = BPF_DYNPTR_SIZE;
15176 		goto out;
15177 	}
15178 
15179 	/* ptr + __sz/__szk pair: the size follows the pointer */
15180 	if (i + 1 < nargs &&
15181 	    (btf_param_match_suffix(btf, &args[i + 1], "__sz") ||
15182 	     btf_param_match_suffix(btf, &args[i + 1], "__szk"))) {
15183 		int size_reg = BPF_REG_1 + arg + 1;
15184 
15185 		if (size_reg <= MAX_BPF_FUNC_REG_ARGS &&
15186 		    (aux->const_reg_mask & BIT(size_reg))) {
15187 			size = aux->const_reg_vals[size_reg];
15188 			goto out;
15189 		}
15190 		/* Unknown size: the read may extend anywhere up to the frame top. */
15191 		return info;
15192 	}
15193 
15194 	/* fixed-size pointed-to type: resolve via BTF */
15195 	ref_t = btf_type_skip_modifiers(btf, t->type, NULL);
15196 	if (!IS_ERR(btf_resolve_size(btf, ref_t, &type_size))) {
15197 		size = type_size;
15198 		goto out;
15199 	}
15200 
15201 	return info;
15202 out:
15203 	info.size = min_t(u64, size, U32_MAX);
15204 	/* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */
15205 	if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW)
15206 		info.may_read = false;
15207 	if (is_kfunc_arg_uninit(btf, &args[i]) &&
15208 	    (is_kfunc_arg_dynptr(btf, &args[i]) || env->allow_uninit_stack))
15209 		info.may_read = false;
15210 	info.must_write = !info.may_read && info.size != U32_MAX;
15211 	return info;
15212 }
15213 
15214 /*
15215  * Called before dead code elimination, some subprograms may be genuinely unreachable.
15216  * Suppress diagnostics so callers can use a conservative result on error.
15217  * The main pass would report errors for reachable subprograms.
15218  */
15219 static bool try_btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
15220 {
15221 	struct bpf_prog *prog = env->prog;
15222 	u32 log_level = env->log.level;
15223 	int err;
15224 
15225 	if (!prog->aux->func_info || !prog->aux->func_info[subprog].type_id)
15226 		return false;
15227 
15228 	env->log.level = 0;
15229 	err = btf_prepare_func_args(env, subprog);
15230 	env->log.level = log_level;
15231 	return !err;
15232 }
15233 
15234 /* Prepare argument metadata for liveness; the main program is handled separately. */
15235 static void prepare_subprog_args(struct bpf_verifier_env *env)
15236 {
15237 	int i;
15238 
15239 	for (i = 1; i < env->subprog_cnt; i++) {
15240 		if (!try_btf_prepare_func_args(env, i) && bpf_subprog_is_global(env, i))
15241 			/* Keep all argument registers live if the prototype is unavailable. */
15242 			env->subprog_info[i].arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS;
15243 	}
15244 }
15245 
15246 /*
15247  * Derive stack effects for a global subprog's argument slot @arg (0-based)
15248  * from its declared type, matching the main pass's argument checks.
15249  */
15250 struct arg_access_info
15251 bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn,
15252 				      int arg, int insn_idx)
15253 {
15254 	struct arg_access_info info = {
15255 		.size = U32_MAX,
15256 		.may_read = true,
15257 		.may_write = true,
15258 	};
15259 	struct bpf_subprog_info *sub;
15260 	int callee;
15261 
15262 	callee = bpf_find_subprog(env, insn_idx + insn->imm + 1);
15263 	if (callee < 0)
15264 		return info;
15265 	sub = subprog_info(env, callee);
15266 	if (!try_btf_prepare_func_args(env, callee))
15267 		return info;
15268 
15269 	switch (base_type(sub->args[arg].arg_type)) {
15270 	case ARG_UNUSED:	/* Not an argument slot. */
15271 	case ARG_SCALAR:	/* A scalar, the callee can't dereference it. */
15272 		return (struct arg_access_info) {};
15273 	case ARG_PTR_TO_BTF_ID:
15274 		/*
15275 		 * Trusted arguments cannot be stack pointers.
15276 		 * Untrusted arguments use ARG_IGNORE:
15277 		 * probe reads need no caller stack state, writes are prohibited.
15278 		 */
15279 		return (struct arg_access_info) {};
15280 	case ARG_PTR_TO_MEM:
15281 		/* mem_size is zero for untrusted pointers of unknown size. */
15282 		if (sub->args[arg].mem_size) {
15283 			info.size = sub->args[arg].mem_size;
15284 			info.must_write = true;
15285 		}
15286 		return info;
15287 	case ARG_PTR_TO_DYNPTR:
15288 		/* The callee may adjust the view, but cannot reinitialize the dynptr. */
15289 		info.size = BPF_DYNPTR_SIZE;
15290 		return info;
15291 	default:
15292 		return info;
15293 	}
15294 }
15295 
15296 /* check special kfuncs and return:
15297  *  1  - not fall-through to 'else' branch, continue verification
15298  *  0  - fall-through to 'else' branch
15299  * < 0 - not fall-through to 'else' branch, return error
15300  */
15301 static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
15302 			       struct bpf_reg_state *regs, struct bpf_insn_aux_data *insn_aux,
15303 			       const struct btf_type *ptr_type, struct btf *desc_btf)
15304 {
15305 	const struct btf_type *ret_t;
15306 	int err = 0;
15307 
15308 	if (meta->btf != btf_vmlinux)
15309 		return 0;
15310 
15311 	if (is_bpf_obj_new_kfunc(meta->func_id) || is_bpf_percpu_obj_new_kfunc(meta->func_id)) {
15312 		struct btf_struct_meta *struct_meta;
15313 		struct btf *ret_btf;
15314 		u32 ret_btf_id;
15315 
15316 		if (is_bpf_obj_new_kfunc(meta->func_id) && !bpf_global_ma_set)
15317 			return -ENOMEM;
15318 
15319 		if (((u64)(u32)meta->arg_constant.value) != meta->arg_constant.value) {
15320 			verbose(env, "local type ID argument must be in range [0, U32_MAX]\n");
15321 			return -EINVAL;
15322 		}
15323 
15324 		ret_btf = env->prog->aux->btf;
15325 		ret_btf_id = meta->arg_constant.value;
15326 
15327 		/* This may be NULL due to user not supplying a BTF */
15328 		if (!ret_btf) {
15329 			verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n");
15330 			return -EINVAL;
15331 		}
15332 
15333 		ret_t = btf_type_by_id(ret_btf, ret_btf_id);
15334 		if (!ret_t || !__btf_type_is_struct(ret_t)) {
15335 			verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n");
15336 			return -EINVAL;
15337 		}
15338 
15339 		if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) {
15340 			if (ret_t->size > BPF_GLOBAL_PERCPU_MA_MAX_SIZE) {
15341 				verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n",
15342 					ret_t->size, BPF_GLOBAL_PERCPU_MA_MAX_SIZE);
15343 				return -EINVAL;
15344 			}
15345 
15346 			if (!bpf_global_percpu_ma_set) {
15347 				mutex_lock(&bpf_percpu_ma_lock);
15348 				if (!bpf_global_percpu_ma_set) {
15349 					/* Charge memory allocated with bpf_global_percpu_ma to
15350 					 * root memcg. The obj_cgroup for root memcg is NULL.
15351 					 */
15352 					err = bpf_mem_alloc_percpu_init(&bpf_global_percpu_ma, NULL);
15353 					if (!err)
15354 						bpf_global_percpu_ma_set = true;
15355 				}
15356 				mutex_unlock(&bpf_percpu_ma_lock);
15357 				if (err)
15358 					return err;
15359 			}
15360 
15361 			mutex_lock(&bpf_percpu_ma_lock);
15362 			err = bpf_mem_alloc_percpu_unit_init(&bpf_global_percpu_ma, ret_t->size);
15363 			mutex_unlock(&bpf_percpu_ma_lock);
15364 			if (err)
15365 				return err;
15366 		}
15367 
15368 		struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id);
15369 		if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) {
15370 			if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) {
15371 				verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n");
15372 				return -EINVAL;
15373 			}
15374 
15375 			if (struct_meta) {
15376 				verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n");
15377 				return -EINVAL;
15378 			}
15379 		}
15380 
15381 		mark_reg_known_zero(env, regs, BPF_REG_0);
15382 		regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
15383 		regs[BPF_REG_0].btf = ret_btf;
15384 		regs[BPF_REG_0].btf_id = ret_btf_id;
15385 		if (is_bpf_percpu_obj_new_kfunc(meta->func_id))
15386 			regs[BPF_REG_0].type |= MEM_PERCPU;
15387 
15388 		insn_aux->obj_new_size = ret_t->size;
15389 		insn_aux->kptr_struct_meta = struct_meta;
15390 	} else if (is_bpf_refcount_acquire_kfunc(meta->func_id)) {
15391 		mark_reg_known_zero(env, regs, BPF_REG_0);
15392 		regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
15393 		regs[BPF_REG_0].btf = meta->arg_btf;
15394 		regs[BPF_REG_0].btf_id = meta->arg_btf_id;
15395 
15396 		insn_aux->kptr_struct_meta =
15397 			btf_find_struct_meta(meta->arg_btf,
15398 					     meta->arg_btf_id);
15399 	} else if (is_list_node_type(ptr_type)) {
15400 		struct btf_field *field = meta->arg_list_head.field;
15401 
15402 		mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
15403 	} else if (is_rbtree_node_type(ptr_type)) {
15404 		struct btf_field *field = meta->arg_rbtree_root.field;
15405 
15406 		mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
15407 	} else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx])) {
15408 		mark_reg_known_zero(env, regs, BPF_REG_0);
15409 		regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED;
15410 		regs[BPF_REG_0].btf = desc_btf;
15411 		regs[BPF_REG_0].btf_id = meta->ret_btf_id;
15412 	} else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_rdonly_cast])) {
15413 		ret_t = btf_type_by_id(desc_btf, meta->arg_constant.value);
15414 		if (!ret_t) {
15415 			verbose(env, "Unknown type ID %lld passed to kfunc bpf_rdonly_cast\n",
15416 				meta->arg_constant.value);
15417 			return -EINVAL;
15418 		} else if (btf_type_is_struct(ret_t)) {
15419 			mark_reg_known_zero(env, regs, BPF_REG_0);
15420 			regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED;
15421 			regs[BPF_REG_0].btf = desc_btf;
15422 			regs[BPF_REG_0].btf_id = meta->arg_constant.value;
15423 		} else if (btf_type_is_void(ret_t)) {
15424 			mark_reg_known_zero(env, regs, BPF_REG_0);
15425 			regs[BPF_REG_0].type = PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED;
15426 			regs[BPF_REG_0].mem_size = 0;
15427 		} else {
15428 			verbose(env,
15429 				"kfunc bpf_rdonly_cast type ID argument must be of a struct or void\n");
15430 			return -EINVAL;
15431 		}
15432 	} else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice]) ||
15433 		   is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice_rdwr])) {
15434 		enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->dynptr.type);
15435 
15436 		mark_reg_known_zero(env, regs, BPF_REG_0);
15437 
15438 		if (!meta->arg_constant.found) {
15439 			verifier_bug(env, "bpf_dynptr_slice(_rdwr) no constant size");
15440 			return -EFAULT;
15441 		}
15442 
15443 		regs[BPF_REG_0].mem_size = meta->arg_constant.value;
15444 
15445 		/* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */
15446 		regs[BPF_REG_0].type = PTR_TO_MEM | type_flag;
15447 
15448 		if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice])) {
15449 			regs[BPF_REG_0].type |= MEM_RDONLY;
15450 		} else {
15451 			/* this will set env->seen_direct_write to true */
15452 			if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) {
15453 				verbose(env, "the prog does not allow writes to packet data\n");
15454 				return -EINVAL;
15455 			}
15456 		}
15457 
15458 		if (!meta->dynptr.id) {
15459 			verifier_bug(env, "no dynptr id");
15460 			return -EFAULT;
15461 		}
15462 		regs[BPF_REG_0].parent_id = meta->dynptr.id;
15463 		regs[BPF_REG_0].buf = meta->slice_buf;
15464 		if (meta->slice_buf.size)
15465 			env->seen_stack_slice = true;
15466 	} else {
15467 		return 0;
15468 	}
15469 
15470 	return 1;
15471 }
15472 
15473 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
15474 
15475 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
15476 			    int *insn_idx_p)
15477 {
15478 	bool sleepable, rcu_lock, rcu_unlock, preempt_disable, preempt_enable;
15479 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
15480 	struct bpf_reg_state *regs = cur_regs(env);
15481 	const char *func_name, *ptr_type_name;
15482 	const struct btf_type *t, *ptr_type;
15483 	struct bpf_call_arg_meta meta;
15484 	struct bpf_insn_aux_data *insn_aux;
15485 	const char *operation;
15486 	int err, insn_idx = *insn_idx_p;
15487 	u32 i, proto_slots, ptr_type_id, ret_nregs = 1;
15488 	struct bpf_kfunc_desc *desc;
15489 	struct btf *desc_btf;
15490 	int id;
15491 
15492 	/* skip for now, but return error when we find this in fixup_kfunc_call */
15493 	if (!insn->imm)
15494 		return 0;
15495 
15496 	err = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta);
15497 	if (err == -EACCES && meta.func_name) {
15498 		verbose(env, "calling kernel function %s is not allowed\n", meta.func_name);
15499 		operation = bpf_diag_fmt(env, "kfunc %s", meta.func_name);
15500 		bpf_diag_policy(
15501 			env, insn_idx, operation, "this program cannot call the kfunc",
15502 			"Use a kfunc allowed for this program type and attach point, or change the program context.");
15503 	}
15504 	if (err)
15505 		return err;
15506 	desc_btf = meta.btf;
15507 	func_name = meta.func_name;
15508 	insn_aux = &env->insn_aux_data[insn_idx];
15509 
15510 	desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
15511 	if (!desc) {
15512 		verifier_bug(env, "kfunc descriptor not found for func_id %u", insn->imm);
15513 		return -EFAULT;
15514 	}
15515 	meta.fn = &desc->proto;
15516 
15517 	insn_aux->is_iter_next = bpf_is_iter_next_kfunc(&meta);
15518 
15519 	if (!insn->off &&
15520 	    (insn->imm == special_kfunc_list[KF_bpf_res_spin_lock] ||
15521 	     insn->imm == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) {
15522 		struct bpf_verifier_state *branch;
15523 		struct bpf_reg_state *regs;
15524 
15525 		branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);
15526 		if (IS_ERR(branch)) {
15527 			verbose(env, "failed to push state for failed lock acquisition\n");
15528 			return PTR_ERR(branch);
15529 		}
15530 
15531 		regs = branch->frame[branch->curframe]->regs;
15532 
15533 		/* Clear r0-r5 registers in forked state */
15534 		for (i = 0; i < CALLER_SAVED_REGS; i++)
15535 			bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);
15536 
15537 		mark_reg_unknown(env, regs, BPF_REG_0);
15538 		err = __mark_reg_s32_range(env, regs, BPF_REG_0, -MAX_ERRNO, -1);
15539 		if (err) {
15540 			verbose(env, "failed to mark s32 range for retval in forked state for lock\n");
15541 			return err;
15542 		}
15543 	} else if (!insn->off && insn->imm == special_kfunc_list[KF___bpf_trap]) {
15544 		verbose(env, "unexpected __bpf_trap() due to uninitialized variable?\n");
15545 		return -EFAULT;
15546 	}
15547 
15548 	if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) {
15549 		verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n");
15550 		operation = bpf_diag_fmt(env, "destructive kfunc %s", meta.func_name);
15551 		bpf_diag_policy(
15552 			env, insn_idx, operation, "destructive kfuncs require CAP_SYS_BOOT",
15553 			"Load the program with CAP_SYS_BOOT, or avoid destructive kfuncs.");
15554 		return -EACCES;
15555 	}
15556 
15557 	if (is_kfunc_perfmon(&meta) && !env->allow_ptr_leaks) {
15558 		verbose(env, "%s is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
15559 			func_name);
15560 		operation = bpf_diag_fmt(env, "kfunc %s", func_name);
15561 		bpf_diag_policy(env, insn_idx, operation, "the kfunc requires CAP_PERFMON",
15562 				"Load the program with CAP_PERFMON, or avoid the kfunc.");
15563 		return -EPERM;
15564 	}
15565 
15566 	sleepable = bpf_is_kfunc_sleepable(&meta);
15567 	if (sleepable && !in_sleepable_context(env)) {
15568 		const char *suggestion;
15569 
15570 		if (in_sleepable(env)) {
15571 			verbose(env, "kernel func %s is sleepable within %s\n",
15572 				func_name, non_sleepable_context_description(env));
15573 			suggestion = "Move the kfunc call outside the critical section, or use a non-sleepable kfunc.";
15574 		} else {
15575 			verbose(env, "program must be sleepable to call sleepable kfunc %s\n",
15576 				func_name);
15577 			suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc.";
15578 		}
15579 		operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name);
15580 		bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion);
15581 		return -EACCES;
15582 	}
15583 
15584 	/* Track non-sleepable context for kfuncs, same as for helpers. */
15585 	if (!in_sleepable_context(env))
15586 		insn_aux->non_sleepable = true;
15587 
15588 	/* Check the arguments */
15589 	err = check_func_args(env, &meta, insn_idx);
15590 	if (err < 0)
15591 		return err;
15592 
15593 	err = mark_raw_stack(env, &meta, insn_idx);
15594 	if (err)
15595 		return err;
15596 
15597 	if ((is_bpf_obj_drop_kfunc(meta.func_id) ||
15598 	     is_bpf_percpu_obj_drop_kfunc(meta.func_id)) && (is_tracing_prog_type(prog_type) ||
15599 	     /* is_tracing_prog_type() for now doesn't cover non-iterator tracing progs. */
15600 	     (prog_type == BPF_PROG_TYPE_TRACING && env->prog->expected_attach_type != BPF_TRACE_ITER
15601 	      && !env->prog->sleepable))) {
15602 		struct btf_struct_meta *struct_meta;
15603 
15604 		struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
15605 		if (struct_meta && btf_record_has_nmi_unsafe_fields(struct_meta->record)) {
15606 			verbose(env, "%s cannot be used in tracing programs on types with NMI unsafe fields\n",
15607 				func_name);
15608 			return -EINVAL;
15609 		}
15610 	}
15611 
15612 	if (is_bpf_rbtree_add_kfunc(meta.func_id)) {
15613 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
15614 					 set_rbtree_add_callback_state);
15615 		if (err) {
15616 			verbose(env, "kfunc %s#%d failed callback verification\n",
15617 				func_name, meta.func_id);
15618 			return err;
15619 		}
15620 	}
15621 
15622 	if (is_bpf_wq_set_callback_kfunc(meta.func_id)) {
15623 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
15624 					 set_timer_callback_state);
15625 		if (err) {
15626 			verbose(env, "kfunc %s#%d failed callback verification\n",
15627 				func_name, meta.func_id);
15628 			return err;
15629 		}
15630 	}
15631 
15632 	if (is_task_work_add_kfunc(meta.func_id)) {
15633 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
15634 					 set_task_work_schedule_callback_state);
15635 		if (err) {
15636 			verbose(env, "kfunc %s#%d failed callback verification\n",
15637 				func_name, meta.func_id);
15638 			return err;
15639 		}
15640 	}
15641 
15642 	if (is_call_rcu_kfunc(meta.func_id)) {
15643 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
15644 					 set_rcu_callback_state);
15645 		if (err) {
15646 			verbose(env, "kfunc %s#%d failed callback verification\n",
15647 				func_name, meta.func_id);
15648 			return err;
15649 		}
15650 	}
15651 
15652 	rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta);
15653 	rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta);
15654 
15655 	preempt_disable = is_kfunc_bpf_preempt_disable(&meta);
15656 	preempt_enable = is_kfunc_bpf_preempt_enable(&meta);
15657 
15658 	if (rcu_lock) {
15659 		env->cur_state->active_rcu_locks++;
15660 		bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, true,
15661 					env->cur_state->active_rcu_locks);
15662 	} else if (rcu_unlock) {
15663 		if (env->cur_state->active_rcu_locks == 0) {
15664 			verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
15665 			bpf_diag_ctx_underflow(
15666 				env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU,
15667 				"Remove the extra bpf_rcu_read_unlock() call, or ensure this path first enters an RCU read lock region.");
15668 			return -EINVAL;
15669 		}
15670 		env->cur_state->active_rcu_locks--;
15671 		bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, false,
15672 					env->cur_state->active_rcu_locks);
15673 		if (!in_rcu_cs(env))
15674 			invalidate_rcu_protected_refs(env);
15675 	} else if (preempt_disable) {
15676 		env->cur_state->active_preempt_locks++;
15677 		bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, true,
15678 					env->cur_state->active_preempt_locks);
15679 	} else if (preempt_enable) {
15680 		if (env->cur_state->active_preempt_locks == 0) {
15681 			verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name);
15682 			bpf_diag_ctx_underflow(
15683 				env, insn_idx, func_name, BPF_DIAG_CONTEXT_PREEMPT,
15684 				"Remove the extra bpf_preempt_enable() call, or ensure this path first disables preemption.");
15685 			return -EINVAL;
15686 		}
15687 		env->cur_state->active_preempt_locks--;
15688 		bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, false,
15689 					env->cur_state->active_preempt_locks);
15690 		if (!in_rcu_cs(env))
15691 			invalidate_rcu_protected_refs(env);
15692 	}
15693 
15694 	if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {
15695 		verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");
15696 		return -EACCES;
15697 	}
15698 
15699 	if (is_kfunc_rcu_protected(&meta) && !in_rcu_cs(env)) {
15700 		verbose(env, "kernel func %s requires RCU critical section protection\n", func_name);
15701 		bpf_diag_ctx_required(
15702 			env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU,
15703 			"Call this kfunc between bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced.");
15704 		return -EACCES;
15705 	}
15706 
15707 	/* In case of release function, we get register number of refcounted
15708 	 * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now.
15709 	 */
15710 	if (meta.release_regno) {
15711 		err = release_reg(env, &regs[meta.release_regno], false, !!meta.dynptr.id);
15712 		if (err)
15713 			return err;
15714 	}
15715 
15716 	if (is_bpf_list_push_kfunc(meta.func_id) || is_bpf_rbtree_add_kfunc(meta.func_id)) {
15717 		id = regs[BPF_REG_2].id;
15718 		insn_aux->insert_off = regs[BPF_REG_2].var_off.value;
15719 		insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
15720 		ref_convert_owning_non_owning(env, id);
15721 	}
15722 
15723 	if ((is_kfunc_call(&meta, special_kfunc_list[KF_bpf_dynptr_slice]) ||
15724 	     is_kfunc_call(&meta, special_kfunc_list[KF_bpf_dynptr_slice_rdwr])) &&
15725 	    dynptr_slice_may_be_buffer(meta.dynptr.type)) {
15726 		struct bpf_reg_state *buf = &regs[BPF_REG_3];
15727 
15728 		if (buf->type == PTR_TO_STACK) {
15729 			meta.slice_buf.frameno = buf->frameno;
15730 			meta.slice_buf.off = reg_smin(buf);
15731 			meta.slice_buf.size = reg_smax(buf) - reg_smin(buf) +
15732 					      meta.arg_constant.value;
15733 		} else if (base_type(buf->type) == PTR_TO_MEM) {
15734 			/* a slice is the buffer of the next one */
15735 			meta.slice_buf = buf->buf;
15736 		}
15737 	}
15738 
15739 	if (meta.func_id == special_kfunc_list[KF_bpf_throw]) {
15740 		if (!bpf_jit_supports_exceptions()) {
15741 			verbose(env, "JIT does not support calling kfunc %s#%d\n",
15742 				func_name, meta.func_id);
15743 			return -ENOTSUPP;
15744 		}
15745 		env->seen_exception = true;
15746 
15747 		/* In the case of the default callback, the cookie value passed
15748 		 * to bpf_throw becomes the return value of the program.
15749 		 */
15750 		if (!env->exception_callback_subprog) {
15751 			err = check_return_code(env, BPF_REG_1, "R1");
15752 			if (err < 0)
15753 				return err;
15754 		}
15755 	}
15756 
15757 	bpf_diag_record_caller_saved(env, regs);
15758 	bpf_diag_mod_begin(env, &regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE);
15759 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
15760 		u32 regno = caller_saved[i];
15761 
15762 		bpf_mark_reg_not_init(env, &regs[regno]);
15763 	}
15764 	invalidate_outgoing_stack_args(env, cur_func(env));
15765 
15766 	/* Check return type */
15767 	t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL);
15768 
15769 	if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) {
15770 		if (meta.btf != btf_vmlinux ||
15771 		    (!is_bpf_obj_new_kfunc(meta.func_id) &&
15772 		     !is_bpf_percpu_obj_new_kfunc(meta.func_id) &&
15773 		     !is_bpf_refcount_acquire_kfunc(meta.func_id))) {
15774 			verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n");
15775 			return -EINVAL;
15776 		}
15777 	}
15778 
15779 	if (btf_type_is_scalar(t)) {
15780 		ret_nregs = mark_kfunc_ret_regs(env, regs, t->size);
15781 		if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] ||
15782 		    meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]))
15783 			__mark_reg_const_zero(env, &regs[BPF_REG_0]);
15784 	} else if (btf_type_is_struct(t)) {
15785 		struct btf_member_path path = {};
15786 		const char *member_note = "";
15787 
15788 		/*
15789 		 * The returned struct comes back as raw register bits modeled
15790 		 * as an unknown scalar, so a pointer member would be laundered
15791 		 * into a scalar and escape provenance and reference tracking.
15792 		 * Only scalars and arena pointers are allowed; see
15793 		 * btf_validate_return_type() in btf.c for why an arena pointer
15794 		 * is safe here.
15795 		 */
15796 		if (!btf_struct_member_walk(env, desc_btf, t,
15797 					    BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) {
15798 			verbose(env,
15799 				"kernel function %s returns %s %s that is not composed of scalars or arena pointers\n",
15800 				func_name, btf_type_str(t),
15801 				btf_name_by_offset(desc_btf, t->name_off));
15802 			if (path.too_deep) {
15803 				member_note = bpf_diag_fmt(
15804 					env, " It nests structs more than %d levels deep.",
15805 					BTF_MEMBER_MAX_DEPTH);
15806 			} else if (path.depth) {
15807 				char bad_name[BTF_MEMBER_PATH_LEN];
15808 
15809 				btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name));
15810 				if (!path.bad_type) {
15811 					verbose(env, "member '%s' is a zero-length array\n",
15812 						bad_name);
15813 					member_note = bpf_diag_fmt(
15814 						env, " Its member '%s' is a zero-length array.",
15815 						bad_name);
15816 				} else {
15817 					verbose(env, "member '%s' has type %s\n", bad_name,
15818 						btf_type_str(path.bad_type));
15819 					member_note = bpf_diag_fmt(
15820 						env,
15821 						" Its member '%s' is %s, not a scalar or an arena pointer.",
15822 						bad_name, btf_type_str(path.bad_type));
15823 				}
15824 			}
15825 			bpf_diag_program_structure(
15826 				env, insn_idx, "unsupported kernel function return type",
15827 				"Call a kernel function that returns only scalars or arena pointers by value.",
15828 				"%s() returns %s %s by value.%s "
15829 				"Only kfuncs returning scalar values or arena pointers, or "
15830 				"structures composed of scalar values or arena pointers are "
15831 				"supported.",
15832 				func_name, btf_type_str(t),
15833 				btf_name_by_offset(desc_btf, t->name_off), member_note);
15834 			return -EINVAL;
15835 		}
15836 		ret_nregs = mark_kfunc_ret_regs(env, regs, t->size);
15837 	} else if (btf_type_is_ptr(t)) {
15838 		ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id);
15839 		err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf);
15840 		if (err) {
15841 			if (err < 0)
15842 				return err;
15843 		} else if (btf_type_is_void(ptr_type)) {
15844 			/* kfunc returning 'void *' is equivalent to returning scalar */
15845 			mark_reg_unknown(env, regs, BPF_REG_0);
15846 		} else if (!__btf_type_is_struct(ptr_type)) {
15847 			if (!meta.ret_mem.found) {
15848 				__u32 sz;
15849 
15850 				if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) {
15851 					meta.ret_mem.found = true;
15852 					meta.ret_mem.size = sz;
15853 					meta.r0_rdonly = true;
15854 				}
15855 
15856 				if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie])
15857 					meta.r0_rdonly = false;
15858 			}
15859 			if (!meta.ret_mem.found) {
15860 				ptr_type_name = btf_name_by_offset(desc_btf,
15861 								   ptr_type->name_off);
15862 				verbose(env,
15863 					"kernel function %s returns pointer type %s %s is not supported\n",
15864 					func_name,
15865 					btf_type_str(ptr_type),
15866 					ptr_type_name);
15867 				return -EINVAL;
15868 			}
15869 
15870 			mark_reg_known_zero(env, regs, BPF_REG_0);
15871 			regs[BPF_REG_0].type = PTR_TO_MEM;
15872 			regs[BPF_REG_0].mem_size = meta.ret_mem.size;
15873 
15874 			if (meta.r0_rdonly)
15875 				regs[BPF_REG_0].type |= MEM_RDONLY;
15876 
15877 			/* Ensures we don't access the memory after a release_reference() */
15878 			if (meta.ref_obj.id) {
15879 				err = validate_ref_obj(env, &meta.ref_obj);
15880 				if (err)
15881 					return err;
15882 				regs[BPF_REG_0].parent_id = meta.ref_obj.id;
15883 			}
15884 
15885 			if (is_kfunc_rcu_protected(&meta))
15886 				regs[BPF_REG_0].type |= MEM_RCU;
15887 		} else {
15888 			enum bpf_reg_type type = PTR_TO_BTF_ID;
15889 
15890 			if (meta.func_id == special_kfunc_list[KF_bpf_get_kmem_cache])
15891 				type |= PTR_UNTRUSTED;
15892 			else if (is_kfunc_rcu_protected(&meta) ||
15893 				 (bpf_is_iter_next_kfunc(&meta) &&
15894 				  (get_iter_from_state(env->cur_state, &meta)
15895 					   ->type & MEM_RCU))) {
15896 				/*
15897 				 * If the iterator's constructor (the _new
15898 				 * function e.g., bpf_iter_task_new) has been
15899 				 * annotated with BPF kfunc flag
15900 				 * KF_RCU_PROTECTED and was called within a RCU
15901 				 * read-side critical section, also propagate
15902 				 * the MEM_RCU flag to the pointer returned from
15903 				 * the iterator's next function (e.g.,
15904 				 * bpf_iter_task_next).
15905 				 */
15906 				type |= MEM_RCU;
15907 			} else {
15908 				/*
15909 				 * Any PTR_TO_BTF_ID that is returned from a BPF
15910 				 * kfunc should by default be treated as
15911 				 * implicitly trusted.
15912 				 */
15913 				type |= PTR_TRUSTED;
15914 			}
15915 
15916 			mark_reg_known_zero(env, regs, BPF_REG_0);
15917 			regs[BPF_REG_0].btf = desc_btf;
15918 			regs[BPF_REG_0].type = type;
15919 			regs[BPF_REG_0].btf_id = ptr_type_id;
15920 			if (iter_next_borrows_struct(&meta) && !(type & MEM_RCU))
15921 				regs[BPF_REG_0].parent_id = meta.ref_obj.id;
15922 		}
15923 
15924 		if (is_kfunc_ret_null(&meta)) {
15925 			regs[BPF_REG_0].type |= PTR_MAYBE_NULL;
15926 			/* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */
15927 			regs[BPF_REG_0].id = ++env->id_gen;
15928 		}
15929 		if (is_kfunc_acquire(&meta)) {
15930 			id = acquire_reference(env, insn_idx, 0);
15931 			if (id < 0)
15932 				return id;
15933 			regs[BPF_REG_0].id = id;
15934 		} else if (is_rbtree_node_type(ptr_type) || is_list_node_type(ptr_type)) {
15935 			ref_set_non_owning(env, &regs[BPF_REG_0]);
15936 		}
15937 
15938 		if (reg_may_point_to_spin_lock(&regs[BPF_REG_0]) && !regs[BPF_REG_0].id)
15939 			regs[BPF_REG_0].id = ++env->id_gen;
15940 	} else if (btf_type_is_void(t)) {
15941 		if (meta.btf == btf_vmlinux) {
15942 			if (is_bpf_obj_drop_kfunc(meta.func_id) ||
15943 			    is_bpf_percpu_obj_drop_kfunc(meta.func_id)) {
15944 				insn_aux->kptr_struct_meta =
15945 					btf_find_struct_meta(meta.arg_btf,
15946 							     meta.arg_btf_id);
15947 			}
15948 		}
15949 	}
15950 
15951 	if (is_kfunc_dynptr_write(&meta) ?
15952 	    dynptr_write_changes_pkt_data(meta.dynptr_write_type) :
15953 	    bpf_is_kfunc_pkt_changing(&meta))
15954 		clear_all_pkt_pointers(env);
15955 
15956 	/*
15957 	 * These kfuncs write to the head of the skb in place if what they write
15958 	 * to is linear. Like a direct write that takes bpf_unclone_prologue().
15959 	 */
15960 	if (is_kfunc_dynptr_write(&meta) &&
15961 	    dynptr_write_changes_pkt_data(meta.dynptr_write_type))
15962 		may_access_direct_pkt_data(env, NULL, BPF_WRITE);
15963 
15964 	proto_slots = kfunc_abi_slots(&desc->func_model);
15965 	if (proto_slots > MAX_BPF_FUNC_REG_ARGS) {
15966 		struct bpf_func_state *caller = cur_func(env);
15967 		struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno];
15968 		u16 out_stack_arg_cnt = proto_slots - MAX_BPF_FUNC_REG_ARGS;
15969 		u16 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + out_stack_arg_cnt;
15970 
15971 		if (stack_arg_cnt > caller_info->stack_arg_cnt)
15972 			caller_info->stack_arg_cnt = stack_arg_cnt;
15973 	}
15974 
15975 	/*
15976 	 * Record R0 before process_iter_next_call() snapshots the alternate
15977 	 * iterator path's diagnostic position.
15978 	 */
15979 	if (ret_nregs == 1)
15980 		bpf_diag_mod_end(env);
15981 
15982 	if (bpf_is_iter_next_kfunc(&meta)) {
15983 		err = process_iter_next_call(env, insn_idx, &meta);
15984 		if (err)
15985 			return err;
15986 	}
15987 
15988 	if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie])
15989 		env->prog->call_session_cookie = true;
15990 
15991 	if (bpf_is_throw_kfunc(insn))
15992 		return process_bpf_exit_full(env, NULL, true);
15993 
15994 	return 0;
15995 }
15996 
15997 static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env,
15998 					 const struct bpf_reg_state *reg,
15999 					 enum bpf_reg_type type)
16000 {
16001 	bool known = tnum_is_const(reg->var_off);
16002 	s64 val = reg->var_off.value;
16003 	s64 smin = reg_smin(reg);
16004 
16005 	if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
16006 		verbose(env, "math between %s pointer and %lld is not allowed\n",
16007 			reg_type_str(env, type), val);
16008 		return false;
16009 	}
16010 
16011 	if (smin == S64_MIN) {
16012 		verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
16013 			reg_type_str(env, type));
16014 		return false;
16015 	}
16016 
16017 	if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
16018 		verbose(env, "value %lld makes %s pointer be out of bounds\n",
16019 			smin, reg_type_str(env, type));
16020 		return false;
16021 	}
16022 
16023 	return true;
16024 }
16025 
16026 static bool check_reg_sane_offset_ptr(struct bpf_verifier_env *env,
16027 				      const struct bpf_reg_state *reg,
16028 				      enum bpf_reg_type type)
16029 {
16030 	bool known = tnum_is_const(reg->var_off);
16031 	s64 val = reg->var_off.value;
16032 	s64 smin = reg_smin(reg);
16033 
16034 	if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
16035 		verbose(env, "%s pointer offset %lld is not allowed\n",
16036 			reg_type_str(env, type), val);
16037 		return false;
16038 	}
16039 
16040 	if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
16041 		verbose(env, "%s pointer offset %lld is not allowed\n",
16042 			reg_type_str(env, type), smin);
16043 		return false;
16044 	}
16045 
16046 	return true;
16047 }
16048 
16049 enum {
16050 	REASON_BOUNDS	= -1,
16051 	REASON_TYPE	= -2,
16052 	REASON_PATHS	= -3,
16053 	REASON_LIMIT	= -4,
16054 	REASON_STACK	= -5,
16055 };
16056 
16057 static int retrieve_ptr_limit(const struct bpf_verifier_env *env,
16058 			      const struct bpf_reg_state *ptr_reg,
16059 			      u32 *alu_limit, bool mask_to_left)
16060 {
16061 	u32 max = 0, ptr_limit = 0;
16062 
16063 	switch (ptr_reg->type) {
16064 	case PTR_TO_STACK:
16065 		/* Offset 0 is out-of-bounds, but acceptable start for the
16066 		 * left direction, see BPF_REG_FP. Also, unknown scalar
16067 		 * offset where we would need to deal with min/max bounds is
16068 		 * currently prohibited for unprivileged.
16069 		 */
16070 		max = env->stack_limit + mask_to_left;
16071 		ptr_limit = -ptr_reg->var_off.value;
16072 		break;
16073 	case PTR_TO_MAP_VALUE:
16074 		max = ptr_reg->map_ptr->value_size;
16075 		ptr_limit = mask_to_left ? reg_smin(ptr_reg) : reg_umax(ptr_reg);
16076 		break;
16077 	default:
16078 		return REASON_TYPE;
16079 	}
16080 
16081 	if (ptr_limit >= max)
16082 		return REASON_LIMIT;
16083 	*alu_limit = ptr_limit;
16084 	return 0;
16085 }
16086 
16087 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
16088 				    const struct bpf_insn *insn)
16089 {
16090 	return env->bypass_spec_v1 ||
16091 		BPF_SRC(insn->code) == BPF_K ||
16092 		cur_aux(env)->nospec;
16093 }
16094 
16095 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
16096 				       u32 alu_state, u32 alu_limit)
16097 {
16098 	/* If we arrived here from different branches with different
16099 	 * state or limits to sanitize, then this won't work.
16100 	 */
16101 	if (aux->alu_state &&
16102 	    (aux->alu_state != alu_state ||
16103 	     aux->alu_limit != alu_limit))
16104 		return REASON_PATHS;
16105 
16106 	/* Corresponding fixup done in do_misc_fixups(). */
16107 	aux->alu_state = alu_state;
16108 	aux->alu_limit = alu_limit;
16109 	return 0;
16110 }
16111 
16112 static int sanitize_val_alu(struct bpf_verifier_env *env,
16113 			    struct bpf_insn *insn)
16114 {
16115 	struct bpf_insn_aux_data *aux = cur_aux(env);
16116 
16117 	if (can_skip_alu_sanitation(env, insn))
16118 		return 0;
16119 
16120 	return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
16121 }
16122 
16123 static bool sanitize_needed(u8 opcode)
16124 {
16125 	return opcode == BPF_ADD || opcode == BPF_SUB;
16126 }
16127 
16128 struct bpf_sanitize_info {
16129 	struct bpf_insn_aux_data aux;
16130 	bool mask_to_left;
16131 };
16132 
16133 static int sanitize_speculative_path(struct bpf_verifier_env *env,
16134 				     const struct bpf_insn *insn,
16135 				     u32 next_idx, u32 curr_idx)
16136 {
16137 	struct bpf_verifier_state *branch;
16138 	struct bpf_reg_state *regs;
16139 
16140 	branch = push_stack(env, next_idx, curr_idx, true);
16141 	if (!IS_ERR(branch) && insn) {
16142 		regs = branch->frame[branch->curframe]->regs;
16143 		if (BPF_SRC(insn->code) == BPF_K) {
16144 			mark_reg_unknown(env, regs, insn->dst_reg);
16145 		} else if (BPF_SRC(insn->code) == BPF_X) {
16146 			mark_reg_unknown(env, regs, insn->dst_reg);
16147 			mark_reg_unknown(env, regs, insn->src_reg);
16148 		}
16149 	}
16150 	return PTR_ERR_OR_ZERO(branch);
16151 }
16152 
16153 static int sanitize_ptr_alu(struct bpf_verifier_env *env,
16154 			    struct bpf_insn *insn,
16155 			    const struct bpf_reg_state *ptr_reg,
16156 			    const struct bpf_reg_state *off_reg,
16157 			    struct bpf_reg_state *dst_reg,
16158 			    struct bpf_sanitize_info *info,
16159 			    const bool commit_window)
16160 {
16161 	struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
16162 	struct bpf_verifier_state *vstate = env->cur_state;
16163 	bool off_is_imm = tnum_is_const(off_reg->var_off);
16164 	bool off_is_neg = reg_smin(off_reg) < 0;
16165 	bool ptr_is_dst_reg = ptr_reg == dst_reg;
16166 	u8 opcode = BPF_OP(insn->code);
16167 	u32 alu_state, alu_limit;
16168 	struct bpf_reg_state tmp;
16169 	int err;
16170 
16171 	if (can_skip_alu_sanitation(env, insn))
16172 		return 0;
16173 
16174 	/* We already marked aux for masking from non-speculative
16175 	 * paths, thus we got here in the first place. We only care
16176 	 * to explore bad access from here.
16177 	 */
16178 	if (vstate->speculative)
16179 		goto do_sim;
16180 
16181 	if (!commit_window) {
16182 		if (!tnum_is_const(off_reg->var_off) &&
16183 		    (reg_smin(off_reg) < 0) != (reg_smax(off_reg) < 0))
16184 			return REASON_BOUNDS;
16185 
16186 		info->mask_to_left = (opcode == BPF_ADD &&  off_is_neg) ||
16187 				     (opcode == BPF_SUB && !off_is_neg);
16188 	}
16189 
16190 	err = retrieve_ptr_limit(env, ptr_reg, &alu_limit, info->mask_to_left);
16191 	if (err < 0)
16192 		return err;
16193 
16194 	if (commit_window) {
16195 		/* In commit phase we narrow the masking window based on
16196 		 * the observed pointer move after the simulated operation.
16197 		 */
16198 		alu_state = info->aux.alu_state;
16199 		alu_limit = abs(info->aux.alu_limit - alu_limit);
16200 	} else {
16201 		alu_state  = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
16202 		alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;
16203 		alu_state |= ptr_is_dst_reg ?
16204 			     BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
16205 
16206 		/* Limit pruning on unknown scalars to enable deep search for
16207 		 * potential masking differences from other program paths.
16208 		 */
16209 		if (!off_is_imm)
16210 			env->explore_alu_limits = true;
16211 	}
16212 
16213 	err = update_alu_sanitation_state(aux, alu_state, alu_limit);
16214 	if (err < 0)
16215 		return err;
16216 do_sim:
16217 	/* If we're in commit phase, we're done here given we already
16218 	 * pushed the truncated dst_reg into the speculative verification
16219 	 * stack.
16220 	 *
16221 	 * Also, when register is a known constant, we rewrite register-based
16222 	 * operation to immediate-based, and thus do not need masking (and as
16223 	 * a consequence, do not need to simulate the zero-truncation either).
16224 	 */
16225 	if (commit_window || off_is_imm)
16226 		return 0;
16227 
16228 	/* Simulate and find potential out-of-bounds access under
16229 	 * speculative execution from truncation as a result of
16230 	 * masking when off was not within expected range. If off
16231 	 * sits in dst, then we temporarily need to move ptr there
16232 	 * to simulate dst (== 0) +/-= ptr. Needed, for example,
16233 	 * for cases where we use K-based arithmetic in one direction
16234 	 * and truncated reg-based in the other in order to explore
16235 	 * bad access.
16236 	 */
16237 	if (!ptr_is_dst_reg) {
16238 		tmp = *dst_reg;
16239 		*dst_reg = *ptr_reg;
16240 	}
16241 	err = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx);
16242 	if (err < 0)
16243 		return REASON_STACK;
16244 	if (!ptr_is_dst_reg)
16245 		*dst_reg = tmp;
16246 	return 0;
16247 }
16248 
16249 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
16250 {
16251 	struct bpf_verifier_state *vstate = env->cur_state;
16252 
16253 	/* If we simulate paths under speculation, we don't update the
16254 	 * insn as 'seen' such that when we verify unreachable paths in
16255 	 * the non-speculative domain, sanitize_dead_code() can still
16256 	 * rewrite/sanitize them.
16257 	 */
16258 	if (!vstate->speculative)
16259 		env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;
16260 }
16261 
16262 static int sanitize_err(struct bpf_verifier_env *env, const struct bpf_insn *insn, int reason)
16263 {
16264 	static const char *err = "pointer arithmetic with it prohibited for !root";
16265 	const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub";
16266 	u32 dst = insn->dst_reg, src = insn->src_reg;
16267 	struct bpf_reg_state *regs = cur_regs(env);
16268 
16269 	switch (reason) {
16270 	case REASON_BOUNDS:
16271 		verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",
16272 			regs[src].type == SCALAR_VALUE ? src : dst, err);
16273 		break;
16274 	case REASON_TYPE:
16275 		verbose(env, "R%d has pointer with unsupported alu operation, %s\n",
16276 			regs[src].type == SCALAR_VALUE ? dst : src, err);
16277 		break;
16278 	case REASON_PATHS:
16279 		verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",
16280 			dst, op, err);
16281 		break;
16282 	case REASON_LIMIT:
16283 		verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",
16284 			dst, op, err);
16285 		break;
16286 	case REASON_STACK:
16287 		verbose(env, "R%d could not be pushed for speculative verification, %s\n",
16288 			dst, err);
16289 		return -ENOMEM;
16290 	default:
16291 		verifier_bug(env, "unknown reason (%d)", reason);
16292 		break;
16293 	}
16294 
16295 	return -EACCES;
16296 }
16297 
16298 /* check that stack access falls within stack limits and that 'reg' doesn't
16299  * have a variable offset.
16300  *
16301  * Variable offset is prohibited for unprivileged mode for simplicity since it
16302  * requires corresponding support in Spectre masking for stack ALU.  See also
16303  * retrieve_ptr_limit().
16304  */
16305 static int check_stack_access_for_ptr_arithmetic(
16306 				struct bpf_verifier_env *env,
16307 				int regno,
16308 				const struct bpf_reg_state *reg,
16309 				int off)
16310 {
16311 	if (!tnum_is_const(reg->var_off)) {
16312 		char tn_buf[48];
16313 
16314 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
16315 		verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
16316 			regno, tn_buf, off);
16317 		return -EACCES;
16318 	}
16319 
16320 	if (off >= 0 || off < -(int)env->stack_limit) {
16321 		verbose(env, "R%d stack pointer arithmetic goes out of range, "
16322 			"prohibited for !root; off=%d\n", regno, off);
16323 		return -EACCES;
16324 	}
16325 
16326 	return 0;
16327 }
16328 
16329 static int sanitize_check_bounds(struct bpf_verifier_env *env,
16330 				 const struct bpf_insn *insn,
16331 				 struct bpf_reg_state *dst_reg)
16332 {
16333 	u32 dst = insn->dst_reg;
16334 
16335 	/* For unprivileged we require that resulting offset must be in bounds
16336 	 * in order to be able to sanitize access later on.
16337 	 */
16338 	if (env->bypass_spec_v1)
16339 		return 0;
16340 
16341 	switch (dst_reg->type) {
16342 	case PTR_TO_STACK:
16343 		if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,
16344 							  dst_reg->var_off.value))
16345 			return -EACCES;
16346 		break;
16347 	case PTR_TO_MAP_VALUE:
16348 		if (check_map_access(env, dst_reg, argno_from_reg(dst), 0, 1, false, ACCESS_HELPER)) {
16349 			verbose(env, "R%d pointer arithmetic of map value goes out of range, "
16350 				"prohibited for !root\n", dst);
16351 			return -EACCES;
16352 		}
16353 		break;
16354 	default:
16355 		return -EOPNOTSUPP;
16356 	}
16357 
16358 	return 0;
16359 }
16360 
16361 static void scalar_step_mul(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg)
16362 {
16363 	if (tnum_is_const(src_reg->var_off))
16364 		scalar_step_scale(dst_reg, src_reg->var_off.value);
16365 	else
16366 		reg_step_reset(dst_reg);
16367 }
16368 
16369 static void scalar_step_lsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg)
16370 {
16371 	u64 amount = src_reg->var_off.value;
16372 
16373 	if (tnum_is_const(src_reg->var_off) && amount < 64)
16374 		scalar_step_scale(dst_reg, 1ULL << amount);
16375 	else
16376 		reg_step_reset(dst_reg);
16377 }
16378 
16379 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
16380  * Caller should also handle BPF_MOV case separately.
16381  * If we return -EACCES, caller may want to try again treating pointer as a
16382  * scalar.  So we only emit a diagnostic if !env->allow_ptr_leaks.
16383  */
16384 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn,
16385 				   u32 ptr_regno, const struct bpf_reg_state *ptr_reg,
16386 				   const struct bpf_reg_state *off_reg)
16387 {
16388 	struct bpf_verifier_state *vstate = env->cur_state;
16389 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
16390 	struct bpf_reg_state *regs = state->regs, *dst_reg;
16391 	bool known = tnum_is_const(off_reg->var_off);
16392 	s64 smin_val = reg_smin(off_reg), smax_val = reg_smax(off_reg);
16393 	u64 umin_val = reg_umin(off_reg), umax_val = reg_umax(off_reg);
16394 	struct bpf_sanitize_info info = {};
16395 	u8 opcode = BPF_OP(insn->code);
16396 	u32 dst = insn->dst_reg;
16397 	const char *reason;
16398 	int ret, bounds_ret;
16399 	bool to_scalar;
16400 
16401 	dst_reg = &regs[dst];
16402 	/*
16403 	 * Only += and -= keep a pointer. Any other op makes a number of it,
16404 	 * which is fine when the program may see values of pointers anyway.
16405 	 */
16406 	to_scalar = opcode != BPF_ADD && opcode != BPF_SUB && env->allow_ptr_leaks;
16407 
16408 	if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
16409 	    smin_val > smax_val || umin_val > umax_val) {
16410 		/* Taint dst register if offset had invalid bounds derived from
16411 		 * e.g. dead branches.
16412 		 */
16413 		__mark_reg_unknown(env, dst_reg);
16414 		return 0;
16415 	}
16416 
16417 	if (BPF_CLASS(insn->code) != BPF_ALU64 && !to_scalar) {
16418 		/* 32-bit ALU ops on pointers produce (meaningless) scalars */
16419 		if (opcode == BPF_SUB && env->allow_ptr_leaks) {
16420 			__mark_reg_unknown(env, dst_reg);
16421 			return 0;
16422 		}
16423 
16424 		verbose(env,
16425 			"R%d 32-bit pointer arithmetic prohibited\n",
16426 			dst);
16427 		reason = bpf_diag_fmt(
16428 			env, "R%d holds %s. 32-bit ALU operations on pointers discard pointer tracking, so the verifier cannot keep the result as a safe pointer.",
16429 			ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type));
16430 		bpf_diag_register_type(
16431 			env, env->insn_idx, ptr_regno, "32-bit pointer arithmetic", reason,
16432 			"Use a 64-bit ALU instruction with an allowed, bounded scalar offset.");
16433 		return -EACCES;
16434 	}
16435 
16436 	if (ptr_reg->type & PTR_MAYBE_NULL) {
16437 		verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
16438 			dst, reg_type_str(env, ptr_reg->type));
16439 		reason = bpf_diag_fmt(
16440 			env, "R%d may be NULL (%s). Pointer arithmetic is allowed only after the program proves the pointer is non-NULL on this path.",
16441 			ptr_regno, reg_type_str(env, ptr_reg->type));
16442 		bpf_diag_register_type(
16443 			env, env->insn_idx, ptr_regno, "pointer arithmetic before NULL check", reason,
16444 			"Make sure that a NULL check precedes any arithmetic performed on the pointer.");
16445 		return -EACCES;
16446 	}
16447 
16448 	switch (base_type(ptr_reg->type)) {
16449 	case PTR_TO_CTX:
16450 	case PTR_TO_MAP_VALUE:
16451 	case PTR_TO_MAP_KEY:
16452 	case PTR_TO_STACK:
16453 	case PTR_TO_PACKET_META:
16454 	case PTR_TO_PACKET:
16455 	case PTR_TO_TP_BUFFER:
16456 	case PTR_TO_BTF_ID:
16457 	case PTR_TO_MEM:
16458 	case PTR_TO_BUF:
16459 	case PTR_TO_FUNC:
16460 	case CONST_PTR_TO_DYNPTR:
16461 		break;
16462 	case PTR_TO_FLOW_KEYS:
16463 		if (known)
16464 			break;
16465 		fallthrough;
16466 	case CONST_PTR_TO_MAP:
16467 		/* smin_val represents the known value */
16468 		if (known && smin_val == 0 && opcode == BPF_ADD)
16469 			break;
16470 		fallthrough;
16471 	default:
16472 		verbose(env, "R%d pointer arithmetic on %s prohibited\n",
16473 			dst, reg_type_str(env, ptr_reg->type));
16474 		reason = bpf_diag_fmt(
16475 			env, "R%d holds %s. This pointer kind does not allow offset arithmetic.",
16476 			ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type));
16477 		bpf_diag_register_type(
16478 			env, env->insn_idx, ptr_regno, "pointer arithmetic is not allowed", reason,
16479 			"Do not change this pointer's offset; use it only in operations accepted for its kind.");
16480 		return -EACCES;
16481 	}
16482 
16483 	if (to_scalar) {
16484 		__mark_reg_unknown(env, dst_reg);
16485 		return 0;
16486 	}
16487 
16488 	/* For 'scalar += pointer', dst_reg inherits the complete pointer
16489 	 * register state. Individual fields may be adjusted later by pointer
16490 	 * arithmetic. Callers guarantee that below does not overwrite off_reg.
16491 	 */
16492 	if (dst_reg != ptr_reg)
16493 		*dst_reg = *ptr_reg;
16494 
16495 	/*
16496 	 * Accesses to untrusted PTR_TO_MEM are done through probe
16497 	 * instructions, hence no need to track offsets.
16498 	 */
16499 	if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED))
16500 		return 0;
16501 
16502 	if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type)) {
16503 		reason = bpf_diag_fmt(
16504 			env, "The scalar offset used with R%d is unbounded or outside the verifier's safe pointer-offset range [-%u, %u].",
16505 			ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF);
16506 		bpf_diag_register_type(
16507 			env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason,
16508 			"Clamp or bounds-check the scalar offset before applying it to the pointer.");
16509 		return -EINVAL;
16510 	}
16511 	if (!check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) {
16512 		reason = bpf_diag_fmt(
16513 			env, "R%d already has an offset outside the verifier's safe range [-%u, %u] for %s.",
16514 			ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF,
16515 			bpf_diag_reg_type_plain(env, ptr_reg->type));
16516 		bpf_diag_register_type(
16517 			env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason,
16518 			"Keep the base pointer within the verifier's allowed offset range before applying more arithmetic.");
16519 		return -EINVAL;
16520 	}
16521 
16522 	if (sanitize_needed(opcode)) {
16523 		ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
16524 				       &info, false);
16525 		if (ret < 0)
16526 			return sanitize_err(env, insn, ret);
16527 	}
16528 
16529 	/*
16530 	 * Pointer types do not carry 32-bit bounds at the moment. Blank r32
16531 	 * only after sanitize_ptr_alu() may have snapshotted dst_reg into a
16532 	 * speculative path: otherwise reg_bounds_sanity_check() might hit some
16533 	 * constraints violations.
16534 	 */
16535 	__mark_reg32_unbounded(dst_reg);
16536 
16537 	switch (opcode) {
16538 	case BPF_ADD:
16539 		/*
16540 		 * dst_reg gets the pointer type and since some positive
16541 		 * integer value was added to the pointer, give it a new 'id'
16542 		 * if it's a PTR_TO_PACKET.
16543 		 * this creates a new 'base' pointer, off_reg (variable) gets
16544 		 * added into the variable offset, and we copy the fixed offset
16545 		 * from ptr_reg.
16546 		 */
16547 		scalar_step_add(dst_reg, ptr_reg, off_reg);
16548 		dst_reg->r64 = cnum64_add(ptr_reg->r64, off_reg->r64);
16549 		dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
16550 		dst_reg->raw = ptr_reg->raw;
16551 		if (reg_is_pkt_pointer(ptr_reg)) {
16552 			if (!known)
16553 				dst_reg->id = ++env->id_gen;
16554 			/*
16555 			 * Clear range for unknown addends since we can't know
16556 			 * where the pkt pointer ended up. Also clear AT_PKT_END /
16557 			 * BEYOND_PKT_END from prior comparison as any pointer
16558 			 * arithmetic invalidates them.
16559 			 */
16560 			if (!known || dst_reg->range < 0)
16561 				memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
16562 		}
16563 		break;
16564 	case BPF_SUB:
16565 		if (dst_reg != ptr_reg) {
16566 			/* scalar -= pointer.  Creates an unknown scalar */
16567 			verbose(env, "R%d tried to subtract pointer from scalar\n",
16568 				dst);
16569 			reason = bpf_diag_fmt(
16570 				env, "This operation subtracts pointer register R%d from scalar register R%d. "
16571 				"The verifier only tracks pointer-minus-scalar arithmetic for allowed pointer types.",
16572 				ptr_regno, dst);
16573 			bpf_diag_register_type(
16574 				env, env->insn_idx, ptr_regno, "pointer subtracted from scalar", reason,
16575 				"Keep the pointer as the base; only add or subtract bounded scalars when permitted.");
16576 			return -EACCES;
16577 		}
16578 		/* We don't allow subtraction from FP, because (according to
16579 		 * test_verifier.c test "invalid fp arithmetic", JITs might not
16580 		 * be able to deal with it.
16581 		 */
16582 		if (ptr_reg->type == PTR_TO_STACK) {
16583 			verbose(env, "R%d subtraction from stack pointer prohibited\n",
16584 				dst);
16585 			reason = bpf_diag_fmt(
16586 				env, "R%d is a stack pointer. The verifier does not allow BPF_SUB to move stack pointers.",
16587 				ptr_regno);
16588 			bpf_diag_register_type(
16589 				env, env->insn_idx, ptr_regno, "subtraction from stack pointer", reason,
16590 				"Use addition from R10 to form stack addresses within the tracked stack frame.");
16591 			return -EACCES;
16592 		}
16593 		dst_reg->r64 = cnum64_add(ptr_reg->r64, cnum64_negate(off_reg->r64));
16594 		dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
16595 		dst_reg->raw = ptr_reg->raw;
16596 		if (reg_is_pkt_pointer(ptr_reg)) {
16597 			if (!known)
16598 				dst_reg->id = ++env->id_gen;
16599 			/*
16600 			 * Clear range if the subtrahend may be negative since
16601 			 * pkt pointer could move past its bounds. A positive
16602 			 * subtrahend moves it backwards keeping positive range
16603 			 * intact. Also clear AT_PKT_END / BEYOND_PKT_END from
16604 			 * prior comparison as arithmetic invalidates them.
16605 			 */
16606 			if ((!known && smin_val < 0) || dst_reg->range < 0)
16607 				memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
16608 		}
16609 		reg_step_reset(dst_reg);
16610 		break;
16611 	case BPF_AND:
16612 	case BPF_OR:
16613 	case BPF_XOR:
16614 		/* bitwise ops on pointers are troublesome, prohibit. */
16615 		verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
16616 			dst, bpf_alu_string[opcode >> 4]);
16617 		reason = bpf_diag_fmt(
16618 			env, "R%d holds %s. Bitwise operator %s would destroy the pointer value the verifier is tracking.",
16619 			ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type),
16620 			bpf_alu_string[opcode >> 4]);
16621 		bpf_diag_register_type(
16622 			env, env->insn_idx, ptr_regno, "bitwise operation on pointer", reason,
16623 			"Do bitwise operations on scalar values, not on pointer-valued registers.");
16624 		return -EACCES;
16625 	default:
16626 		/* other operators (e.g. MUL,LSH) produce non-pointer results */
16627 		verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
16628 			dst, bpf_alu_string[opcode >> 4]);
16629 		reason = bpf_diag_fmt(
16630 			env, "R%d holds %s. Operator %s is not one of the limited pointer arithmetic operations the verifier can track.",
16631 			ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type),
16632 			bpf_alu_string[opcode >> 4]);
16633 		bpf_diag_register_type(
16634 			env, env->insn_idx, ptr_regno, "invalid pointer arithmetic operator", reason,
16635 			"Use only verifier-supported addition or subtraction with a bounded scalar offset, or perform this operation on a scalar value.");
16636 		return -EACCES;
16637 	}
16638 
16639 	if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) {
16640 		reason = bpf_diag_fmt(
16641 			env, "After this arithmetic, R%d would be outside the verifier's safe offset range [-%u, %u] for %s.",
16642 			dst, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF,
16643 			bpf_diag_reg_type_plain(env, ptr_reg->type));
16644 		bpf_diag_register_type(
16645 			env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason,
16646 			"Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range.");
16647 		return -EINVAL;
16648 	}
16649 	/*
16650 	 * A packet pointer that keeps its id or range is checked against a
16651 	 * range set from the checked pointer's umax, so var_off must not tighten
16652 	 * its umax. r32 must still match var_off for reg_bounds_sanity_check().
16653 	 */
16654 	if (reg_is_pkt_pointer(dst_reg) && (known || dst_reg->range > 0))
16655 		__update_reg32_bounds(dst_reg);
16656 	else
16657 		reg_bounds_sync(dst_reg);
16658 	bounds_ret = sanitize_check_bounds(env, insn, dst_reg);
16659 	if (bounds_ret == -EACCES)
16660 		return bounds_ret;
16661 	if (sanitize_needed(opcode)) {
16662 		ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,
16663 				       &info, true);
16664 		if (verifier_bug_if(!can_skip_alu_sanitation(env, insn)
16665 				    && !env->cur_state->speculative
16666 				    && bounds_ret
16667 				    && !ret,
16668 				    env, "Pointer type unsupported by sanitize_check_bounds() not rejected by retrieve_ptr_limit() as required")) {
16669 			return -EFAULT;
16670 		}
16671 		if (ret < 0)
16672 			return sanitize_err(env, insn, ret);
16673 	}
16674 
16675 	return 0;
16676 }
16677 
16678 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,
16679 				 struct bpf_reg_state *src_reg)
16680 {
16681 	dst_reg->r32 = cnum32_add(dst_reg->r32, src_reg->r32);
16682 }
16683 
16684 static void scalar_min_max_add(struct bpf_reg_state *dst_reg,
16685 			       struct bpf_reg_state *src_reg)
16686 {
16687 	dst_reg->r64 = cnum64_add(dst_reg->r64, src_reg->r64);
16688 }
16689 
16690 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,
16691 				 struct bpf_reg_state *src_reg)
16692 {
16693 	dst_reg->r32 = cnum32_add(dst_reg->r32, cnum32_negate(src_reg->r32));
16694 }
16695 
16696 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,
16697 			       struct bpf_reg_state *src_reg)
16698 {
16699 	dst_reg->r64 = cnum64_add(dst_reg->r64, cnum64_negate(src_reg->r64));
16700 }
16701 
16702 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,
16703 				 struct bpf_reg_state *src_reg)
16704 {
16705 	s32 smin = reg_s32_min(dst_reg);
16706 	s32 smax = reg_s32_max(dst_reg);
16707 	u32 umin = reg_u32_min(dst_reg);
16708 	u32 umax = reg_u32_max(dst_reg);
16709 	s32 tmp_prod[4];
16710 
16711 	if (check_mul_overflow(umax, reg_u32_max(src_reg), &umax) ||
16712 	    check_mul_overflow(umin, reg_u32_min(src_reg), &umin)) {
16713 		/* Overflow possible, we know nothing */
16714 		umin = 0;
16715 		umax = U32_MAX;
16716 	}
16717 	if (check_mul_overflow(smin, reg_s32_min(src_reg), &tmp_prod[0]) ||
16718 	    check_mul_overflow(smin, reg_s32_max(src_reg), &tmp_prod[1]) ||
16719 	    check_mul_overflow(smax, reg_s32_min(src_reg), &tmp_prod[2]) ||
16720 	    check_mul_overflow(smax, reg_s32_max(src_reg), &tmp_prod[3])) {
16721 		/* Overflow possible, we know nothing */
16722 		smin = S32_MIN;
16723 		smax = S32_MAX;
16724 	} else {
16725 		smin = min_array(tmp_prod, 4);
16726 		smax = max_array(tmp_prod, 4);
16727 	}
16728 
16729 	dst_reg->r32 = cnum32_intersect(cnum32_from_urange(umin, umax),
16730 					cnum32_from_srange(smin, smax));
16731 }
16732 
16733 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,
16734 			       struct bpf_reg_state *src_reg)
16735 {
16736 	s64 smin = reg_smin(dst_reg);
16737 	s64 smax = reg_smax(dst_reg);
16738 	u64 umin = reg_umin(dst_reg);
16739 	u64 umax = reg_umax(dst_reg);
16740 	s64 tmp_prod[4];
16741 
16742 	if (check_mul_overflow(umax, reg_umax(src_reg), &umax) ||
16743 	    check_mul_overflow(umin, reg_umin(src_reg), &umin)) {
16744 		/* Overflow possible, we know nothing */
16745 		umin = 0;
16746 		umax = U64_MAX;
16747 	}
16748 	if (check_mul_overflow(smin, reg_smin(src_reg), &tmp_prod[0]) ||
16749 	    check_mul_overflow(smin, reg_smax(src_reg), &tmp_prod[1]) ||
16750 	    check_mul_overflow(smax, reg_smin(src_reg), &tmp_prod[2]) ||
16751 	    check_mul_overflow(smax, reg_smax(src_reg), &tmp_prod[3])) {
16752 		/* Overflow possible, we know nothing */
16753 		smin = S64_MIN;
16754 		smax = S64_MAX;
16755 	} else {
16756 		smin = min_array(tmp_prod, 4);
16757 		smax = max_array(tmp_prod, 4);
16758 	}
16759 
16760 	dst_reg->r64 = cnum64_intersect(cnum64_from_urange(umin, umax),
16761 					cnum64_from_srange(smin, smax));
16762 }
16763 
16764 static void scalar32_min_max_udiv(struct bpf_reg_state *dst_reg,
16765 				  struct bpf_reg_state *src_reg)
16766 {
16767 	u32 src_val = reg_u32_min(src_reg); /* non-zero, const divisor */
16768 
16769 	reg_set_urange32(dst_reg, reg_u32_min(dst_reg) / src_val,
16770 			 reg_u32_max(dst_reg) / src_val);
16771 
16772 	/* Reset other ranges/tnum to unbounded/unknown. */
16773 	reset_reg64_and_tnum(dst_reg);
16774 }
16775 
16776 static void scalar_min_max_udiv(struct bpf_reg_state *dst_reg,
16777 				struct bpf_reg_state *src_reg)
16778 {
16779 	u64 src_val = reg_umin(src_reg); /* non-zero, const divisor */
16780 
16781 	reg_set_urange64(dst_reg, div64_u64(reg_umin(dst_reg), src_val),
16782 			 div64_u64(reg_umax(dst_reg), src_val));
16783 
16784 	/* Reset other ranges/tnum to unbounded/unknown. */
16785 	reset_reg32_and_tnum(dst_reg);
16786 }
16787 
16788 static void scalar32_min_max_sdiv(struct bpf_reg_state *dst_reg,
16789 				  struct bpf_reg_state *src_reg)
16790 {
16791 	s32 smin = reg_s32_min(dst_reg);
16792 	s32 smax = reg_s32_max(dst_reg);
16793 	s32 src_val = reg_s32_min(src_reg); /* non-zero, const divisor */
16794 	s32 res1, res2;
16795 
16796 	/* BPF div specification: S32_MIN / -1 = S32_MIN */
16797 	if (smin == S32_MIN && src_val == -1) {
16798 		/*
16799 		 * If the dividend range contains more than just S32_MIN,
16800 		 * we cannot precisely track the result, so it becomes unbounded.
16801 		 * e.g., [S32_MIN, S32_MIN+10]/(-1),
16802 		 *     = {S32_MIN} U [-(S32_MIN+10), -(S32_MIN+1)]
16803 		 *     = {S32_MIN} U [S32_MAX-9, S32_MAX] = [S32_MIN, S32_MAX]
16804 		 * Otherwise (if dividend is exactly S32_MIN), result remains S32_MIN.
16805 		 */
16806 		if (smax != S32_MIN) {
16807 			smin = S32_MIN;
16808 			smax = S32_MAX;
16809 		}
16810 		goto reset;
16811 	}
16812 
16813 	res1 = smin / src_val;
16814 	res2 = smax / src_val;
16815 	smin = min(res1, res2);
16816 	smax = max(res1, res2);
16817 
16818 reset:
16819 	reg_set_srange32(dst_reg, smin, smax);
16820 	/* Reset other ranges/tnum to unbounded/unknown. */
16821 	reset_reg64_and_tnum(dst_reg);
16822 }
16823 
16824 static void scalar_min_max_sdiv(struct bpf_reg_state *dst_reg,
16825 				struct bpf_reg_state *src_reg)
16826 {
16827 	s64 smin = reg_smin(dst_reg);
16828 	s64 smax = reg_smax(dst_reg);
16829 	s64 src_val = reg_smin(src_reg); /* non-zero, const divisor */
16830 	s64 res1, res2;
16831 
16832 	/* BPF div specification: S64_MIN / -1 = S64_MIN */
16833 	if (smin == S64_MIN && src_val == -1) {
16834 		/*
16835 		 * If the dividend range contains more than just S64_MIN,
16836 		 * we cannot precisely track the result, so it becomes unbounded.
16837 		 * e.g., [S64_MIN, S64_MIN+10]/(-1),
16838 		 *     = {S64_MIN} U [-(S64_MIN+10), -(S64_MIN+1)]
16839 		 *     = {S64_MIN} U [S64_MAX-9, S64_MAX] = [S64_MIN, S64_MAX]
16840 		 * Otherwise (if dividend is exactly S64_MIN), result remains S64_MIN.
16841 		 */
16842 		if (smax != S64_MIN) {
16843 			smin = S64_MIN;
16844 			smax = S64_MAX;
16845 		}
16846 		goto reset;
16847 	}
16848 
16849 	res1 = div64_s64(smin, src_val);
16850 	res2 = div64_s64(smax, src_val);
16851 	smin = min(res1, res2);
16852 	smax = max(res1, res2);
16853 
16854 reset:
16855 	reg_set_srange64(dst_reg, smin, smax);
16856 	/* Reset other ranges/tnum to unbounded/unknown. */
16857 	reset_reg32_and_tnum(dst_reg);
16858 }
16859 
16860 static void scalar32_min_max_umod(struct bpf_reg_state *dst_reg,
16861 				  struct bpf_reg_state *src_reg)
16862 {
16863 	u32 src_val = reg_u32_min(src_reg); /* non-zero, const divisor */
16864 	u32 res_max = src_val - 1;
16865 
16866 	/*
16867 	 * If dst_umax <= res_max, the result remains unchanged.
16868 	 * e.g., [2, 5] % 10 = [2, 5].
16869 	 */
16870 	if (reg_u32_max(dst_reg) <= res_max)
16871 		return;
16872 
16873 	reg_set_urange32(dst_reg, 0, min(reg_u32_max(dst_reg), res_max));
16874 
16875 	/* Reset other ranges/tnum to unbounded/unknown. */
16876 	reset_reg64_and_tnum(dst_reg);
16877 }
16878 
16879 static void scalar_min_max_umod(struct bpf_reg_state *dst_reg,
16880 				struct bpf_reg_state *src_reg)
16881 {
16882 	u64 src_val = reg_umin(src_reg); /* non-zero, const divisor */
16883 	u64 res_max = src_val - 1;
16884 
16885 	/*
16886 	 * If dst_umax <= res_max, the result remains unchanged.
16887 	 * e.g., [2, 5] % 10 = [2, 5].
16888 	 */
16889 	if (reg_umax(dst_reg) <= res_max)
16890 		return;
16891 
16892 	reg_set_urange64(dst_reg, 0, min(reg_umax(dst_reg), res_max));
16893 
16894 	/* Reset other ranges/tnum to unbounded/unknown. */
16895 	reset_reg32_and_tnum(dst_reg);
16896 }
16897 
16898 static void scalar32_min_max_smod(struct bpf_reg_state *dst_reg,
16899 				  struct bpf_reg_state *src_reg)
16900 {
16901 	s32 src_val = reg_s32_min(src_reg); /* non-zero, const divisor */
16902 
16903 	/*
16904 	 * Safe absolute value calculation:
16905 	 * If src_val == S32_MIN (-2147483648), src_abs becomes 2147483648.
16906 	 * Here use unsigned integer to avoid overflow.
16907 	 */
16908 	u32 src_abs = (src_val > 0) ? (u32)src_val : -(u32)src_val;
16909 
16910 	/*
16911 	 * Calculate the maximum possible absolute value of the result.
16912 	 * Even if src_abs is 2147483648 (S32_MIN), subtracting 1 gives
16913 	 * 2147483647 (S32_MAX), which fits perfectly in s32.
16914 	 */
16915 	s32 res_max_abs = src_abs - 1;
16916 
16917 	/*
16918 	 * If the dividend is already within the result range,
16919 	 * the result remains unchanged. e.g., [-2, 5] % 10 = [-2, 5].
16920 	 */
16921 	if (reg_s32_min(dst_reg) >= -res_max_abs && reg_s32_max(dst_reg) <= res_max_abs)
16922 		return;
16923 
16924 	/* General case: result has the same sign as the dividend. */
16925 	if (reg_s32_min(dst_reg) >= 0) {
16926 		reg_set_srange32(dst_reg, 0, min(reg_s32_max(dst_reg), res_max_abs));
16927 	} else if (reg_s32_max(dst_reg) <= 0) {
16928 		reg_set_srange32(dst_reg, max(reg_s32_min(dst_reg), -res_max_abs), 0);
16929 	} else {
16930 		reg_set_srange32(dst_reg, -res_max_abs, res_max_abs);
16931 	}
16932 
16933 	/* Reset other ranges/tnum to unbounded/unknown. */
16934 	reset_reg64_and_tnum(dst_reg);
16935 }
16936 
16937 static void scalar_min_max_smod(struct bpf_reg_state *dst_reg,
16938 				struct bpf_reg_state *src_reg)
16939 {
16940 	s64 src_val = reg_smin(src_reg); /* non-zero, const divisor */
16941 
16942 	/*
16943 	 * Safe absolute value calculation:
16944 	 * If src_val == S64_MIN (-2^63), src_abs becomes 2^63.
16945 	 * Here use unsigned integer to avoid overflow.
16946 	 */
16947 	u64 src_abs = (src_val > 0) ? (u64)src_val : -(u64)src_val;
16948 
16949 	/*
16950 	 * Calculate the maximum possible absolute value of the result.
16951 	 * Even if src_abs is 2^63 (S64_MIN), subtracting 1 gives
16952 	 * 2^63 - 1 (S64_MAX), which fits perfectly in s64.
16953 	 */
16954 	s64 res_max_abs = src_abs - 1;
16955 
16956 	/*
16957 	 * If the dividend is already within the result range,
16958 	 * the result remains unchanged. e.g., [-2, 5] % 10 = [-2, 5].
16959 	 */
16960 	if (reg_smin(dst_reg) >= -res_max_abs && reg_smax(dst_reg) <= res_max_abs)
16961 		return;
16962 
16963 	/* General case: result has the same sign as the dividend. */
16964 	if (reg_smin(dst_reg) >= 0) {
16965 		reg_set_srange64(dst_reg, 0, min(reg_smax(dst_reg), res_max_abs));
16966 	} else if (reg_smax(dst_reg) <= 0) {
16967 		reg_set_srange64(dst_reg, max(reg_smin(dst_reg), -res_max_abs), 0);
16968 	} else {
16969 		reg_set_srange64(dst_reg, -res_max_abs, res_max_abs);
16970 	}
16971 
16972 	/* Reset other ranges/tnum to unbounded/unknown. */
16973 	reset_reg32_and_tnum(dst_reg);
16974 }
16975 
16976 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,
16977 				 struct bpf_reg_state *src_reg)
16978 {
16979 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
16980 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
16981 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
16982 	u32 umax_val = reg_u32_max(src_reg);
16983 
16984 	if (src_known && dst_known) {
16985 		__mark_reg32_known(dst_reg, var32_off.value);
16986 		return;
16987 	}
16988 
16989 	/* We get our minimum from the var_off, since that's inherently
16990 	 * bitwise.  Our maximum is the minimum of the operands' maxima.
16991 	 */
16992 	reg_set_urange32(dst_reg,
16993 			 var32_off.value,
16994 			 min(reg_u32_max(dst_reg), umax_val));
16995 }
16996 
16997 static void scalar_min_max_and(struct bpf_reg_state *dst_reg,
16998 			       struct bpf_reg_state *src_reg)
16999 {
17000 	bool src_known = tnum_is_const(src_reg->var_off);
17001 	bool dst_known = tnum_is_const(dst_reg->var_off);
17002 	u64 umax_val = reg_umax(src_reg);
17003 
17004 	if (src_known && dst_known) {
17005 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
17006 		return;
17007 	}
17008 
17009 	/* We get our minimum from the var_off, since that's inherently
17010 	 * bitwise.  Our maximum is the minimum of the operands' maxima.
17011 	 */
17012 	reg_set_urange64(dst_reg,
17013 			 dst_reg->var_off.value,
17014 			 min(reg_umax(dst_reg), umax_val));
17015 
17016 	/* We may learn something more from the var_off */
17017 	__update_reg_bounds(dst_reg);
17018 }
17019 
17020 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,
17021 				struct bpf_reg_state *src_reg)
17022 {
17023 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
17024 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
17025 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
17026 	u32 umin_val = reg_u32_min(src_reg);
17027 
17028 	if (src_known && dst_known) {
17029 		__mark_reg32_known(dst_reg, var32_off.value);
17030 		return;
17031 	}
17032 
17033 	/* We get our maximum from the var_off, and our minimum is the
17034 	 * maximum of the operands' minima
17035 	 */
17036 	reg_set_urange32(dst_reg,
17037 			 max(reg_u32_min(dst_reg), umin_val),
17038 			 var32_off.value | var32_off.mask);
17039 }
17040 
17041 static void scalar_min_max_or(struct bpf_reg_state *dst_reg,
17042 			      struct bpf_reg_state *src_reg)
17043 {
17044 	bool src_known = tnum_is_const(src_reg->var_off);
17045 	bool dst_known = tnum_is_const(dst_reg->var_off);
17046 	u64 umin_val = reg_umin(src_reg);
17047 
17048 	if (src_known && dst_known) {
17049 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
17050 		return;
17051 	}
17052 
17053 	/* We get our maximum from the var_off, and our minimum is the
17054 	 * maximum of the operands' minima
17055 	 */
17056 	reg_set_urange64(dst_reg,
17057 			 max(reg_umin(dst_reg), umin_val),
17058 			 dst_reg->var_off.value | dst_reg->var_off.mask);
17059 
17060 	/* We may learn something more from the var_off */
17061 	__update_reg_bounds(dst_reg);
17062 }
17063 
17064 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,
17065 				 struct bpf_reg_state *src_reg)
17066 {
17067 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
17068 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
17069 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
17070 
17071 	if (src_known && dst_known) {
17072 		__mark_reg32_known(dst_reg, var32_off.value);
17073 		return;
17074 	}
17075 
17076 	/* We get both minimum and maximum from the var32_off. */
17077 	reg_set_urange32(dst_reg, var32_off.value, var32_off.value | var32_off.mask);
17078 }
17079 
17080 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
17081 			       struct bpf_reg_state *src_reg)
17082 {
17083 	bool src_known = tnum_is_const(src_reg->var_off);
17084 	bool dst_known = tnum_is_const(dst_reg->var_off);
17085 
17086 	if (src_known && dst_known) {
17087 		/* dst_reg->var_off.value has been updated earlier */
17088 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
17089 		return;
17090 	}
17091 
17092 	/* We get both minimum and maximum from the var_off. */
17093 	reg_set_urange64(dst_reg,
17094 			 dst_reg->var_off.value,
17095 			 dst_reg->var_off.value | dst_reg->var_off.mask);
17096 }
17097 
17098 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
17099 				   u64 umin_val, u64 umax_val)
17100 {
17101 	/* If we might shift our top bit out, then we know nothing */
17102 	if (umax_val > 31 || reg_u32_max(dst_reg) > 1ULL << (31 - umax_val))
17103 		reg_set_urange32(dst_reg, 0, U32_MAX);
17104 	else
17105 		/* We lose all sign bit information (except what we can pick
17106 		 * up from var_off)
17107 		 */
17108 		reg_set_urange32(dst_reg, reg_u32_min(dst_reg) << umin_val,
17109 				 reg_u32_max(dst_reg) << umax_val);
17110 }
17111 
17112 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
17113 				 struct bpf_reg_state *src_reg)
17114 {
17115 	u32 umax_val = reg_u32_max(src_reg);
17116 	u32 umin_val = reg_u32_min(src_reg);
17117 	/* u32 alu operation will zext upper bits */
17118 	struct tnum subreg = tnum_subreg(dst_reg->var_off);
17119 
17120 	__scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
17121 	dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val));
17122 	/* Not required but being careful mark reg64 bounds as unknown so
17123 	 * that we are forced to pick them up from tnum and zext later and
17124 	 * if some path skips this step we are still safe.
17125 	 */
17126 	__mark_reg64_unbounded(dst_reg);
17127 	__update_reg32_bounds(dst_reg);
17128 }
17129 
17130 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,
17131 				   u64 umin_val, u64 umax_val)
17132 {
17133 	struct cnum64 u, s;
17134 
17135 	/* Special case <<32 because it is a common compiler pattern to sign
17136 	 * extend subreg by doing <<32 s>>32. smin/smax assignments are correct
17137 	 * because s32 bounds don't flip sign when shifting to the left by
17138 	 * 32bits.
17139 	 */
17140 	if (umin_val == 32 && umax_val == 32)
17141 		s = cnum64_from_srange((s64)reg_s32_min(dst_reg) << 32,
17142 				       (s64)reg_s32_max(dst_reg) << 32);
17143 	else
17144 		s = CNUM64_UNBOUNDED;
17145 
17146 	/* If we might shift our top bit out, then we know nothing */
17147 	if (reg_umax(dst_reg) > 1ULL << (63 - umax_val))
17148 		u = CNUM64_UNBOUNDED;
17149 	else
17150 		u = cnum64_from_urange(reg_umin(dst_reg) << umin_val,
17151 				       reg_umax(dst_reg) << umax_val);
17152 
17153 	dst_reg->r64 = cnum64_intersect(u, s);
17154 }
17155 
17156 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,
17157 			       struct bpf_reg_state *src_reg)
17158 {
17159 	u64 umax_val = reg_umax(src_reg);
17160 	u64 umin_val = reg_umin(src_reg);
17161 
17162 	/* scalar64 calc uses 32bit unshifted bounds so must be called first */
17163 	__scalar64_min_max_lsh(dst_reg, umin_val, umax_val);
17164 	__scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
17165 
17166 	dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
17167 	/* We may learn something more from the var_off */
17168 	__update_reg_bounds(dst_reg);
17169 }
17170 
17171 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,
17172 				 struct bpf_reg_state *src_reg)
17173 {
17174 	struct tnum subreg = tnum_subreg(dst_reg->var_off);
17175 	u32 umax_val = reg_u32_max(src_reg);
17176 	u32 umin_val = reg_u32_min(src_reg);
17177 
17178 	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
17179 	 * be negative, then either:
17180 	 * 1) src_reg might be zero, so the sign bit of the result is
17181 	 *    unknown, so we lose our signed bounds
17182 	 * 2) it's known negative, thus the unsigned bounds capture the
17183 	 *    signed bounds
17184 	 * 3) the signed bounds cross zero, so they tell us nothing
17185 	 *    about the result
17186 	 * If the value in dst_reg is known nonnegative, then again the
17187 	 * unsigned bounds capture the signed bounds.
17188 	 * Thus, in all cases it suffices to blow away our signed bounds
17189 	 * and rely on inferring new ones from the unsigned bounds and
17190 	 * var_off of the result.
17191 	 */
17192 
17193 	dst_reg->var_off = tnum_rshift(subreg, umin_val);
17194 	reg_set_urange32(dst_reg, reg_u32_min(dst_reg) >> umax_val,
17195 			 reg_u32_max(dst_reg) >> umin_val);
17196 
17197 	__mark_reg64_unbounded(dst_reg);
17198 	__update_reg32_bounds(dst_reg);
17199 }
17200 
17201 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,
17202 			       struct bpf_reg_state *src_reg)
17203 {
17204 	u64 umax_val = reg_umax(src_reg);
17205 	u64 umin_val = reg_umin(src_reg);
17206 
17207 	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
17208 	 * be negative, then either:
17209 	 * 1) src_reg might be zero, so the sign bit of the result is
17210 	 *    unknown, so we lose our signed bounds
17211 	 * 2) it's known negative, thus the unsigned bounds capture the
17212 	 *    signed bounds
17213 	 * 3) the signed bounds cross zero, so they tell us nothing
17214 	 *    about the result
17215 	 * If the value in dst_reg is known nonnegative, then again the
17216 	 * unsigned bounds capture the signed bounds.
17217 	 * Thus, in all cases it suffices to blow away our signed bounds
17218 	 * and rely on inferring new ones from the unsigned bounds and
17219 	 * var_off of the result.
17220 	 */
17221 	dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
17222 	reg_set_urange64(dst_reg, reg_umin(dst_reg) >> umax_val,
17223 			 reg_umax(dst_reg) >> umin_val);
17224 
17225 	/* Its not easy to operate on alu32 bounds here because it depends
17226 	 * on bits being shifted in. Take easy way out and mark unbounded
17227 	 * so we can recalculate later from tnum.
17228 	 */
17229 	__mark_reg32_unbounded(dst_reg);
17230 	__update_reg_bounds(dst_reg);
17231 }
17232 
17233 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,
17234 				  struct bpf_reg_state *src_reg)
17235 {
17236 	u64 umin_val = reg_u32_min(src_reg);
17237 
17238 	/* Upon reaching here, src_known is true and
17239 	 * umax_val is equal to umin_val.
17240 	 * Blow away the dst_reg umin_value/umax_value and rely on
17241 	 * dst_reg var_off to refine the result.
17242 	 */
17243 	reg_set_srange32(dst_reg,
17244 			 (u32)(((s32)reg_s32_min(dst_reg)) >> umin_val),
17245 			 (u32)(((s32)reg_s32_max(dst_reg)) >> umin_val));
17246 
17247 	dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32);
17248 
17249 	__mark_reg64_unbounded(dst_reg);
17250 	__update_reg32_bounds(dst_reg);
17251 }
17252 
17253 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,
17254 				struct bpf_reg_state *src_reg)
17255 {
17256 	u64 umin_val = reg_umin(src_reg);
17257 
17258 	/* Upon reaching here, src_known is true and umax_val is equal
17259 	 * to umin_val.
17260 	 */
17261 	reg_set_srange64(dst_reg, reg_smin(dst_reg) >> umin_val,
17262 			 reg_smax(dst_reg) >> umin_val);
17263 
17264 	dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64);
17265 
17266 	/* Its not easy to operate on alu32 bounds here because it depends
17267 	 * on bits being shifted in from upper 32-bits. Take easy way out
17268 	 * and mark unbounded so we can recalculate later from tnum.
17269 	 */
17270 	__mark_reg32_unbounded(dst_reg);
17271 	__update_reg_bounds(dst_reg);
17272 }
17273 
17274 static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *insn)
17275 {
17276 	/*
17277 	 * Byte swap operation - update var_off using tnum_bswap.
17278 	 * Three cases:
17279 	 * 1. bswap(16|32|64): opcode=0xd7 (BPF_END | BPF_ALU64 | BPF_TO_LE)
17280 	 *    unconditional swap
17281 	 * 2. to_le(16|32|64): opcode=0xd4 (BPF_END | BPF_ALU | BPF_TO_LE)
17282 	 *    swap on big-endian, truncation or no-op on little-endian
17283 	 * 3. to_be(16|32|64): opcode=0xdc (BPF_END | BPF_ALU | BPF_TO_BE)
17284 	 *    swap on little-endian, truncation or no-op on big-endian
17285 	 */
17286 
17287 	bool alu64 = BPF_CLASS(insn->code) == BPF_ALU64;
17288 	bool to_le = BPF_SRC(insn->code) == BPF_TO_LE;
17289 	bool is_big_endian;
17290 #ifdef CONFIG_CPU_BIG_ENDIAN
17291 	is_big_endian = true;
17292 #else
17293 	is_big_endian = false;
17294 #endif
17295 	/* Apply bswap if alu64 or switch between big-endian and little-endian machines */
17296 	bool need_bswap = alu64 || (to_le == is_big_endian);
17297 
17298 	/*
17299 	 * If the register is mutated, manually reset its scalar ID to break
17300 	 * any existing ties and avoid incorrect bounds propagation.
17301 	 */
17302 	if (need_bswap || insn->imm == 16 || insn->imm == 32)
17303 		clear_scalar_id(dst_reg);
17304 
17305 	if (need_bswap) {
17306 		if (insn->imm == 16)
17307 			dst_reg->var_off = tnum_bswap16(dst_reg->var_off);
17308 		else if (insn->imm == 32)
17309 			dst_reg->var_off = tnum_bswap32(dst_reg->var_off);
17310 		else if (insn->imm == 64)
17311 			dst_reg->var_off = tnum_bswap64(dst_reg->var_off);
17312 		/*
17313 		 * Byteswap scrambles the range, so we must reset bounds.
17314 		 * Bounds will be re-derived from the new tnum later.
17315 		 */
17316 		__mark_reg_unbounded(dst_reg);
17317 		reg_step_reset(dst_reg);
17318 	}
17319 	/* For bswap16/32, truncate dst register to match the swapped size */
17320 	if (insn->imm == 16 || insn->imm == 32)
17321 		coerce_reg_to_size(dst_reg, insn->imm / 8);
17322 }
17323 
17324 static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn,
17325 					     const struct bpf_reg_state *src_reg)
17326 {
17327 	bool src_is_const = false;
17328 	u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
17329 
17330 	if (insn_bitness == 32) {
17331 		if (tnum_subreg_is_const(src_reg->var_off)
17332 		    && reg_s32_min(src_reg) == reg_s32_max(src_reg)
17333 		    && reg_u32_min(src_reg) == reg_u32_max(src_reg))
17334 			src_is_const = true;
17335 	} else {
17336 		if (tnum_is_const(src_reg->var_off)
17337 		    && reg_smin(src_reg) == reg_smax(src_reg)
17338 		    && reg_umin(src_reg) == reg_umax(src_reg))
17339 			src_is_const = true;
17340 	}
17341 
17342 	switch (BPF_OP(insn->code)) {
17343 	case BPF_ADD:
17344 	case BPF_SUB:
17345 	case BPF_NEG:
17346 	case BPF_AND:
17347 	case BPF_XOR:
17348 	case BPF_OR:
17349 	case BPF_MUL:
17350 	case BPF_END:
17351 		return true;
17352 
17353 	/*
17354 	 * Division and modulo operators range is only safe to compute when the
17355 	 * divisor is a constant.
17356 	 */
17357 	case BPF_DIV:
17358 	case BPF_MOD:
17359 		return src_is_const;
17360 
17361 	/* Shift operators range is only computable if shift dimension operand
17362 	 * is a constant. Shifts greater than 31 or 63 are undefined. This
17363 	 * includes shifts by a negative number.
17364 	 */
17365 	case BPF_LSH:
17366 	case BPF_RSH:
17367 	case BPF_ARSH:
17368 		return (src_is_const && reg_umax(src_reg) < insn_bitness);
17369 	default:
17370 		return false;
17371 	}
17372 }
17373 
17374 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn,
17375 				      struct bpf_reg_state *dst_reg, struct bpf_reg_state src_reg);
17376 
17377 static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn,
17378 			      struct bpf_reg_state *dst_reg, const struct bpf_reg_state *src_reg)
17379 {
17380 	struct bpf_verifier_state *branch;
17381 	struct bpf_reg_state *reg;
17382 	bool alu32;
17383 
17384 	if (reg_smin(dst_reg) == -1 && reg_smax(dst_reg) == 0)
17385 		alu32 = false;
17386 	else if (reg_s32_min(dst_reg) == -1 && reg_s32_max(dst_reg) == 0)
17387 		alu32 = true;
17388 	else
17389 		return 0;
17390 
17391 	branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);
17392 	if (IS_ERR(branch))
17393 		return PTR_ERR(branch);
17394 
17395 	reg = &branch->frame[branch->curframe]->regs[insn->dst_reg];
17396 	if (alu32) {
17397 		__mark_reg32_known(reg, 0);
17398 		__mark_reg32_known(dst_reg, -1ull);
17399 	} else {
17400 		__mark_reg_known(reg, 0);
17401 		__mark_reg_known(dst_reg, -1ull);
17402 	}
17403 	/*
17404 	 * The insn is done with in the branch too, and what gets to the next insn is a state
17405 	 * like any other. At this insn the branch is a part of the state that is looked at:
17406 	 * it must not be compared with it.
17407 	 */
17408 	clear_scalar_id(reg);
17409 	return adjust_scalar_min_max_vals(env, insn, reg, *src_reg);
17410 }
17411 
17412 /* WARNING: This function does calculations on 64-bit values, but the actual
17413  * execution may occur on 32-bit values. Therefore, things like bitshifts
17414  * need extra checks in the 32-bit case.
17415  */
17416 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
17417 				      struct bpf_insn *insn,
17418 				      struct bpf_reg_state *dst_reg,
17419 				      struct bpf_reg_state src_reg)
17420 {
17421 	u8 opcode = BPF_OP(insn->code);
17422 	s16 off = insn->off;
17423 	bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
17424 	int ret;
17425 
17426 	if (!is_safe_to_compute_dst_reg_range(insn, &src_reg)) {
17427 		__mark_reg_unknown(env, dst_reg);
17428 		return 0;
17429 	}
17430 
17431 	if (sanitize_needed(opcode)) {
17432 		ret = sanitize_val_alu(env, insn);
17433 		if (ret < 0)
17434 			return sanitize_err(env, insn, ret);
17435 	}
17436 
17437 	/* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops.
17438 	 * There are two classes of instructions: The first class we track both
17439 	 * alu32 and alu64 sign/unsigned bounds independently this provides the
17440 	 * greatest amount of precision when alu operations are mixed with jmp32
17441 	 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD,
17442 	 * and BPF_OR. This is possible because these ops have fairly easy to
17443 	 * understand and calculate behavior in both 32-bit and 64-bit alu ops.
17444 	 * See alu32 verifier tests for examples. The second class of
17445 	 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy
17446 	 * with regards to tracking sign/unsigned bounds because the bits may
17447 	 * cross subreg boundaries in the alu64 case. When this happens we mark
17448 	 * the reg unbounded in the subreg bound space and use the resulting
17449 	 * tnum to calculate an approximation of the sign/unsigned bounds.
17450 	 */
17451 	switch (opcode) {
17452 	case BPF_ADD:
17453 		scalar_step_add(dst_reg, dst_reg, &src_reg);
17454 		scalar32_min_max_add(dst_reg, &src_reg);
17455 		scalar_min_max_add(dst_reg, &src_reg);
17456 		dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
17457 		break;
17458 	case BPF_SUB:
17459 		scalar32_min_max_sub(dst_reg, &src_reg);
17460 		scalar_min_max_sub(dst_reg, &src_reg);
17461 		dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
17462 		reg_step_reset(dst_reg);
17463 		break;
17464 	case BPF_NEG:
17465 		env->fake_reg[0] = *dst_reg;
17466 		__mark_reg_known(dst_reg, 0);
17467 		scalar32_min_max_sub(dst_reg, &env->fake_reg[0]);
17468 		scalar_min_max_sub(dst_reg, &env->fake_reg[0]);
17469 		dst_reg->var_off = tnum_neg(env->fake_reg[0].var_off);
17470 		reg_step_reset(dst_reg);
17471 		break;
17472 	case BPF_MUL:
17473 		scalar_step_mul(dst_reg, &src_reg);
17474 		dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
17475 		scalar32_min_max_mul(dst_reg, &src_reg);
17476 		scalar_min_max_mul(dst_reg, &src_reg);
17477 		break;
17478 	case BPF_DIV:
17479 		/* BPF div specification: x / 0 = 0 */
17480 		if ((alu32 && reg_u32_min(&src_reg) == 0) || (!alu32 && reg_umin(&src_reg) == 0)) {
17481 			___mark_reg_known(dst_reg, 0);
17482 			break;
17483 		}
17484 		if (alu32)
17485 			if (off == 1)
17486 				scalar32_min_max_sdiv(dst_reg, &src_reg);
17487 			else
17488 				scalar32_min_max_udiv(dst_reg, &src_reg);
17489 		else
17490 			if (off == 1)
17491 				scalar_min_max_sdiv(dst_reg, &src_reg);
17492 			else
17493 				scalar_min_max_udiv(dst_reg, &src_reg);
17494 		reg_step_reset(dst_reg);
17495 		break;
17496 	case BPF_MOD:
17497 		/* BPF mod specification: x % 0 = x */
17498 		if ((alu32 && reg_u32_min(&src_reg) == 0) || (!alu32 && reg_umin(&src_reg) == 0))
17499 			break;
17500 		if (alu32)
17501 			if (off == 1)
17502 				scalar32_min_max_smod(dst_reg, &src_reg);
17503 			else
17504 				scalar32_min_max_umod(dst_reg, &src_reg);
17505 		else
17506 			if (off == 1)
17507 				scalar_min_max_smod(dst_reg, &src_reg);
17508 			else
17509 				scalar_min_max_umod(dst_reg, &src_reg);
17510 		reg_step_reset(dst_reg);
17511 		break;
17512 	case BPF_AND:
17513 		if (tnum_is_const(src_reg.var_off)) {
17514 			ret = maybe_fork_scalars(env, insn, dst_reg, &src_reg);
17515 			if (ret)
17516 				return ret;
17517 		}
17518 		dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
17519 		scalar32_min_max_and(dst_reg, &src_reg);
17520 		scalar_min_max_and(dst_reg, &src_reg);
17521 		reg_step_reset(dst_reg);
17522 		break;
17523 	case BPF_OR:
17524 		if (tnum_is_const(src_reg.var_off)) {
17525 			ret = maybe_fork_scalars(env, insn, dst_reg, &src_reg);
17526 			if (ret)
17527 				return ret;
17528 		}
17529 		dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
17530 		scalar32_min_max_or(dst_reg, &src_reg);
17531 		scalar_min_max_or(dst_reg, &src_reg);
17532 		reg_step_reset(dst_reg);
17533 		break;
17534 	case BPF_XOR:
17535 		dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off);
17536 		scalar32_min_max_xor(dst_reg, &src_reg);
17537 		scalar_min_max_xor(dst_reg, &src_reg);
17538 		reg_step_reset(dst_reg);
17539 		break;
17540 	case BPF_LSH:
17541 		scalar_step_lsh(dst_reg, &src_reg);
17542 		if (alu32)
17543 			scalar32_min_max_lsh(dst_reg, &src_reg);
17544 		else
17545 			scalar_min_max_lsh(dst_reg, &src_reg);
17546 		break;
17547 	case BPF_RSH:
17548 		if (alu32)
17549 			scalar32_min_max_rsh(dst_reg, &src_reg);
17550 		else
17551 			scalar_min_max_rsh(dst_reg, &src_reg);
17552 		reg_step_reset(dst_reg);
17553 		break;
17554 	case BPF_ARSH:
17555 		if (alu32)
17556 			scalar32_min_max_arsh(dst_reg, &src_reg);
17557 		else
17558 			scalar_min_max_arsh(dst_reg, &src_reg);
17559 		reg_step_reset(dst_reg);
17560 		break;
17561 	case BPF_END:
17562 		scalar_byte_swap(dst_reg, insn);
17563 		reg_step_reset(dst_reg);
17564 		break;
17565 	default:
17566 		break;
17567 	}
17568 
17569 	/*
17570 	 * ALU32 ops are zero extended into 64bit register.
17571 	 *
17572 	 * BPF_END is already handled inside the helper (truncation),
17573 	 * so skip zext here to avoid unexpected zero extension.
17574 	 * e.g., le64: opcode=(BPF_END|BPF_ALU|BPF_TO_LE), imm=0x40
17575 	 * This is a 64bit byte swap operation with alu32==true,
17576 	 * but we should not zero extend the result.
17577 	 */
17578 	if (alu32 && opcode != BPF_END)
17579 		zext_32_to_64(dst_reg);
17580 	reg_bounds_sync(dst_reg);
17581 	return 0;
17582 }
17583 
17584 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max
17585  * and var_off.
17586  */
17587 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
17588 				   struct bpf_insn *insn)
17589 {
17590 	struct bpf_verifier_state *vstate = env->cur_state;
17591 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
17592 	struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
17593 	struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
17594 	bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
17595 	struct bpf_insn_aux_data *aux = cur_aux(env);
17596 	u8 opcode = BPF_OP(insn->code);
17597 	int err;
17598 
17599 	dst_reg = &regs[insn->dst_reg];
17600 	if (BPF_SRC(insn->code) == BPF_X)
17601 		src_reg = &regs[insn->src_reg];
17602 	else
17603 		src_reg = NULL;
17604 
17605 	/* Case where at least one operand is an arena. */
17606 	if (dst_reg->type == PTR_TO_ARENA || (src_reg && src_reg->type == PTR_TO_ARENA)) {
17607 
17608 		if (dst_reg->type != PTR_TO_ARENA)
17609 			*dst_reg = *src_reg;
17610 
17611 		if (BPF_CLASS(insn->code) == BPF_ALU64) {
17612 			/*
17613 			 * Only arena pointers set needs_zext, but doing so
17614 			 * modifies the instruction at fixup time to an ALU32
17615 			 * and makes it unsuitable for 64-bit scalar args. We
17616 			 * prevent zext from being set if the instruction has
17617 			 * been previously called with non-arena registers.
17618 			 */
17619 			if (aux->prevent_zext) {
17620 				verbose(env, "same insn cannot be used with and without arena pointer\n");
17621 				return -EINVAL;
17622 			}
17623 
17624 			/*
17625 			 * 32-bit operations zero upper bits automatically.
17626 			 * 64-bit operations need to be converted to 32.
17627 			 */
17628 			aux->needs_zext = true;
17629 			aux->zext_dst = true;
17630 		}
17631 
17632 		/* Any arithmetic operations are allowed on arena pointers */
17633 		return 0;
17634 	}
17635 
17636 	/* Prevent the instruction from being used with arena pointers (see above). */
17637 	if (env->prog->aux->arena && BPF_CLASS(insn->code) == BPF_ALU64) {
17638 		if (aux->needs_zext) {
17639 			verbose(env, "same insn cannot be used with and without arena pointer\n");
17640 			return -EINVAL;
17641 		}
17642 
17643 		aux->prevent_zext = true;
17644 	}
17645 
17646 	if (dst_reg->type != SCALAR_VALUE)
17647 		ptr_reg = dst_reg;
17648 
17649 	if (BPF_SRC(insn->code) == BPF_X) {
17650 		if (src_reg->type != SCALAR_VALUE) {
17651 			if (dst_reg->type != SCALAR_VALUE) {
17652 				/* Combining two pointers by any ALU op yields
17653 				 * an arbitrary scalar. Disallow all math except
17654 				 * pointer subtraction
17655 				 */
17656 				if (opcode == BPF_SUB && env->allow_ptr_leaks) {
17657 					mark_reg_unknown(env, regs, insn->dst_reg);
17658 					return 0;
17659 				}
17660 				verbose(env, "R%d pointer %s pointer prohibited\n",
17661 					insn->dst_reg,
17662 					bpf_alu_string[opcode >> 4]);
17663 				return -EACCES;
17664 			} else {
17665 				/* scalar += pointer
17666 				 * This is legal, but we have to reverse our
17667 				 * src/dest handling in computing the range
17668 				 */
17669 				err = mark_chain_precision(env, insn->dst_reg);
17670 				if (err)
17671 					return err;
17672 				off_reg = *dst_reg;
17673 				return adjust_ptr_min_max_vals(env, insn, insn->src_reg, src_reg,
17674 							       &off_reg);
17675 			}
17676 		} else if (ptr_reg) {
17677 			/* pointer += scalar */
17678 			err = mark_chain_precision(env, insn->src_reg);
17679 			if (err)
17680 				return err;
17681 			return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, dst_reg, src_reg);
17682 		} else if (dst_reg->precise) {
17683 			/* if dst_reg is precise, src_reg should be precise as well */
17684 			err = mark_chain_precision(env, insn->src_reg);
17685 			if (err)
17686 				return err;
17687 		}
17688 	} else {
17689 		/* Pretend the src is a reg with a known value, since we only
17690 		 * need to be able to read from this state.
17691 		 */
17692 		off_reg.type = SCALAR_VALUE;
17693 		__mark_reg_known(&off_reg, insn->imm);
17694 		src_reg = &off_reg;
17695 		if (ptr_reg) /* pointer += K */
17696 			return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, ptr_reg, src_reg);
17697 	}
17698 
17699 	/* Got here implies adding two SCALAR_VALUEs */
17700 	if (WARN_ON_ONCE(ptr_reg)) {
17701 		print_verifier_state(env, vstate, vstate->curframe, true);
17702 		verbose(env, "verifier internal error: unexpected ptr_reg\n");
17703 		return -EFAULT;
17704 	}
17705 	if (WARN_ON(!src_reg)) {
17706 		print_verifier_state(env, vstate, vstate->curframe, true);
17707 		verbose(env, "verifier internal error: no src_reg\n");
17708 		return -EFAULT;
17709 	}
17710 	/*
17711 	 * For alu32 linked register tracking, we need to check dst_reg's
17712 	 * umax_value before the ALU operation. After adjust_scalar_min_max_vals(),
17713 	 * alu32 ops will have zero-extended the result, making umax_value <= U32_MAX.
17714 	 */
17715 	u64 dst_umax = reg_umax(dst_reg);
17716 
17717 	err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
17718 	if (err)
17719 		return err;
17720 	/*
17721 	 * Compilers can generate the code
17722 	 * r1 = r2
17723 	 * r1 += 0x1
17724 	 * if r2 < 1000 goto ...
17725 	 * use r1 in memory access
17726 	 * So remember constant delta between r2 and r1 and update r1 after
17727 	 * 'if' condition.
17728 	 */
17729 	if (env->bpf_capable &&
17730 	    (BPF_OP(insn->code) == BPF_ADD || BPF_OP(insn->code) == BPF_SUB) &&
17731 	    dst_reg->id && is_reg_const(src_reg, alu32) &&
17732 	    !(BPF_SRC(insn->code) == BPF_X && insn->src_reg == insn->dst_reg)) {
17733 		u64 val = reg_const_value(src_reg, alu32);
17734 		s32 off;
17735 
17736 		if (!alu32 && ((s64)val < S32_MIN || (s64)val > S32_MAX))
17737 			goto clear_id;
17738 
17739 		if (alu32 && (dst_umax > U32_MAX))
17740 			goto clear_id;
17741 
17742 		off = (s32)val;
17743 
17744 		if (BPF_OP(insn->code) == BPF_SUB) {
17745 			/* Negating S32_MIN would overflow */
17746 			if (off == S32_MIN)
17747 				goto clear_id;
17748 			off = -off;
17749 		}
17750 
17751 		if (dst_reg->id & BPF_ADD_CONST) {
17752 			/*
17753 			 * If the register already went through rX += val
17754 			 * we cannot accumulate another val into rx->off.
17755 			 */
17756 clear_id:
17757 			clear_scalar_id(dst_reg);
17758 		} else {
17759 			if (alu32)
17760 				dst_reg->id |= BPF_ADD_CONST32;
17761 			else
17762 				dst_reg->id |= BPF_ADD_CONST64;
17763 			dst_reg->delta = off;
17764 		}
17765 	} else {
17766 		/*
17767 		 * Make sure ID is cleared otherwise dst_reg min/max could be
17768 		 * incorrectly propagated into other registers by sync_linked_regs()
17769 		 */
17770 		clear_scalar_id(dst_reg);
17771 	}
17772 	return 0;
17773 }
17774 
17775 /*
17776  * Many checks done by this function are quite conservative.
17777  * This is because main verification pass does not maintain
17778  * enough information to track object identities for some
17779  * of the interesting types, e.g. PTR_TO_MEM.
17780  */
17781 bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a,
17782 			    const struct bpf_reg_state *reg_b)
17783 {
17784 	if (reg_a == reg_b)
17785 		return true;
17786 	/* Require matching flags and base types. */
17787 	if (reg_a->type != reg_b->type)
17788 		return false;
17789 	/* NULL can't be compared to some base+offset pointer. */
17790 	if (type_may_be_null(reg_a->type))
17791 		return false;
17792 
17793 	switch (base_type(reg_a->type)) {
17794 	case PTR_TO_STACK:
17795 		return reg_a->frameno == reg_b->frameno;
17796 	case PTR_TO_MAP_VALUE:
17797 		if (reg_a->map_ptr != reg_b->map_ptr || reg_a->map_uid != reg_b->map_uid)
17798 			return false;
17799 		if (reg_a->id && reg_a->id == reg_b->id)
17800 			return true;
17801 		/*
17802 		 * A plain single-element array has one stable value address,
17803 		 * but not when BPF_F_INNER_MAP is set, see arraymap.c:array_map_meta_equal().
17804 		 */
17805 		if (reg_a->map_ptr->map_type == BPF_MAP_TYPE_ARRAY &&
17806 		    reg_a->map_ptr->max_entries == 1 &&
17807 		    !(reg_a->map_ptr->map_flags & BPF_F_INNER_MAP))
17808 			return true;
17809 		/*
17810 		 * The rules above can be simplified / relaxed if:
17811 		 * - fresh IDs would always be assigned for map-value lookups;
17812 		 * - direct map value loads would always have and ID of zero.
17813 		 */
17814 		return false;
17815 	case PTR_TO_MEM:
17816 	case PTR_TO_BUF:
17817 		return reg_a->id && reg_a->id == reg_b->id;
17818 
17819 	default:
17820 		return false;
17821 	}
17822 }
17823 
17824 /*
17825  * 'range - smax' bytes from a packet pointer are in the packet. reg_bounds_sync() can make smax
17826  * smaller than @smax: with a step of 8, or with 3 zero bits in var_off, 58 becomes 56. Make the
17827  * range smaller by the same number, so that the number of bytes does not grow. umax is not
17828  * sufficient: it is U64_MAX when the offset can be negative.
17829  */
17830 void bpf_trim_pkt_range(struct bpf_reg_state *reg, s64 smax)
17831 {
17832 	if (type_is_pkt_pointer(reg->type) && reg->range > 0)
17833 		reg->range = max_t(s64, reg->range - (smax - reg_smax(reg)), 0);
17834 }
17835 
17836 int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
17837 		      struct cnum64 r64, struct cnum32 r32, u16 base, u16 step)
17838 {
17839 	reg->r64 = r64;
17840 	reg->r32 = r32;
17841 	reg->step = step;
17842 	reg->base = base;
17843 	reg->var_off = tnum_unknown;
17844 	reg_bounds_sync(reg); /* this should infer the tnum alignment */
17845 	bpf_trim_pkt_range(reg, cnum64_smax(r64));
17846 	return reg_bounds_sanity_check(env, reg, "bpf_set_reg_range");
17847 }
17848 
17849 /*
17850  * Checks if pointer and packet metadata matches between two registers.
17851  * For use with bpf_reg_union().
17852  */
17853 bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b)
17854 {
17855 	if (a->type != b->type)
17856 		return false;
17857 	if (a->type == SCALAR_VALUE)
17858 		return true;
17859 	return !memcmp(a, b, offsetof(struct bpf_reg_state, var_off)) &&
17860 	       a->id == b->id && a->parent_id == b->parent_id && a->map_uid == b->map_uid;
17861 }
17862 
17863 /*
17864  * Extends acc's scalar range to include src's scalar range.
17865  * Caller must ensure that other kinds of metadata is compatible.
17866  * Caller must clear acc's scalar ID.
17867  */
17868 int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc,
17869 		  const struct bpf_reg_state *src)
17870 {
17871 	u16 base, step;
17872 
17873 	if (!bpf_reg_union_compatible(acc, src)) {
17874 		verifier_bug(env, "union of incompatible registers");
17875 		return -EFAULT;
17876 	}
17877 	acc->r64 = cnum64_union(acc->r64, src->r64);
17878 	acc->r32 = cnum32_union(acc->r32, src->r32);
17879 	acc->var_off = tnum_union(acc->var_off, src->var_off);
17880 
17881 	/* Retain a common congruence if the bases agree modulo the gcd. */
17882 	step = gcd(acc->step, src->step);
17883 	base = acc->base % step;
17884 	if (base != src->base % step) {
17885 		reg_step_reset(acc);
17886 	} else {
17887 		acc->base = base;
17888 		acc->step = step;
17889 	}
17890 	reg_bounds_sync(acc);
17891 	return reg_bounds_sanity_check(env, acc, "bpf_reg_union");
17892 }
17893 
17894 /* check validity of 32-bit and 64-bit arithmetic operations */
17895 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
17896 {
17897 	struct bpf_reg_state *regs = cur_regs(env);
17898 	u8 opcode = BPF_OP(insn->code);
17899 	int err;
17900 
17901 	bpf_diag_mod_begin(env, &regs[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE);
17902 
17903 	if (opcode == BPF_END || opcode == BPF_NEG) {
17904 		/* check src operand */
17905 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
17906 		if (err)
17907 			return err;
17908 
17909 		if (is_pointer_value(env, insn->dst_reg)) {
17910 			verbose(env, "R%d pointer arithmetic prohibited\n",
17911 				insn->dst_reg);
17912 			return -EACCES;
17913 		}
17914 
17915 		/* check dest operand */
17916 		if (regs[insn->dst_reg].type == SCALAR_VALUE) {
17917 			err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
17918 			err = err ?: adjust_scalar_min_max_vals(env, insn,
17919 							 &regs[insn->dst_reg],
17920 							 regs[insn->dst_reg]);
17921 		} else {
17922 			err = check_reg_arg(env, insn->dst_reg, DST_OP);
17923 		}
17924 		if (err)
17925 			return err;
17926 
17927 	} else if (opcode == BPF_MOV) {
17928 
17929 		if (BPF_SRC(insn->code) == BPF_X) {
17930 			if (insn->off == BPF_ADDR_SPACE_CAST) {
17931 				if (!env->prog->aux->arena) {
17932 					verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n");
17933 					return -EINVAL;
17934 				}
17935 			}
17936 
17937 			/* check src operand */
17938 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
17939 			if (err)
17940 				return err;
17941 		}
17942 
17943 		/* check dest operand, mark as required later */
17944 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
17945 		if (err)
17946 			return err;
17947 
17948 		if (BPF_SRC(insn->code) == BPF_X) {
17949 			struct bpf_reg_state *src_reg = regs + insn->src_reg;
17950 			struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
17951 
17952 			if (BPF_CLASS(insn->code) == BPF_ALU64) {
17953 				if (insn->imm) {
17954 					/* off == BPF_ADDR_SPACE_CAST */
17955 					mark_reg_unknown(env, regs, insn->dst_reg);
17956 					if (insn->imm == 1) /* cast from as(1) to as(0) */
17957 						dst_reg->type = PTR_TO_ARENA;
17958 				} else if (insn->off == 0) {
17959 					/* case: R1 = R2
17960 					 * copy register state to dest reg
17961 					 */
17962 					assign_scalar_id_before_mov(env, src_reg);
17963 					*dst_reg = *src_reg;
17964 				} else {
17965 					/* case: R1 = (s8, s16 s32)R2 */
17966 					if (is_pointer_value(env, insn->src_reg)) {
17967 						verbose(env,
17968 							"R%d sign-extension part of pointer\n",
17969 							insn->src_reg);
17970 						return -EACCES;
17971 					} else if (src_reg->type == SCALAR_VALUE) {
17972 						bool no_sext;
17973 
17974 						no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1));
17975 						if (no_sext)
17976 							assign_scalar_id_before_mov(env, src_reg);
17977 						*dst_reg = *src_reg;
17978 						if (!no_sext)
17979 							clear_scalar_id(dst_reg);
17980 						coerce_reg_to_size_sx(dst_reg, insn->off >> 3);
17981 					} else {
17982 						mark_reg_unknown(env, regs, insn->dst_reg);
17983 					}
17984 				}
17985 			} else {
17986 				/* R1 = (u32) R2 */
17987 				if (is_pointer_value(env, insn->src_reg)) {
17988 					verbose(env,
17989 						"R%d partial copy of pointer\n",
17990 						insn->src_reg);
17991 					return -EACCES;
17992 				} else if (src_reg->type == SCALAR_VALUE) {
17993 					if (insn->off == 0) {
17994 						bool is_src_reg_u32 = get_reg_width(src_reg) <= 32;
17995 
17996 						if (is_src_reg_u32)
17997 							assign_scalar_id_before_mov(env, src_reg);
17998 						*dst_reg = *src_reg;
17999 						/* Make sure ID is cleared if src_reg is not in u32
18000 						 * range otherwise dst_reg min/max could be incorrectly
18001 						 * propagated into src_reg by sync_linked_regs()
18002 						 */
18003 						if (!is_src_reg_u32)
18004 							clear_scalar_id(dst_reg);
18005 					} else {
18006 						/* case: W1 = (s8, s16)W2 */
18007 						bool no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1));
18008 
18009 						if (no_sext)
18010 							assign_scalar_id_before_mov(env, src_reg);
18011 						*dst_reg = *src_reg;
18012 						if (!no_sext)
18013 							clear_scalar_id(dst_reg);
18014 						coerce_subreg_to_size_sx(dst_reg, insn->off >> 3);
18015 					}
18016 				} else {
18017 					mark_reg_unknown(env, regs,
18018 							 insn->dst_reg);
18019 				}
18020 				zext_32_to_64(dst_reg);
18021 				reg_bounds_sync(dst_reg);
18022 			}
18023 		} else {
18024 			/* case: R = imm
18025 			 * remember the value we stored into this reg
18026 			 */
18027 			/* clear any state __mark_reg_known doesn't set */
18028 			mark_reg_unknown(env, regs, insn->dst_reg);
18029 			regs[insn->dst_reg].type = SCALAR_VALUE;
18030 			if (BPF_CLASS(insn->code) == BPF_ALU64) {
18031 				__mark_reg_known(regs + insn->dst_reg,
18032 						 insn->imm);
18033 			} else {
18034 				__mark_reg_known(regs + insn->dst_reg,
18035 						 (u32)insn->imm);
18036 			}
18037 		}
18038 
18039 	} else {	/* all other ALU ops: and, sub, xor, add, ... */
18040 
18041 		if (BPF_SRC(insn->code) == BPF_X) {
18042 			/* check src1 operand */
18043 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
18044 			if (err)
18045 				return err;
18046 		}
18047 
18048 		/* check src2 operand */
18049 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
18050 		if (err)
18051 			return err;
18052 
18053 		if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
18054 		    BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
18055 			verbose(env, "div by zero\n");
18056 			return -EINVAL;
18057 		}
18058 
18059 		if ((opcode == BPF_LSH || opcode == BPF_RSH ||
18060 		     opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
18061 			int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
18062 
18063 			if (insn->imm < 0 || insn->imm >= size) {
18064 				verbose(env, "invalid shift %d\n", insn->imm);
18065 				return -EINVAL;
18066 			}
18067 		}
18068 
18069 		/* check dest operand */
18070 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
18071 		err = err ?: adjust_reg_min_max_vals(env, insn);
18072 		if (err)
18073 			return err;
18074 	}
18075 
18076 	err = reg_bounds_sanity_check(env, &regs[insn->dst_reg], "alu");
18077 	if (err)
18078 		return err;
18079 
18080 	bpf_diag_mod_end(env);
18081 	return 0;
18082 }
18083 
18084 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
18085 				   struct bpf_reg_state *dst_reg,
18086 				   enum bpf_reg_type type,
18087 				   bool range_right_open)
18088 {
18089 	struct bpf_func_state *state;
18090 	struct bpf_reg_state *reg;
18091 	int new_range;
18092 
18093 	if (reg_umax(dst_reg) == 0 && range_right_open)
18094 		/* This doesn't give us any range */
18095 		return;
18096 
18097 	if (reg_umax(dst_reg) > MAX_PACKET_OFF)
18098 		/* Risk of overflow.  For instance, ptr + (1<<63) may be less
18099 		 * than pkt_end, but that's because it's also less than pkt.
18100 		 */
18101 		return;
18102 
18103 	new_range = reg_umax(dst_reg);
18104 	if (range_right_open)
18105 		new_range++;
18106 
18107 	/* Examples for register markings:
18108 	 *
18109 	 * pkt_data in dst register:
18110 	 *
18111 	 *   r2 = r3;
18112 	 *   r2 += 8;
18113 	 *   if (r2 > pkt_end) goto <handle exception>
18114 	 *   <access okay>
18115 	 *
18116 	 *   r2 = r3;
18117 	 *   r2 += 8;
18118 	 *   if (r2 < pkt_end) goto <access okay>
18119 	 *   <handle exception>
18120 	 *
18121 	 *   Where:
18122 	 *     r2 == dst_reg, pkt_end == src_reg
18123 	 *     r2=pkt(id=n,off=8,r=0)
18124 	 *     r3=pkt(id=n,off=0,r=0)
18125 	 *
18126 	 * pkt_data in src register:
18127 	 *
18128 	 *   r2 = r3;
18129 	 *   r2 += 8;
18130 	 *   if (pkt_end >= r2) goto <access okay>
18131 	 *   <handle exception>
18132 	 *
18133 	 *   r2 = r3;
18134 	 *   r2 += 8;
18135 	 *   if (pkt_end <= r2) goto <handle exception>
18136 	 *   <access okay>
18137 	 *
18138 	 *   Where:
18139 	 *     pkt_end == dst_reg, r2 == src_reg
18140 	 *     r2=pkt(id=n,off=8,r=0)
18141 	 *     r3=pkt(id=n,off=0,r=0)
18142 	 *
18143 	 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8)
18144 	 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8)
18145 	 * and [r3, r3 + 8-1) respectively is safe to access depending on
18146 	 * the check.
18147 	 */
18148 
18149 	/* If our ids match, then we must have the same max_value.  And we
18150 	 * don't care about the other reg's fixed offset, since if it's too big
18151 	 * the range won't allow anything.
18152 	 * reg_umax(dst_reg) is known < MAX_PACKET_OFF, therefore it fits in a u16.
18153 	 */
18154 	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
18155 		if (reg->type == type && reg->id == dst_reg->id)
18156 			/* keep the maximum range already checked */
18157 			reg->range = max(reg->range, new_range);
18158 	}));
18159 }
18160 
18161 static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
18162 				u8 opcode, bool is_jmp32);
18163 /*
18164  * Learn more information about live branches by simulating refinement on both branches.
18165  * regs_refine_cond_op() is sound, so producing ill-formed register bounds for the branch means
18166  * that branch is dead.
18167  */
18168 static int simulate_both_branches_taken(struct bpf_verifier_env *env, u8 opcode, bool is_jmp32)
18169 {
18170 	/* Fallthrough (FALSE) branch */
18171 	regs_refine_cond_op(&env->false_reg1, &env->false_reg2, bpf_rev_opcode(opcode), is_jmp32);
18172 	reg_bounds_sync(&env->false_reg1);
18173 	reg_bounds_sync(&env->false_reg2);
18174 	/*
18175 	 * If there is a range bounds violation in *any* of the abstract values in either
18176 	 * reg_states in the FALSE branch (i.e. reg1, reg2), the FALSE branch must be dead. Only
18177 	 * TRUE branch will be taken.
18178 	 */
18179 	if (range_bounds_violation(&env->false_reg1) || range_bounds_violation(&env->false_reg2))
18180 		return 1;
18181 
18182 	/* Jump (TRUE) branch */
18183 	regs_refine_cond_op(&env->true_reg1, &env->true_reg2, opcode, is_jmp32);
18184 	reg_bounds_sync(&env->true_reg1);
18185 	reg_bounds_sync(&env->true_reg2);
18186 	/*
18187 	 * If there is a range bounds violation in *any* of the abstract values in either
18188 	 * reg_states in the TRUE branch (i.e. true_reg1, true_reg2), the TRUE branch must be dead.
18189 	 * Only FALSE branch will be taken.
18190 	 */
18191 	if (range_bounds_violation(&env->true_reg1) || range_bounds_violation(&env->true_reg2))
18192 		return 0;
18193 
18194 	/* Both branches are possible, we can't determine which one will be taken. */
18195 	return -1;
18196 }
18197 
18198 /*
18199  * <reg1> <op> <reg2>, currently assuming reg2 is a constant
18200  */
18201 static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *reg1,
18202 				  struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32)
18203 {
18204 	struct tnum t1 = is_jmp32 ? tnum_subreg(reg1->var_off) : reg1->var_off;
18205 	struct tnum t2 = is_jmp32 ? tnum_subreg(reg2->var_off) : reg2->var_off;
18206 	u64 umin1 = is_jmp32 ? (u64)reg_u32_min(reg1) : reg_umin(reg1);
18207 	u64 umax1 = is_jmp32 ? (u64)reg_u32_max(reg1) : reg_umax(reg1);
18208 	s64 smin1 = is_jmp32 ? (s64)reg_s32_min(reg1) : reg_smin(reg1);
18209 	s64 smax1 = is_jmp32 ? (s64)reg_s32_max(reg1) : reg_smax(reg1);
18210 	u64 umin2 = is_jmp32 ? (u64)reg_u32_min(reg2) : reg_umin(reg2);
18211 	u64 umax2 = is_jmp32 ? (u64)reg_u32_max(reg2) : reg_umax(reg2);
18212 	s64 smin2 = is_jmp32 ? (s64)reg_s32_min(reg2) : reg_smin(reg2);
18213 	s64 smax2 = is_jmp32 ? (s64)reg_s32_max(reg2) : reg_smax(reg2);
18214 
18215 	if (reg1 == reg2) {
18216 		switch (opcode) {
18217 		case BPF_JGE:
18218 		case BPF_JLE:
18219 		case BPF_JSGE:
18220 		case BPF_JSLE:
18221 		case BPF_JEQ:
18222 			return 1;
18223 		case BPF_JGT:
18224 		case BPF_JLT:
18225 		case BPF_JSGT:
18226 		case BPF_JSLT:
18227 		case BPF_JNE:
18228 			return 0;
18229 		case BPF_JSET:
18230 			if (tnum_is_const(t1))
18231 				return t1.value != 0;
18232 			else
18233 				return (smin1 <= 0 && smax1 >= 0) ? -1 : 1;
18234 		default:
18235 			return -1;
18236 		}
18237 	}
18238 
18239 	switch (opcode) {
18240 	case BPF_JEQ:
18241 		/* constants, umin/umax and smin/smax checks would be
18242 		 * redundant in this case because they all should match
18243 		 */
18244 		if (tnum_is_const(t1) && tnum_is_const(t2))
18245 			return t1.value == t2.value;
18246 		if (!tnum_overlap(t1, t2))
18247 			return 0;
18248 		/* non-overlapping ranges */
18249 		if (umin1 > umax2 || umax1 < umin2)
18250 			return 0;
18251 		if (smin1 > smax2 || smax1 < smin2)
18252 			return 0;
18253 		if (!is_jmp32) {
18254 			/* if 64-bit ranges are inconclusive, see if we can
18255 			 * utilize 32-bit subrange knowledge to eliminate
18256 			 * branches that can't be taken a priori
18257 			 */
18258 			if (reg_u32_min(reg1) > reg_u32_max(reg2) ||
18259 			    reg_u32_max(reg1) < reg_u32_min(reg2))
18260 				return 0;
18261 			if (reg_s32_min(reg1) > reg_s32_max(reg2) ||
18262 			    reg_s32_max(reg1) < reg_s32_min(reg2))
18263 				return 0;
18264 		}
18265 		break;
18266 	case BPF_JNE:
18267 		/* constants, umin/umax and smin/smax checks would be
18268 		 * redundant in this case because they all should match
18269 		 */
18270 		if (tnum_is_const(t1) && tnum_is_const(t2))
18271 			return t1.value != t2.value;
18272 		if (!tnum_overlap(t1, t2))
18273 			return 1;
18274 		/* non-overlapping ranges */
18275 		if (umin1 > umax2 || umax1 < umin2)
18276 			return 1;
18277 		if (smin1 > smax2 || smax1 < smin2)
18278 			return 1;
18279 		if (!is_jmp32) {
18280 			/* if 64-bit ranges are inconclusive, see if we can
18281 			 * utilize 32-bit subrange knowledge to eliminate
18282 			 * branches that can't be taken a priori
18283 			 */
18284 			if (reg_u32_min(reg1) > reg_u32_max(reg2) ||
18285 			    reg_u32_max(reg1) < reg_u32_min(reg2))
18286 				return 1;
18287 			if (reg_s32_min(reg1) > reg_s32_max(reg2) ||
18288 			    reg_s32_max(reg1) < reg_s32_min(reg2))
18289 				return 1;
18290 		}
18291 		break;
18292 	case BPF_JSET:
18293 		if (!is_reg_const(reg2, is_jmp32)) {
18294 			swap(reg1, reg2);
18295 			swap(t1, t2);
18296 		}
18297 		if (!is_reg_const(reg2, is_jmp32))
18298 			return -1;
18299 		if ((~t1.mask & t1.value) & t2.value)
18300 			return 1;
18301 		if (!((t1.mask | t1.value) & t2.value))
18302 			return 0;
18303 		break;
18304 	case BPF_JGT:
18305 		if (umin1 > umax2)
18306 			return 1;
18307 		else if (umax1 <= umin2)
18308 			return 0;
18309 		break;
18310 	case BPF_JSGT:
18311 		if (smin1 > smax2)
18312 			return 1;
18313 		else if (smax1 <= smin2)
18314 			return 0;
18315 		break;
18316 	case BPF_JLT:
18317 		if (umax1 < umin2)
18318 			return 1;
18319 		else if (umin1 >= umax2)
18320 			return 0;
18321 		break;
18322 	case BPF_JSLT:
18323 		if (smax1 < smin2)
18324 			return 1;
18325 		else if (smin1 >= smax2)
18326 			return 0;
18327 		break;
18328 	case BPF_JGE:
18329 		if (umin1 >= umax2)
18330 			return 1;
18331 		else if (umax1 < umin2)
18332 			return 0;
18333 		break;
18334 	case BPF_JSGE:
18335 		if (smin1 >= smax2)
18336 			return 1;
18337 		else if (smax1 < smin2)
18338 			return 0;
18339 		break;
18340 	case BPF_JLE:
18341 		if (umax1 <= umin2)
18342 			return 1;
18343 		else if (umin1 > umax2)
18344 			return 0;
18345 		break;
18346 	case BPF_JSLE:
18347 		if (smax1 <= smin2)
18348 			return 1;
18349 		else if (smin1 > smax2)
18350 			return 0;
18351 		break;
18352 	}
18353 
18354 	return simulate_both_branches_taken(env, opcode, is_jmp32);
18355 }
18356 
18357 int bpf_flip_opcode(u32 opcode)
18358 {
18359 	/* How can we transform "a <op> b" into "b <op> a"? */
18360 	static const u8 opcode_flip[16] = {
18361 		/* these stay the same */
18362 		[BPF_JEQ  >> 4] = BPF_JEQ,
18363 		[BPF_JNE  >> 4] = BPF_JNE,
18364 		[BPF_JSET >> 4] = BPF_JSET,
18365 		/* these swap "lesser" and "greater" (L and G in the opcodes) */
18366 		[BPF_JGE  >> 4] = BPF_JLE,
18367 		[BPF_JGT  >> 4] = BPF_JLT,
18368 		[BPF_JLE  >> 4] = BPF_JGE,
18369 		[BPF_JLT  >> 4] = BPF_JGT,
18370 		[BPF_JSGE >> 4] = BPF_JSLE,
18371 		[BPF_JSGT >> 4] = BPF_JSLT,
18372 		[BPF_JSLE >> 4] = BPF_JSGE,
18373 		[BPF_JSLT >> 4] = BPF_JSGT
18374 	};
18375 	return opcode_flip[opcode >> 4];
18376 }
18377 
18378 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,
18379 				   struct bpf_reg_state *src_reg,
18380 				   u8 opcode)
18381 {
18382 	struct bpf_reg_state *pkt;
18383 
18384 	if (src_reg->type == PTR_TO_PACKET_END) {
18385 		pkt = dst_reg;
18386 	} else if (dst_reg->type == PTR_TO_PACKET_END) {
18387 		pkt = src_reg;
18388 		opcode = bpf_flip_opcode(opcode);
18389 	} else {
18390 		return -1;
18391 	}
18392 
18393 	if (pkt->range >= 0)
18394 		return -1;
18395 
18396 	switch (opcode) {
18397 	case BPF_JLE:
18398 		/* pkt <= pkt_end */
18399 		fallthrough;
18400 	case BPF_JGT:
18401 		/* pkt > pkt_end */
18402 		if (pkt->range == BEYOND_PKT_END)
18403 			/* pkt has at last one extra byte beyond pkt_end */
18404 			return opcode == BPF_JGT;
18405 		break;
18406 	case BPF_JLT:
18407 		/* pkt < pkt_end */
18408 		fallthrough;
18409 	case BPF_JGE:
18410 		/* pkt >= pkt_end */
18411 		if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END)
18412 			return opcode == BPF_JGE;
18413 		break;
18414 	}
18415 	return -1;
18416 }
18417 
18418 /* compute branch direction of the expression "if (<reg1> opcode <reg2>) goto target;"
18419  * and return:
18420  *  1 - branch will be taken and "goto target" will be executed
18421  *  0 - branch will not be taken and fall-through to next insn
18422  * -1 - unknown. Example: "if (reg1 < 5)" is unknown when register value
18423  *      range [0,10]
18424  */
18425 static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *reg1,
18426 			   struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32)
18427 {
18428 	if (reg_is_pkt_pointer_any(reg1) && reg_is_pkt_pointer_any(reg2) && !is_jmp32)
18429 		return is_pkt_ptr_branch_taken(reg1, reg2, opcode);
18430 
18431 	if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) {
18432 		u64 val;
18433 
18434 		/*
18435 		 * The low 32 bits of a valid pointer may well be zero, hence
18436 		 * nothing below applies to a 32-bit comparison.
18437 		 */
18438 		if (is_jmp32)
18439 			return -1;
18440 
18441 		/* arrange that reg2 is a scalar, and reg1 is a pointer */
18442 		if (!is_reg_const(reg2, is_jmp32)) {
18443 			opcode = bpf_flip_opcode(opcode);
18444 			swap(reg1, reg2);
18445 		}
18446 		/* and ensure that reg2 is a constant */
18447 		if (!is_reg_const(reg2, is_jmp32))
18448 			return -1;
18449 
18450 		if (!reg_not_null(env, reg1))
18451 			return -1;
18452 
18453 		/* If pointer is valid tests against zero will fail so we can
18454 		 * use this to direct branch taken.
18455 		 */
18456 		val = reg_const_value(reg2, is_jmp32);
18457 		if (val != 0)
18458 			return -1;
18459 
18460 		switch (opcode) {
18461 		case BPF_JEQ:
18462 			return 0;
18463 		case BPF_JNE:
18464 			return 1;
18465 		default:
18466 			return -1;
18467 		}
18468 	}
18469 
18470 	/* now deal with two scalars, but not necessarily constants */
18471 	return is_scalar_branch_taken(env, reg1, reg2, opcode, is_jmp32);
18472 }
18473 
18474 /* Opcode that corresponds to a *false* branch condition.
18475  * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2
18476  */
18477 u8 bpf_rev_opcode(u8 opcode)
18478 {
18479 	switch (opcode) {
18480 	case BPF_JEQ:		return BPF_JNE;
18481 	case BPF_JNE:		return BPF_JEQ;
18482 	/* JSET doesn't have it's reverse opcode in BPF, so add
18483 	 * BPF_X flag to denote the reverse of that operation
18484 	 */
18485 	case BPF_JSET:		return BPF_JSET | BPF_X;
18486 	case BPF_JSET | BPF_X:	return BPF_JSET;
18487 	case BPF_JGE:		return BPF_JLT;
18488 	case BPF_JGT:		return BPF_JLE;
18489 	case BPF_JLE:		return BPF_JGT;
18490 	case BPF_JLT:		return BPF_JGE;
18491 	case BPF_JSGE:		return BPF_JSLT;
18492 	case BPF_JSGT:		return BPF_JSLE;
18493 	case BPF_JSLE:		return BPF_JSGT;
18494 	case BPF_JSLT:		return BPF_JSGE;
18495 	default:		return 0;
18496 	}
18497 }
18498 
18499 /* Refine range knowledge for <reg1> <op> <reg>2 conditional operation. */
18500 static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
18501 				u8 opcode, bool is_jmp32)
18502 {
18503 	struct tnum t;
18504 	u64 val;
18505 
18506 	/* In case of GE/GT/SGE/JST, reuse LE/LT/SLE/SLT logic from below */
18507 	switch (opcode) {
18508 	case BPF_JGE:
18509 	case BPF_JGT:
18510 	case BPF_JSGE:
18511 	case BPF_JSGT:
18512 		opcode = bpf_flip_opcode(opcode);
18513 		swap(reg1, reg2);
18514 		break;
18515 	default:
18516 		break;
18517 	}
18518 
18519 	switch (opcode) {
18520 	case BPF_JEQ:
18521 		if (is_jmp32) {
18522 			reg1->r32 = cnum32_intersect(reg1->r32, reg2->r32);
18523 			reg2->r32 = reg1->r32;
18524 
18525 			t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off));
18526 			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
18527 			reg2->var_off = tnum_with_subreg(reg2->var_off, t);
18528 		} else {
18529 			reg1->r64 = cnum64_intersect(reg1->r64, reg2->r64);
18530 			reg2->r64 = reg1->r64;
18531 
18532 			reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off);
18533 			reg2->var_off = reg1->var_off;
18534 		}
18535 		break;
18536 	case BPF_JNE:
18537 		if (!is_reg_const(reg2, is_jmp32))
18538 			swap(reg1, reg2);
18539 		if (!is_reg_const(reg2, is_jmp32))
18540 			break;
18541 
18542 		/* try to recompute the bound of reg1 if reg2 is a const and
18543 		 * is exactly the edge of reg1.
18544 		 */
18545 		val = reg_const_value(reg2, is_jmp32);
18546 		if (is_jmp32) {
18547 			/* Complement of the range [val, val] as cnum32. */
18548 			cnum32_intersect_with(&reg1->r32, (struct cnum32){ val + 1, U32_MAX - 1 });
18549 		} else {
18550 			/* Complement of the range [val, val] as cnum64. */
18551 			cnum64_intersect_with(&reg1->r64, (struct cnum64){ val + 1, U64_MAX - 1 });
18552 		}
18553 		break;
18554 	case BPF_JSET:
18555 		if (!is_reg_const(reg2, is_jmp32))
18556 			swap(reg1, reg2);
18557 		if (!is_reg_const(reg2, is_jmp32))
18558 			break;
18559 		val = reg_const_value(reg2, is_jmp32);
18560 		/* BPF_JSET (i.e., TRUE branch, *not* BPF_JSET | BPF_X)
18561 		 * requires single bit to learn something useful. E.g., if we
18562 		 * know that `r1 & 0x3` is true, then which bits (0, 1, or both)
18563 		 * are actually set? We can learn something definite only if
18564 		 * it's a single-bit value to begin with.
18565 		 *
18566 		 * BPF_JSET | BPF_X (i.e., negation of BPF_JSET) doesn't have
18567 		 * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor
18568 		 * bit 1 is set, which we can readily use in adjustments.
18569 		 */
18570 		if (hweight64(val) != 1)
18571 			break;
18572 		if (is_jmp32) {
18573 			t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val));
18574 			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
18575 		} else {
18576 			reg1->var_off = tnum_or(reg1->var_off, tnum_const(val));
18577 		}
18578 		break;
18579 	case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see bpf_rev_opcode() */
18580 		if (!is_reg_const(reg2, is_jmp32))
18581 			swap(reg1, reg2);
18582 		if (!is_reg_const(reg2, is_jmp32))
18583 			break;
18584 		val = reg_const_value(reg2, is_jmp32);
18585 		/* Forget the ranges before narrowing tnums, to avoid invariant
18586 		 * violations if we're on a dead branch.
18587 		 */
18588 		__mark_reg_unbounded(reg1);
18589 		reg_step_reset(reg1);
18590 		if (is_jmp32) {
18591 			t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val));
18592 			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
18593 		} else {
18594 			reg1->var_off = tnum_and(reg1->var_off, tnum_const(~val));
18595 		}
18596 		break;
18597 	case BPF_JLE:
18598 		if (is_jmp32) {
18599 			cnum32_intersect_with_urange(&reg1->r32, 0, reg_u32_max(reg2));
18600 			cnum32_intersect_with_urange(&reg2->r32, reg_u32_min(reg1), U32_MAX);
18601 		} else {
18602 			cnum64_intersect_with_urange(&reg1->r64, 0, reg_umax(reg2));
18603 			cnum64_intersect_with_urange(&reg2->r64, reg_umin(reg1), U64_MAX);
18604 		}
18605 		break;
18606 	case BPF_JLT:
18607 		if (is_jmp32) {
18608 			cnum32_intersect_with_urange(&reg1->r32, 0, reg_u32_max(reg2) - 1);
18609 			cnum32_intersect_with_urange(&reg2->r32, reg_u32_min(reg1) + 1, U32_MAX);
18610 		} else {
18611 			cnum64_intersect_with_urange(&reg1->r64, 0, reg_umax(reg2) - 1);
18612 			cnum64_intersect_with_urange(&reg2->r64, reg_umin(reg1) + 1, U64_MAX);
18613 		}
18614 		break;
18615 	case BPF_JSLE:
18616 		if (is_jmp32) {
18617 			cnum32_intersect_with_srange(&reg1->r32, S32_MIN, reg_s32_max(reg2));
18618 			cnum32_intersect_with_srange(&reg2->r32, reg_s32_min(reg1), S32_MAX);
18619 		} else {
18620 			cnum64_intersect_with_srange(&reg1->r64, S64_MIN, reg_smax(reg2));
18621 			cnum64_intersect_with_srange(&reg2->r64, reg_smin(reg1), S64_MAX);
18622 		}
18623 		break;
18624 	case BPF_JSLT:
18625 		if (is_jmp32) {
18626 			cnum32_intersect_with_srange(&reg1->r32, S32_MIN, reg_s32_max(reg2) - 1);
18627 			cnum32_intersect_with_srange(&reg2->r32, reg_s32_min(reg1) + 1, S32_MAX);
18628 		} else {
18629 			cnum64_intersect_with_srange(&reg1->r64, S64_MIN, reg_smax(reg2) - 1);
18630 			cnum64_intersect_with_srange(&reg2->r64, reg_smin(reg1) + 1, S64_MAX);
18631 		}
18632 		break;
18633 	default:
18634 		return;
18635 	}
18636 }
18637 
18638 /* Check for invariant violations on the registers for both branches of a condition */
18639 static int regs_bounds_sanity_check_branches(struct bpf_verifier_env *env)
18640 {
18641 	int err;
18642 
18643 	err = reg_bounds_sanity_check(env, &env->true_reg1, "true_reg1");
18644 	err = err ?: reg_bounds_sanity_check(env, &env->true_reg2, "true_reg2");
18645 	err = err ?: reg_bounds_sanity_check(env, &env->false_reg1, "false_reg1");
18646 	err = err ?: reg_bounds_sanity_check(env, &env->false_reg2, "false_reg2");
18647 	return err;
18648 }
18649 
18650 static void mark_ptr_or_null_reg(struct bpf_func_state *state,
18651 				 struct bpf_reg_state *reg, u32 id,
18652 				 bool is_null)
18653 {
18654 	if (type_may_be_null(reg->type) && reg->id == id &&
18655 	    (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) {
18656 		/* Old offset should have been known-zero, because we don't
18657 		 * allow pointer arithmetic on pointers that might be NULL.
18658 		 * If we see this happening, don't convert the register.
18659 		 *
18660 		 * But in some cases, some helpers that return local kptrs
18661 		 * advance offset for the returned pointer. In those cases,
18662 		 * it is fine to expect to see reg->var_off.
18663 		 */
18664 		if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) &&
18665 		    WARN_ON_ONCE(!tnum_equals_const(reg->var_off, 0)))
18666 			return;
18667 		if (is_null) {
18668 			/* We don't need id from this point
18669 			 * onwards anymore, thus we should better reset it,
18670 			 * so that state pruning has chances to take effect.
18671 			 */
18672 			__mark_reg_known_zero(reg);
18673 			reg->type = SCALAR_VALUE;
18674 
18675 			return;
18676 		}
18677 
18678 		mark_ptr_not_null_reg(reg);
18679 
18680 		/*
18681 		 * reg->id is preserved for object relationship tracking
18682 		 * and spin_lock lock state tracking
18683 		 */
18684 	}
18685 }
18686 
18687 /* The logic is similar to find_good_pkt_pointers(), both could eventually
18688  * be folded together at some point.
18689  */
18690 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
18691 				  bool is_null)
18692 {
18693 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
18694 	struct bpf_reg_state *regs = state->regs, *reg;
18695 	u32 id = regs[regno].id;
18696 
18697 	if (is_null && find_reference_state(vstate, id))
18698 		/* regs[regno] is in the " == NULL" branch.
18699 		 * No one could have freed the reference state before
18700 		 * doing the NULL check.
18701 		 */
18702 		WARN_ON_ONCE(__release_reference_nomark(vstate, id));
18703 
18704 	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
18705 		mark_ptr_or_null_reg(state, reg, id, is_null);
18706 	}));
18707 }
18708 
18709 static bool try_match_pkt_pointers(const struct bpf_insn *insn,
18710 				   struct bpf_reg_state *dst_reg,
18711 				   struct bpf_reg_state *src_reg,
18712 				   struct bpf_verifier_state *this_branch,
18713 				   struct bpf_verifier_state *other_branch)
18714 {
18715 	if (BPF_SRC(insn->code) != BPF_X)
18716 		return false;
18717 
18718 	/* Pointers are always 64-bit. */
18719 	if (BPF_CLASS(insn->code) == BPF_JMP32)
18720 		return false;
18721 
18722 	switch (BPF_OP(insn->code)) {
18723 	case BPF_JGT:
18724 		if ((dst_reg->type == PTR_TO_PACKET &&
18725 		     src_reg->type == PTR_TO_PACKET_END) ||
18726 		    (dst_reg->type == PTR_TO_PACKET_META &&
18727 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
18728 			/* pkt_data' > pkt_end, pkt_meta' > pkt_data */
18729 			find_good_pkt_pointers(this_branch, dst_reg,
18730 					       dst_reg->type, false);
18731 			mark_pkt_end(other_branch, insn->dst_reg, true);
18732 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
18733 			    src_reg->type == PTR_TO_PACKET) ||
18734 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
18735 			    src_reg->type == PTR_TO_PACKET_META)) {
18736 			/* pkt_end > pkt_data', pkt_data > pkt_meta' */
18737 			find_good_pkt_pointers(other_branch, src_reg,
18738 					       src_reg->type, true);
18739 			mark_pkt_end(this_branch, insn->src_reg, false);
18740 		} else {
18741 			return false;
18742 		}
18743 		break;
18744 	case BPF_JLT:
18745 		if ((dst_reg->type == PTR_TO_PACKET &&
18746 		     src_reg->type == PTR_TO_PACKET_END) ||
18747 		    (dst_reg->type == PTR_TO_PACKET_META &&
18748 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
18749 			/* pkt_data' < pkt_end, pkt_meta' < pkt_data */
18750 			find_good_pkt_pointers(other_branch, dst_reg,
18751 					       dst_reg->type, true);
18752 			mark_pkt_end(this_branch, insn->dst_reg, false);
18753 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
18754 			    src_reg->type == PTR_TO_PACKET) ||
18755 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
18756 			    src_reg->type == PTR_TO_PACKET_META)) {
18757 			/* pkt_end < pkt_data', pkt_data > pkt_meta' */
18758 			find_good_pkt_pointers(this_branch, src_reg,
18759 					       src_reg->type, false);
18760 			mark_pkt_end(other_branch, insn->src_reg, true);
18761 		} else {
18762 			return false;
18763 		}
18764 		break;
18765 	case BPF_JGE:
18766 		if ((dst_reg->type == PTR_TO_PACKET &&
18767 		     src_reg->type == PTR_TO_PACKET_END) ||
18768 		    (dst_reg->type == PTR_TO_PACKET_META &&
18769 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
18770 			/* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */
18771 			find_good_pkt_pointers(this_branch, dst_reg,
18772 					       dst_reg->type, true);
18773 			mark_pkt_end(other_branch, insn->dst_reg, false);
18774 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
18775 			    src_reg->type == PTR_TO_PACKET) ||
18776 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
18777 			    src_reg->type == PTR_TO_PACKET_META)) {
18778 			/* pkt_end >= pkt_data', pkt_data >= pkt_meta' */
18779 			find_good_pkt_pointers(other_branch, src_reg,
18780 					       src_reg->type, false);
18781 			mark_pkt_end(this_branch, insn->src_reg, true);
18782 		} else {
18783 			return false;
18784 		}
18785 		break;
18786 	case BPF_JLE:
18787 		if ((dst_reg->type == PTR_TO_PACKET &&
18788 		     src_reg->type == PTR_TO_PACKET_END) ||
18789 		    (dst_reg->type == PTR_TO_PACKET_META &&
18790 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
18791 			/* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */
18792 			find_good_pkt_pointers(other_branch, dst_reg,
18793 					       dst_reg->type, false);
18794 			mark_pkt_end(this_branch, insn->dst_reg, true);
18795 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
18796 			    src_reg->type == PTR_TO_PACKET) ||
18797 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
18798 			    src_reg->type == PTR_TO_PACKET_META)) {
18799 			/* pkt_end <= pkt_data', pkt_data <= pkt_meta' */
18800 			find_good_pkt_pointers(this_branch, src_reg,
18801 					       src_reg->type, true);
18802 			mark_pkt_end(other_branch, insn->src_reg, false);
18803 		} else {
18804 			return false;
18805 		}
18806 		break;
18807 	default:
18808 		return false;
18809 	}
18810 
18811 	return true;
18812 }
18813 
18814 static void __collect_linked_regs(struct linked_regs *reg_set, struct bpf_reg_state *reg,
18815 				  u32 id, u32 frameno, u32 spi_or_reg, bool is_reg)
18816 {
18817 	struct linked_reg *e;
18818 
18819 	if (reg->type != SCALAR_VALUE || (reg->id & ~BPF_ADD_CONST) != id)
18820 		return;
18821 
18822 	e = linked_regs_push(reg_set);
18823 	if (e) {
18824 		e->frameno = frameno;
18825 		e->is_reg = is_reg;
18826 		e->regno = spi_or_reg;
18827 	} else {
18828 		clear_scalar_id(reg);
18829 	}
18830 }
18831 
18832 /* For all R being scalar registers or spilled scalar registers
18833  * in verifier state, save R in linked_regs if R->id == id.
18834  * If there are too many Rs sharing same id, reset id for leftover Rs.
18835  */
18836 static void collect_linked_regs(struct bpf_verifier_env *env,
18837 				struct bpf_verifier_state *vstate,
18838 				u32 id,
18839 				struct linked_regs *linked_regs)
18840 {
18841 	struct bpf_insn_aux_data *aux = env->insn_aux_data;
18842 	struct bpf_func_state *func;
18843 	struct bpf_reg_state *reg;
18844 	u16 live_regs;
18845 	int i, j;
18846 
18847 	id = id & ~BPF_ADD_CONST;
18848 	for (i = vstate->curframe; i >= 0; i--) {
18849 		live_regs = aux[bpf_frame_insn_idx(vstate, i)].live_regs_before;
18850 		func = vstate->frame[i];
18851 		for (j = 0; j < BPF_REG_FP; j++) {
18852 			if (!(live_regs & BIT(j)))
18853 				continue;
18854 			reg = &func->regs[j];
18855 			__collect_linked_regs(linked_regs, reg, id, i, j, true);
18856 		}
18857 		for (j = 0; j < bpf_stack_nr_slots(func); j++) {
18858 			struct bpf_stack_state *ss = bpf_stack_slot(func, j);
18859 
18860 			if (!bpf_is_spilled_reg(ss))
18861 				continue;
18862 			reg = &ss->spilled_ptr;
18863 			__collect_linked_regs(linked_regs, reg, id, i, j, false);
18864 		}
18865 	}
18866 }
18867 
18868 /* For all R in linked_regs, copy known_reg range into R
18869  * if R->id == known_reg->id.
18870  */
18871 static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *vstate,
18872 			     struct bpf_reg_state *known_reg, struct linked_regs *linked_regs)
18873 {
18874 	struct bpf_reg_state fake_reg;
18875 	struct bpf_reg_state *reg;
18876 	struct linked_reg *e;
18877 	int i;
18878 
18879 	for (i = 0; i < linked_regs->cnt; ++i) {
18880 		e = &linked_regs->entries[i];
18881 		reg = e->is_reg ? &vstate->frame[e->frameno]->regs[e->regno]
18882 				: &bpf_stack_slot(vstate->frame[e->frameno], e->spi)->spilled_ptr;
18883 		if (reg->type != SCALAR_VALUE || reg == known_reg)
18884 			continue;
18885 		if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST))
18886 			continue;
18887 		/*
18888 		 * Skip mixed 32/64-bit links: the delta relationship doesn't
18889 		 * hold across different ALU widths.
18890 		 */
18891 		if (((reg->id ^ known_reg->id) & BPF_ADD_CONST) == BPF_ADD_CONST)
18892 			continue;
18893 		if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) ||
18894 		    reg->delta == known_reg->delta) {
18895 			*reg = *known_reg;
18896 		} else {
18897 			s32 saved_off = reg->delta;
18898 			u32 saved_id = reg->id;
18899 
18900 			fake_reg.type = SCALAR_VALUE;
18901 			__mark_reg_known(&fake_reg, (s64)reg->delta - (s64)known_reg->delta);
18902 
18903 			/* reg = known_reg; reg += delta */
18904 			*reg = *known_reg;
18905 			/*
18906 			 * Must preserve off and id, otherwise another sync_linked_regs()
18907 			 * will be incorrect.
18908 			 */
18909 			reg->delta = saved_off;
18910 			reg->id = saved_id;
18911 
18912 			scalar_step_add(reg, reg, &fake_reg);
18913 			scalar32_min_max_add(reg, &fake_reg);
18914 			scalar_min_max_add(reg, &fake_reg);
18915 			reg->var_off = tnum_add(reg->var_off, fake_reg.var_off);
18916 			if ((reg->id | known_reg->id) & BPF_ADD_CONST32)
18917 				zext_32_to_64(reg);
18918 			reg_bounds_sync(reg);
18919 		}
18920 		if (e->is_reg)
18921 			mark_reg_scratched(env, e->regno);
18922 		else
18923 			mark_stack_slot_scratched(env, e->spi);
18924 	}
18925 }
18926 
18927 static int check_cond_jmp_op(struct bpf_verifier_env *env,
18928 			     struct bpf_insn *insn, int *insn_idx)
18929 {
18930 	struct bpf_verifier_state *this_branch = env->cur_state;
18931 	struct bpf_verifier_state *other_branch;
18932 	struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
18933 	struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;
18934 	struct bpf_reg_state *eq_branch_regs;
18935 	struct linked_regs linked_regs = {};
18936 	u8 opcode = BPF_OP(insn->code);
18937 	int insn_flags = 0;
18938 	bool is_jmp32;
18939 	int pred = -1;
18940 	int err;
18941 
18942 	/* Only conditional jumps are expected to reach here. */
18943 	if (opcode == BPF_JA || opcode > BPF_JCOND) {
18944 		verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
18945 		return -EINVAL;
18946 	}
18947 
18948 	if (bpf_is_unwind_goto(insn)) {
18949 		if (!env->cur_state->unwound)
18950 			return 0;
18951 		env->cur_state->unwound = false;
18952 		/*
18953 		 * unwind_exit changed its return address, so the return is
18954 		 * mispredicted and what follows unwind_goto runs speculatively.
18955 		 */
18956 		if (!env->bypass_spec_v1) {
18957 			err = sanitize_speculative_path(env, NULL, *insn_idx + 1, *insn_idx);
18958 			if (err)
18959 				return err;
18960 		}
18961 		*insn_idx += insn->off;
18962 		return 0;
18963 	}
18964 
18965 	if (opcode == BPF_JCOND) {
18966 		struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;
18967 		int idx = *insn_idx;
18968 
18969 		prev_st = find_prev_entry(env, cur_st->parent, idx);
18970 
18971 		/* branch out 'fallthrough' insn as a new state to explore */
18972 		queued_st = push_stack(env, idx + 1, idx, false);
18973 		if (IS_ERR(queued_st))
18974 			return PTR_ERR(queued_st);
18975 
18976 		queued_st->may_goto_depth++;
18977 		if (prev_st)
18978 			widen_imprecise_scalars(env, prev_st, queued_st);
18979 		*insn_idx += insn->off;
18980 		return 0;
18981 	}
18982 
18983 	/* check src2 operand */
18984 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
18985 	if (err)
18986 		return err;
18987 
18988 	dst_reg = &regs[insn->dst_reg];
18989 	if (BPF_SRC(insn->code) == BPF_X) {
18990 		/* check src1 operand */
18991 		err = check_reg_arg(env, insn->src_reg, SRC_OP);
18992 		if (err)
18993 			return err;
18994 
18995 		src_reg = &regs[insn->src_reg];
18996 		if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) &&
18997 		    is_pointer_value(env, insn->src_reg)) {
18998 			verbose(env, "R%d pointer comparison prohibited\n",
18999 				insn->src_reg);
19000 			return -EACCES;
19001 		}
19002 
19003 		if (src_reg->type == PTR_TO_STACK)
19004 			insn_flags |= INSN_F_SRC_REG_STACK;
19005 		if (dst_reg->type == PTR_TO_STACK)
19006 			insn_flags |= INSN_F_DST_REG_STACK;
19007 	} else {
19008 		src_reg = &env->fake_reg[0];
19009 		memset(src_reg, 0, sizeof(*src_reg));
19010 		src_reg->type = SCALAR_VALUE;
19011 		__mark_reg_known(src_reg, insn->imm);
19012 
19013 		if (dst_reg->type == PTR_TO_STACK)
19014 			insn_flags |= INSN_F_DST_REG_STACK;
19015 	}
19016 
19017 	if (insn_flags) {
19018 		err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, NULL, 0);
19019 		if (err)
19020 			return err;
19021 	}
19022 
19023 	/*
19024 	 * Collect the linked registers before env->{true,false}_reg{1,2} setup,
19025 	 * otherwise ids dropped by collect_linked_regs() would be resurrected
19026 	 * when env->{true,false}_reg{1,2} are copied back.
19027 	 */
19028 	if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id)
19029 		collect_linked_regs(env, this_branch, src_reg->id, &linked_regs);
19030 	if (dst_reg->type == SCALAR_VALUE && dst_reg->id)
19031 		collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs);
19032 
19033 	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
19034 	env->false_reg1 = *dst_reg;
19035 	env->false_reg2 = *src_reg;
19036 	env->true_reg1 = *dst_reg;
19037 	env->true_reg2 = *src_reg;
19038 	pred = is_branch_taken(env, dst_reg, src_reg, opcode, is_jmp32);
19039 	if (pred >= 0) {
19040 		/* If we get here with a dst_reg pointer type it is because
19041 		 * above is_branch_taken() special cased the 0 comparison.
19042 		 */
19043 		if (!__is_pointer_value(false, dst_reg))
19044 			err = mark_chain_precision(env, insn->dst_reg);
19045 		if (BPF_SRC(insn->code) == BPF_X && !err &&
19046 		    !__is_pointer_value(false, src_reg))
19047 			err = mark_chain_precision(env, insn->src_reg);
19048 		if (err)
19049 			return err;
19050 	}
19051 
19052 	if (pred == 1) {
19053 		/* Only follow the goto, ignore fall-through. If needed, push
19054 		 * the fall-through branch for simulation under speculative
19055 		 * execution.
19056 		 */
19057 		if (!env->bypass_spec_v1) {
19058 			err = sanitize_speculative_path(env, insn, *insn_idx + 1, *insn_idx);
19059 			if (err < 0)
19060 				return err;
19061 		}
19062 		if (env->log.level & BPF_LOG_LEVEL)
19063 			print_insn_state(env, this_branch, this_branch->curframe);
19064 		*insn_idx += insn->off;
19065 		return 0;
19066 	} else if (pred == 0) {
19067 		/* Only follow the fall-through branch, since that's where the
19068 		 * program will go. If needed, push the goto branch for
19069 		 * simulation under speculative execution.
19070 		 */
19071 		if (!env->bypass_spec_v1) {
19072 			err = sanitize_speculative_path(env, insn, *insn_idx + insn->off + 1,
19073 							*insn_idx);
19074 			if (err < 0)
19075 				return err;
19076 		}
19077 		if (env->log.level & BPF_LOG_LEVEL)
19078 			print_insn_state(env, this_branch, this_branch->curframe);
19079 		return 0;
19080 	}
19081 
19082 	/* Push scalar registers sharing same ID to jump history,
19083 	 * do this before creating 'other_branch', so that both
19084 	 * 'this_branch' and 'other_branch' share this history
19085 	 * if parent state is created.
19086 	 */
19087 	if (linked_regs.cnt > 1) {
19088 		u16 packed[LINKED_REGS_MAX];
19089 
19090 		linked_regs_pack(&linked_regs, packed);
19091 		err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, packed, linked_regs.cnt);
19092 		if (err)
19093 			return err;
19094 	}
19095 
19096 	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, false);
19097 	if (IS_ERR(other_branch))
19098 		return PTR_ERR(other_branch);
19099 	other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
19100 
19101 	err = regs_bounds_sanity_check_branches(env);
19102 	if (err)
19103 		return err;
19104 
19105 	*dst_reg = env->false_reg1;
19106 	*src_reg = env->false_reg2;
19107 	other_branch_regs[insn->dst_reg] = env->true_reg1;
19108 	if (BPF_SRC(insn->code) == BPF_X)
19109 		other_branch_regs[insn->src_reg] = env->true_reg2;
19110 
19111 	if (BPF_SRC(insn->code) == BPF_X &&
19112 	    src_reg->type == SCALAR_VALUE && src_reg->id &&
19113 	    !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) {
19114 		sync_linked_regs(env, this_branch, src_reg, &linked_regs);
19115 		sync_linked_regs(env, other_branch, &other_branch_regs[insn->src_reg],
19116 				 &linked_regs);
19117 	}
19118 	if (dst_reg->type == SCALAR_VALUE && dst_reg->id &&
19119 	    !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) {
19120 		sync_linked_regs(env, this_branch, dst_reg, &linked_regs);
19121 		sync_linked_regs(env, other_branch, &other_branch_regs[insn->dst_reg],
19122 				 &linked_regs);
19123 	}
19124 
19125 	/* if one pointer register is compared to another pointer
19126 	 * register check if PTR_MAYBE_NULL could be lifted.
19127 	 * E.g. register A - maybe null
19128 	 *      register B - not null
19129 	 * for JNE A, B, ... - A is not null in the false branch;
19130 	 * for JEQ A, B, ... - A is not null in the true branch.
19131 	 *
19132 	 * Since PTR_TO_BTF_ID points to a kernel struct that does
19133 	 * not need to be null checked by the BPF program, i.e.,
19134 	 * could be null even without PTR_MAYBE_NULL marking, so
19135 	 * only propagate nullness when neither reg is that type.
19136 	 */
19137 	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X &&
19138 	    __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) &&
19139 	    base_type(src_reg->type) != PTR_TO_BTF_ID &&
19140 	    base_type(dst_reg->type) != PTR_TO_BTF_ID) {
19141 		eq_branch_regs = NULL;
19142 		switch (opcode) {
19143 		case BPF_JEQ:
19144 			eq_branch_regs = other_branch_regs;
19145 			break;
19146 		case BPF_JNE:
19147 			eq_branch_regs = regs;
19148 			break;
19149 		default:
19150 			/* do nothing */
19151 			break;
19152 		}
19153 		if (eq_branch_regs) {
19154 			/* src == dst && dst != NULL => src != NULL */
19155 			if (reg_not_null(env, dst_reg) && type_may_be_null(src_reg->type))
19156 				mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]);
19157 			/* src == dst && src != NULL => dst != NULL */
19158 			if (reg_not_null(env, src_reg) && type_may_be_null(dst_reg->type))
19159 				mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]);
19160 		}
19161 	}
19162 
19163 	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().
19164 	 * Also does the same detection for a register whose the value is
19165 	 * known to be 0.
19166 	 * NOTE: these optimizations below are related with pointer comparison
19167 	 *       which will never be JMP32.
19168 	 */
19169 	if (!is_jmp32 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
19170 	    type_may_be_null(dst_reg->type) &&
19171 	    ((BPF_SRC(insn->code) == BPF_K && insn->imm == 0) ||
19172 	     (BPF_SRC(insn->code) == BPF_X && bpf_register_is_null(src_reg)))) {
19173 		/*
19174 		 * For BPF_X the zero is a property of this execution path,
19175 		 * hence src_reg has to be precise.
19176 		 */
19177 		if (BPF_SRC(insn->code) == BPF_X) {
19178 			err = mark_chain_precision(env, insn->src_reg);
19179 			if (err)
19180 				return err;
19181 		}
19182 		/* Mark all identical registers in each branch as either
19183 		 * safe or unknown depending R == 0 or R != 0 conditional.
19184 		 */
19185 		mark_ptr_or_null_regs(this_branch, insn->dst_reg,
19186 				      opcode == BPF_JNE);
19187 		mark_ptr_or_null_regs(other_branch, insn->dst_reg,
19188 				      opcode == BPF_JEQ);
19189 	} else if (!try_match_pkt_pointers(insn, dst_reg, &regs[insn->src_reg],
19190 					   this_branch, other_branch) &&
19191 		   is_pointer_value(env, insn->dst_reg)) {
19192 		verbose(env, "R%d pointer comparison prohibited\n",
19193 			insn->dst_reg);
19194 		return -EACCES;
19195 	}
19196 	if (env->log.level & BPF_LOG_LEVEL)
19197 		print_insn_state(env, this_branch, this_branch->curframe);
19198 	return 0;
19199 }
19200 
19201 /* verify BPF_LD_IMM64 instruction */
19202 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
19203 {
19204 	struct bpf_insn_aux_data *aux = cur_aux(env);
19205 	struct bpf_reg_state *regs = cur_regs(env);
19206 	struct bpf_reg_state *dst_reg;
19207 	struct bpf_map *map;
19208 	int err;
19209 
19210 	if (BPF_SIZE(insn->code) != BPF_DW) {
19211 		verbose(env, "invalid BPF_LD_IMM insn\n");
19212 		return -EINVAL;
19213 	}
19214 
19215 	err = check_reg_arg(env, insn->dst_reg, DST_OP);
19216 	if (err)
19217 		return err;
19218 
19219 	dst_reg = &regs[insn->dst_reg];
19220 	bpf_diag_mod_begin(env, dst_reg, NULL, BPF_DIAG_MOD_WRITE);
19221 	if (insn->src_reg == 0) {
19222 		u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
19223 
19224 		dst_reg->type = SCALAR_VALUE;
19225 		__mark_reg_known(&regs[insn->dst_reg], imm);
19226 		bpf_diag_mod_end(env);
19227 		return 0;
19228 	}
19229 
19230 	/* All special src_reg cases are listed below. From this point onwards
19231 	 * we either succeed and assign a corresponding dst_reg->type after
19232 	 * zeroing the offset, or fail and reject the program.
19233 	 */
19234 	mark_reg_known_zero(env, regs, insn->dst_reg);
19235 
19236 	if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
19237 		dst_reg->type = aux->btf_var.reg_type;
19238 		switch (base_type(dst_reg->type)) {
19239 		case PTR_TO_MEM:
19240 			dst_reg->mem_size = aux->btf_var.mem_size;
19241 			break;
19242 		case PTR_TO_BTF_ID:
19243 			dst_reg->btf = aux->btf_var.btf;
19244 			dst_reg->btf_id = aux->btf_var.btf_id;
19245 			break;
19246 		default:
19247 			verifier_bug(env, "pseudo btf id: unexpected dst reg type");
19248 			return -EFAULT;
19249 		}
19250 		bpf_diag_mod_end(env);
19251 		return 0;
19252 	}
19253 
19254 	if (insn->src_reg == BPF_PSEUDO_FUNC) {
19255 		struct bpf_prog_aux *aux = env->prog->aux;
19256 		u32 subprogno = bpf_find_subprog(env,
19257 						 env->insn_idx + insn->imm + 1);
19258 
19259 		if (!aux->func_info) {
19260 			verbose(env, "missing btf func_info\n");
19261 			return -EINVAL;
19262 		}
19263 		if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) {
19264 			verbose(env, "callback function not static\n");
19265 			return -EINVAL;
19266 		}
19267 		/*
19268 		 * When env->subprog_cnt == 1 this instruction won't be rewritten
19269 		 * to hold a real function address. Assume that no usable program
19270 		 * combines e.g. main and timer callback and just reject here.
19271 		 */
19272 		if (subprogno == 0) {
19273 			verbose(env, "callback function cannot be the main program\n");
19274 			return -EINVAL;
19275 		}
19276 
19277 		dst_reg->type = PTR_TO_FUNC;
19278 		dst_reg->subprogno = subprogno;
19279 		bpf_diag_mod_end(env);
19280 		return 0;
19281 	}
19282 
19283 	map = env->used_maps[aux->map_index];
19284 
19285 	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
19286 	    insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
19287 		if (map->map_type == BPF_MAP_TYPE_ARENA) {
19288 			__mark_reg_unknown(env, dst_reg);
19289 			dst_reg->map_ptr = map;
19290 			bpf_diag_mod_end(env);
19291 			return 0;
19292 		}
19293 		__mark_reg_known(dst_reg, aux->map_off);
19294 		dst_reg->type = PTR_TO_MAP_VALUE;
19295 		dst_reg->map_ptr = map;
19296 		WARN_ON_ONCE(map->map_type != BPF_MAP_TYPE_INSN_ARRAY &&
19297 			     map->max_entries != 1);
19298 		/* We want reg->id to be same (0) as map_value is not distinct */
19299 	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD ||
19300 		   insn->src_reg == BPF_PSEUDO_MAP_IDX) {
19301 		dst_reg->type = CONST_PTR_TO_MAP;
19302 		dst_reg->map_ptr = map;
19303 	} else {
19304 		verifier_bug(env, "unexpected src reg value for ldimm64");
19305 		return -EFAULT;
19306 	}
19307 
19308 	bpf_diag_mod_end(env);
19309 	return 0;
19310 }
19311 
19312 static bool may_access_skb(enum bpf_prog_type type)
19313 {
19314 	switch (type) {
19315 	case BPF_PROG_TYPE_SOCKET_FILTER:
19316 	case BPF_PROG_TYPE_SCHED_CLS:
19317 	case BPF_PROG_TYPE_SCHED_ACT:
19318 		return true;
19319 	default:
19320 		return false;
19321 	}
19322 }
19323 
19324 /* verify safety of LD_ABS|LD_IND instructions:
19325  * - they can only appear in the programs where ctx == skb
19326  * - since they are wrappers of function calls, they scratch R1-R5 registers,
19327  *   preserve R6-R9, and store return value into R0
19328  *
19329  * Implicit input:
19330  *   ctx == skb == R6 == CTX
19331  *
19332  * Explicit input:
19333  *   SRC == any register
19334  *   IMM == 32-bit immediate
19335  *
19336  * Output:
19337  *   R0 - 8/16/32-bit skb data converted to cpu endianness
19338  */
19339 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
19340 {
19341 	struct bpf_verifier_state *state = env->cur_state;
19342 	struct bpf_reg_state *regs = cur_regs(env);
19343 	static const int ctx_reg = BPF_REG_6;
19344 	u8 mode = BPF_MODE(insn->code);
19345 	int i, err;
19346 
19347 	if (!may_access_skb(resolve_prog_type(env->prog))) {
19348 		verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
19349 		return -EINVAL;
19350 	}
19351 
19352 	for (i = state->curframe; i; i--) {
19353 		if (state->frame[i]->in_callback_fn) {
19354 			verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n");
19355 			return -EINVAL;
19356 		}
19357 	}
19358 
19359 	if (!env->ops->gen_ld_abs) {
19360 		verifier_bug(env, "gen_ld_abs is null");
19361 		return -EFAULT;
19362 	}
19363 
19364 	/* check whether implicit source operand (register R6) is readable */
19365 	err = check_reg_arg(env, ctx_reg, SRC_OP);
19366 	if (err)
19367 		return err;
19368 
19369 	/* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as
19370 	 * gen_ld_abs() may terminate the program at runtime, leading to
19371 	 * reference leak.
19372 	 */
19373 	err = check_resource_leak(env, false, true, "BPF_LD_[ABS|IND]");
19374 	if (err)
19375 		return err;
19376 
19377 	if (regs[ctx_reg].type != PTR_TO_CTX) {
19378 		verbose(env,
19379 			"at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
19380 		return -EINVAL;
19381 	}
19382 
19383 	if (mode == BPF_IND) {
19384 		/* check explicit source operand */
19385 		err = check_reg_arg(env, insn->src_reg, SRC_OP);
19386 		if (err)
19387 			return err;
19388 	}
19389 
19390 	err = check_ptr_off_reg(env, &regs[ctx_reg], ctx_reg);
19391 	if (err < 0)
19392 		return err;
19393 
19394 	/* reset caller saved regs to unreadable */
19395 	bpf_diag_record_caller_saved(env, regs);
19396 	bpf_diag_mod_begin(env, &regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE);
19397 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
19398 		bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);
19399 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
19400 	}
19401 
19402 	/* mark destination R0 register as readable, since it contains
19403 	 * the value fetched from the packet.
19404 	 * Already marked as written above.
19405 	 */
19406 	mark_reg_unknown(env, regs, BPF_REG_0);
19407 	bpf_diag_mod_end(env);
19408 	/*
19409 	 * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=0
19410 	 * which must be explored by the verifier when in a subprog.
19411 	 */
19412 	if (env->cur_state->curframe) {
19413 		struct bpf_verifier_state *branch;
19414 
19415 		mark_reg_scratched(env, BPF_REG_0);
19416 		branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);
19417 		if (IS_ERR(branch))
19418 			return PTR_ERR(branch);
19419 		mark_reg_known_zero(env, regs, BPF_REG_0);
19420 		err = prepare_func_exit(env, &env->insn_idx);
19421 		if (err)
19422 			return err;
19423 		env->insn_idx--;
19424 	}
19425 	return 0;
19426 }
19427 
19428 static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_range *range)
19429 {
19430 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
19431 
19432 	/* Default return value range. */
19433 	*range = retval_range(0, 1);
19434 
19435 	switch (prog_type) {
19436 	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
19437 		switch (env->prog->expected_attach_type) {
19438 		case BPF_CGROUP_UDP4_RECVMSG:
19439 		case BPF_CGROUP_UDP6_RECVMSG:
19440 		case BPF_CGROUP_UNIX_RECVMSG:
19441 		case BPF_CGROUP_INET4_GETPEERNAME:
19442 		case BPF_CGROUP_INET6_GETPEERNAME:
19443 		case BPF_CGROUP_UNIX_GETPEERNAME:
19444 		case BPF_CGROUP_INET4_GETSOCKNAME:
19445 		case BPF_CGROUP_INET6_GETSOCKNAME:
19446 		case BPF_CGROUP_UNIX_GETSOCKNAME:
19447 			*range = retval_range(1, 1);
19448 			break;
19449 		case BPF_CGROUP_INET4_BIND:
19450 		case BPF_CGROUP_INET6_BIND:
19451 			*range = retval_range(0, 3);
19452 			break;
19453 		default:
19454 			break;
19455 		}
19456 		break;
19457 	case BPF_PROG_TYPE_CGROUP_SKB:
19458 		if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS)
19459 			*range = retval_range(0, 3);
19460 		break;
19461 	case BPF_PROG_TYPE_CGROUP_SOCK:
19462 	case BPF_PROG_TYPE_SOCK_OPS:
19463 	case BPF_PROG_TYPE_CGROUP_DEVICE:
19464 	case BPF_PROG_TYPE_CGROUP_SYSCTL:
19465 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
19466 		break;
19467 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
19468 		if (!env->prog->aux->attach_btf_id)
19469 			return false;
19470 		*range = retval_range(0, 0);
19471 		break;
19472 	case BPF_PROG_TYPE_TRACING:
19473 		switch (env->prog->expected_attach_type) {
19474 		case BPF_TRACE_FENTRY:
19475 		case BPF_TRACE_FEXIT:
19476 		case BPF_TRACE_FSESSION:
19477 		case BPF_TRACE_FENTRY_MULTI:
19478 		case BPF_TRACE_FEXIT_MULTI:
19479 		case BPF_TRACE_FSESSION_MULTI:
19480 			*range = retval_range(0, 0);
19481 			break;
19482 		case BPF_TRACE_RAW_TP:
19483 		case BPF_MODIFY_RETURN:
19484 			return false;
19485 		case BPF_TRACE_ITER:
19486 		default:
19487 			break;
19488 		}
19489 		break;
19490 	case BPF_PROG_TYPE_KPROBE:
19491 		switch (env->prog->expected_attach_type) {
19492 		case BPF_TRACE_KPROBE_SESSION:
19493 		case BPF_TRACE_UPROBE_SESSION:
19494 			break;
19495 		default:
19496 			return false;
19497 		}
19498 		break;
19499 	case BPF_PROG_TYPE_SK_LOOKUP:
19500 		*range = retval_range(SK_DROP, SK_PASS);
19501 		break;
19502 
19503 	case BPF_PROG_TYPE_LSM:
19504 		if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {
19505 			/* no range found, any return value is allowed */
19506 			if (!get_func_retval_range(env->prog, range))
19507 				return false;
19508 			/* no restricted range, any return value is allowed */
19509 			if (range->minval == S32_MIN && range->maxval == S32_MAX)
19510 				return false;
19511 			range->return_32bit = true;
19512 		} else if (!env->prog->aux->attach_func_proto->type) {
19513 			/* Make sure programs that attach to void
19514 			 * hooks don't try to modify return value.
19515 			 */
19516 			*range = retval_range(1, 1);
19517 		}
19518 		break;
19519 
19520 	case BPF_PROG_TYPE_NETFILTER:
19521 		*range = retval_range(NF_DROP, NF_ACCEPT);
19522 		break;
19523 	case BPF_PROG_TYPE_STRUCT_OPS:
19524 		*range = retval_range(0, 0);
19525 		break;
19526 	case BPF_PROG_TYPE_EXT:
19527 		/* freplace program can return anything as its return value
19528 		 * depends on the to-be-replaced kernel func or bpf program.
19529 		 */
19530 	default:
19531 		return false;
19532 	}
19533 
19534 	/* Continue calculating. */
19535 
19536 	return true;
19537 }
19538 
19539 static bool program_returns_void(struct bpf_verifier_env *env)
19540 {
19541 	const struct bpf_prog *prog = env->prog;
19542 	enum bpf_prog_type prog_type = prog->type;
19543 
19544 	switch (prog_type) {
19545 	case BPF_PROG_TYPE_LSM:
19546 		/* See return_retval_range, for BPF_LSM_CGROUP can be 0 or 0-1 depending on hook. */
19547 		if (prog->expected_attach_type != BPF_LSM_CGROUP &&
19548 		    !prog->aux->attach_func_proto->type)
19549 			return true;
19550 		break;
19551 	case BPF_PROG_TYPE_STRUCT_OPS:
19552 		if (!prog->aux->attach_func_proto->type)
19553 			return true;
19554 		break;
19555 	case BPF_PROG_TYPE_EXT:
19556 		/*
19557 		 * If the actual program is an extension, let it
19558 		 * return void - attaching will succeed only if the
19559 		 * program being replaced also returns void, and since
19560 		 * it has passed verification its actual type doesn't matter.
19561 		 */
19562 		if (subprog_returns_void(env, 0))
19563 			return true;
19564 		break;
19565 	default:
19566 		break;
19567 	}
19568 	return false;
19569 }
19570 
19571 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name)
19572 {
19573 	const char *exit_ctx = "At program exit";
19574 	struct tnum enforce_attach_type_range = tnum_unknown;
19575 	const struct bpf_prog *prog = env->prog;
19576 	struct bpf_reg_state *reg = reg_state(env, regno);
19577 	struct bpf_retval_range range = retval_range(0, 1);
19578 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
19579 	struct bpf_func_state *frame = env->cur_state->frame[0];
19580 	const struct btf_type *reg_type, *ret_type = NULL;
19581 	int err;
19582 
19583 	/* LSM and struct_ops func-ptr's return type could be "void" */
19584 	if (!frame->in_async_callback_fn && program_returns_void(env))
19585 		return 0;
19586 
19587 	if (prog_type == BPF_PROG_TYPE_STRUCT_OPS) {
19588 		/* Allow a struct_ops program to return a referenced kptr if it
19589 		 * matches the operator's return type and is in its unmodified
19590 		 * form. A scalar zero (i.e., a null pointer) is also allowed.
19591 		 */
19592 		reg_type = reg->btf ? btf_type_by_id(reg->btf, reg->btf_id) : NULL;
19593 		ret_type = btf_type_resolve_ptr(prog->aux->attach_btf,
19594 						prog->aux->attach_func_proto->type,
19595 						NULL);
19596 		if (ret_type && ret_type == reg_type && reg_is_referenced(env, reg))
19597 			return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false);
19598 	}
19599 
19600 	/* eBPF calling convention is such that R0 is used
19601 	 * to return the value from eBPF program.
19602 	 * Make sure that it's readable at this time
19603 	 * of bpf_exit, which means that program wrote
19604 	 * something into it earlier
19605 	 */
19606 	err = check_reg_arg(env, regno, SRC_OP);
19607 	if (err)
19608 		return err;
19609 
19610 	if (is_pointer_value(env, regno)) {
19611 		verbose(env, "R%d leaks addr as return value\n", regno);
19612 		return -EACCES;
19613 	}
19614 
19615 	if (frame->in_async_callback_fn) {
19616 		exit_ctx = "At async callback return";
19617 		range = frame->callback_ret_range;
19618 		goto enforce_retval;
19619 	}
19620 
19621 	if (prog_type == BPF_PROG_TYPE_STRUCT_OPS && !ret_type)
19622 		return 0;
19623 
19624 	if (prog_type == BPF_PROG_TYPE_CGROUP_SKB && (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS))
19625 		enforce_attach_type_range = tnum_range(2, 3);
19626 
19627 	if (!return_retval_range(env, &range))
19628 		return 0;
19629 
19630 enforce_retval:
19631 	if (reg->type != SCALAR_VALUE) {
19632 		verbose(env, "%s the register R%d is not a known value (%s)\n",
19633 			exit_ctx, regno, reg_type_str(env, reg->type));
19634 		return -EINVAL;
19635 	}
19636 
19637 	err = mark_chain_precision(env, regno);
19638 	if (err)
19639 		return err;
19640 
19641 	if (!retval_range_within(range, reg)) {
19642 		verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name);
19643 		if (prog->expected_attach_type == BPF_LSM_CGROUP &&
19644 		    prog_type == BPF_PROG_TYPE_LSM &&
19645 		    !prog->aux->attach_func_proto->type)
19646 			verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
19647 		return -EINVAL;
19648 	}
19649 
19650 	if (!tnum_is_unknown(enforce_attach_type_range) &&
19651 	    tnum_in(enforce_attach_type_range, reg->var_off))
19652 		env->prog->enforce_expected_attach_type = 1;
19653 	return 0;
19654 }
19655 
19656 static int check_global_ret_scalar_reg(struct bpf_verifier_env *env, u32 regno)
19657 {
19658 	struct bpf_reg_state *reg;
19659 	int err;
19660 
19661 	err = check_reg_arg(env, regno, SRC_OP);
19662 	if (err)
19663 		return err;
19664 
19665 	/* Pointers to arena are safe to pass between subprograms. */
19666 	if (is_arena_reg(env, regno))
19667 		return 0;
19668 
19669 	if (is_pointer_value(env, regno)) {
19670 		verbose(env, "R%d leaks addr as return value\n", regno);
19671 		return -EACCES;
19672 	}
19673 
19674 	reg = reg_state(env, regno);
19675 	if (reg->type != SCALAR_VALUE) {
19676 		verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n",
19677 			regno, reg_type_str(env, reg->type));
19678 		return -EINVAL;
19679 	}
19680 
19681 	return 0;
19682 }
19683 
19684 static int check_global_subprog_return_code(struct bpf_verifier_env *env)
19685 {
19686 	struct bpf_func_state *cur_frame = cur_func(env);
19687 	u32 subprog = cur_frame->subprogno;
19688 	u32 i, nregs;
19689 	int err;
19690 
19691 	if (subprog_returns_void(env, subprog))
19692 		return 0;
19693 
19694 	nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1;
19695 	for (i = 0; i < nregs; i++) {
19696 		err = check_global_ret_scalar_reg(env, ret_regs[i]);
19697 		if (err)
19698 			return err;
19699 	}
19700 
19701 	return 0;
19702 }
19703 
19704 /* Bitmask with 1s for all caller saved registers */
19705 #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)
19706 
19707 /* True if do_misc_fixups() replaces calls to helper number 'imm',
19708  * replacement patch is presumed to follow bpf_fastcall contract
19709  * (see mark_fastcall_pattern_for_call() below).
19710  */
19711 bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
19712 {
19713 	switch (imm) {
19714 #ifdef CONFIG_X86_64
19715 	case BPF_FUNC_get_smp_processor_id:
19716 #ifdef CONFIG_SMP
19717 	case BPF_FUNC_get_current_task_btf:
19718 	case BPF_FUNC_get_current_task:
19719 #endif
19720 		return env->prog->jit_requested && bpf_jit_supports_percpu_insn();
19721 #endif
19722 	default:
19723 		return false;
19724 	}
19725 }
19726 
19727 /* If @call is a kfunc or helper call, fills @cs and returns true,
19728  * otherwise returns false.
19729  */
19730 bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
19731 			  struct bpf_call_summary *cs)
19732 {
19733 	struct bpf_call_arg_meta meta;
19734 	const struct bpf_func_proto *fn;
19735 	int i;
19736 
19737 	if (bpf_helper_call(call)) {
19738 		if (bpf_get_helper_proto(env, call->imm, &fn) < 0)
19739 			/* error would be reported later */
19740 			return false;
19741 		cs->fastcall = fn->allow_fastcall &&
19742 			       (bpf_verifier_inlines_helper_call(env, call->imm) ||
19743 				bpf_jit_inlines_helper_call(call->imm));
19744 		cs->is_void = fn->ret_type == RET_VOID;
19745 		cs->arg_slot_cnt = 0;
19746 		for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) {
19747 			if (fn->arg_type[i] == ARG_UNUSED)
19748 				break;
19749 			cs->arg_slot_cnt++;
19750 		}
19751 		return true;
19752 	}
19753 
19754 	if (bpf_pseudo_kfunc_call(call)) {
19755 		int err;
19756 
19757 		err = bpf_fetch_kfunc_arg_meta(env, call->imm, call->off, &meta);
19758 		if (err < 0)
19759 			/* error would be reported later */
19760 			return false;
19761 		cs->arg_slot_cnt = btf_proto_slots(meta.btf, meta.func_proto);
19762 		cs->fastcall = meta.kfunc_flags & KF_FASTCALL;
19763 		cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type));
19764 		return true;
19765 	}
19766 
19767 	return false;
19768 }
19769 
19770 /* LLVM define a bpf_fastcall function attribute.
19771  * This attribute means that function scratches only some of
19772  * the caller saved registers defined by ABI.
19773  * For BPF the set of such registers could be defined as follows:
19774  * - R0 is scratched only if function is non-void;
19775  * - R1-R5 are scratched only if corresponding parameter type is defined
19776  *   in the function prototype.
19777  *
19778  * The contract between kernel and clang allows to simultaneously use
19779  * such functions and maintain backwards compatibility with old
19780  * kernels that don't understand bpf_fastcall calls:
19781  *
19782  * - for bpf_fastcall calls clang allocates registers as-if relevant r0-r5
19783  *   registers are not scratched by the call;
19784  *
19785  * - as a post-processing step, clang visits each bpf_fastcall call and adds
19786  *   spill/fill for every live r0-r5;
19787  *
19788  * - stack offsets used for the spill/fill are allocated as lowest
19789  *   stack offsets in whole function and are not used for any other
19790  *   purposes;
19791  *
19792  * - when kernel loads a program, it looks for such patterns
19793  *   (bpf_fastcall function surrounded by spills/fills) and checks if
19794  *   spill/fill stack offsets are used exclusively in fastcall patterns;
19795  *
19796  * - if so, and if verifier or current JIT inlines the call to the
19797  *   bpf_fastcall function (e.g. a helper call), kernel removes unnecessary
19798  *   spill/fill pairs;
19799  *
19800  * - when old kernel loads a program, presence of spill/fill pairs
19801  *   keeps BPF program valid, albeit slightly less efficient.
19802  *
19803  * For example:
19804  *
19805  *   r1 = 1;
19806  *   r2 = 2;
19807  *   *(u64 *)(r10 - 8)  = r1;            r1 = 1;
19808  *   *(u64 *)(r10 - 16) = r2;            r2 = 2;
19809  *   call %[to_be_inlined]         -->   call %[to_be_inlined]
19810  *   r2 = *(u64 *)(r10 - 16);            r0 = r1;
19811  *   r1 = *(u64 *)(r10 - 8);             r0 += r2;
19812  *   r0 = r1;                            exit;
19813  *   r0 += r2;
19814  *   exit;
19815  *
19816  * The purpose of mark_fastcall_pattern_for_call is to:
19817  * - look for such patterns;
19818  * - mark spill and fill instructions in env->insn_aux_data[*].fastcall_pattern;
19819  * - mark set env->insn_aux_data[*].fastcall_spills_num for call instruction;
19820  * - update env->subprog_info[*]->fastcall_stack_off to find an offset
19821  *   at which bpf_fastcall spill/fill stack slots start;
19822  * - update env->subprog_info[*]->keep_fastcall_stack.
19823  *
19824  * The .fastcall_pattern and .fastcall_stack_off are used by
19825  * check_fastcall_stack_contract() to check if every stack access to
19826  * fastcall spill/fill stack slot originates from spill/fill
19827  * instructions, members of fastcall patterns.
19828  *
19829  * If such condition holds true for a subprogram, fastcall patterns could
19830  * be rewritten by remove_fastcall_spills_fills().
19831  * Otherwise bpf_fastcall patterns are not changed in the subprogram
19832  * (code, presumably, generated by an older clang version).
19833  *
19834  * For example, it is *not* safe to remove spill/fill below:
19835  *
19836  *   r1 = 1;
19837  *   *(u64 *)(r10 - 8)  = r1;            r1 = 1;
19838  *   call %[to_be_inlined]         -->   call %[to_be_inlined]
19839  *   r1 = *(u64 *)(r10 - 8);             r0 = *(u64 *)(r10 - 8);  <---- wrong !!!
19840  *   r0 = *(u64 *)(r10 - 8);             r0 += r1;
19841  *   r0 += r1;                           exit;
19842  *   exit;
19843  *
19844  * Both uses of the marks assume that a pattern is entered at its first
19845  * spill and thus executes as a unit, hence a pattern is not grown past
19846  * an instruction targeted by a jump.
19847  */
19848 static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
19849 					   struct bpf_subprog_info *subprog,
19850 					   int insn_idx, s16 lowest_off)
19851 {
19852 	struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx;
19853 	struct bpf_insn *call = &env->prog->insnsi[insn_idx];
19854 	u32 clobbered_regs_mask;
19855 	struct bpf_call_summary cs;
19856 	u32 expected_regs_mask;
19857 	s16 off;
19858 	int i;
19859 
19860 	if (!bpf_get_call_summary(env, call, &cs))
19861 		return;
19862 
19863 	/* A bitmask specifying which caller saved registers are clobbered
19864 	 * by a call to a helper/kfunc *as if* this helper/kfunc follows
19865 	 * bpf_fastcall contract:
19866 	 * - includes R0 if function is non-void;
19867 	 * - includes R1-R5 if corresponding parameter has is described
19868 	 *   in the function prototype.
19869 	 */
19870 	clobbered_regs_mask = GENMASK(cs.arg_slot_cnt, cs.is_void ? 1 : 0);
19871 	/* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */
19872 	expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS;
19873 
19874 	/* match pairs of form:
19875 	 *
19876 	 * *(u64 *)(r10 - Y) = rX   (where Y % 8 == 0)
19877 	 * ...
19878 	 * call %[to_be_inlined]
19879 	 * ...
19880 	 * rX = *(u64 *)(r10 - Y)
19881 	 */
19882 	for (i = 1, off = lowest_off; i <= ARRAY_SIZE(caller_saved); ++i, off += BPF_REG_SIZE) {
19883 		if (insn_idx - i < 0 || insn_idx + i >= env->prog->len)
19884 			break;
19885 		/* stx/ldx/call must not be a jump targets, a jump to the first stx is fine */
19886 		if (bpf_is_jump_target(env, insn_idx - i + 1) ||
19887 		    bpf_is_jump_target(env, insn_idx + i))
19888 			break;
19889 		stx = &insns[insn_idx - i];
19890 		ldx = &insns[insn_idx + i];
19891 		/* must be a stack spill/fill pair */
19892 		if (stx->code != (BPF_STX | BPF_MEM | BPF_DW) ||
19893 		    ldx->code != (BPF_LDX | BPF_MEM | BPF_DW) ||
19894 		    stx->dst_reg != BPF_REG_10 ||
19895 		    ldx->src_reg != BPF_REG_10)
19896 			break;
19897 		/* must be a spill/fill for the same reg */
19898 		if (stx->src_reg != ldx->dst_reg)
19899 			break;
19900 		/* must be one of the previously unseen registers */
19901 		if ((BIT(stx->src_reg) & expected_regs_mask) == 0)
19902 			break;
19903 		/* must be a spill/fill for the same expected offset,
19904 		 * no need to check offset alignment, BPF_DW stack access
19905 		 * is always 8-byte aligned.
19906 		 */
19907 		if (stx->off != off || ldx->off != off)
19908 			break;
19909 		expected_regs_mask &= ~BIT(stx->src_reg);
19910 		env->insn_aux_data[insn_idx - i].fastcall_pattern = 1;
19911 		env->insn_aux_data[insn_idx + i].fastcall_pattern = 1;
19912 	}
19913 	if (i == 1)
19914 		return;
19915 
19916 	/* Conditionally set 'fastcall_spills_num' to allow forward
19917 	 * compatibility when more helper functions are marked as
19918 	 * bpf_fastcall at compile time than current kernel supports, e.g:
19919 	 *
19920 	 *   1: *(u64 *)(r10 - 8) = r1
19921 	 *   2: call A                  ;; assume A is bpf_fastcall for current kernel
19922 	 *   3: r1 = *(u64 *)(r10 - 8)
19923 	 *   4: *(u64 *)(r10 - 8) = r1
19924 	 *   5: call B                  ;; assume B is not bpf_fastcall for current kernel
19925 	 *   6: r1 = *(u64 *)(r10 - 8)
19926 	 *
19927 	 * There is no need to block bpf_fastcall rewrite for such program.
19928 	 * Set 'fastcall_pattern' for both calls to keep check_fastcall_stack_contract() happy,
19929 	 * don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills()
19930 	 * does not remove spill/fill pair {4,6}.
19931 	 */
19932 	if (cs.fastcall)
19933 		env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;
19934 	else
19935 		subprog->keep_fastcall_stack = 1;
19936 	subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off);
19937 }
19938 
19939 static int mark_fastcall_patterns(struct bpf_verifier_env *env)
19940 {
19941 	struct bpf_subprog_info *subprog = env->subprog_info;
19942 	struct bpf_insn *insn;
19943 	s16 lowest_off;
19944 	int s, i;
19945 
19946 	for (s = 0; s < env->subprog_cnt; ++s, ++subprog) {
19947 		/* find lowest stack spill offset used in this subprog */
19948 		lowest_off = 0;
19949 		for (i = subprog->start; i < (subprog + 1)->start; ++i) {
19950 			insn = env->prog->insnsi + i;
19951 			if (insn->code != (BPF_STX | BPF_MEM | BPF_DW) ||
19952 			    insn->dst_reg != BPF_REG_10)
19953 				continue;
19954 			lowest_off = min(lowest_off, insn->off);
19955 		}
19956 		/* use this offset to find fastcall patterns */
19957 		for (i = subprog->start; i < (subprog + 1)->start; ++i) {
19958 			insn = env->prog->insnsi + i;
19959 			if (insn->code != (BPF_JMP | BPF_CALL))
19960 				continue;
19961 			mark_fastcall_pattern_for_call(env, subprog, i, lowest_off);
19962 		}
19963 	}
19964 	return 0;
19965 }
19966 
19967 static void adjust_btf_func(struct bpf_verifier_env *env)
19968 {
19969 	struct bpf_prog_aux *aux = env->prog->aux;
19970 	int i;
19971 
19972 	if (!aux->func_info)
19973 		return;
19974 
19975 	/* func_info is not available for hidden subprogs */
19976 	for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++)
19977 		aux->func_info[i].insn_off = env->subprog_info[i].start;
19978 }
19979 
19980 /* Find id in idset and increment its count, or add new entry */
19981 static void idset_cnt_inc(struct bpf_idset *idset, u32 id)
19982 {
19983 	u32 i;
19984 
19985 	for (i = 0; i < idset->num_ids; i++) {
19986 		if (idset->entries[i].id == id) {
19987 			idset->entries[i].cnt++;
19988 			return;
19989 		}
19990 	}
19991 	/* New id */
19992 	if (idset->num_ids < BPF_ID_MAP_SIZE) {
19993 		idset->entries[idset->num_ids].id = id;
19994 		idset->entries[idset->num_ids].cnt = 1;
19995 		idset->num_ids++;
19996 	}
19997 }
19998 
19999 /* Find id in idset and return its count, or 0 if not found */
20000 static u32 idset_cnt_get(struct bpf_idset *idset, u32 id)
20001 {
20002 	u32 i;
20003 
20004 	for (i = 0; i < idset->num_ids; i++) {
20005 		if (idset->entries[i].id == id)
20006 			return idset->entries[i].cnt;
20007 	}
20008 	return 0;
20009 }
20010 
20011 /*
20012  * Clear singular scalar ids in a state.
20013  * A register with a non-zero id is called singular if no other register shares
20014  * the same base id. Such registers can be treated as independent (id=0).
20015  */
20016 void bpf_clear_singular_ids(struct bpf_verifier_env *env,
20017 			    struct bpf_verifier_state *st)
20018 {
20019 	struct bpf_idset *idset = &env->idset_scratch;
20020 	struct bpf_func_state *func;
20021 	struct bpf_reg_state *reg;
20022 
20023 	idset->num_ids = 0;
20024 
20025 	bpf_for_each_reg_in_vstate(st, func, reg, ({
20026 		if (reg->type != SCALAR_VALUE)
20027 			continue;
20028 		if (!reg->id)
20029 			continue;
20030 		idset_cnt_inc(idset, reg->id & ~BPF_ADD_CONST);
20031 	}));
20032 
20033 	bpf_for_each_reg_in_vstate(st, func, reg, ({
20034 		if (reg->type != SCALAR_VALUE)
20035 			continue;
20036 		if (!reg->id)
20037 			continue;
20038 		if (idset_cnt_get(idset, reg->id & ~BPF_ADD_CONST) == 1)
20039 			clear_scalar_id(reg);
20040 	}));
20041 }
20042 
20043 /* Return true if it's OK to have the same insn return a different type. */
20044 static bool reg_type_mismatch_ok(enum bpf_reg_type type)
20045 {
20046 	switch (base_type(type)) {
20047 	case PTR_TO_CTX:
20048 	case PTR_TO_SOCKET:
20049 	case PTR_TO_SOCK_COMMON:
20050 	case PTR_TO_TCP_SOCK:
20051 	case PTR_TO_XDP_SOCK:
20052 	case PTR_TO_BTF_ID:
20053 	case PTR_TO_ARENA:
20054 		return false;
20055 	case PTR_TO_MEM:
20056 		return !bpf_may_fault_on_deref(type);
20057 	default:
20058 		return true;
20059 	}
20060 }
20061 
20062 /* If an instruction was previously used with particular pointer types, then we
20063  * need to be careful to avoid cases such as the below, where it may be ok
20064  * for one branch accessing the pointer, but not ok for the other branch:
20065  *
20066  * R1 = sock_ptr
20067  * goto X;
20068  * ...
20069  * R1 = some_other_valid_ptr;
20070  * goto X;
20071  * ...
20072  * R2 = *(u32 *)(R1 + 0);
20073  */
20074 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
20075 {
20076 	return src != prev && (!reg_type_mismatch_ok(src) ||
20077 			       !reg_type_mismatch_ok(prev));
20078 }
20079 
20080 static bool is_ptr_to_mem(enum bpf_reg_type type)
20081 {
20082 	return base_type(type) == PTR_TO_MEM;
20083 }
20084 
20085 static enum bpf_reg_type merge_ptr_types(enum bpf_reg_type type_a,
20086 					 enum bpf_reg_type type_b)
20087 {
20088 	bool to_mem = is_ptr_to_mem(type_a) || is_ptr_to_mem(type_b);
20089 	enum bpf_reg_type type_merged = to_mem ? PTR_TO_MEM : PTR_TO_BTF_ID;
20090 
20091 	if (bpf_may_fault_on_deref(type_a) || bpf_may_fault_on_deref(type_b))
20092 		type_merged |= to_mem ? MEM_RDONLY | PTR_UNTRUSTED :
20093 					PTR_UNTRUSTED;
20094 	else
20095 		type_merged |= ((type_a | type_b) & MEM_RDONLY);
20096 	return type_merged;
20097 }
20098 
20099 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
20100 			     bool allow_trust_mismatch)
20101 {
20102 	enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type;
20103 
20104 	if (*prev_type == NOT_INIT) {
20105 		/* Saw a valid insn
20106 		 * dst_reg = *(u32 *)(src_reg + off)
20107 		 * save type to validate intersecting paths
20108 		 */
20109 		*prev_type = type;
20110 	} else if (reg_type_mismatch(type, *prev_type)) {
20111 		/* Abuser program is trying to use the same insn
20112 		 * dst_reg = *(u32*) (src_reg + off)
20113 		 * with different pointer types:
20114 		 * src_reg == ctx in one branch and
20115 		 * src_reg == stack|map in some other branch.
20116 		 * Reject it.
20117 		 */
20118 		if (allow_trust_mismatch &&
20119 		    bpf_is_ptr_to_mem_or_btf_id(type) &&
20120 		    bpf_is_ptr_to_mem_or_btf_id(*prev_type)) {
20121 			/*
20122 			 * Have to support a use case when one path through
20123 			 * the program yields a TRUSTED pointer while another
20124 			 * is UNTRUSTED. Merge them into a type which keeps
20125 			 * the BPF_PROBE_MEM/BPF_PROBE_MEMSX rewrite when
20126 			 * either side needs it.
20127 			 */
20128 			*prev_type = merge_ptr_types(type, *prev_type);
20129 		} else {
20130 			verbose(env, "same insn cannot be used with different pointers\n");
20131 			return -EINVAL;
20132 		}
20133 	}
20134 
20135 	return 0;
20136 }
20137 
20138 enum {
20139 	PROCESS_BPF_EXIT = 1,
20140 	INSN_IDX_UPDATED = 2,
20141 };
20142 
20143 static int process_bpf_exit_full(struct bpf_verifier_env *env,
20144 				 bool *do_print_state,
20145 				 bool exception_exit)
20146 {
20147 	struct bpf_func_state *cur_frame = cur_func(env);
20148 
20149 	/* We must do check_reference_leak here before
20150 	 * prepare_func_exit to handle the case when
20151 	 * state->curframe > 0, it may be a callback function,
20152 	 * for which reference_state must match caller reference
20153 	 * state when it exits.
20154 	 */
20155 	int err = check_resource_leak(env, exception_exit,
20156 				      exception_exit || !env->cur_state->curframe,
20157 				      exception_exit ? "bpf_throw" :
20158 				      "BPF_EXIT instruction in main prog");
20159 	if (err)
20160 		return err;
20161 
20162 	/* The side effect of the prepare_func_exit which is
20163 	 * being skipped is that it frees bpf_func_state.
20164 	 * Typically, process_bpf_exit will only be hit with
20165 	 * outermost exit. copy_verifier_state in pop_stack will
20166 	 * handle freeing of any extra bpf_func_state left over
20167 	 * from not processing all nested function exits. We
20168 	 * also skip return code checks as they are not needed
20169 	 * for exceptional exits.
20170 	 */
20171 	if (exception_exit)
20172 		return PROCESS_BPF_EXIT;
20173 
20174 	if (env->cur_state->curframe) {
20175 		/* exit from nested function */
20176 		err = prepare_func_exit(env, &env->insn_idx);
20177 		if (err)
20178 			return err;
20179 		*do_print_state = true;
20180 		return INSN_IDX_UPDATED;
20181 	}
20182 
20183 	/*
20184 	 * Return from a regular global subprogram differs from return
20185 	 * from the main program or async/exception callback.
20186 	 * Main program exit implies return code restrictions
20187 	 * that depend on program type.
20188 	 * Exit from exception callback is equivalent to main program exit.
20189 	 * Exit from async callback implies return code restrictions
20190 	 * that depend on async scheduling mechanism.
20191 	 */
20192 	if (cur_frame->subprogno &&
20193 	    !cur_frame->in_async_callback_fn &&
20194 	    !cur_frame->in_exception_callback_fn)
20195 		err = check_global_subprog_return_code(env);
20196 	else
20197 		err = check_return_code(env, BPF_REG_0, "R0");
20198 	if (err)
20199 		return err;
20200 	return PROCESS_BPF_EXIT;
20201 }
20202 
20203 static int indirect_jump_min_max_index(struct bpf_verifier_env *env,
20204 				       int regno,
20205 				       struct bpf_map *map,
20206 				       u32 *pmin_index, u32 *pmax_index)
20207 {
20208 	struct bpf_reg_state *reg = reg_state(env, regno);
20209 	u64 min_index = reg_umin(reg);
20210 	u64 max_index = reg_umax(reg);
20211 	const u32 size = 8;
20212 
20213 	if (min_index > (u64) U32_MAX * size) {
20214 		verbose(env, "the sum of R%u umin_value %llu is too big\n", regno, reg_umin(reg));
20215 		return -ERANGE;
20216 	}
20217 	if (max_index > (u64) U32_MAX * size) {
20218 		verbose(env, "the sum of R%u umax_value %llu is too big\n", regno, reg_umax(reg));
20219 		return -ERANGE;
20220 	}
20221 
20222 	min_index /= size;
20223 	max_index /= size;
20224 
20225 	if (max_index >= map->max_entries) {
20226 		verbose(env, "R%u points to outside of jump table: [%llu,%llu] max_entries %u\n",
20227 			     regno, min_index, max_index, map->max_entries);
20228 		return -EINVAL;
20229 	}
20230 
20231 	*pmin_index = min_index;
20232 	*pmax_index = max_index;
20233 	return 0;
20234 }
20235 
20236 static int reject_gotox_out_of_subprog(struct bpf_verifier_env *env, u32 target,
20237 				       u32 subprog_start, u32 subprog_end)
20238 {
20239 	verbose(env, "indirect jump from insn %d to %u leaves the subprog [%u,%u)\n",
20240 		env->insn_idx, target, subprog_start, subprog_end);
20241 	bpf_diag_program_structure(
20242 		env, env->insn_idx, "indirect jump leaves subprogram",
20243 		"Keep every reachable jump-table target inside the subprogram of the indirect jump.",
20244 		"Instruction %d can jump indirectly to instruction %u, which is outside its own subprogram [%u,%u).",
20245 		env->insn_idx, target, subprog_start, subprog_end);
20246 	return -EINVAL;
20247 }
20248 
20249 /* gotox *dst_reg */
20250 static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *insn)
20251 {
20252 	struct bpf_verifier_state *other_branch;
20253 	struct bpf_subprog_info *subprog;
20254 	u32 subprog_start, subprog_end;
20255 	struct bpf_reg_state *dst_reg;
20256 	struct bpf_map *map;
20257 	u32 min_index, max_index;
20258 	int err = 0;
20259 	int n;
20260 	int i;
20261 
20262 	dst_reg = reg_state(env, insn->dst_reg);
20263 	if (dst_reg->type != PTR_TO_INSN) {
20264 		verbose(env, "R%d has type %s, expected PTR_TO_INSN\n",
20265 			     insn->dst_reg, reg_type_str(env, dst_reg->type));
20266 		return -EINVAL;
20267 	}
20268 
20269 	map = dst_reg->map_ptr;
20270 	if (verifier_bug_if(!map, env, "R%d has an empty map pointer", insn->dst_reg))
20271 		return -EFAULT;
20272 
20273 	if (verifier_bug_if(map->map_type != BPF_MAP_TYPE_INSN_ARRAY, env,
20274 			    "R%d has incorrect map type %d", insn->dst_reg, map->map_type))
20275 		return -EFAULT;
20276 
20277 	err = indirect_jump_min_max_index(env, insn->dst_reg, map, &min_index, &max_index);
20278 	if (err)
20279 		return err;
20280 
20281 	/* Ensure that the buffer is large enough */
20282 	if (!env->gotox_tmp_buf || env->gotox_tmp_buf->cnt < max_index - min_index + 1) {
20283 		env->gotox_tmp_buf = bpf_iarray_realloc(env->gotox_tmp_buf,
20284 						        max_index - min_index + 1);
20285 		if (!env->gotox_tmp_buf)
20286 			return -ENOMEM;
20287 	}
20288 
20289 	n = bpf_copy_insn_array_uniq(map, min_index, max_index, env->gotox_tmp_buf->items);
20290 	if (n < 0)
20291 		return n;
20292 	if (n == 0) {
20293 		verbose(env, "register R%d doesn't point to any offset in map id=%d\n",
20294 			     insn->dst_reg, map->id);
20295 		return -EINVAL;
20296 	}
20297 
20298 	subprog = bpf_find_containing_subprog(env, env->insn_idx);
20299 	if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", env->insn_idx))
20300 		return -EFAULT;
20301 	subprog_start = subprog->start;
20302 	subprog_end = (subprog + 1)->start;
20303 
20304 	/*
20305 	 * Maps are confined to a subprog, see compute_subprog_jts(), and the
20306 	 * targets are sorted and free of duplicates, see sort_insn_array_uniq(),
20307 	 * so either all of them belong to this subprog or none of them does.
20308 	 */
20309 	if (env->gotox_tmp_buf->items[0] < subprog_start)
20310 		return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[0],
20311 						   subprog_start, subprog_end);
20312 	if (env->gotox_tmp_buf->items[n - 1] >= subprog_end)
20313 		return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[n - 1],
20314 						   subprog_start, subprog_end);
20315 
20316 	for (i = 0; i < n - 1; i++) {
20317 		mark_indirect_target(env, env->gotox_tmp_buf->items[i]);
20318 		other_branch = push_stack(env, env->gotox_tmp_buf->items[i],
20319 					  env->insn_idx, env->cur_state->speculative);
20320 		if (IS_ERR(other_branch))
20321 			return PTR_ERR(other_branch);
20322 	}
20323 	env->insn_idx = env->gotox_tmp_buf->items[n-1];
20324 	mark_indirect_target(env, env->insn_idx);
20325 	return INSN_IDX_UPDATED;
20326 }
20327 
20328 static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
20329 {
20330 	int err;
20331 	struct bpf_insn *insn = &env->prog->insnsi[env->insn_idx];
20332 	u8 class = BPF_CLASS(insn->code);
20333 
20334 	switch (class) {
20335 	case BPF_ALU:
20336 	case BPF_ALU64:
20337 		return check_alu_op(env, insn);
20338 
20339 	case BPF_LDX:
20340 		return check_load_mem(env, insn, false,
20341 				      BPF_MODE(insn->code) == BPF_MEMSX,
20342 				      true, "ldx");
20343 
20344 	case BPF_STX:
20345 		if (BPF_MODE(insn->code) == BPF_ATOMIC)
20346 			return check_atomic(env, insn);
20347 		return check_store_reg(env, insn, false);
20348 
20349 	case BPF_ST: {
20350 		/* Handle stack arg write (store immediate) */
20351 		if (is_stack_arg_st(insn)) {
20352 			struct bpf_verifier_state *vstate = env->cur_state;
20353 			struct bpf_func_state *state = vstate->frame[vstate->curframe];
20354 
20355 			return check_stack_arg_write(env, state, insn->off, NULL);
20356 		}
20357 
20358 		enum bpf_reg_type dst_reg_type;
20359 
20360 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
20361 		if (err)
20362 			return err;
20363 
20364 		dst_reg_type = cur_regs(env)[insn->dst_reg].type;
20365 
20366 		err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg,
20367 				       argno_from_reg(insn->dst_reg),
20368 				       insn->off, BPF_SIZE(insn->code),
20369 				       BPF_WRITE, -1, false, false);
20370 		if (err)
20371 			return err;
20372 
20373 		return save_aux_ptr_type(env, dst_reg_type, false);
20374 	}
20375 	case BPF_JMP:
20376 	case BPF_JMP32: {
20377 		u8 opcode = BPF_OP(insn->code);
20378 
20379 		env->jmps_processed++;
20380 		if (opcode == BPF_CALL) {
20381 			if (env->cur_state->active_locks) {
20382 				/* similar to static subprog calls callx is allowed under a lock */
20383 				if (!bpf_is_callx(insn) &&
20384 				    ((insn->src_reg == BPF_REG_0 &&
20385 				      insn->imm != BPF_FUNC_spin_unlock &&
20386 				      insn->imm != BPF_FUNC_kptr_xchg) ||
20387 				     (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
20388 				      !kfunc_spin_allowed(env, insn->imm, insn->off)))) {
20389 					verbose(env,
20390 						"function calls are not allowed while holding a lock\n");
20391 					bpf_diag_ctx_active(
20392 						env, env->insn_idx,
20393 						"function call", BPF_DIAG_CONTEXT_LOCK,
20394 						"Release the BPF spin lock before making this call, or move the call outside the locked region.");
20395 					return -EINVAL;
20396 				}
20397 			}
20398 			mark_reg_scratched(env, BPF_REG_0);
20399 			if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno]))
20400 				cur_func(env)->no_stack_arg_load = true;
20401 			if (bpf_is_callx(insn))
20402 				return check_func_callx(env, insn, &env->insn_idx);
20403 			if (insn->src_reg == BPF_PSEUDO_CALL)
20404 				return check_func_call(env, insn, &env->insn_idx);
20405 			if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
20406 				return check_kfunc_call(env, insn, &env->insn_idx);
20407 			return check_helper_call(env, insn, &env->insn_idx);
20408 		} else if (opcode == BPF_JA) {
20409 			if (BPF_SRC(insn->code) == BPF_X)
20410 				return check_indirect_jump(env, insn);
20411 
20412 			if (class == BPF_JMP)
20413 				env->insn_idx += insn->off + 1;
20414 			else
20415 				env->insn_idx += insn->imm + 1;
20416 			return INSN_IDX_UPDATED;
20417 		} else if (opcode == BPF_EXIT) {
20418 			return process_bpf_exit_full(env, do_print_state, false);
20419 		}
20420 		return check_cond_jmp_op(env, insn, &env->insn_idx);
20421 	}
20422 	case BPF_LD: {
20423 		u8 mode = BPF_MODE(insn->code);
20424 
20425 		if (mode == BPF_ABS || mode == BPF_IND)
20426 			return check_ld_abs(env, insn);
20427 
20428 		if (mode == BPF_IMM) {
20429 			err = check_ld_imm(env, insn);
20430 			if (err)
20431 				return err;
20432 
20433 			env->insn_idx++;
20434 			sanitize_mark_insn_seen(env);
20435 		}
20436 		return 0;
20437 	}
20438 	}
20439 	/* all class values are handled above. silence compiler warning */
20440 	return -EFAULT;
20441 }
20442 
20443 /*
20444  * Push the loop entered at insn_idx onto the stack. Usually this adds a single
20445  * entry on top of its enclosing loop. However, an edge may enter an inner loop
20446  * directly, bypassing the header(s) of its enclosing loop(s), e.g.:
20447  *
20448  *   1: for (...):       // enclosing loop, header at 1
20449  *   2:   for (...):     // inner loop, header at 2
20450  *        ...
20451  *   3: if ...:
20452  *        goto 2b;       // enters loop 2 without going through header 1
20453  *
20454  * Such a bypass makes the enclosing loop irreducible, so its header is missing
20455  * from the stack and both headers (1) and (2) need to be pushed onto stack.
20456  * Only the innermost loop (the one actually entered at insn_idx) carries SCEV bounds;
20457  * the bypassed ancestors are irreducible and pushed as non-terminating.
20458  *
20459  * Assumes loop_stack_pop() has already truncated the stack to the common ancestor
20460  * of the bpf_loop_at_index(env->insn_idx) and whatever was at the top of the loop stack.
20461  */
20462 static int loop_stack_push(struct bpf_verifier_env *env, bool *pushed)
20463 {
20464 	struct bpf_insn_aux_data *aux = env->insn_aux_data;
20465 	struct bpf_func_state *frame = cur_func(env);
20466 	struct loop_stack_entry *loop_stack = frame->loop_stack;
20467 	u32 missing_headers[LOOP_STACK_SIZE];
20468 	u32 cnt = frame->loop_stack_cnt;
20469 	u32 num_missing = 0;
20470 	int h;
20471 
20472 	*pushed = false;
20473 
20474 	for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header, num_missing++) {
20475 		if (cnt && loop_stack[cnt - 1].loop_id == h)
20476 			break;
20477 		if (num_missing == LOOP_STACK_SIZE)
20478 			goto e2big;
20479 		missing_headers[num_missing] = h;
20480 	}
20481 
20482 	if (num_missing == 0)
20483 		return 0;
20484 	if (cnt + num_missing > LOOP_STACK_SIZE)
20485 		goto e2big;
20486 	frame->loop_stack = realloc_array(frame->loop_stack, cnt, cnt + num_missing,
20487 					 sizeof(*frame->loop_stack));
20488 	if (!frame->loop_stack)
20489 		return -ENOMEM;
20490 	loop_stack = frame->loop_stack;
20491 
20492 	for (; num_missing; num_missing--, cnt++) {
20493 		h = missing_headers[num_missing - 1];
20494 		loop_stack[cnt] = (struct loop_stack_entry){ .loop_id = h };
20495 		if (env->log.level & BPF_LOG_LEVEL2)
20496 			verbose(env, "entering loop %d\n", h);
20497 		*pushed = true;
20498 	}
20499 	frame->loop_stack_cnt = cnt;
20500 	return 0;
20501 
20502 e2big:
20503 	verbose(env, "Too many nested loops (%d/%d) at %d\n", cnt, num_missing, env->insn_idx);
20504 	return -E2BIG;
20505 }
20506 
20507 /*
20508  * An exit from a loop can cross several nested loops, e.g.:
20509  *
20510  *   1: for (...):
20511  *   2:   for (...):
20512  *   3:     for (...):
20513  *            if ...:    // before goto the loop stack is [1, 2, 3 <top>]
20514  *              goto 1b; // after goto it should become [1]
20515  */
20516 static int loop_stack_pop(struct bpf_verifier_env *env)
20517 {
20518 	struct bpf_insn_aux_data *aux = env->insn_aux_data;
20519 	struct bpf_func_state *frame = cur_func(env);
20520 	int new_cnt = 0;
20521 	int h, i;
20522 
20523 	/*
20524 	 * Walk the loop nest of insn_idx outwards (innermost first) and stop at
20525 	 * the first loop that is present on the stack - that loop is the common
20526 	 * ancestor and becomes the new top. If no enclosing loop is on the
20527 	 * stack, drop all loops.
20528 	 */
20529 	for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header) {
20530 		for (i = frame->loop_stack_cnt - 1; i >= 0; i--) {
20531 			if (frame->loop_stack[i].loop_id == h) {
20532 				new_cnt = i + 1;
20533 				goto pop;
20534 			}
20535 		}
20536 	}
20537 
20538 pop:
20539 	if (env->log.level & BPF_LOG_LEVEL2)
20540 		for (i = frame->loop_stack_cnt - 1; i >= new_cnt; i--)
20541 			verbose(env, "exiting loop %d\n", frame->loop_stack[i].loop_id);
20542 	frame->loop_stack_cnt = new_cnt;
20543 	return 0;
20544 }
20545 
20546 static int maybe_clamp_scev_regs(struct bpf_verifier_env *env)
20547 {
20548 	struct bpf_func_state *frame = cur_func(env);
20549 	struct loop_stack_entry *entry;
20550 
20551 	if (frame->loop_stack_cnt == 0) {
20552 		verifier_bug(env, "%s: loop stack empty at %d", __FUNCTION__, env->insn_idx);
20553 		return -EFAULT;
20554 	}
20555 
20556 	entry = &frame->loop_stack[frame->loop_stack_cnt - 1];
20557 	if (!entry->terminates)
20558 		return 0;
20559 
20560 	return bpf_clamp_scev_regs(env, frame, env->insn_idx, entry->entry_state,
20561 				   entry->max_header_count);
20562 }
20563 
20564 static int handle_loop_entry_exit(struct bpf_verifier_env *env)
20565 {
20566 	struct bpf_verifier_state *cur = env->cur_state;
20567 	struct bpf_func_state *frame = cur_func(env);
20568 	struct loop_stack_entry *entry;
20569 	struct bpf_loop_iters iters;
20570 	bool pushed, terminates;
20571 	int err;
20572 
20573 	err = loop_stack_pop(env);
20574 	if (err)
20575 		return err;
20576 
20577 	err = loop_stack_push(env, &pushed);
20578 	if (err)
20579 		return err;
20580 
20581 	if (frame->loop_stack_cnt == 0)
20582 		return 0;
20583 
20584 	entry = &frame->loop_stack[frame->loop_stack_cnt - 1];
20585 	if (env->insn_idx != entry->loop_id)
20586 		return 0;
20587 
20588 	/* If nothing was pushed and env->insn_idx is a loop header, we've taken a backedge. */
20589 	if (!pushed)
20590 		return maybe_clamp_scev_regs(env);
20591 
20592 	err = bpf_compute_loop_iters(env, env->cur_state, &iters);
20593 	if (err < 0)
20594 		return err;
20595 
20596 	terminates = err == 1;
20597 	if (!terminates)
20598 		return 0;
20599 
20600 	/* Create a loop_entry checkpoint before widening */
20601 	err = bpf_split_cur_state(env);
20602 	if (err)
20603 		return err;
20604 
20605 	entry->min_header_count = iters.min_header_count;
20606 	entry->max_header_count = iters.max_header_count;
20607 	entry->entry_state = cur->parent;
20608 	entry->terminates = true;
20609 	err = bpf_widen_scev_regs(env, cur, cur->parent, &iters);
20610 	if (err < 0)
20611 		return err;
20612 	return 0;
20613 }
20614 
20615 static int do_check(struct bpf_verifier_env *env)
20616 {
20617 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
20618 	struct bpf_verifier_state *state = env->cur_state;
20619 	struct bpf_insn *insns = env->prog->insnsi;
20620 	int insn_cnt = env->prog->len;
20621 	bool do_print_state = false;
20622 	int prev_insn_idx = -1;
20623 
20624 	for (;;) {
20625 		struct bpf_insn *insn;
20626 		struct bpf_insn_aux_data *insn_aux;
20627 		int err;
20628 
20629 		if (signal_pending(current))
20630 			return -EAGAIN;
20631 
20632 		cond_resched_tasks_rcu_qs();
20633 
20634 		/* reset current history entry on each new instruction */
20635 		env->cur_hist_ent = NULL;
20636 
20637 		env->prev_insn_idx = prev_insn_idx;
20638 		if (env->insn_idx >= insn_cnt) {
20639 			verbose(env, "invalid insn idx %d insn_cnt %d\n",
20640 				env->insn_idx, insn_cnt);
20641 			return -EFAULT;
20642 		}
20643 
20644 		insn = &insns[env->insn_idx];
20645 		insn_aux = &env->insn_aux_data[env->insn_idx];
20646 
20647 		account_processed_insn(env);
20648 
20649 		if (env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
20650 			verbose(env,
20651 				"BPF program is too large. Processed %d insn\n",
20652 				env->insn_processed);
20653 			return -E2BIG;
20654 		}
20655 
20656 		state->last_insn_idx = env->prev_insn_idx;
20657 		state->insn_idx = env->insn_idx;
20658 		/*
20659 		 * Record the incoming edge so active and queued paths use the same
20660 		 * branch-recording path. A zero-offset conditional has identical
20661 		 * successors, so its outcome cannot be reconstructed from the edge.
20662 		 */
20663 		if (!state->speculative && prev_insn_idx >= 0 && prev_insn_idx < insn_cnt) {
20664 			struct bpf_insn *prev_insn = &insns[prev_insn_idx];
20665 			int fallthrough_idx = prev_insn_idx + 1;
20666 			int branch_idx = prev_insn_idx + bpf_jmp_offset(prev_insn) + 1;
20667 			u8 class = BPF_CLASS(prev_insn->code);
20668 			u8 opcode = BPF_OP(prev_insn->code);
20669 
20670 			if ((class == BPF_JMP || class == BPF_JMP32) &&
20671 			    opcode != BPF_JA && opcode != BPF_CALL && opcode != BPF_EXIT &&
20672 			    opcode <= BPF_JCOND && branch_idx != fallthrough_idx) {
20673 				if (env->insn_idx == branch_idx)
20674 					bpf_diag_record_branch(env, prev_insn_idx, true);
20675 				else if (env->insn_idx == fallthrough_idx)
20676 					bpf_diag_record_branch(env, prev_insn_idx, false);
20677 			}
20678 		}
20679 
20680 		/*
20681 		 * Possibly widen the registers before creating a checkpoint
20682 		 * in bpf_is_state_visited(). The next loop iteration will
20683 		 * have a chance to hit this checkpoint and converge.
20684 		 */
20685 		err = handle_loop_entry_exit(env);
20686 		if (err)
20687 			return err;
20688 
20689 		if (bpf_is_prune_point(env, env->insn_idx)) {
20690 			err = bpf_is_state_visited(env, env->insn_idx);
20691 			if (err < 0)
20692 				return err;
20693 			if (err == 1) {
20694 				/* found equivalent state, can prune the search */
20695 				if (env->log.level & BPF_LOG_LEVEL) {
20696 					if (do_print_state)
20697 						verbose(env, "\nfrom %d to %d%s: safe\n",
20698 							env->prev_insn_idx, env->insn_idx,
20699 							env->cur_state->speculative ?
20700 							" (speculative execution)" : "");
20701 					else
20702 						verbose(env, "%d: safe\n", env->insn_idx);
20703 				}
20704 				goto process_bpf_exit;
20705 			}
20706 		}
20707 
20708 		if (bpf_is_jmp_point(env, env->insn_idx)) {
20709 			err = bpf_push_jmp_history(env, state, 0, 0, 0, NULL, 0);
20710 			if (err)
20711 				return err;
20712 		}
20713 
20714 		if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {
20715 			verbose(env, "\nfrom %d to %d%s:",
20716 				env->prev_insn_idx, env->insn_idx,
20717 				env->cur_state->speculative ?
20718 				" (speculative execution)" : "");
20719 			print_verifier_state(env, state, state->curframe, true);
20720 			do_print_state = false;
20721 		}
20722 
20723 		if (bpf_loop_at_index(env, env->insn_idx) >= 0 &&
20724 		    cur_func(env)->loop_stack_cnt == 0) {
20725 			verifier_bug(env, "loop stack empty at %d, while inside the loop %d\n",
20726 				     env->insn_idx, bpf_loop_at_index(env, env->insn_idx));
20727 			return -EFAULT;
20728 		}
20729 
20730 		if (env->log.level & BPF_LOG_LEVEL) {
20731 			if (verifier_state_scratched(env))
20732 				print_insn_state(env, state, state->curframe);
20733 
20734 			verbose_linfo(env, env->insn_idx, "; ");
20735 			env->prev_log_pos = env->log.end_pos;
20736 			verbose(env, "%d: ", env->insn_idx);
20737 			bpf_verbose_insn(env, insn);
20738 			verbose(env, "\n");
20739 			env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;
20740 			env->prev_log_pos = env->log.end_pos;
20741 		}
20742 
20743 		if (bpf_prog_is_offloaded(env->prog->aux)) {
20744 			err = bpf_prog_offload_verify_insn(env, env->insn_idx,
20745 							   env->prev_insn_idx);
20746 			if (err)
20747 				return err;
20748 		}
20749 
20750 		sanitize_mark_insn_seen(env);
20751 		prev_insn_idx = env->insn_idx;
20752 
20753 		/* Sanity check: precomputed constants must match verifier state */
20754 		if (!state->speculative && insn_aux->const_reg_mask) {
20755 			struct bpf_reg_state *regs = cur_regs(env);
20756 			u16 mask = insn_aux->const_reg_mask;
20757 
20758 			for (int r = 0; r < ARRAY_SIZE(insn_aux->const_reg_vals); r++) {
20759 				u32 cval = insn_aux->const_reg_vals[r];
20760 
20761 				if (!(mask & BIT(r)))
20762 					continue;
20763 				if (regs[r].type != SCALAR_VALUE)
20764 					continue;
20765 				if (!tnum_is_const(regs[r].var_off))
20766 					continue;
20767 				if (verifier_bug_if((u32)regs[r].var_off.value != cval,
20768 						    env, "const R%d: %u != %llu",
20769 						    r, cval, regs[r].var_off.value))
20770 					return -EFAULT;
20771 			}
20772 		}
20773 
20774 		/* Reduce verification complexity by stopping speculative path
20775 		 * verification when a nospec is encountered.
20776 		 */
20777 		if (state->speculative && insn_aux->nospec)
20778 			goto process_bpf_exit;
20779 
20780 		err = do_check_insn(env, &do_print_state);
20781 		if (error_recoverable_with_nospec(err) && state->speculative) {
20782 			/* Prevent this speculative path from ever reaching the
20783 			 * insn that would have been unsafe to execute.
20784 			 */
20785 			insn_aux->nospec = true;
20786 			/* If it was an ADD/SUB insn, potentially remove any
20787 			 * markings for alu sanitization.
20788 			 */
20789 			insn_aux->alu_state = 0;
20790 			goto process_bpf_exit;
20791 		} else if (err < 0) {
20792 			return err;
20793 		} else if (err == PROCESS_BPF_EXIT) {
20794 			goto process_bpf_exit;
20795 		} else if (err == INSN_IDX_UPDATED) {
20796 		} else if (err == 0) {
20797 			env->insn_idx++;
20798 		}
20799 
20800 		if (state->speculative && insn_aux->nospec_result) {
20801 			/* If we are on a path that performed a jump-op, this
20802 			 * may skip a nospec patched-in after the jump. This can
20803 			 * currently never happen because nospec_result is only
20804 			 * used for the write-ops
20805 			 * `*(size*)(dst_reg+off)=src_reg|imm32` and helper
20806 			 * calls. These must never skip the following insn
20807 			 * (i.e., bpf_insn_successors()'s opcode_info.can_jump
20808 			 * is false). Still, add a warning to document this in
20809 			 * case nospec_result is used elsewhere in the future.
20810 			 *
20811 			 * All non-branch instructions have a single
20812 			 * fall-through edge. For these, nospec_result should
20813 			 * already work.
20814 			 */
20815 			if (verifier_bug_if((BPF_CLASS(insn->code) == BPF_JMP ||
20816 					     BPF_CLASS(insn->code) == BPF_JMP32) &&
20817 					    BPF_OP(insn->code) != BPF_CALL, env,
20818 					    "speculation barrier after jump instruction may not have the desired effect"))
20819 				return -EFAULT;
20820 process_bpf_exit:
20821 			account_current_path(env);
20822 			mark_verifier_state_scratched(env);
20823 			err = bpf_update_branch_counts(env, env->cur_state);
20824 			if (err)
20825 				return err;
20826 			err = pop_stack(env, &prev_insn_idx, &env->insn_idx,
20827 					pop_log);
20828 			if (err < 0) {
20829 				if (err != -ENOENT)
20830 					return err;
20831 				break;
20832 			} else {
20833 				do_print_state = true;
20834 				continue;
20835 			}
20836 		}
20837 	}
20838 
20839 	return 0;
20840 }
20841 
20842 static int find_btf_percpu_datasec(struct btf *btf)
20843 {
20844 	const struct btf_type *t;
20845 	const char *tname;
20846 	int i, n;
20847 
20848 	/*
20849 	 * Both vmlinux and module each have their own ".data..percpu"
20850 	 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF
20851 	 * types to look at only module's own BTF types.
20852 	 */
20853 	n = btf_nr_types(btf);
20854 	for (i = btf_named_start_id(btf, true); i < n; i++) {
20855 		t = btf_type_by_id(btf, i);
20856 		if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC)
20857 			continue;
20858 
20859 		tname = btf_name_by_offset(btf, t->name_off);
20860 		if (!strcmp(tname, ".data..percpu"))
20861 			return i;
20862 	}
20863 
20864 	return -ENOENT;
20865 }
20866 
20867 /*
20868  * Add btf to the env->used_btfs array. If needed, refcount the
20869  * corresponding kernel module. To simplify caller's logic
20870  * in case of error or if btf was added before the function
20871  * decreases the btf refcount.
20872  */
20873 static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf)
20874 {
20875 	struct btf_mod_pair *btf_mod;
20876 	int ret = 0;
20877 	int i;
20878 
20879 	/* check whether we recorded this BTF (and maybe module) already */
20880 	for (i = 0; i < env->used_btf_cnt; i++)
20881 		if (env->used_btfs[i].btf == btf)
20882 			goto ret_put;
20883 
20884 	if (env->signature) {
20885 		verbose(env, "signed program cannot bind any BTF\n");
20886 		ret = -EACCES;
20887 		goto ret_put;
20888 	}
20889 	if (env->used_btf_cnt >= MAX_USED_BTFS) {
20890 		verbose(env, "The total number of btfs per program has reached the limit of %u\n",
20891 			MAX_USED_BTFS);
20892 		ret = -E2BIG;
20893 		goto ret_put;
20894 	}
20895 
20896 	btf_mod = &env->used_btfs[env->used_btf_cnt];
20897 	btf_mod->btf = btf;
20898 	btf_mod->module = NULL;
20899 
20900 	/* if we reference variables from kernel module, bump its refcount */
20901 	if (btf_is_module(btf)) {
20902 		btf_mod->module = btf_try_get_module(btf);
20903 		if (!btf_mod->module) {
20904 			ret = -ENXIO;
20905 			goto ret_put;
20906 		}
20907 	}
20908 
20909 	env->used_btf_cnt++;
20910 	return 0;
20911 
20912 ret_put:
20913 	/* Either error or this BTF was already added */
20914 	btf_put(btf);
20915 	return ret;
20916 }
20917 
20918 /* replace pseudo btf_id with kernel symbol address */
20919 static int __check_pseudo_btf_id(struct bpf_verifier_env *env,
20920 				 struct bpf_insn *insn,
20921 				 struct bpf_insn_aux_data *aux,
20922 				 struct btf *btf)
20923 {
20924 	const struct btf_var_secinfo *vsi;
20925 	const struct btf_type *datasec;
20926 	const struct btf_type *t;
20927 	const char *sym_name;
20928 	bool percpu = false;
20929 	u32 type, id = insn->imm;
20930 	s32 datasec_id;
20931 	u64 addr;
20932 	int i;
20933 
20934 	t = btf_type_by_id(btf, id);
20935 	if (!t) {
20936 		verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
20937 		return -ENOENT;
20938 	}
20939 
20940 	if (!btf_type_is_var(t) && !btf_type_is_func(t)) {
20941 		verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id);
20942 		return -EINVAL;
20943 	}
20944 
20945 	sym_name = btf_name_by_offset(btf, t->name_off);
20946 	addr = kallsyms_lookup_name(sym_name);
20947 	if (!addr) {
20948 		verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
20949 			sym_name);
20950 		return -ENOENT;
20951 	}
20952 	insn[0].imm = (u32)addr;
20953 	insn[1].imm = addr >> 32;
20954 
20955 	if (btf_type_is_func(t)) {
20956 		aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
20957 		aux->btf_var.mem_size = 0;
20958 		return 0;
20959 	}
20960 
20961 	datasec_id = find_btf_percpu_datasec(btf);
20962 	if (datasec_id > 0) {
20963 		datasec = btf_type_by_id(btf, datasec_id);
20964 		for_each_vsi(i, datasec, vsi) {
20965 			if (vsi->type == id) {
20966 				percpu = true;
20967 				break;
20968 			}
20969 		}
20970 	}
20971 
20972 	type = t->type;
20973 	t = btf_type_skip_modifiers(btf, type, NULL);
20974 	if (percpu) {
20975 		aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU;
20976 		aux->btf_var.btf = btf;
20977 		aux->btf_var.btf_id = type;
20978 	} else if (!btf_type_is_struct(t)) {
20979 		const struct btf_type *ret;
20980 		const char *tname;
20981 		u32 tsize;
20982 
20983 		/* resolve the type size of ksym. */
20984 		ret = btf_resolve_size(btf, t, &tsize);
20985 		if (IS_ERR(ret)) {
20986 			tname = btf_name_by_offset(btf, t->name_off);
20987 			verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
20988 				tname, PTR_ERR(ret));
20989 			return -EINVAL;
20990 		}
20991 		aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
20992 		aux->btf_var.mem_size = tsize;
20993 	} else {
20994 		aux->btf_var.reg_type = PTR_TO_BTF_ID;
20995 		aux->btf_var.btf = btf;
20996 		aux->btf_var.btf_id = type;
20997 	}
20998 
20999 	return 0;
21000 }
21001 
21002 static int check_pseudo_btf_id(struct bpf_verifier_env *env,
21003 			       struct bpf_insn *insn,
21004 			       struct bpf_insn_aux_data *aux)
21005 {
21006 	struct btf *btf;
21007 	int btf_fd;
21008 	int err;
21009 
21010 	btf_fd = insn[1].imm;
21011 	if (btf_fd) {
21012 		btf = btf_get_by_fd(btf_fd);
21013 		if (IS_ERR(btf)) {
21014 			verbose(env, "invalid module BTF object FD specified.\n");
21015 			return -EINVAL;
21016 		}
21017 	} else {
21018 		if (!btf_vmlinux) {
21019 			verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
21020 			return -EINVAL;
21021 		}
21022 		btf_get(btf_vmlinux);
21023 		btf = btf_vmlinux;
21024 	}
21025 
21026 	err = __check_pseudo_btf_id(env, insn, aux, btf);
21027 	if (err) {
21028 		btf_put(btf);
21029 		return err;
21030 	}
21031 
21032 	return __add_used_btf(env, btf);
21033 }
21034 
21035 static bool is_tracing_prog_type(enum bpf_prog_type type)
21036 {
21037 	switch (type) {
21038 	case BPF_PROG_TYPE_KPROBE:
21039 	case BPF_PROG_TYPE_TRACEPOINT:
21040 	case BPF_PROG_TYPE_PERF_EVENT:
21041 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
21042 	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
21043 		return true;
21044 	default:
21045 		return false;
21046 	}
21047 }
21048 
21049 static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
21050 {
21051 	return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
21052 		map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
21053 }
21054 
21055 static int check_map_prog_compatibility(struct bpf_verifier_env *env,
21056 					struct bpf_map *map,
21057 					struct bpf_prog *prog)
21058 
21059 {
21060 	enum bpf_prog_type prog_type = resolve_prog_type(prog);
21061 
21062 	if (btf_record_has_field(map->record, BPF_LIST_HEAD) ||
21063 	    btf_record_has_field(map->record, BPF_RB_ROOT)) {
21064 		if (is_tracing_prog_type(prog_type)) {
21065 			verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n");
21066 			return -EINVAL;
21067 		}
21068 	}
21069 
21070 	if (btf_record_has_field(map->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) {
21071 		if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) {
21072 			verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");
21073 			return -EINVAL;
21074 		}
21075 	}
21076 
21077 	if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
21078 		if (is_tracing_prog_type(prog_type)) {
21079 			verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
21080 			return -EINVAL;
21081 		}
21082 	}
21083 
21084 	if ((bpf_prog_is_offloaded(prog->aux) || bpf_map_is_offloaded(map)) &&
21085 	    !bpf_offload_prog_map_match(prog, map)) {
21086 		verbose(env, "offload device mismatch between prog and map\n");
21087 		return -EINVAL;
21088 	}
21089 
21090 	if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
21091 		verbose(env, "bpf_struct_ops map cannot be used in prog\n");
21092 		return -EINVAL;
21093 	}
21094 
21095 	if (prog->sleepable && !map_needs_no_rcu(map)) {
21096 		verbose(env,
21097 			"Sleepable programs can only use array, hash, ringbuf and local storage maps\n");
21098 		return -EINVAL;
21099 	}
21100 
21101 	if (bpf_map_is_cgroup_storage(map) &&
21102 	    bpf_cgroup_storage_assign(env->prog->aux, map)) {
21103 		verbose(env, "only one cgroup storage of each type is allowed\n");
21104 		return -EBUSY;
21105 	}
21106 
21107 	if (map->map_type == BPF_MAP_TYPE_ARENA) {
21108 		if (env->prog->aux->arena) {
21109 			verbose(env, "Only one arena per program\n");
21110 			return -EBUSY;
21111 		}
21112 		if (!env->allow_ptr_leaks || !env->bpf_capable) {
21113 			verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n");
21114 			return -EPERM;
21115 		}
21116 		if (!env->prog->jit_requested) {
21117 			verbose(env, "JIT is required to use arena\n");
21118 			return -EOPNOTSUPP;
21119 		}
21120 		if (!bpf_jit_supports_arena()) {
21121 			verbose(env, "JIT doesn't support arena\n");
21122 			return -EOPNOTSUPP;
21123 		}
21124 		env->prog->aux->arena = (void *)map;
21125 		env->prog->jit_required = true;
21126 		if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) {
21127 			verbose(env, "arena's user address must be set via map_extra or mmap()\n");
21128 			return -EINVAL;
21129 		}
21130 	}
21131 
21132 	return 0;
21133 }
21134 
21135 /*
21136  * Keep track of whether the map is used by one program only. Such program may
21137  * store the addresses of its functions into the map when it's frozen, see
21138  * resolve_func_ptrs(), since nothing else relies on what the map has. After
21139  * that the map is not available to other programs.
21140  */
21141 static int bpf_map_claim(struct bpf_verifier_env *env, struct bpf_map *map)
21142 {
21143 	unsigned long me = (unsigned long)env->prog->aux, old;
21144 
21145 	for (;;) {
21146 		old = READ_ONCE(map->user);
21147 		if (old == me || old == BPF_MAP_USER_MANY)
21148 			return 0;
21149 		if (old & BPF_MAP_USER_PATCHED) {
21150 			verbose(env, "map '%s' has addresses of functions of another program\n",
21151 				map->name);
21152 			return -EBUSY;
21153 		}
21154 		if (cmpxchg(&map->user, old, old ? BPF_MAP_USER_MANY : me) == old)
21155 			return 0;
21156 	}
21157 }
21158 
21159 /*
21160  * An exclusive map is bound to the digest of its program. The digest covers
21161  * the insns as they are verified, after the CO-RE relocations the kernel
21162  * applies, so it exists only once check_and_resolve_insns() has computed it,
21163  * and the maps are checked against it there, whether an insn referred to
21164  * them or a continuous fd_array bound them before.
21165  */
21166 static int check_excl_maps(struct bpf_verifier_env *env)
21167 {
21168 	int i;
21169 
21170 	for (i = 0; i < env->used_map_cnt; i++) {
21171 		struct bpf_map *map = env->used_maps[i];
21172 
21173 		if (map->excl_prog_sha &&
21174 		    memcmp(map->excl_prog_sha, env->prog->digest, SHA256_DIGEST_SIZE)) {
21175 			verbose(env, "program's hash doesn't match map's excl_prog_hash\n");
21176 			return -EACCES;
21177 		}
21178 	}
21179 	return 0;
21180 }
21181 
21182 /*
21183  * The program may use every map it referred to, see check_excl_maps(). Only now
21184  * does it leave a trace on them: it claims each as its user, see
21185  * bpf_map_claim(), and takes over its insn arrays. A program rejected before
21186  * this point, one that is not the owner of an exclusive map in particular,
21187  * leaves the maps as it found them. Nothing consults a claim before
21188  * resolve_func_ptrs() and the fixup passes, which run later.
21189  */
21190 static int claim_used_maps(struct bpf_verifier_env *env)
21191 {
21192 	int i, err;
21193 
21194 	for (i = 0; i < env->used_map_cnt; i++) {
21195 		struct bpf_map *map = env->used_maps[i];
21196 
21197 		err = bpf_map_claim(env, map);
21198 		if (err)
21199 			return err;
21200 
21201 		if (map->map_type != BPF_MAP_TYPE_INSN_ARRAY)
21202 			continue;
21203 		err = bpf_insn_array_init(map, env->prog);
21204 		if (err) {
21205 			verbose(env, "Failed to properly initialize insn array\n");
21206 			return err;
21207 		}
21208 		env->insn_array_maps[env->insn_array_map_cnt++] = map;
21209 		env->prog->jit_required = true;
21210 	}
21211 	return 0;
21212 }
21213 
21214 static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map)
21215 {
21216 	int i, err;
21217 
21218 	/* check whether we recorded this map already */
21219 	for (i = 0; i < env->used_map_cnt; i++)
21220 		if (env->used_maps[i] == map)
21221 			return i;
21222 
21223 	if (env->signature &&
21224 	    env->prog->aux->sig.verdict == BPF_SIG_VERIFIED) {
21225 		verbose(env, "signed program cannot bind map '%s' not covered by the signature\n",
21226 			map->name);
21227 		return -EACCES;
21228 	}
21229 	if (env->used_map_cnt >= MAX_USED_MAPS) {
21230 		verbose(env, "The total number of maps per program has reached the limit of %u\n",
21231 			MAX_USED_MAPS);
21232 		return -E2BIG;
21233 	}
21234 
21235 	err = check_map_prog_compatibility(env, map, env->prog);
21236 	if (err)
21237 		return err;
21238 
21239 	if (bpf_prog_runs_sleepable(env->prog))
21240 		atomic64_inc(&map->sleepable_refcnt);
21241 
21242 	/* hold the map. If the program is rejected by verifier,
21243 	 * the map will be released by release_maps() or it
21244 	 * will be used by the valid program until it's unloaded
21245 	 * and all maps are released in bpf_free_used_maps()
21246 	 */
21247 	bpf_map_inc(map);
21248 
21249 	env->used_maps[env->used_map_cnt++] = map;
21250 
21251 	return env->used_map_cnt - 1;
21252 }
21253 
21254 /* Add map behind fd to used maps list, if it's not already there, and return
21255  * its index.
21256  * Returns <0 on error, or >= 0 index, on success.
21257  */
21258 static int add_used_map(struct bpf_verifier_env *env, int fd)
21259 {
21260 	struct bpf_map *map;
21261 	CLASS(fd, f)(fd);
21262 
21263 	map = __bpf_map_get(f);
21264 	if (IS_ERR(map)) {
21265 		verbose(env, "fd %d is not pointing to valid bpf_map\n", fd);
21266 		return PTR_ERR(map);
21267 	}
21268 
21269 	return __add_used_map(env, map);
21270 }
21271 
21272 static int fd_array_get_map_idx_continuous(struct bpf_verifier_env *env, u32 idx)
21273 {
21274 	struct bpf_map *map;
21275 
21276 	if (idx >= env->fd_array_cnt) {
21277 		verbose(env, "fd_idx %u out of bounds, fd_array_cnt %u\n",
21278 			idx, env->fd_array_cnt);
21279 		return -EINVAL;
21280 	}
21281 	map = fd_slot_map(env->fd_array[idx]);
21282 	if (!map) {
21283 		verbose(env, "fd_idx %u is not a map\n", idx);
21284 		return -EINVAL;
21285 	}
21286 	return __add_used_map(env, map);
21287 }
21288 
21289 static int fd_array_get_map_idx_sparse(struct bpf_verifier_env *env, u32 idx)
21290 {
21291 	int fd;
21292 
21293 	if (copy_from_bpfptr_offset(&fd, env->fd_array_raw,
21294 				    (size_t)idx * sizeof(fd), sizeof(fd)))
21295 		return -EFAULT;
21296 	return add_used_map(env, fd);
21297 }
21298 
21299 static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx)
21300 {
21301 	if (env->fd_array)
21302 		return fd_array_get_map_idx_continuous(env, idx);
21303 	if (env->signature) {
21304 		verbose(env, "signed program must bind maps via a continuous fd_array (fd_array_cnt)\n");
21305 		return -EACCES;
21306 	}
21307 	if (!bpfptr_is_null(env->fd_array_raw))
21308 		return fd_array_get_map_idx_sparse(env, idx);
21309 
21310 	verbose(env, "fd_idx without fd_array is invalid\n");
21311 	return -EPROTO;
21312 }
21313 
21314 static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
21315 {
21316 	u8 class = BPF_CLASS(insn->code);
21317 	u8 opcode = BPF_OP(insn->code);
21318 
21319 	switch (opcode) {
21320 	case BPF_NEG:
21321 		if (BPF_SRC(insn->code) != BPF_K || insn->src_reg != BPF_REG_0 ||
21322 		    insn->off != 0 || insn->imm != 0) {
21323 			verbose(env, "BPF_NEG uses reserved fields\n");
21324 			return -EINVAL;
21325 		}
21326 		return 0;
21327 	case BPF_END:
21328 		if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
21329 		    (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
21330 		    (class == BPF_ALU64 && BPF_SRC(insn->code) != BPF_TO_LE)) {
21331 			verbose(env, "BPF_END uses reserved fields\n");
21332 			return -EINVAL;
21333 		}
21334 		return 0;
21335 	case BPF_MOV:
21336 		if (BPF_SRC(insn->code) == BPF_X) {
21337 			if (class == BPF_ALU) {
21338 				if ((insn->off != 0 && insn->off != 8 && insn->off != 16) ||
21339 				    insn->imm) {
21340 					verbose(env, "BPF_MOV uses reserved fields\n");
21341 					return -EINVAL;
21342 				}
21343 			} else if (insn->off == BPF_ADDR_SPACE_CAST) {
21344 				if (insn->imm != 1 && insn->imm != 1u << 16) {
21345 					verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n");
21346 					return -EINVAL;
21347 				}
21348 			} else if ((insn->off != 0 && insn->off != 8 &&
21349 				    insn->off != 16 && insn->off != 32) || insn->imm) {
21350 				verbose(env, "BPF_MOV uses reserved fields\n");
21351 				return -EINVAL;
21352 			}
21353 		} else if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
21354 			verbose(env, "BPF_MOV uses reserved fields\n");
21355 			return -EINVAL;
21356 		}
21357 		return 0;
21358 	case BPF_ADD:
21359 	case BPF_SUB:
21360 	case BPF_AND:
21361 	case BPF_OR:
21362 	case BPF_XOR:
21363 	case BPF_LSH:
21364 	case BPF_RSH:
21365 	case BPF_ARSH:
21366 	case BPF_MUL:
21367 	case BPF_DIV:
21368 	case BPF_MOD:
21369 		if (BPF_SRC(insn->code) == BPF_X) {
21370 			if (insn->imm != 0 || (insn->off != 0 && insn->off != 1) ||
21371 			    (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
21372 				verbose(env, "BPF_ALU uses reserved fields\n");
21373 				return -EINVAL;
21374 			}
21375 		} else if (insn->src_reg != BPF_REG_0 ||
21376 			   (insn->off != 0 && insn->off != 1) ||
21377 			   (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
21378 			verbose(env, "BPF_ALU uses reserved fields\n");
21379 			return -EINVAL;
21380 		}
21381 		return 0;
21382 	default:
21383 		verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
21384 		return -EINVAL;
21385 	}
21386 }
21387 
21388 static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
21389 {
21390 	u8 class = BPF_CLASS(insn->code);
21391 	u8 opcode = BPF_OP(insn->code);
21392 
21393 	switch (opcode) {
21394 	case BPF_CALL:
21395 		if (bpf_is_callx(insn)) {
21396 			/* callx dst_reg */
21397 			if (insn->src_reg != BPF_REG_0 || insn->imm != 0 || insn->off != 0) {
21398 				verbose(env, "BPF_CALL|BPF_X uses reserved fields\n");
21399 				return -EINVAL;
21400 			}
21401 			return 0;
21402 		}
21403 		if (BPF_SRC(insn->code) != BPF_K ||
21404 		    (insn->src_reg != BPF_PSEUDO_KFUNC_CALL && insn->off != 0) ||
21405 		    (insn->src_reg != BPF_REG_0 && insn->src_reg != BPF_PSEUDO_CALL &&
21406 		     insn->src_reg != BPF_PSEUDO_KFUNC_CALL) ||
21407 		    insn->dst_reg != BPF_REG_0 || class == BPF_JMP32) {
21408 			verbose(env, "BPF_CALL uses reserved fields\n");
21409 			return -EINVAL;
21410 		}
21411 		return 0;
21412 	case BPF_JA:
21413 		if (BPF_SRC(insn->code) == BPF_X) {
21414 			if (insn->src_reg != BPF_REG_0 || insn->imm != 0 || insn->off != 0) {
21415 				verbose(env, "BPF_JA|BPF_X uses reserved fields\n");
21416 				return -EINVAL;
21417 			}
21418 		} else if (insn->src_reg != BPF_REG_0 || insn->dst_reg != BPF_REG_0 ||
21419 			   (class == BPF_JMP && insn->imm != 0) ||
21420 			   (class == BPF_JMP32 && insn->off != 0)) {
21421 			verbose(env, "BPF_JA uses reserved fields\n");
21422 			return -EINVAL;
21423 		}
21424 		return 0;
21425 	case BPF_EXIT:
21426 		if (BPF_SRC(insn->code) != BPF_K || insn->imm != 0 ||
21427 		    insn->src_reg > BPF_UNWIND_EXIT || insn->dst_reg != BPF_REG_0 ||
21428 		    class == BPF_JMP32) {
21429 			verbose(env, "BPF_EXIT uses reserved fields\n");
21430 			return -EINVAL;
21431 		}
21432 		return 0;
21433 	case BPF_JCOND:
21434 		if (insn->code != (BPF_JMP | BPF_JCOND) || insn->src_reg > BPF_UNWIND_GOTO ||
21435 		    insn->dst_reg || insn->imm) {
21436 			verbose(env, "invalid may_goto imm %d\n", insn->imm);
21437 			return -EINVAL;
21438 		}
21439 		return 0;
21440 	default:
21441 		if (BPF_SRC(insn->code) == BPF_X) {
21442 			if (insn->imm != 0) {
21443 				verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
21444 				return -EINVAL;
21445 			}
21446 		} else if (insn->src_reg != BPF_REG_0) {
21447 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
21448 			return -EINVAL;
21449 		}
21450 		return 0;
21451 	}
21452 }
21453 
21454 static int check_insn_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
21455 {
21456 	switch (BPF_CLASS(insn->code)) {
21457 	case BPF_ALU:
21458 	case BPF_ALU64:
21459 		return check_alu_fields(env, insn);
21460 	case BPF_LDX:
21461 		if ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) ||
21462 		    insn->imm != 0) {
21463 			verbose(env, "BPF_LDX uses reserved fields\n");
21464 			return -EINVAL;
21465 		}
21466 		return 0;
21467 	case BPF_STX:
21468 		if (BPF_MODE(insn->code) == BPF_ATOMIC)
21469 			return 0;
21470 		if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) {
21471 			verbose(env, "BPF_STX uses reserved fields\n");
21472 			return -EINVAL;
21473 		}
21474 		return 0;
21475 	case BPF_ST:
21476 		if (BPF_MODE(insn->code) != BPF_MEM || insn->src_reg != BPF_REG_0) {
21477 			verbose(env, "BPF_ST uses reserved fields\n");
21478 			return -EINVAL;
21479 		}
21480 		return 0;
21481 	case BPF_JMP:
21482 	case BPF_JMP32:
21483 		return check_jmp_fields(env, insn);
21484 	case BPF_LD: {
21485 		u8 mode = BPF_MODE(insn->code);
21486 
21487 		if (mode == BPF_ABS || mode == BPF_IND) {
21488 			if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
21489 			    BPF_SIZE(insn->code) == BPF_DW ||
21490 			    (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
21491 				verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
21492 				return -EINVAL;
21493 			}
21494 		} else if (mode != BPF_IMM) {
21495 			verbose(env, "invalid BPF_LD mode\n");
21496 			return -EINVAL;
21497 		}
21498 		return 0;
21499 	}
21500 	default:
21501 		verbose(env, "unknown insn class %d\n", BPF_CLASS(insn->code));
21502 		return -EINVAL;
21503 	}
21504 }
21505 
21506 /*
21507  * Check that insns are sane and rewrite pseudo imm in ld_imm64 instructions:
21508  *
21509  * 1. if it accesses map FD, replace it with actual map pointer.
21510  * 2. if it accesses btf_id of a VAR, replace it with pointer to the var.
21511  *
21512  * NOTE: btf_vmlinux is required for converting pseudo btf_id.
21513  */
21514 static int check_and_resolve_insns(struct bpf_verifier_env *env)
21515 {
21516 	struct bpf_insn *insn = env->prog->insnsi;
21517 	int insn_cnt = env->prog->len;
21518 	int i, err;
21519 
21520 	err = bpf_prog_calc_tag(env->prog);
21521 	if (err)
21522 		return err;
21523 
21524 	for (i = 0; i < insn_cnt; i++, insn++) {
21525 		if (insn->dst_reg >= MAX_BPF_REG &&
21526 		    !is_stack_arg_st(insn) && !is_stack_arg_stx(insn)) {
21527 			verbose(env, "R%d is invalid\n", insn->dst_reg);
21528 			return -EINVAL;
21529 		}
21530 		if (insn->src_reg >= MAX_BPF_REG && !is_stack_arg_ldx(insn)) {
21531 			verbose(env, "R%d is invalid\n", insn->src_reg);
21532 			return -EINVAL;
21533 		}
21534 		if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
21535 			struct bpf_insn_aux_data *aux;
21536 			struct bpf_map *map;
21537 			int map_idx;
21538 			u64 addr;
21539 
21540 			if (i == insn_cnt - 1 || insn[1].code != 0 ||
21541 			    insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
21542 			    insn[1].off != 0) {
21543 				verbose(env, "invalid bpf_ld_imm64 insn\n");
21544 				return -EINVAL;
21545 			}
21546 
21547 			if (insn[0].off != 0) {
21548 				verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
21549 				return -EINVAL;
21550 			}
21551 
21552 			if (insn[0].src_reg == 0)
21553 				/* valid generic load 64-bit imm */
21554 				goto next_insn;
21555 
21556 			if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
21557 				aux = &env->insn_aux_data[i];
21558 				err = check_pseudo_btf_id(env, insn, aux);
21559 				if (err)
21560 					return err;
21561 				goto next_insn;
21562 			}
21563 
21564 			if (insn[0].src_reg == BPF_PSEUDO_FUNC) {
21565 				aux = &env->insn_aux_data[i];
21566 				aux->ptr_type = PTR_TO_FUNC;
21567 				goto next_insn;
21568 			}
21569 
21570 			/* In final convert_pseudo_ld_imm64() step, this is
21571 			 * converted into regular 64-bit imm load insn.
21572 			 */
21573 			switch (insn[0].src_reg) {
21574 			case BPF_PSEUDO_MAP_VALUE:
21575 			case BPF_PSEUDO_MAP_IDX_VALUE:
21576 				break;
21577 			case BPF_PSEUDO_MAP_FD:
21578 			case BPF_PSEUDO_MAP_IDX:
21579 				if (insn[1].imm == 0)
21580 					break;
21581 				fallthrough;
21582 			default:
21583 				verbose(env, "unrecognized bpf_ld_imm64 insn\n");
21584 				return -EINVAL;
21585 			}
21586 
21587 			switch (insn[0].src_reg) {
21588 			case BPF_PSEUDO_MAP_IDX_VALUE:
21589 			case BPF_PSEUDO_MAP_IDX:
21590 				map_idx = fd_array_get_map_idx(env, insn[0].imm);
21591 				break;
21592 			default:
21593 				if (env->signature) {
21594 					verbose(env, "signed program cannot reference a map by fd, only via fd_array index\n");
21595 					return -EINVAL;
21596 				}
21597 				map_idx = add_used_map(env, insn[0].imm);
21598 				break;
21599 			}
21600 
21601 			if (map_idx < 0)
21602 				return map_idx;
21603 			map = env->used_maps[map_idx];
21604 
21605 			aux = &env->insn_aux_data[i];
21606 			aux->map_index = map_idx;
21607 
21608 			if (insn[0].src_reg == BPF_PSEUDO_MAP_FD ||
21609 			    insn[0].src_reg == BPF_PSEUDO_MAP_IDX) {
21610 				addr = (unsigned long)map;
21611 			} else {
21612 				u32 off = insn[1].imm;
21613 
21614 				if (!map->ops->map_direct_value_addr) {
21615 					verbose(env, "no direct value access support for this map type\n");
21616 					return -EINVAL;
21617 				}
21618 
21619 				err = map->ops->map_direct_value_addr(map, &addr, off);
21620 				if (err) {
21621 					verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",
21622 						map->value_size, off);
21623 					return err;
21624 				}
21625 
21626 				aux->map_off = off;
21627 				addr += off;
21628 			}
21629 
21630 			insn[0].imm = (u32)addr;
21631 			insn[1].imm = addr >> 32;
21632 
21633 next_insn:
21634 			insn++;
21635 			i++;
21636 			continue;
21637 		}
21638 
21639 		/* Basic sanity check before we invest more work here. */
21640 		if (!bpf_opcode_in_insntable(insn->code)) {
21641 			verbose(env, "unknown opcode %02x\n", insn->code);
21642 			return -EINVAL;
21643 		}
21644 
21645 		err = check_insn_fields(env, insn);
21646 		if (err)
21647 			return err;
21648 	}
21649 
21650 	/* now all pseudo BPF_LD_IMM64 instructions load valid
21651 	 * 'struct bpf_map *' into a register instead of user map_fd.
21652 	 * These pointers will be used later by verifier to validate map access.
21653 	 */
21654 	err = check_excl_maps(env);
21655 	if (err)
21656 		return err;
21657 	return claim_used_maps(env);
21658 }
21659 
21660 /* that many pointers to functions are recognized in the maps of a program at most */
21661 #define BPF_MAX_FUNC_PTRS (1 << 20)
21662 
21663 static int add_func_ptr(struct bpf_verifier_env *env, struct bpf_map *map, u32 map_off,
21664 			u32 xlated_off)
21665 {
21666 	struct bpf_func_ptr *ptrs;
21667 
21668 	if (env->func_ptr_cnt == BPF_MAX_FUNC_PTRS) {
21669 		verbose(env, "too many pointers to functions in maps, the limit is %d\n",
21670 			BPF_MAX_FUNC_PTRS);
21671 		return -E2BIG;
21672 	}
21673 
21674 	/* grow by doubling, the array is sorted and searched later */
21675 	if (!(env->func_ptr_cnt & (env->func_ptr_cnt - 1))) {
21676 		ptrs = kvrealloc(env->func_ptrs,
21677 				 array_size(max(2 * env->func_ptr_cnt, 16U), sizeof(*ptrs)),
21678 				 GFP_KERNEL_ACCOUNT);
21679 		if (!ptrs)
21680 			return -ENOMEM;
21681 		env->func_ptrs = ptrs;
21682 	}
21683 	env->func_ptrs[env->func_ptr_cnt++] = (struct bpf_func_ptr){
21684 		.map = map,
21685 		.map_off = map_off,
21686 		.orig_off = xlated_off,
21687 		.xlated_off = xlated_off,
21688 	};
21689 	return 0;
21690 }
21691 
21692 /*
21693  * Compilers put pointers to functions into read-only data: tables of functions,
21694  * structures of operations, vtables, where they are mixed with other data.
21695  * The loader stores such data in a frozen read-only array map and resolves
21696  * a pointer to a static function to the offset in bytes of its first
21697  * instruction in the program: the address of the function in the program.
21698  *
21699  * Find 64-bit values that look like that in the maps of a program that uses
21700  * callx. It's a guess. When the value is not a pointer, the program either
21701  * fails to load, because it does with a pointer what can be done with
21702  * a number only, or it sees the address of a function instead of the number.
21703  * It's known before the control flow graph of the program is built and the main
21704  * verification pass begins which functions may be called via callx.
21705  *
21706  * When the program is jitted the offsets are replaced with the addresses of
21707  * the functions in the map itself, see jit_subprogs(). Hence the program has to
21708  * be the only user of the map, see bpf_map_claim(): nothing else may rely on
21709  * what the map had.
21710  *
21711  * The program reads the addresses of its functions from there like any other
21712  * data, so it has to be allowed to leak pointers.
21713  */
21714 static int resolve_func_ptrs(struct bpf_verifier_env *env)
21715 {
21716 	int insn_cnt = env->prog->len;
21717 	int i, err, subprog;
21718 	struct bpf_map *map;
21719 	u64 addr, val;
21720 	u32 off;
21721 
21722 	if (!env->has_callx || !env->allow_ptr_leaks)
21723 		return 0;
21724 
21725 	for (i = 0; i < env->used_map_cnt; i++) {
21726 		map = env->used_maps[i];
21727 		/* coincidences in maps that are shared with other programs don't matter */
21728 		if (READ_ONCE(map->user) != (unsigned long)env->prog->aux)
21729 			continue;
21730 		if (map->map_type != BPF_MAP_TYPE_ARRAY || map->max_entries != 1 ||
21731 		    !bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr ||
21732 		    !IS_ERR_OR_NULL(map->record))
21733 			continue;
21734 		if (map->ops->map_direct_value_addr(map, &addr, 0))
21735 			continue;
21736 
21737 		for (off = 0; off + sizeof(u64) <= map->value_size; off += sizeof(u64)) {
21738 			if (!(off % SZ_64K))
21739 				cond_resched();
21740 			val = *(u64 *)(unsigned long)(addr + off);
21741 			if (!val || val % sizeof(struct bpf_insn) ||
21742 			    val / sizeof(struct bpf_insn) >= insn_cnt)
21743 				continue;
21744 			subprog = bpf_find_subprog(env, val / sizeof(struct bpf_insn));
21745 			if (subprog <= 0 || bpf_subprog_is_global(env, subprog))
21746 				continue;
21747 			err = add_func_ptr(env, map, off, val / sizeof(struct bpf_insn));
21748 			if (err)
21749 				return err;
21750 		}
21751 	}
21752 	if (env->func_ptr_cnt)
21753 		sort(env->func_ptrs, env->func_ptr_cnt, sizeof(*env->func_ptrs),
21754 		     cmp_func_ptrs, NULL);
21755 	return 0;
21756 }
21757 
21758 /* drop refcnt of maps used by the rejected program */
21759 static void release_maps(struct bpf_verifier_env *env)
21760 {
21761 	__bpf_free_used_maps(env->prog->aux, env->used_maps,
21762 			     env->used_map_cnt);
21763 }
21764 
21765 /* drop refcnt of maps used by the rejected program */
21766 static void release_btfs(struct bpf_verifier_env *env)
21767 {
21768 	__bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt);
21769 }
21770 
21771 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */
21772 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
21773 {
21774 	struct bpf_insn *insn = env->prog->insnsi;
21775 	int insn_cnt = env->prog->len;
21776 	int i;
21777 
21778 	for (i = 0; i < insn_cnt; i++, insn++) {
21779 		if (insn->code != (BPF_LD | BPF_IMM | BPF_DW))
21780 			continue;
21781 		if (insn->src_reg == BPF_PSEUDO_FUNC)
21782 			continue;
21783 		insn->src_reg = 0;
21784 	}
21785 }
21786 
21787 static void release_insn_arrays(struct bpf_verifier_env *env)
21788 {
21789 	int i;
21790 
21791 	for (i = 0; i < env->insn_array_map_cnt; i++)
21792 		bpf_insn_array_release(env->insn_array_maps[i]);
21793 }
21794 
21795 /* The verifier does more data flow analysis than llvm and will not
21796  * explore branches that are dead at run time. Malicious programs can
21797  * have dead code too. Therefore replace all dead at-run-time code
21798  * with 'ja -1'.
21799  *
21800  * Just nops are not optimal, e.g. if they would sit at the end of the
21801  * program and through another bug we would manage to jump there, then
21802  * we'd execute beyond program memory otherwise. Returning exception
21803  * code also wouldn't work since we can have subprogs where the dead
21804  * code could be located.
21805  */
21806 static void sanitize_dead_code(struct bpf_verifier_env *env)
21807 {
21808 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
21809 	struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
21810 	struct bpf_insn *insn = env->prog->insnsi;
21811 	const int insn_cnt = env->prog->len;
21812 	int i;
21813 
21814 	for (i = 0; i < insn_cnt; i++) {
21815 		if (aux_data[i].seen)
21816 			continue;
21817 		memcpy(insn + i, &trap, sizeof(trap));
21818 		aux_data[i].zext_dst = false;
21819 	}
21820 }
21821 
21822 static void free_states(struct bpf_verifier_env *env)
21823 {
21824 	struct bpf_verifier_state_list *sl;
21825 	struct list_head *head, *pos, *tmp;
21826 	struct bpf_scc_info *info;
21827 	int i, j;
21828 
21829 	bpf_free_verifier_state(env->cur_state, true);
21830 	env->cur_state = NULL;
21831 	while (!pop_stack(env, NULL, NULL, false));
21832 
21833 	list_for_each_safe(pos, tmp, &env->free_list) {
21834 		sl = container_of(pos, struct bpf_verifier_state_list, node);
21835 		bpf_free_verifier_state(&sl->state, false);
21836 		kfree(sl);
21837 	}
21838 	INIT_LIST_HEAD(&env->free_list);
21839 
21840 	for (i = 0; i < env->scc_cnt; ++i) {
21841 		info = env->scc_info[i];
21842 		if (!info)
21843 			continue;
21844 		for (j = 0; j < info->num_visits; j++)
21845 			bpf_free_backedges(&info->visits[j]);
21846 		kvfree(info);
21847 		env->scc_info[i] = NULL;
21848 	}
21849 
21850 	if (!env->explored_states)
21851 		return;
21852 
21853 	for (i = 0; i < state_htab_size(env); i++) {
21854 		head = &env->explored_states[i];
21855 
21856 		list_for_each_safe(pos, tmp, head) {
21857 			sl = container_of(pos, struct bpf_verifier_state_list, node);
21858 			bpf_free_verifier_state(&sl->state, false);
21859 			kfree(sl);
21860 		}
21861 		INIT_LIST_HEAD(&env->explored_states[i]);
21862 	}
21863 }
21864 
21865 static int do_check_common(struct bpf_verifier_env *env, int subprog,
21866 			   enum bpf_func_context context)
21867 {
21868 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
21869 	struct bpf_subprog_info *sub = subprog_info(env, subprog);
21870 	struct bpf_prog_aux *aux = env->prog->aux;
21871 	struct bpf_verifier_state *state;
21872 	struct bpf_reg_state *regs;
21873 	u32 old_insns_total = sub->insns_total;
21874 	u32 insn_processed = env->insn_processed;
21875 	int ret, i;
21876 
21877 	env->prev_linfo = NULL;
21878 	env->pass_cnt++;
21879 
21880 	state = kzalloc_obj(struct bpf_verifier_state, GFP_KERNEL_ACCOUNT);
21881 	if (!state)
21882 		return -ENOMEM;
21883 	state->curframe = 0;
21884 	state->speculative = false;
21885 	state->branches = 1;
21886 	state->in_sleepable = context == BPF_FUNC_CTX_SLEEPABLE;
21887 	env->no_rcu_read_lock = context == BPF_FUNC_CTX_NO_RCU_LOCK;
21888 	state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT);
21889 	if (!state->frame[0]) {
21890 		kfree(state);
21891 		return -ENOMEM;
21892 	}
21893 	env->cur_state = state;
21894 	init_func_state(env, state->frame[0],
21895 			BPF_MAIN_FUNC /* callsite */,
21896 			0 /* frameno */,
21897 			subprog);
21898 	state->first_insn_idx = env->subprog_info[subprog].start;
21899 	state->last_insn_idx = -1;
21900 
21901 	regs = state->frame[state->curframe]->regs;
21902 	if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {
21903 		const char *sub_name = bpf_subprog_name(env, subprog);
21904 		struct bpf_subprog_arg_info *arg;
21905 		struct bpf_reg_state *reg;
21906 
21907 		if (env->log.level & BPF_LOG_LEVEL)
21908 			verbose(env, "Validating %s() func#%d...\n", sub_name, subprog);
21909 		ret = btf_prepare_func_args(env, subprog);
21910 		if (ret)
21911 			goto out;
21912 
21913 		if (subprog_is_exc_cb(env, subprog)) {
21914 			state->frame[0]->in_exception_callback_fn = true;
21915 
21916 			/*
21917 			 * Global functions are scalar or void, make sure
21918 			 * we return a scalar.
21919 			 */
21920 			if (subprog_returns_void(env, subprog)) {
21921 				verbose(env, "exception cb cannot return void\n");
21922 				ret = -EINVAL;
21923 				goto out;
21924 			}
21925 
21926 			/* Also ensure the callback only has a single scalar argument. */
21927 			if (sub->arg_slot_cnt != 1 || sub->args[0].arg_type != ARG_SCALAR) {
21928 				verbose(env, "exception cb only supports single integer argument\n");
21929 				ret = -EINVAL;
21930 				goto out;
21931 			}
21932 		}
21933 		for (i = BPF_REG_1;
21934 		     i <= min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) {
21935 			arg = &sub->args[i - BPF_REG_1];
21936 			reg = &regs[i];
21937 
21938 			if (arg->arg_type == ARG_PTR_TO_CTX) {
21939 				reg->type = PTR_TO_CTX;
21940 				mark_reg_known_zero(env, regs, i);
21941 			} else if (arg->arg_type == ARG_SCALAR) {
21942 				reg->type = SCALAR_VALUE;
21943 				mark_reg_unknown(env, regs, i);
21944 			} else if (arg->arg_type == ARG_PTR_TO_DYNPTR) {
21945 				/*
21946 				 * The callers may hand in a dynptr of any type.
21947 				 * Take it for that of an skb, so that a write
21948 				 * through it and its slices are treated with care.
21949 				 */
21950 				__mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_SKB, true, ++env->id_gen, 0);
21951 			} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
21952 				reg->type = PTR_TO_MEM;
21953 				reg->type |= arg->arg_type &
21954 					     (PTR_MAYBE_NULL | PTR_UNTRUSTED | MEM_RDONLY);
21955 				mark_reg_known_zero(env, regs, i);
21956 				reg->mem_size = arg->mem_size;
21957 				if (arg->arg_type & PTR_MAYBE_NULL)
21958 					reg->id = ++env->id_gen;
21959 			} else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
21960 				reg->type = PTR_TO_BTF_ID;
21961 				if (arg->arg_type & PTR_MAYBE_NULL)
21962 					reg->type |= PTR_MAYBE_NULL;
21963 				if (arg->arg_type & PTR_UNTRUSTED)
21964 					reg->type |= PTR_UNTRUSTED;
21965 				if (arg->arg_type & PTR_TRUSTED)
21966 					reg->type |= PTR_TRUSTED;
21967 				mark_reg_known_zero(env, regs, i);
21968 				reg->btf = bpf_get_btf_vmlinux(); /* can't fail at this point */
21969 				reg->btf_id = arg->btf_id;
21970 				reg->id = ++env->id_gen;
21971 			} else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) {
21972 				/* caller can pass either PTR_TO_ARENA or SCALAR */
21973 				mark_reg_unknown(env, regs, i);
21974 			} else {
21975 				verifier_bug(env, "unhandled arg#%d type %d",
21976 					     i - BPF_REG_1 + 1, arg->arg_type);
21977 				ret = -EFAULT;
21978 				goto out;
21979 			}
21980 		}
21981 		if (env->prog->type == BPF_PROG_TYPE_EXT &&
21982 		    sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) {
21983 			verbose(env, "freplace programs with >%d args not supported yet\n",
21984 				MAX_BPF_FUNC_REG_ARGS);
21985 			ret = -EINVAL;
21986 			goto out;
21987 		}
21988 	} else {
21989 		/* if main BPF program has associated BTF info, validate that
21990 		 * it's matching expected signature, and otherwise mark BTF
21991 		 * info for main program as unreliable
21992 		 */
21993 		if (env->prog->aux->func_info_aux) {
21994 			ret = btf_prepare_func_args(env, 0);
21995 			if (ret || sub->arg_slot_cnt != 1 ||
21996 			    sub->args[0].arg_type != ARG_PTR_TO_CTX) {
21997 				env->prog->aux->func_info_aux[0].unreliable = true;
21998 				sub->arg_slot_cnt = 1;
21999 				sub->stack_arg_cnt = 0;
22000 			}
22001 		}
22002 
22003 		/* 1st arg to a function */
22004 		regs[BPF_REG_1].type = PTR_TO_CTX;
22005 		mark_reg_known_zero(env, regs, BPF_REG_1);
22006 	}
22007 
22008 	/* Acquire references for struct_ops program arguments tagged with "__ref" */
22009 	if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
22010 		for (i = 0; i < aux->ctx_arg_info_size; i++) {
22011 			ret = aux->ctx_arg_info[i].refcounted ? acquire_reference(env, 0, 0) : 0;
22012 			if (ret < 0)
22013 				goto out;
22014 
22015 			aux->ctx_arg_info[i].ref_id = ret;
22016 		}
22017 	}
22018 
22019 	ret = do_check(env);
22020 out:
22021 	account_current_path(env);
22022 	if (!ret) {
22023 		if (pop_log)
22024 			bpf_vlog_reset(&env->log, 0);
22025 		bpf_diag_event_log_restore(env, 0);
22026 	}
22027 	free_states(env);
22028 
22029 	/*
22030 	 * The override is needed to account for async subprograms, which
22031 	 * are verified with their own set of stack frames and thus are
22032 	 * not accounted as callees by account_current_path().
22033 	 * Accumulate their total counts as total counts of the main or
22034 	 * global subprog hosting the async call.
22035 	 * Start from the saved total of earlier contexts: adding to the current
22036 	 * total would count this pass's synchronous paths twice.
22037 	 */
22038 	sub->insns_total = old_insns_total + (env->insn_processed - insn_processed);
22039 	return ret;
22040 }
22041 
22042 /* Lazily verify all global functions based on their BTF, if they are called
22043  * from main BPF program or any of subprograms transitively.
22044  * BPF global subprogs called from dead code are not validated.
22045  * All callable global functions must pass verification.
22046  * Otherwise the whole program is rejected.
22047  * Consider:
22048  * int bar(int);
22049  * int foo(int f)
22050  * {
22051  *    return bar(f);
22052  * }
22053  * int bar(int b)
22054  * {
22055  *    ...
22056  * }
22057  * foo() will be verified first for R1=any_scalar_value. During verification it
22058  * will be assumed that bar() already verified successfully and call to bar()
22059  * from foo() will be checked for type match only. Later bar() will be verified
22060  * independently to check that it's safe for R1=any_scalar_value.
22061  */
22062 static int do_check_subprogs(struct bpf_verifier_env *env)
22063 {
22064 	struct bpf_prog_aux *aux = env->prog->aux;
22065 	struct bpf_func_info_aux *sub_aux;
22066 	int context, i, ret, new_cnt;
22067 
22068 	if (!aux->func_info)
22069 		return 0;
22070 
22071 	/*
22072 	 * Callbacks cannot throw, so the exception callback always runs in the
22073 	 * main program's context. It is presumed to be always called.
22074 	 */
22075 	if (env->exception_callback_subprog) {
22076 		sub_aux = subprog_aux(env, env->exception_callback_subprog);
22077 		sub_aux->called[env->prog->sleepable] = true;
22078 	}
22079 
22080 again:
22081 	new_cnt = 0;
22082 	for (i = 1; i < env->subprog_cnt; i++) {
22083 		if (!bpf_subprog_is_global(env, i))
22084 			continue;
22085 
22086 		sub_aux = subprog_aux(env, i);
22087 		for (context = 0; context < ARRAY_SIZE(sub_aux->called); context++) {
22088 			if (!sub_aux->called[context] || sub_aux->verified[context])
22089 				continue;
22090 
22091 			env->insn_idx = env->subprog_info[i].start;
22092 			WARN_ON_ONCE(env->insn_idx == 0);
22093 			ret = do_check_common(env, i, context);
22094 			if (ret)
22095 				return ret;
22096 			if (env->log.level & BPF_LOG_LEVEL)
22097 				verbose(env, "Func#%d ('%s') is safe for any args "
22098 					"that match its prototype\n",
22099 					i, bpf_subprog_name(env, i));
22100 
22101 			sub_aux->verified[context] = true;
22102 			new_cnt++;
22103 		}
22104 	}
22105 
22106 	/*
22107 	 * We can't loop forever as each pass verifies at least one new context,
22108 	 * and there are only three contexts per global subprog.
22109 	 */
22110 	if (new_cnt)
22111 		goto again;
22112 
22113 	return 0;
22114 }
22115 
22116 static int do_check_main(struct bpf_verifier_env *env)
22117 {
22118 	int ret;
22119 
22120 	env->insn_idx = 0;
22121 	ret = do_check_common(env, 0, env->prog->sleepable);
22122 	if (!ret)
22123 		env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
22124 	return ret;
22125 }
22126 
22127 static void print_verification_stats(struct bpf_verifier_env *env)
22128 {
22129 	/* Skip over hidden subprogs which are not verified. */
22130 	int i, subprog_cnt = env->subprog_cnt - env->hidden_subprog_cnt;
22131 
22132 	if (env->log.level & BPF_LOG_STATS) {
22133 		verbose(env, "verification time %lld usec\n",
22134 			div_u64(env->verification_time, 1000));
22135 		verbose(env, "stack depth max %d\n", env->max_stack_depth);
22136 		for (i = 0; i < subprog_cnt; i++) {
22137 			const char *name = env->subprog_info[i].name;
22138 			const char *kind;
22139 
22140 			if (!name || !name[0])
22141 				name = "<unknown>";
22142 			kind = i == 0 ? "main" :
22143 			       bpf_subprog_is_global(env, i) ? "global" : "static";
22144 			verbose(env, "subprog %d (%s) %s insns_self %d insns_total %d stack %d\n",
22145 				i, name, kind, env->subprog_info[i].insns_self,
22146 				env->subprog_info[i].insns_total,
22147 				env->subprog_info[i].stack_depth);
22148 		}
22149 	}
22150 	verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
22151 		"total_states %d peak_states %d mark_read %d\n",
22152 		env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,
22153 		env->max_states_per_insn, env->total_states,
22154 		env->peak_states, env->longest_mark_read_walk);
22155 }
22156 
22157 int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
22158 			       const struct bpf_ctx_arg_aux *info, u32 cnt)
22159 {
22160 	prog->aux->ctx_arg_info = kmemdup_array(info, cnt, sizeof(*info), GFP_KERNEL_ACCOUNT);
22161 	prog->aux->ctx_arg_info_size = cnt;
22162 
22163 	return prog->aux->ctx_arg_info ? 0 : -ENOMEM;
22164 }
22165 
22166 static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
22167 {
22168 	const struct btf_type *t, *func_proto;
22169 	const struct bpf_struct_ops_desc *st_ops_desc;
22170 	const struct bpf_struct_ops_arg_info *arg_info;
22171 	const struct bpf_struct_ops *st_ops;
22172 	const struct btf_member *member;
22173 	struct bpf_prog *prog = env->prog;
22174 	bool has_refcounted_arg = false;
22175 	u32 btf_id, member_idx, member_off;
22176 	struct btf *btf;
22177 	const char *mname;
22178 	int i, err;
22179 
22180 	if (!prog->gpl_compatible) {
22181 		verbose(env, "struct ops programs must have a GPL compatible license\n");
22182 		return -EINVAL;
22183 	}
22184 
22185 	if (!prog->aux->attach_btf_id)
22186 		return -ENOTSUPP;
22187 
22188 	btf = prog->aux->attach_btf;
22189 	if (btf_is_module(btf)) {
22190 		/* Make sure st_ops is valid through the lifetime of env */
22191 		env->attach_btf_mod = btf_try_get_module(btf);
22192 		if (!env->attach_btf_mod) {
22193 			verbose(env, "struct_ops module %s is not found\n",
22194 				btf_get_name(btf));
22195 			return -ENOTSUPP;
22196 		}
22197 	}
22198 
22199 	btf_id = prog->aux->attach_btf_id;
22200 	st_ops_desc = bpf_struct_ops_find(btf, btf_id);
22201 	if (!st_ops_desc) {
22202 		verbose(env, "attach_btf_id %u is not a supported struct\n",
22203 			btf_id);
22204 		return -ENOTSUPP;
22205 	}
22206 	st_ops = st_ops_desc->st_ops;
22207 
22208 	t = st_ops_desc->type;
22209 	member_idx = prog->expected_attach_type;
22210 	if (member_idx >= btf_type_vlen(t)) {
22211 		verbose(env, "attach to invalid member idx %u of struct %s\n",
22212 			member_idx, st_ops->name);
22213 		return -EINVAL;
22214 	}
22215 
22216 	member = &btf_type_member(t)[member_idx];
22217 	mname = btf_name_by_offset(btf, member->name_off);
22218 	func_proto = btf_type_resolve_func_ptr(btf, member->type,
22219 					       NULL);
22220 	if (!func_proto) {
22221 		verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",
22222 			mname, member_idx, st_ops->name);
22223 		return -EINVAL;
22224 	}
22225 
22226 	member_off = __btf_member_bit_offset(t, member) / 8;
22227 	err = bpf_struct_ops_supported(st_ops, member_off);
22228 	if (err) {
22229 		verbose(env, "attach to unsupported member %s of struct %s\n",
22230 			mname, st_ops->name);
22231 		return err;
22232 	}
22233 
22234 	if (st_ops->check_member) {
22235 		err = st_ops->check_member(t, member, prog);
22236 
22237 		if (err) {
22238 			verbose(env, "attach to unsupported member %s of struct %s\n",
22239 				mname, st_ops->name);
22240 			return err;
22241 		}
22242 	}
22243 
22244 	if (prog->aux->priv_stack_requested && !bpf_jit_supports_private_stack()) {
22245 		verbose(env, "Private stack not supported by jit\n");
22246 		return -EACCES;
22247 	}
22248 
22249 	arg_info = &st_ops_desc->arg_info[member_idx];
22250 	for (i = 0; i < arg_info->cnt; i++) {
22251 		const struct bpf_ctx_arg_aux *info = &arg_info->info[i];
22252 
22253 		if (info->refcounted)
22254 			has_refcounted_arg = true;
22255 		if (base_type(info->reg_type) == PTR_TO_ARENA) {
22256 			if (!bpf_jit_supports_arena_args()) {
22257 				verbose(env, "JIT does not support arena arguments\n");
22258 				return -ENOTSUPP;
22259 			}
22260 			if (!prog->aux->arena) {
22261 				verbose(env,
22262 					"arena argument of %s requires a program with an associated arena\n",
22263 					mname);
22264 				return -EINVAL;
22265 			}
22266 		}
22267 	}
22268 
22269 	/* Tail call is not allowed for programs with refcounted arguments since we
22270 	 * cannot guarantee that valid refcounted kptrs will be passed to the callee.
22271 	 */
22272 	for (i = 0; i < env->subprog_cnt; i++) {
22273 		if (has_refcounted_arg && env->subprog_info[i].has_tail_call) {
22274 			verbose(env, "program with __ref argument cannot tail call\n");
22275 			return -EINVAL;
22276 		}
22277 	}
22278 
22279 	prog->aux->st_ops = st_ops;
22280 	prog->aux->attach_st_ops_member_off = member_off;
22281 
22282 	prog->aux->attach_func_proto = func_proto;
22283 	prog->aux->attach_func_name = mname;
22284 	env->ops = st_ops->verifier_ops;
22285 
22286 	return bpf_prog_ctx_arg_info_init(prog, arg_info->info, arg_info->cnt);
22287 }
22288 #define SECURITY_PREFIX "security_"
22289 
22290 #ifdef CONFIG_FUNCTION_ERROR_INJECTION
22291 
22292 /* list of non-sleepable functions that are otherwise on
22293  * ALLOW_ERROR_INJECTION list
22294  */
22295 BTF_SET_START(btf_non_sleepable_error_inject)
22296 /* Three functions below can be called from sleepable and non-sleepable context.
22297  * Assume non-sleepable from bpf safety point of view.
22298  */
22299 BTF_ID(func, __filemap_add_folio)
22300 #ifdef CONFIG_FAIL_PAGE_ALLOC
22301 BTF_ID(func, should_fail_alloc_page)
22302 #endif
22303 #ifdef CONFIG_FAILSLAB
22304 BTF_ID(func, should_failslab)
22305 #endif
22306 BTF_SET_END(btf_non_sleepable_error_inject)
22307 
22308 static int check_non_sleepable_error_inject(u32 btf_id)
22309 {
22310 	return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
22311 }
22312 
22313 static int check_attach_sleepable(u32 btf_id, unsigned long addr, const char *func_name)
22314 {
22315 	/* fentry/fexit/fmod_ret progs can be sleepable if they are
22316 	 * attached to ALLOW_ERROR_INJECTION and are not in denylist.
22317 	 */
22318 	if (!check_non_sleepable_error_inject(btf_id) &&
22319 	    within_error_injection_list(addr))
22320 		return 0;
22321 
22322 	return -EINVAL;
22323 }
22324 
22325 static int check_attach_modify_return(unsigned long addr, const char *func_name)
22326 {
22327 	if (within_error_injection_list(addr) ||
22328 	    !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
22329 		return 0;
22330 
22331 	return -EINVAL;
22332 }
22333 
22334 #else
22335 
22336 /* Unfortunately, the arch-specific prefixes are hard-coded in arch syscall code
22337  * so we need to hard-code them, too. Ftrace has arch_syscall_match_sym_name()
22338  * but that just compares two concrete function names.
22339  */
22340 static bool has_arch_syscall_prefix(const char *func_name)
22341 {
22342 #if defined(__x86_64__)
22343 	return !strncmp(func_name, "__x64_", 6);
22344 #elif defined(__i386__)
22345 	return !strncmp(func_name, "__ia32_", 7);
22346 #elif defined(__s390x__)
22347 	return !strncmp(func_name, "__s390x_", 8);
22348 #elif defined(__aarch64__)
22349 	return !strncmp(func_name, "__arm64_", 8);
22350 #elif defined(__riscv)
22351 	return !strncmp(func_name, "__riscv_", 8);
22352 #elif defined(__powerpc__) || defined(__powerpc64__)
22353 	return !strncmp(func_name, "sys_", 4);
22354 #elif defined(__loongarch__)
22355 	return !strncmp(func_name, "sys_", 4);
22356 #else
22357 	return false;
22358 #endif
22359 }
22360 
22361 /* Without error injection, allow sleepable and fmod_ret progs on syscalls. */
22362 
22363 static int check_attach_sleepable(u32 btf_id, unsigned long addr, const char *func_name)
22364 {
22365 	if (has_arch_syscall_prefix(func_name))
22366 		return 0;
22367 
22368 	return -EINVAL;
22369 }
22370 
22371 static int check_attach_modify_return(unsigned long addr, const char *func_name)
22372 {
22373 	if (has_arch_syscall_prefix(func_name) ||
22374 	    !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
22375 		return 0;
22376 
22377 	return -EINVAL;
22378 }
22379 
22380 #endif /* CONFIG_FUNCTION_ERROR_INJECTION */
22381 
22382 static bool is_tracing_multi_id(const struct bpf_prog *prog, u32 btf_id)
22383 {
22384 	return is_tracing_multi(prog->expected_attach_type) && bpf_multi_func_btf_id[0] == btf_id;
22385 }
22386 
22387 static int btf_id_allow_sleepable(u32 btf_id, unsigned long addr, const struct bpf_prog *prog,
22388 				  const struct btf *btf)
22389 {
22390 	const struct btf_type *t;
22391 	const char *tname;
22392 
22393 	if (!btf_is_kernel(btf))
22394 		return -EINVAL;
22395 
22396 	switch (prog->type) {
22397 	case BPF_PROG_TYPE_TRACING:
22398 		t = btf_type_by_id(btf, btf_id);
22399 		if (!t)
22400 			return -EINVAL;
22401 		tname = btf_name_by_offset(btf, t->name_off);
22402 		if (!tname)
22403 			return -EINVAL;
22404 
22405 		/*
22406 		 * *.multi sleepable programs will pass initial sleepable check,
22407 		 * the actual attached btf ids are checked later during the link
22408 		 * attachment.
22409 		 */
22410 		if (is_tracing_multi_id(prog, btf_id))
22411 			return 0;
22412 		if (!check_attach_sleepable(btf_id, addr, tname))
22413 			return 0;
22414 		/*
22415 		 * fentry/fexit/fmod_ret progs can also be sleepable if they are
22416 		 * in the fmodret id set with the KF_SLEEPABLE flag.
22417 		 */
22418 		else {
22419 			u32 *flags = btf_kfunc_is_modify_return(btf, btf_id, prog);
22420 
22421 			if (flags && (*flags & KF_SLEEPABLE))
22422 				return 0;
22423 		}
22424 		break;
22425 	case BPF_PROG_TYPE_LSM:
22426 		/*
22427 		 * LSM progs check that they are attached to bpf_lsm_*() funcs.
22428 		 * Only some of them are sleepable.
22429 		 */
22430 		if (bpf_lsm_is_sleepable_hook(btf_id))
22431 			return 0;
22432 		break;
22433 	default:
22434 		break;
22435 	}
22436 	return -EINVAL;
22437 }
22438 
22439 /*
22440  * Resolve the prototype describing a trace target's real ABI. A
22441  * KF_IMPLICIT_ARGS kfunc has its injected args stripped from the public
22442  * prototype, so use the _impl prototype; other targets use their own.
22443  */
22444 static const struct btf_type *
22445 btf_attach_func_proto(struct bpf_verifier_log *log, struct btf *btf, u32 func_id)
22446 {
22447 	const struct btf_type *func;
22448 	struct module *mod = NULL;
22449 	const char *name;
22450 	int implicit;
22451 
22452 	func = btf_type_by_id(btf, func_id);
22453 	if (!func || !btf_type_is_func(func))
22454 		return NULL;
22455 	name = btf_name_by_offset(btf, func->name_off);
22456 
22457 	/*
22458 	 * btf_kfunc_check_flag() reads kfunc_set_tab, which for a module is
22459 	 * stable only once it is live; hold a module ref across the read to
22460 	 * exclude a concurrent module load.
22461 	 */
22462 	if (btf_is_module(btf)) {
22463 		mod = btf_try_get_module(btf);
22464 		if (!mod)
22465 			return NULL;
22466 	}
22467 	implicit = btf_kfunc_check_flag(btf, func_id, KF_IMPLICIT_ARGS);
22468 	module_put(mod);
22469 
22470 	if (implicit == -EINVAL) {
22471 		bpf_log(log, "kfunc %s has inconsistent KF_IMPLICIT_ARGS\n", name);
22472 		return NULL;
22473 	}
22474 	if (implicit > 0)
22475 		return find_kfunc_impl_proto(log, btf, name);
22476 
22477 	return btf_type_by_id(btf, func->type);
22478 }
22479 
22480 static bool attach_uses_trampoline_retval(enum bpf_attach_type type)
22481 {
22482 	switch (type) {
22483 	case BPF_MODIFY_RETURN:
22484 	case BPF_TRACE_FEXIT:
22485 	case BPF_TRACE_FEXIT_MULTI:
22486 	case BPF_TRACE_FSESSION:
22487 	case BPF_TRACE_FSESSION_MULTI:
22488 		return true;
22489 	default:
22490 		return false;
22491 	}
22492 }
22493 
22494 /*
22495  * BPF code passes argument slot N where the kernel convention passes its Nth
22496  * u64 argument, so describe a BPF function to its trampoline as one u64 per
22497  * slot, and every arch places its arguments right.
22498  */
22499 static int flatten_bpf_func_model(struct bpf_verifier_log *log, const char *tname,
22500 				  struct btf_func_model *m)
22501 {
22502 	u32 i, nslots = 0;
22503 
22504 	for (i = 0; i < m->nr_args; i++)
22505 		nslots += btf_func_model_arg_slots(m, i);
22506 	if (nslots > MAX_BPF_FUNC_ARGS) {
22507 		bpf_log(log, "The function %s has %u argument slots. Too many.\n",
22508 			tname, nslots);
22509 		return -EINVAL;
22510 	}
22511 
22512 	for (i = 0; i < nslots; i++) {
22513 		m->arg_size[i] = sizeof(u64);
22514 		m->arg_flags[i] = 0;
22515 	}
22516 	m->nr_args = nslots;
22517 	return 0;
22518 }
22519 
22520 /*
22521  * The verifier lets a static function use stack arguments counted from its
22522  * BTF, even if that BTF turns out unreliable. A trampoline on the function is
22523  * then given only MAX_BPF_FUNC_REG_ARGS u64 arguments and would lose the
22524  * stack arguments, so such a function cannot be attached to.
22525  */
22526 static int check_unreliable_func_args(struct bpf_verifier_log *log, struct btf *btf,
22527 				      const struct btf_type *func_proto, const char *tname)
22528 {
22529 	struct btf_func_model m;
22530 	u32 i, nslots = 0;
22531 
22532 	if (btf_distill_func_proto(NULL, btf, func_proto, tname, &m)) {
22533 		/* at least a slot for each argument */
22534 		nslots = btf_type_vlen(func_proto);
22535 	} else {
22536 		for (i = 0; i < m.nr_args; i++)
22537 			nslots += btf_func_model_arg_slots(&m, i);
22538 	}
22539 	if (nslots <= MAX_BPF_FUNC_REG_ARGS)
22540 		return 0;
22541 
22542 	bpf_log(log, "Cannot attach to static function %s: unreliable BTF and stack arguments\n",
22543 		tname);
22544 	return -EINVAL;
22545 }
22546 
22547 /* Functions may share a type id. The first of them is attached to. */
22548 static int find_tgt_subprog(const struct bpf_prog *tgt_prog, u32 btf_id)
22549 {
22550 	const struct bpf_prog_aux *aux = tgt_prog->aux;
22551 	int i;
22552 
22553 	for (i = 0; i < aux->func_info_cnt; i++)
22554 		if (aux->func_info[i].type_id == btf_id)
22555 			return i;
22556 
22557 	return -1;
22558 }
22559 
22560 /*
22561  * The prologue of an extension gives a cloned skb a new head, unless the
22562  * prologue of the target program has done that. The callers of the function
22563  * that it replaces must not hold on to the old head.
22564  */
22565 int bpf_check_ext_prologue(struct bpf_verifier_log *log,
22566 			   const struct bpf_prog *tgt_prog, u32 btf_id)
22567 {
22568 	const struct bpf_prog_aux *aux = tgt_prog->aux;
22569 	int subprog = find_tgt_subprog(tgt_prog, btf_id);
22570 
22571 	if (aux->has_prologue)
22572 		return 0;
22573 
22574 	if (subprog >= 0 && !aux->func_info_aux[subprog].called_with_pkt_ptrs)
22575 		return 0;
22576 
22577 	bpf_log(log,
22578 		"Extension writing packet data cannot replace a function whose callers keep packet pointers\n");
22579 	return -EINVAL;
22580 }
22581 
22582 /*
22583  * An extension is verified as a program that runs under the RCU read lock. Does
22584  * the function that it is to replace?
22585  */
22586 static bool func_has_rcu_read_lock(const struct bpf_prog *prog, int subprog)
22587 {
22588 	const struct bpf_func_info_aux *sub_aux = &prog->aux->func_info_aux[subprog];
22589 
22590 	if (!subprog)
22591 		return !prog->sleepable;
22592 
22593 	return !sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] &&
22594 	       !sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK];
22595 }
22596 
22597 /*
22598  * The callers of a global function are checked against the arguments that its
22599  * BTF describes, and an extension is verified with those of its own BTF. It
22600  * must not expect more than the callers of the function provide.
22601  *
22602  * While an extension is loaded, bpf_check_attach_target() runs before its
22603  * arguments are known. check_attach_btf_id() does the check when they are.
22604  */
22605 static int check_ext_args(struct bpf_verifier_log *log, const struct bpf_prog *prog,
22606 			  const struct bpf_prog *tgt_prog, int subprog)
22607 {
22608 	const struct bpf_subprog_arg_info *tgt = tgt_prog->aux->func_info_aux[subprog].args;
22609 	const struct bpf_subprog_arg_info *ext = prog->aux->func_info_aux[0].args;
22610 	int i;
22611 
22612 	for (i = 0; i < MAX_BPF_FUNC_REG_ARGS && ext[i].arg_type != ARG_UNUSED; i++) {
22613 		/* An extension may accept NULL where the callers never pass it. */
22614 		if ((ext[i].arg_type == tgt[i].arg_type ||
22615 		     ext[i].arg_type == (tgt[i].arg_type | PTR_MAYBE_NULL)) &&
22616 		    (base_type(ext[i].arg_type) == ARG_PTR_TO_MEM ?
22617 		     ext[i].mem_size <= tgt[i].mem_size : ext[i].btf_id == tgt[i].btf_id))
22618 			continue;
22619 
22620 		bpf_log(log, "Extension program's argument in R%d is incompatible with original\n",
22621 			i + 1);
22622 		return -EINVAL;
22623 	}
22624 
22625 	return 0;
22626 }
22627 
22628 int bpf_check_attach_target(struct bpf_verifier_log *log,
22629 			    const struct bpf_prog *prog,
22630 			    const struct bpf_prog *tgt_prog,
22631 			    u32 btf_id,
22632 			    struct bpf_attach_target_info *tgt_info)
22633 {
22634 	bool prog_extension = prog->type == BPF_PROG_TYPE_EXT;
22635 	bool prog_tracing = prog->type == BPF_PROG_TYPE_TRACING;
22636 	char trace_symbol[KSYM_SYMBOL_LEN];
22637 	const char prefix[] = "btf_trace_";
22638 	struct bpf_raw_event_map *btp;
22639 	int ret = 0, subprog = -1;
22640 	const struct btf_type *t;
22641 	bool conservative = true;
22642 	const char *tname, *fname;
22643 	struct btf *btf;
22644 	long addr = 0;
22645 	struct module *mod = NULL;
22646 
22647 	if (!btf_id) {
22648 		bpf_log(log, "Tracing programs must provide btf_id\n");
22649 		return -EINVAL;
22650 	}
22651 	btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf;
22652 	if (!btf) {
22653 		bpf_log(log,
22654 			"Tracing program can only be attached to another program annotated with BTF\n");
22655 		return -EINVAL;
22656 	}
22657 	t = btf_type_by_id(btf, btf_id);
22658 	if (!t) {
22659 		bpf_log(log, "attach_btf_id %u is invalid\n", btf_id);
22660 		return -EINVAL;
22661 	}
22662 	tname = btf_name_by_offset(btf, t->name_off);
22663 	if (!tname) {
22664 		bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id);
22665 		return -EINVAL;
22666 	}
22667 	if (tgt_prog) {
22668 		struct bpf_prog_aux *aux = tgt_prog->aux;
22669 		bool tgt_changes_pkt_data;
22670 		bool tgt_might_sleep;
22671 		bool tgt_can_unwind;
22672 
22673 		if (bpf_prog_is_dev_bound(prog->aux) &&
22674 		    !bpf_prog_dev_bound_match(prog, tgt_prog)) {
22675 			bpf_log(log, "Target program bound device mismatch");
22676 			return -EINVAL;
22677 		}
22678 
22679 		subprog = find_tgt_subprog(tgt_prog, btf_id);
22680 		if (subprog == -1) {
22681 			bpf_log(log, "Subprog %s doesn't exist\n", tname);
22682 			return -EINVAL;
22683 		}
22684 		/*
22685 		 * A struct_ops indirect trampoline converts arena arguments
22686 		 * before invoking its program. A tracing or extension program
22687 		 * attached to the main program would see the converted offset as a
22688 		 * regular BTF pointer.
22689 		 */
22690 		if (subprog == 0 && bpf_prog_has_arena_ctx_arg(tgt_prog)) {
22691 			bpf_log(log, "Cannot attach to a target with arena context arguments\n");
22692 			return -EOPNOTSUPP;
22693 		}
22694 		if (aux->func && aux->func[subprog]->aux->exception_cb) {
22695 			bpf_log(log,
22696 				"%s programs cannot attach to exception callback\n",
22697 				prog_extension ? "Extension" : "Tracing");
22698 			return -EINVAL;
22699 		}
22700 		/* unwind_exit of the main prog is exit, unless it's an extension */
22701 		tgt_can_unwind = (subprog || tgt_prog->type == BPF_PROG_TYPE_EXT) &&
22702 				 (aux->func ? aux->func[subprog]->aux->can_unwind
22703 					    : aux->can_unwind);
22704 		/* unwind_exit would return to the trampoline that called the target */
22705 		if (tgt_can_unwind && !prog_extension &&
22706 		    prog->expected_attach_type != BPF_TRACE_FENTRY) {
22707 			bpf_log(log, "Only fentry can attach to %s, which can unwind\n", tname);
22708 			return -EINVAL;
22709 		}
22710 		conservative = aux->func_info_aux[subprog].unreliable;
22711 		if (prog_extension) {
22712 			if (aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL ||
22713 			    conservative) {
22714 				bpf_log(log,
22715 					"Cannot replace static functions\n");
22716 				return -EINVAL;
22717 			}
22718 			if (!prog->jit_requested) {
22719 				bpf_log(log,
22720 					"Extension programs should be JITed\n");
22721 				return -EINVAL;
22722 			}
22723 			/*
22724 			 * How much of the buffer of the tracepoint is used
22725 			 * is checked for the program that is attached to it.
22726 			 */
22727 			if (tgt_prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) {
22728 				bpf_log(log,
22729 					"Cannot extend a writable raw tracepoint program\n");
22730 				return -EINVAL;
22731 			}
22732 			if (!func_has_rcu_read_lock(tgt_prog, subprog)) {
22733 				bpf_log(log,
22734 					"Cannot extend a function that runs without RCU read lock\n");
22735 				return -EINVAL;
22736 			}
22737 			tgt_changes_pkt_data = aux->func
22738 					       ? aux->func[subprog]->aux->changes_pkt_data
22739 					       : aux->changes_pkt_data;
22740 			if (prog->aux->changes_pkt_data && !tgt_changes_pkt_data) {
22741 				bpf_log(log,
22742 					"Extension program changes packet data, while original does not\n");
22743 				return -EINVAL;
22744 			}
22745 			if (prog->aux->has_prologue &&
22746 			    bpf_check_ext_prologue(log, tgt_prog, btf_id))
22747 				return -EINVAL;
22748 
22749 			tgt_might_sleep = aux->func
22750 					  ? aux->func[subprog]->aux->might_sleep
22751 					  : aux->might_sleep;
22752 			if (prog->aux->might_sleep && !tgt_might_sleep) {
22753 				bpf_log(log,
22754 					"Extension program may sleep, while original does not\n");
22755 				return -EINVAL;
22756 			}
22757 			/* There is no unwind_goto after the calls of the original */
22758 			if (prog->aux->can_unwind && !tgt_can_unwind) {
22759 				bpf_log(log,
22760 					"Extension program can unwind, while original does not\n");
22761 				return -EINVAL;
22762 			}
22763 		}
22764 		if (!tgt_prog->jited) {
22765 			bpf_log(log, "Can attach to only JITed progs\n");
22766 			return -EINVAL;
22767 		}
22768 		if (prog_tracing) {
22769 			if (aux->attach_tracing_prog) {
22770 				/*
22771 				 * Target program is an fentry/fexit which is already attached
22772 				 * to another tracing program. More levels of nesting
22773 				 * attachment are not allowed.
22774 				 */
22775 				bpf_log(log, "Cannot nest tracing program attach more than once\n");
22776 				return -EINVAL;
22777 			}
22778 		} else if (tgt_prog->type == prog->type) {
22779 			/*
22780 			 * To avoid potential call chain cycles, prevent attaching of a
22781 			 * program extension to another extension. It's ok to attach
22782 			 * fentry/fexit to extension program.
22783 			 */
22784 			bpf_log(log, "Cannot recursively attach\n");
22785 			return -EINVAL;
22786 		}
22787 		if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&
22788 		    prog_extension &&
22789 		    (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY ||
22790 		     tgt_prog->expected_attach_type == BPF_TRACE_FEXIT ||
22791 		     tgt_prog->expected_attach_type == BPF_TRACE_FENTRY_MULTI ||
22792 		     tgt_prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI ||
22793 		     tgt_prog->expected_attach_type == BPF_TRACE_FSESSION ||
22794 		     tgt_prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) {
22795 			/* Program extensions can extend all program types
22796 			 * except fentry/fexit. The reason is the following.
22797 			 * The fentry/fexit programs are used for performance
22798 			 * analysis, stats and can be attached to any program
22799 			 * type. When extension program is replacing XDP function
22800 			 * it is necessary to allow performance analysis of all
22801 			 * functions. Both original XDP program and its program
22802 			 * extension. Hence attaching fentry/fexit to
22803 			 * BPF_PROG_TYPE_EXT is allowed. If extending of
22804 			 * fentry/fexit was allowed it would be possible to create
22805 			 * long call chain fentry->extension->fentry->extension
22806 			 * beyond reasonable stack size. Hence extending fentry
22807 			 * is not allowed.
22808 			 */
22809 			bpf_log(log, "Cannot extend fentry/fexit/fsession\n");
22810 			return -EINVAL;
22811 		}
22812 	} else {
22813 		if (prog_extension) {
22814 			bpf_log(log, "Cannot replace kernel functions\n");
22815 			return -EINVAL;
22816 		}
22817 	}
22818 
22819 	switch (prog->expected_attach_type) {
22820 	case BPF_TRACE_RAW_TP:
22821 		if (tgt_prog) {
22822 			bpf_log(log,
22823 				"Only FENTRY/FEXIT/FSESSION progs are attachable to another BPF prog\n");
22824 			return -EINVAL;
22825 		}
22826 		if (!btf_type_is_typedef(t)) {
22827 			bpf_log(log, "attach_btf_id %u is not a typedef\n",
22828 				btf_id);
22829 			return -EINVAL;
22830 		}
22831 		if (strncmp(prefix, tname, sizeof(prefix) - 1)) {
22832 			bpf_log(log, "attach_btf_id %u points to wrong type name %s\n",
22833 				btf_id, tname);
22834 			return -EINVAL;
22835 		}
22836 		tname += sizeof(prefix) - 1;
22837 
22838 		/* The func_proto of "btf_trace_##tname" is generated from typedef without argument
22839 		 * names. Thus using bpf_raw_event_map to get argument names.
22840 		 */
22841 		btp = bpf_get_raw_tracepoint(tname);
22842 		if (!btp)
22843 			return -EINVAL;
22844 		if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) {
22845 			bpf_log(log, "Sleepable program cannot attach to non-faultable tracepoint %s\n",
22846 				tname);
22847 			bpf_put_raw_tracepoint(btp);
22848 			return -EINVAL;
22849 		}
22850 		fname = kallsyms_lookup((unsigned long)btp->bpf_func, NULL, NULL, NULL,
22851 					trace_symbol);
22852 		bpf_put_raw_tracepoint(btp);
22853 
22854 		if (fname)
22855 			ret = btf_find_by_name_kind(btf, fname, BTF_KIND_FUNC);
22856 
22857 		if (!fname || ret < 0) {
22858 			bpf_log(log, "Cannot find btf of tracepoint template, fall back to %s%s.\n",
22859 				prefix, tname);
22860 			t = btf_type_by_id(btf, t->type);
22861 			if (!btf_type_is_ptr(t))
22862 				/* should never happen in valid vmlinux build */
22863 				return -EINVAL;
22864 		} else {
22865 			t = btf_type_by_id(btf, ret);
22866 			if (!btf_type_is_func(t))
22867 				/* should never happen in valid vmlinux build */
22868 				return -EINVAL;
22869 		}
22870 
22871 		t = btf_type_by_id(btf, t->type);
22872 		if (!btf_type_is_func_proto(t))
22873 			/* should never happen in valid vmlinux build */
22874 			return -EINVAL;
22875 
22876 		break;
22877 	case BPF_TRACE_ITER:
22878 		if (!btf_type_is_func(t)) {
22879 			bpf_log(log, "attach_btf_id %u is not a function\n",
22880 				btf_id);
22881 			return -EINVAL;
22882 		}
22883 		t = btf_type_by_id(btf, t->type);
22884 		if (!btf_type_is_func_proto(t))
22885 			return -EINVAL;
22886 		ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
22887 		if (ret)
22888 			return ret;
22889 		break;
22890 	default:
22891 		if (!prog_extension)
22892 			return -EINVAL;
22893 		fallthrough;
22894 	case BPF_MODIFY_RETURN:
22895 	case BPF_LSM_MAC:
22896 	case BPF_LSM_CGROUP:
22897 	case BPF_TRACE_FENTRY:
22898 	case BPF_TRACE_FEXIT:
22899 	case BPF_TRACE_FSESSION:
22900 	case BPF_TRACE_FSESSION_MULTI:
22901 	case BPF_TRACE_FENTRY_MULTI:
22902 	case BPF_TRACE_FEXIT_MULTI:
22903 		if ((prog->expected_attach_type == BPF_TRACE_FSESSION ||
22904 		    prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) &&
22905 		    !bpf_jit_supports_fsession()) {
22906 			bpf_log(log, "JIT does not support fsession\n");
22907 			return -EOPNOTSUPP;
22908 		}
22909 		if (!btf_type_is_func(t)) {
22910 			bpf_log(log, "attach_btf_id %u is not a function\n",
22911 				btf_id);
22912 			return -EINVAL;
22913 		}
22914 		if (prog_extension &&
22915 		    (btf_check_type_match(log, prog, btf, t) ||
22916 		     check_ext_args(log, prog, tgt_prog, subprog)))
22917 			return -EINVAL;
22918 		t = btf_attach_func_proto(log, btf, btf_id);
22919 		if (!t || !btf_type_is_func_proto(t))
22920 			return -EINVAL;
22921 
22922 		if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
22923 		    (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type ||
22924 		     prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type))
22925 			return -EINVAL;
22926 
22927 		if (tgt_prog && conservative) {
22928 			if (subprog > 0) {
22929 				ret = check_unreliable_func_args(log, btf, t, tname);
22930 				if (ret)
22931 					return ret;
22932 			}
22933 			t = NULL;
22934 		}
22935 
22936 		ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
22937 		if (ret < 0)
22938 			return ret;
22939 		if (tgt_prog) {
22940 			ret = flatten_bpf_func_model(log, tname, &tgt_info->fmodel);
22941 			if (ret < 0)
22942 				return ret;
22943 		}
22944 
22945 		if (tgt_info->fmodel.ret_size > 8 &&
22946 		    attach_uses_trampoline_retval(prog->expected_attach_type)) {
22947 			bpf_log(log,
22948 				"Attach to function %s with a >8 byte return value is not supported for this attach type\n",
22949 				tname);
22950 			return -EOPNOTSUPP;
22951 		}
22952 
22953 		/*
22954 		 * *.multi programs don't need an address during program
22955 		 * verification, we just take the module ref if needed.
22956 		 */
22957 		if (is_tracing_multi_id(prog, btf_id)) {
22958 			if (btf_is_module(btf)) {
22959 				mod = btf_try_get_module(btf);
22960 				if (!mod)
22961 					return -ENOENT;
22962 			}
22963 			addr = 0;
22964 		} else if (tgt_prog) {
22965 			if (subprog == 0)
22966 				addr = (long) tgt_prog->bpf_func;
22967 			else
22968 				addr = (long) tgt_prog->aux->func[subprog]->bpf_func;
22969 		} else {
22970 			if (btf_is_module(btf)) {
22971 				mod = btf_try_get_module(btf);
22972 				if (mod)
22973 					addr = find_kallsyms_symbol_value(mod, tname);
22974 				else
22975 					addr = 0;
22976 			} else {
22977 				addr = kallsyms_lookup_name(tname);
22978 			}
22979 			if (!addr) {
22980 				module_put(mod);
22981 				bpf_log(log,
22982 					"The address of function %s cannot be found\n",
22983 					tname);
22984 				return -ENOENT;
22985 			}
22986 		}
22987 
22988 		if (prog->sleepable) {
22989 			ret = btf_id_allow_sleepable(btf_id, addr, prog, btf);
22990 			if (ret) {
22991 				module_put(mod);
22992 				bpf_log(log, "%s is not sleepable\n", tname);
22993 				return ret;
22994 			}
22995 		} else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
22996 			if (tgt_prog) {
22997 				module_put(mod);
22998 				bpf_log(log, "can't modify return codes of BPF programs\n");
22999 				return -EINVAL;
23000 			}
23001 			ret = -EINVAL;
23002 			if (btf_kfunc_is_modify_return(btf, btf_id, prog) ||
23003 			    !check_attach_modify_return(addr, tname))
23004 				ret = 0;
23005 			if (ret) {
23006 				module_put(mod);
23007 				bpf_log(log, "%s() is not modifiable\n", tname);
23008 				return ret;
23009 			}
23010 		}
23011 
23012 		break;
23013 	}
23014 	tgt_info->tgt_addr = addr;
23015 	tgt_info->tgt_name = tname;
23016 	tgt_info->tgt_type = t;
23017 	tgt_info->tgt_mod = mod;
23018 	return 0;
23019 }
23020 
23021 BTF_SET_START(btf_id_deny)
23022 BTF_ID_UNUSED
23023 #ifdef CONFIG_SMP
23024 BTF_ID(func, ___migrate_enable)
23025 BTF_ID(func, migrate_disable)
23026 BTF_ID(func, migrate_enable)
23027 #endif
23028 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU
23029 BTF_ID(func, rcu_read_unlock_strict)
23030 #endif
23031 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE)
23032 BTF_ID(func, preempt_count_add)
23033 BTF_ID(func, preempt_count_sub)
23034 #endif
23035 #ifdef CONFIG_PREEMPT_RCU
23036 BTF_ID(func, __rcu_read_lock)
23037 BTF_ID(func, __rcu_read_unlock)
23038 #endif
23039 BTF_SET_END(btf_id_deny)
23040 
23041 /* fexit and fmod_ret can't be used to attach to __noreturn functions.
23042  * Currently, we must manually list all __noreturn functions here. Once a more
23043  * robust solution is implemented, this workaround can be removed.
23044  */
23045 BTF_SET_START(noreturn_deny)
23046 #ifdef CONFIG_IA32_EMULATION
23047 BTF_ID(func, __ia32_sys_exit)
23048 BTF_ID(func, __ia32_sys_exit_group)
23049 #endif
23050 #ifdef CONFIG_KUNIT
23051 BTF_ID(func, __kunit_abort)
23052 BTF_ID(func, kunit_try_catch_throw)
23053 #endif
23054 #ifdef CONFIG_MODULES
23055 BTF_ID(func, __module_put_and_kthread_exit)
23056 #endif
23057 #ifdef CONFIG_X86_64
23058 BTF_ID(func, __x64_sys_exit)
23059 BTF_ID(func, __x64_sys_exit_group)
23060 #endif
23061 BTF_ID(func, do_exit)
23062 BTF_ID(func, do_group_exit)
23063 BTF_ID(func, kthread_complete_and_exit)
23064 BTF_ID(func, make_task_dead)
23065 BTF_SET_END(noreturn_deny)
23066 
23067 static bool can_be_sleepable(struct bpf_prog *prog)
23068 {
23069 	if (prog->type == BPF_PROG_TYPE_TRACING) {
23070 		switch (prog->expected_attach_type) {
23071 		case BPF_TRACE_FENTRY:
23072 		case BPF_TRACE_FEXIT:
23073 		case BPF_MODIFY_RETURN:
23074 		case BPF_TRACE_ITER:
23075 		case BPF_TRACE_FSESSION:
23076 		case BPF_TRACE_RAW_TP:
23077 		case BPF_TRACE_FENTRY_MULTI:
23078 		case BPF_TRACE_FEXIT_MULTI:
23079 		case BPF_TRACE_FSESSION_MULTI:
23080 			return true;
23081 		default:
23082 			return false;
23083 		}
23084 	}
23085 	if (prog->type == BPF_PROG_TYPE_LSM)
23086 		return prog->expected_attach_type != BPF_LSM_CGROUP;
23087 
23088 	return prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ ||
23089 	       prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
23090 	       prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT ||
23091 	       prog->type == BPF_PROG_TYPE_TRACEPOINT;
23092 }
23093 
23094 static int check_attach_btf_id(struct bpf_verifier_env *env)
23095 {
23096 	struct bpf_prog *prog = env->prog;
23097 	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
23098 	struct bpf_attach_target_info tgt_info = {};
23099 	u32 btf_id = prog->aux->attach_btf_id;
23100 	struct bpf_trampoline *tr;
23101 	int ret;
23102 	u64 key;
23103 
23104 	if (prog->type == BPF_PROG_TYPE_SYSCALL) {
23105 		if (prog->sleepable)
23106 			/* attach_btf_id checked to be zero already */
23107 			return 0;
23108 		verbose(env, "Syscall programs can only be sleepable\n");
23109 		return -EINVAL;
23110 	}
23111 
23112 	if (prog->sleepable && !can_be_sleepable(prog)) {
23113 		verbose(env, "Program of this type cannot be sleepable\n");
23114 		return -EINVAL;
23115 	}
23116 
23117 	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
23118 		return check_struct_ops_btf_id(env);
23119 
23120 	if (prog->type != BPF_PROG_TYPE_TRACING &&
23121 	    prog->type != BPF_PROG_TYPE_LSM &&
23122 	    prog->type != BPF_PROG_TYPE_EXT)
23123 		return 0;
23124 
23125 	ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);
23126 	if (ret)
23127 		return ret;
23128 
23129 	if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) {
23130 		/* to make freplace equivalent to their targets, they need to
23131 		 * inherit env->ops and expected_attach_type for the rest of the
23132 		 * verification
23133 		 */
23134 		env->ops = bpf_verifier_ops[tgt_prog->type];
23135 		prog->expected_attach_type = tgt_prog->expected_attach_type;
23136 
23137 		/* only now it is known what the arguments of prog stand for */
23138 		ret = btf_prepare_func_args(env, 0);
23139 		if (ret)
23140 			return ret;
23141 		ret = check_ext_args(&env->log, prog, tgt_prog,
23142 				     find_tgt_subprog(tgt_prog, btf_id));
23143 		if (ret)
23144 			return ret;
23145 	}
23146 
23147 	/* store info about the attachment target that will be used later */
23148 	prog->aux->attach_func_proto = tgt_info.tgt_type;
23149 	prog->aux->attach_func_name = tgt_info.tgt_name;
23150 	prog->aux->mod = tgt_info.tgt_mod;
23151 
23152 	if (tgt_prog) {
23153 		prog->aux->saved_dst_prog_type = tgt_prog->type;
23154 		prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type;
23155 	}
23156 
23157 	if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
23158 		prog->aux->attach_btf_trace = true;
23159 		return 0;
23160 	} else if (prog->expected_attach_type == BPF_TRACE_ITER) {
23161 		return bpf_iter_prog_supported(prog);
23162 	}
23163 
23164 	if (prog->type == BPF_PROG_TYPE_LSM) {
23165 		ret = bpf_lsm_verify_prog(&env->log, prog);
23166 		if (ret < 0)
23167 			return ret;
23168 	} else if (prog->type == BPF_PROG_TYPE_TRACING &&
23169 		   btf_id_set_contains(&btf_id_deny, btf_id)) {
23170 		verbose(env, "Attaching tracing programs to function '%s' is rejected.\n",
23171 			tgt_info.tgt_name);
23172 		return -EINVAL;
23173 	} else if ((prog->expected_attach_type == BPF_TRACE_FEXIT ||
23174 		   prog->expected_attach_type == BPF_TRACE_FSESSION ||
23175 		   prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI ||
23176 		   prog->expected_attach_type == BPF_MODIFY_RETURN) &&
23177 		   btf_id_set_contains(&noreturn_deny, btf_id)) {
23178 		verbose(env, "Attaching fexit/fsession/fmod_ret to __noreturn function '%s' is rejected.\n",
23179 			tgt_info.tgt_name);
23180 		return -EINVAL;
23181 	}
23182 
23183 	/*
23184 	 * We don't get trampoline for tracing_multi programs at this point,
23185 	 * it's done when tracing_multi link is created.
23186 	 */
23187 	if (prog->type == BPF_PROG_TYPE_TRACING &&
23188 	    is_tracing_multi(prog->expected_attach_type))
23189 		return 0;
23190 
23191 	key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
23192 	tr = bpf_trampoline_get(key, &tgt_info);
23193 	if (!tr)
23194 		return -ENOMEM;
23195 
23196 	if (tgt_prog && tgt_prog->aux->tail_call_reachable)
23197 		bpf_trampoline_set_flags(tr, BPF_TRAMP_F_TAIL_CALL_CTX);
23198 
23199 	prog->aux->dst_trampoline = tr;
23200 	return 0;
23201 }
23202 
23203 int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 btf_id,
23204 				  struct bpf_attach_target_info *tgt_info)
23205 {
23206 	const struct btf_type *t;
23207 	unsigned long addr;
23208 	const char *tname;
23209 	int err;
23210 
23211 	if (!btf_id || !btf)
23212 		return -EINVAL;
23213 
23214 	/* Check noreturn attachment. */
23215 	if ((prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI ||
23216 	     prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) &&
23217 	     btf_id_set_contains(&noreturn_deny, btf_id))
23218 		return -EINVAL;
23219 	/* Check denied attachment. */
23220 	if (btf_id_set_contains(&btf_id_deny, btf_id))
23221 		return -EINVAL;
23222 
23223 	/* Check and get function target data. */
23224 	t = btf_type_by_id(btf, btf_id);
23225 	if (!t)
23226 		return -EINVAL;
23227 	tname = btf_name_by_offset(btf, t->name_off);
23228 	if (!tname)
23229 		return -EINVAL;
23230 	t = btf_attach_func_proto(NULL, btf, btf_id);
23231 	if (!t || !btf_type_is_func_proto(t))
23232 		return -EINVAL;
23233 	err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel);
23234 	if (err < 0)
23235 		return err;
23236 	if (tgt_info->fmodel.ret_size > 8 &&
23237 	    attach_uses_trampoline_retval(prog->expected_attach_type))
23238 		return -EOPNOTSUPP;
23239 	if (btf_is_module(btf)) {
23240 		/* The bpf program already holds reference to module. */
23241 		if (WARN_ON_ONCE(!prog->aux->mod))
23242 			return -EINVAL;
23243 		addr = find_kallsyms_symbol_value(prog->aux->mod, tname);
23244 	} else {
23245 		addr = kallsyms_lookup_name(tname);
23246 	}
23247 	if (!addr || !ftrace_location(addr))
23248 		return -ENOENT;
23249 
23250 	/* Check sleepable program attachment. */
23251 	if (prog->sleepable) {
23252 		err = btf_id_allow_sleepable(btf_id, addr, prog, btf);
23253 		if (err)
23254 			return err;
23255 	}
23256 	tgt_info->tgt_addr = addr;
23257 	return 0;
23258 }
23259 
23260 struct btf *bpf_get_btf_vmlinux(void)
23261 {
23262 	/* Pairs with the smp_store_release() on the parse path below. */
23263 	struct btf *btf = smp_load_acquire(&btf_vmlinux);
23264 
23265 	if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
23266 		mutex_lock(&btf_vmlinux_lock);
23267 		btf = btf_vmlinux;
23268 		if (!btf) {
23269 			btf = btf_parse_vmlinux();
23270 			/*
23271 			 * Order the parsed BTF contents and the globals the
23272 			 * parse populated (e.g. bpf_ctx_convert.t) before
23273 			 * the pointer publication. Pairs with the acquire
23274 			 * on the lockless fast path above.
23275 			 */
23276 			smp_store_release(&btf_vmlinux, btf);
23277 		}
23278 		mutex_unlock(&btf_vmlinux_lock);
23279 	}
23280 	return btf;
23281 }
23282 
23283 /*
23284  * The add_fd_from_fd_array() is executed only if fd_array_cnt is non-zero. In
23285  * this case expect that every file descriptor in the array is either a map or
23286  * a BTF. Everything else is considered to be trash.
23287  */
23288 static int add_fd_from_fd_array(struct bpf_verifier_env *env, u32 idx, int fd)
23289 {
23290 	struct bpf_map *map;
23291 	struct btf *btf;
23292 	CLASS(fd, f)(fd);
23293 	int err;
23294 
23295 	map = __bpf_map_get(f);
23296 	if (!IS_ERR(map)) {
23297 		err = __add_used_map(env, map);
23298 		if (err < 0)
23299 			return err;
23300 		fd_slot_set_map(&env->fd_array[idx], map);
23301 		return 0;
23302 	}
23303 
23304 	btf = __btf_get_by_fd(f);
23305 	if (!IS_ERR(btf)) {
23306 		btf_get(btf);
23307 		err = __add_used_btf(env, btf);
23308 		if (err < 0)
23309 			return err;
23310 		fd_slot_set_btf(&env->fd_array[idx], btf);
23311 		return 0;
23312 	}
23313 
23314 	verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd);
23315 	return PTR_ERR(map);
23316 }
23317 
23318 /*
23319  * A continuous fd_array is resolved into an in-memory cache with one slot
23320  * per entry. The bound here is deliberately generous and not derived from
23321  * the per-program object limits: Duplicate entries /are/ permitted, and
23322  * the number of distinct maps and BTFs a program can bind is enforced when
23323  * each entry is resolved by __add_used_map() and __add_used_btf().
23324  */
23325 #define MAX_FD_ARRAY_CNT 4096
23326 
23327 static int process_fd_array_continuous(struct bpf_verifier_env *env,
23328 				       bpfptr_t fd_array, u32 cnt)
23329 {
23330 	int fd, ret;
23331 	u32 i;
23332 
23333 	if (cnt > MAX_FD_ARRAY_CNT) {
23334 		verbose(env, "fd_array has too many entries (%u, max %u)\n",
23335 			cnt, MAX_FD_ARRAY_CNT);
23336 		return -E2BIG;
23337 	}
23338 
23339 	env->fd_array = kvzalloc_objs(*env->fd_array, cnt, GFP_KERNEL_ACCOUNT);
23340 	if (!env->fd_array)
23341 		return -ENOMEM;
23342 	env->fd_array_cnt = cnt;
23343 	for (i = 0; i < cnt; i++) {
23344 		if (copy_from_bpfptr_offset(&fd, fd_array,
23345 					    (size_t)i * sizeof(fd), sizeof(fd)))
23346 			return -EFAULT;
23347 		ret = add_fd_from_fd_array(env, i, fd);
23348 		if (ret)
23349 			return ret;
23350 	}
23351 	return 0;
23352 }
23353 
23354 static int process_fd_array(struct bpf_verifier_env *env,
23355 			    union bpf_attr *attr, bpfptr_t uattr)
23356 {
23357 	bpfptr_t fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
23358 
23359 	if (bpfptr_is_null(fd_array)) {
23360 		if (attr->fd_array_cnt) {
23361 			verbose(env, "fd_array_cnt %u without fd_array is invalid\n",
23362 				attr->fd_array_cnt);
23363 			return -EINVAL;
23364 		}
23365 		return 0;
23366 	}
23367 	/*
23368 	 * New API: the caller passes fd_array_cnt and a continuous array that
23369 	 * is resolved and bound up front. Legacy API (no fd_array_cnt): keep
23370 	 * the caller's array and resolve entries on the spot at each reference.
23371 	 */
23372 	if (attr->fd_array_cnt)
23373 		return process_fd_array_continuous(env, fd_array,
23374 						   attr->fd_array_cnt);
23375 	env->fd_array_raw = fd_array;
23376 	return 0;
23377 }
23378 
23379 /* replace a generic kfunc with a specialized version if necessary */
23380 static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc, int insn_idx)
23381 {
23382 	struct bpf_prog *prog = env->prog;
23383 	bool seen_direct_write;
23384 	void *xdp_kfunc;
23385 	bool is_rdonly;
23386 	u32 func_id = desc->func_id;
23387 	u16 offset = desc->offset;
23388 	unsigned long addr = desc->addr;
23389 
23390 	if (offset) /* return if module BTF is used */
23391 		return 0;
23392 
23393 	if (bpf_dev_bound_kfunc_id(func_id)) {
23394 		xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id);
23395 		if (xdp_kfunc)
23396 			addr = (unsigned long)xdp_kfunc;
23397 		/* fallback to default kfunc when not supported by netdev */
23398 	} else if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
23399 		seen_direct_write = env->seen_direct_write;
23400 		is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
23401 
23402 		if (is_rdonly)
23403 			addr = (unsigned long)bpf_dynptr_from_skb_rdonly;
23404 
23405 		/* restore env->seen_direct_write to its original value, since
23406 		 * may_access_direct_pkt_data mutates it
23407 		 */
23408 		env->seen_direct_write = seen_direct_write;
23409 	} else if (func_id == special_kfunc_list[KF_bpf_set_dentry_xattr]) {
23410 		if (bpf_lsm_has_d_inode_locked(prog))
23411 			addr = (unsigned long)bpf_set_dentry_xattr_locked;
23412 	} else if (func_id == special_kfunc_list[KF_bpf_remove_dentry_xattr]) {
23413 		if (bpf_lsm_has_d_inode_locked(prog))
23414 			addr = (unsigned long)bpf_remove_dentry_xattr_locked;
23415 	} else if (func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) {
23416 		if (!env->insn_aux_data[insn_idx].non_sleepable)
23417 			addr = (unsigned long)bpf_dynptr_from_file_sleepable;
23418 	} else if (func_id == special_kfunc_list[KF_bpf_arena_alloc_pages]) {
23419 		if (env->insn_aux_data[insn_idx].non_sleepable)
23420 			addr = (unsigned long)bpf_arena_alloc_pages_non_sleepable;
23421 	} else if (func_id == special_kfunc_list[KF_bpf_arena_free_pages]) {
23422 		if (env->insn_aux_data[insn_idx].non_sleepable)
23423 			addr = (unsigned long)bpf_arena_free_pages_non_sleepable;
23424 	}
23425 	desc->addr = addr;
23426 	return 0;
23427 }
23428 
23429 static int add_kfunc_desc_target(struct bpf_verifier_env *env,
23430 				 const struct bpf_kfunc_desc *target_desc,
23431 				 u16 base_desc_idx, u16 *desc_idx)
23432 {
23433 	struct bpf_kfunc_desc desc = *target_desc;
23434 	struct bpf_kfunc_desc_tab *new_tab;
23435 	struct bpf_kfunc_desc_tab *tab;
23436 	struct bpf_prog_aux *prog_aux;
23437 	u32 i;
23438 
23439 	prog_aux = env->prog->aux;
23440 	tab = prog_aux->kfunc_tab;
23441 
23442 	if (base_desc_idx >= tab->nr_base_descs) {
23443 		verifier_bug(env, "kfunc desc_idx %d out of bounds (%d descriptors)",
23444 				base_desc_idx, tab->nr_base_descs);
23445 		return -EFAULT;
23446 	}
23447 	if (tab->descs[base_desc_idx].func_id != desc.func_id) {
23448 		verifier_bug(env, "kfunc desc %d has invalid id (expected %d, got %d)",
23449 				base_desc_idx, tab->descs[base_desc_idx].func_id, desc.func_id);
23450 		return -EFAULT;
23451 	}
23452 	if (tab->descs[base_desc_idx].offset != desc.offset) {
23453 		verifier_bug(env, "kfunc desc %d has invalid offset (expected %d, got %d)",
23454 				base_desc_idx, tab->descs[base_desc_idx].offset, desc.offset);
23455 		return -EFAULT;
23456 	}
23457 
23458 	if (tab->descs[base_desc_idx].addr == desc.addr) {
23459 		*desc_idx = base_desc_idx;
23460 		return 0;
23461 	}
23462 
23463 	for (i = tab->nr_base_descs; i < tab->nr_descs; i++) {
23464 		if (tab->descs[i].func_id == desc.func_id &&
23465 		    tab->descs[i].offset == desc.offset &&
23466 		    tab->descs[i].addr == desc.addr) {
23467 			*desc_idx = i;
23468 			return 0;
23469 		}
23470 	}
23471 
23472 	if (tab->nr_descs == MAX_KFUNC_CALL_DESCS) {
23473 		verbose(env, "too many different kernel function call targets\n");
23474 		return -E2BIG;
23475 	}
23476 
23477 	new_tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1),
23478 			   GFP_KERNEL_ACCOUNT);
23479 	if (!new_tab)
23480 		return -ENOMEM;
23481 	tab = new_tab;
23482 	prog_aux->kfunc_tab = tab;
23483 
23484 	*desc_idx = tab->nr_descs;
23485 	tab->descs[tab->nr_descs++] = desc;
23486 	return 0;
23487 }
23488 
23489 static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
23490 					    u16 struct_meta_reg,
23491 					    u16 node_offset_reg,
23492 					    struct bpf_insn *insn,
23493 					    struct bpf_insn *insn_buf,
23494 					    int *cnt)
23495 {
23496 	struct btf_struct_meta *kptr_struct_meta = insn_aux->kptr_struct_meta;
23497 	struct bpf_insn addr[2] = { BPF_LD_IMM64(struct_meta_reg, (long)kptr_struct_meta) };
23498 
23499 	insn_buf[0] = addr[0];
23500 	insn_buf[1] = addr[1];
23501 	insn_buf[2] = BPF_MOV64_IMM(node_offset_reg, insn_aux->insert_off);
23502 	insn_buf[3] = *insn;
23503 	*cnt = 4;
23504 }
23505 
23506 int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
23507 		     struct bpf_insn *insn_buf, int insn_idx, int *cnt)
23508 {
23509 	struct bpf_kfunc_desc desc_copy;
23510 	struct bpf_kfunc_desc *desc;
23511 	unsigned long call_imm;
23512 	u16 base_desc_idx, desc_idx;
23513 	bool near_call;
23514 	int err;
23515 
23516 	if (!insn->imm) {
23517 		verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
23518 		return -EINVAL;
23519 	}
23520 
23521 	*cnt = 0;
23522 
23523 	/* insn->imm has the btf func_id. Replace it with an offset relative to
23524 	 * __bpf_call_base, unless the JIT needs to call functions that are
23525 	 * further than 32 bits away (bpf_jit_supports_far_kfunc_call()).
23526 	 */
23527 	desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
23528 	if (!desc) {
23529 		verifier_bug(env, "kernel function descriptor not found for func_id %u",
23530 			     insn->imm);
23531 		return -EFAULT;
23532 	}
23533 	base_desc_idx = desc - env->prog->aux->kfunc_tab->descs;
23534 
23535 	near_call = !bpf_jit_supports_far_kfunc_call();
23536 	desc_copy = *desc;
23537 	desc = &desc_copy;
23538 
23539 	err = specialize_kfunc(env, desc, insn_idx);
23540 	if (err)
23541 		return err;
23542 
23543 	if (near_call) {
23544 		call_imm = BPF_CALL_IMM(desc->addr);
23545 		if ((unsigned long)(s32)call_imm != call_imm) {
23546 			verbose(env, "address of kernel func_id %u is out of range\n",
23547 				desc->func_id);
23548 			return -EINVAL;
23549 		}
23550 		insn->imm = call_imm;
23551 	}
23552 
23553 	err = add_kfunc_desc_target(env, desc, base_desc_idx, &desc_idx);
23554 	if (err)
23555 		return err;
23556 	insn->off = desc_idx;
23557 
23558 	if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) {
23559 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
23560 		struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };
23561 		u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size;
23562 
23563 		if (is_bpf_percpu_obj_new_kfunc(desc->func_id) && kptr_struct_meta) {
23564 			verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d",
23565 				     insn_idx);
23566 			return -EFAULT;
23567 		}
23568 
23569 		insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size);
23570 		insn_buf[1] = addr[0];
23571 		insn_buf[2] = addr[1];
23572 		insn_buf[3] = *insn;
23573 		*cnt = 4;
23574 	} else if (is_bpf_obj_drop_kfunc(desc->func_id) ||
23575 		   is_bpf_percpu_obj_drop_kfunc(desc->func_id) ||
23576 		   is_bpf_refcount_acquire_kfunc(desc->func_id)) {
23577 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
23578 		struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };
23579 
23580 		if (is_bpf_percpu_obj_drop_kfunc(desc->func_id) && kptr_struct_meta) {
23581 			verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d",
23582 				     insn_idx);
23583 			return -EFAULT;
23584 		}
23585 
23586 		if (is_bpf_refcount_acquire_kfunc(desc->func_id) && !kptr_struct_meta) {
23587 			verifier_bug(env, "kptr_struct_meta expected at insn_idx %d",
23588 				     insn_idx);
23589 			return -EFAULT;
23590 		}
23591 
23592 		insn_buf[0] = addr[0];
23593 		insn_buf[1] = addr[1];
23594 		insn_buf[2] = *insn;
23595 		*cnt = 3;
23596 	} else if (is_bpf_list_push_kfunc(desc->func_id) ||
23597 		   is_bpf_rbtree_add_kfunc(desc->func_id)) {
23598 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
23599 		int struct_meta_reg = BPF_REG_3;
23600 		int node_offset_reg = BPF_REG_4;
23601 
23602 		/* list_add/rbtree_add have an extra arg (prev/less),
23603 		 * so args-to-fixup are in diff regs.
23604 		 */
23605 		if (desc->func_id == special_kfunc_list[KF_bpf_list_add] ||
23606 		    is_bpf_rbtree_add_kfunc(desc->func_id)) {
23607 			struct_meta_reg = BPF_REG_4;
23608 			node_offset_reg = BPF_REG_5;
23609 		}
23610 
23611 		if (!kptr_struct_meta) {
23612 			verifier_bug(env, "kptr_struct_meta expected at insn_idx %d",
23613 				     insn_idx);
23614 			return -EFAULT;
23615 		}
23616 
23617 		__fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg,
23618 						node_offset_reg, insn, insn_buf, cnt);
23619 	} else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
23620 		   desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
23621 		insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
23622 		*cnt = 1;
23623 	} else if (desc->func_id == special_kfunc_list[KF_bpf_session_is_return] &&
23624 		   (env->prog->expected_attach_type == BPF_TRACE_FSESSION ||
23625 		    env->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) {
23626 
23627 		/*
23628 		 * inline the bpf_session_is_return() for fsession:
23629 		 *   bool bpf_session_is_return(void *ctx)
23630 		 *   {
23631 		 *       return (((u64 *)ctx)[-1] >> BPF_TRAMP_IS_RETURN_SHIFT) & 1;
23632 		 *   }
23633 		 */
23634 		insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
23635 		insn_buf[1] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_0, BPF_TRAMP_IS_RETURN_SHIFT);
23636 		insn_buf[2] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 1);
23637 		*cnt = 3;
23638 	} else if (desc->func_id == special_kfunc_list[KF_bpf_session_cookie] &&
23639 		   (env->prog->expected_attach_type == BPF_TRACE_FSESSION ||
23640 		    env->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) {
23641 		/*
23642 		 * inline bpf_session_cookie() for fsession:
23643 		 *   __u64 *bpf_session_cookie(void *ctx)
23644 		 *   {
23645 		 *       u64 off = (((u64 *)ctx)[-1] >> BPF_TRAMP_COOKIE_INDEX_SHIFT) & 0xFF;
23646 		 *       return &((u64 *)ctx)[-off];
23647 		 *   }
23648 		 */
23649 		insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
23650 		insn_buf[1] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_0, BPF_TRAMP_COOKIE_INDEX_SHIFT);
23651 		insn_buf[2] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF);
23652 		insn_buf[3] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
23653 		insn_buf[4] = BPF_ALU64_REG(BPF_SUB, BPF_REG_0, BPF_REG_1);
23654 		insn_buf[5] = BPF_ALU64_IMM(BPF_NEG, BPF_REG_0, 0);
23655 		*cnt = 6;
23656 	} else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_new]) {
23657 		/* inline bpf_iter_num_new(&it, start, end); R1=&it, R2=start, R3=end */
23658 		int i = 0;
23659 
23660 		/* if (start > end) goto einval; */
23661 		insn_buf[i++] = BPF_JMP32_REG(BPF_JSGT, BPF_REG_2, BPF_REG_3, 8);
23662 		/* r0 = (u32)end - (u32)start; if (r0 > BPF_MAX_LOOPS) goto e2big; */
23663 		insn_buf[i++] = BPF_MOV32_REG(BPF_REG_0, BPF_REG_3);
23664 		insn_buf[i++] = BPF_ALU32_REG(BPF_SUB, BPF_REG_0, BPF_REG_2);
23665 		insn_buf[i++] = BPF_JMP_IMM(BPF_JGT, BPF_REG_0, BPF_MAX_LOOPS, 8);
23666 		/* s->cur = start - 1; s->end = end; return 0; */
23667 		insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_2, -1);
23668 		insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_2, 0);
23669 		insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, 4);
23670 		insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0);
23671 		insn_buf[i++] = BPF_JMP_A(5);
23672 		/* einval: s->cur = s->end = 0; return -EINVAL; */
23673 		insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0);
23674 		insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
23675 		insn_buf[i++] = BPF_JMP_A(2);
23676 		/* e2big: s->cur = s->end = 0; return -E2BIG; */
23677 		insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0);
23678 		insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -E2BIG);
23679 		*cnt = i;
23680 	} else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_next]) {
23681 		/* inline bpf_iter_num_next(&it); R1=&it, returns &s->cur or NULL */
23682 		int i = 0;
23683 
23684 		/* r0 = s->cur + 1; if ((s32)r0 >= s->end) goto done; */
23685 		insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0);
23686 		insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_0, 1);
23687 		insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 4);
23688 		insn_buf[i++] = BPF_JMP32_REG(BPF_JSGE, BPF_REG_0, BPF_REG_2, 3);
23689 		/* s->cur = r0; return &s->cur; */
23690 		insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0);
23691 		insn_buf[i++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
23692 		insn_buf[i++] = BPF_JMP_A(2);
23693 		/* done: s->cur = s->end = 0; return NULL; */
23694 		insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0);
23695 		insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0);
23696 		*cnt = i;
23697 	} else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_destroy]) {
23698 		/* bpf_iter_num_destroy() is a no-op; emit a nop to drop the call */
23699 		insn_buf[0] = BPF_JMP_A(0);
23700 		*cnt = 1;
23701 	}
23702 
23703 	if (env->insn_aux_data[insn_idx].arg_prog) {
23704 		u32 regno = env->insn_aux_data[insn_idx].arg_prog;
23705 		struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(regno, (long)env->prog->aux) };
23706 		int idx = *cnt;
23707 
23708 		insn_buf[idx++] = ld_addrs[0];
23709 		insn_buf[idx++] = ld_addrs[1];
23710 		insn_buf[idx++] = *insn;
23711 		*cnt = idx;
23712 	}
23713 	return 0;
23714 }
23715 
23716 /*
23717  * Upper bound on the PKCS#7 signature blob passed with a program. Comfortably
23718  * above the largest signature the kernel can verify, and far below anything
23719  * that would make rejecting a load expensive. Deliberately a fixed number so
23720  * that what the syscall accepts does not depend on PAGE_SIZE.
23721  */
23722 #define BPF_PROG_MAX_SIGNATURE_SIZE	(64 * 1024)
23723 
23724 static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id)
23725 {
23726 	switch (keyring_id) {
23727 	case 0:
23728 		return BPF_SIG_KEYRING_BUILTIN;
23729 	case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING:
23730 		return BPF_SIG_KEYRING_SECONDARY;
23731 	case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING:
23732 		return BPF_SIG_KEYRING_PLATFORM;
23733 	case KEY_SPEC_BPF_KEYRING:
23734 		return BPF_SIG_KEYRING_BPF;
23735 	default:
23736 		return BPF_SIG_KEYRING_USER;
23737 	}
23738 }
23739 
23740 /*
23741  * Verify the PKCS#7 signature of a loaded program. Called from bpf_check()
23742  * once the program's metadata maps have been resolved into used_maps, so
23743  * the exact maps folded into the signature are the ones the program binds.
23744  *
23745  * The signature covers the instructions followed by the frozen contents of
23746  * each map, in @maps order: insns || map_0 || map_1 || [...]. On success the
23747  * verdict and keyring info are recorded on prog->aux.
23748  */
23749 static int bpf_prog_verify_signature(struct bpf_verifier_env *env,
23750 				     union bpf_attr *attr, bool is_kernel)
23751 {
23752 	bpfptr_t usig = make_bpfptr(attr->signature, is_kernel);
23753 	struct bpf_dynptr_kern sig_ptr, data_ptr;
23754 	struct bpf_prog *prog = env->prog;
23755 	struct bpf_map **maps = env->used_maps;
23756 	struct bpf_key *key = NULL;
23757 	void *sig, *data = NULL;
23758 	u32 map_cnt = env->used_map_cnt;
23759 	u32 i, off, insns_sz;
23760 	u64 data_sz;
23761 	int err = 0;
23762 
23763 	/*
23764 	 * The signature covers the insns as submitted. CO-RE relocations would
23765 	 * have the kernel rewrite them afterwards, with values the loader picks
23766 	 * through the BTF it supplies, see bpf_check_core_relo().
23767 	 */
23768 	if (attr->core_relo_cnt) {
23769 		verbose(env, "signed program cannot carry CO-RE relocations\n");
23770 		return -EACCES;
23771 	}
23772 	if (!attr->signature_size ||
23773 	    attr->signature_size > BPF_PROG_MAX_SIGNATURE_SIZE)
23774 		return -EINVAL;
23775 
23776 	if (system_keyring_id_check(attr->keyring_id) == 0) {
23777 		key = bpf_lookup_system_key(attr->keyring_id);
23778 	} else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) {
23779 		key = bpf_lookup_keyring();
23780 	} else if (bpf_keyring_enforced()) {
23781 		verbose(env, "caller-supplied keyring refused, use bpf keyring\n");
23782 		return -EPERM;
23783 	} else {
23784 		key = bpf_lookup_user_key(attr->keyring_id, 0);
23785 	}
23786 	if (!key) {
23787 		if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) {
23788 			verbose(env, "the bpf keyring is empty or has not been restricted\n");
23789 			return -ENOKEY;
23790 		}
23791 		verbose(env, "cannot resolve signing keyring with keyring_id %d\n",
23792 			attr->keyring_id);
23793 		return -EINVAL;
23794 	}
23795 
23796 	sig = kvmemdup_bpfptr(usig, attr->signature_size);
23797 	if (IS_ERR(sig)) {
23798 		bpf_key_put(key);
23799 		return PTR_ERR(sig);
23800 	}
23801 
23802 	insns_sz = prog->len * sizeof(struct bpf_insn);
23803 	data_sz = insns_sz;
23804 	for (i = 0; i < map_cnt; i++) {
23805 		struct bpf_map *map = maps[i];
23806 
23807 		if (map->map_type != BPF_MAP_TYPE_ARRAY ||
23808 		    !map->ops->map_direct_value_addr) {
23809 			verbose(env, "signed program metadata map '%s' must be an array\n",
23810 				map->name);
23811 			err = -EINVAL;
23812 			goto out;
23813 		}
23814 		if (!READ_ONCE(map->frozen)) {
23815 			verbose(env, "signed program metadata map '%s' must be frozen\n",
23816 				map->name);
23817 			err = -EPERM;
23818 			goto out;
23819 		}
23820 		if (bpf_map_write_active(map)) {
23821 			verbose(env, "signed program metadata map '%s' has active writers\n",
23822 				map->name);
23823 			err = -EBUSY;
23824 			goto out;
23825 		}
23826 		if (!map->excl_prog_sha) {
23827 			verbose(env, "signed program metadata map '%s' must be exclusive\n",
23828 				map->name);
23829 			err = -EPERM;
23830 			goto out;
23831 		}
23832 		data_sz += map->value_size;
23833 	}
23834 	if (bpf_dynptr_check_size(data_sz)) {
23835 		verbose(env, "signed payload too large: %llu bytes\n", data_sz);
23836 		err = -E2BIG;
23837 		goto out;
23838 	}
23839 	data = kvmalloc(data_sz, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
23840 	if (!data) {
23841 		err = -ENOMEM;
23842 		goto out;
23843 	}
23844 	memcpy(data, prog->insnsi, insns_sz);
23845 	off = insns_sz;
23846 	for (i = 0; i < map_cnt; i++) {
23847 		struct bpf_map *map = maps[i];
23848 		u64 addr;
23849 
23850 		err = map->ops->map_direct_value_addr(map, &addr, 0);
23851 		if (err) {
23852 			verbose(env, "failed to read signed metadata map '%s': %d\n",
23853 				map->name, err);
23854 			goto out;
23855 		}
23856 		memcpy(data + off, (void *)(unsigned long)addr,
23857 		       map->value_size);
23858 		off += map->value_size;
23859 	}
23860 
23861 	bpf_dynptr_init(&data_ptr, data, BPF_DYNPTR_TYPE_LOCAL, 0, data_sz);
23862 	bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0,
23863 			attr->signature_size);
23864 
23865 	err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&data_ptr,
23866 					 (struct bpf_dynptr *)&sig_ptr, key);
23867 	if (err) {
23868 		verbose(env, "signature verification failed: %d\n", err);
23869 	} else {
23870 		verbose(env, "signature verification passed\n");
23871 		prog->aux->sig.keyring_serial = bpf_key_serial(key);
23872 		prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id);
23873 		prog->aux->sig.verdict = BPF_SIG_VERIFIED;
23874 	}
23875 out:
23876 	kvfree(data);
23877 	bpf_key_put(key);
23878 	kvfree(sig);
23879 	return err;
23880 }
23881 
23882 /*
23883  * bpf_clone_redirect() transmits from inside the program, so a tc egress or
23884  * lwt_xmit program that redirects to its own device runs again on top of its
23885  * own frame, and again from there, until the datapath's recursion limit drops
23886  * the packet: ten frames deep. Ten MAX_BPF_STACK frames fit the kernel stack
23887  * as they always did; ten MAX_BPF_STACK_JIT frames would not, so a program
23888  * calling it keeps the smaller budget. The redirect helpers that transmit
23889  * after the program has returned leave no frame behind.
23890  */
23891 static bool bpf_prog_reenters_datapath(const struct bpf_prog *prog)
23892 {
23893 	const struct bpf_insn *insn = prog->insnsi;
23894 	int i;
23895 
23896 	for (i = 0; i < prog->len; i++, insn++)
23897 		if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_clone_redirect)
23898 			return true;
23899 	return false;
23900 }
23901 
23902 /* Various log level 2 information about the program */
23903 static void log_program(struct bpf_verifier_env *env)
23904 {
23905 	struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;
23906 	struct bpf_insn *insns = env->prog->insnsi;
23907 	u32 insn_cnt = env->prog->len;
23908 	u64 pos, insn_pos;
23909 	u32 i, j;
23910 
23911 	verbose(env, "Program dump (scc? loop_header? idom insn#: live_regs_before):\n");
23912 	for (i = 0; i < insn_cnt; ++i) {
23913 		verbose_linfo(env, i, "    ; ");
23914 		if (env->insn_aux_data[i].scc)
23915 			verbose(env, "%3d ", env->insn_aux_data[i].scc);
23916 		else
23917 			verbose(env, "    ");
23918 		if (env->insn_aux_data[i].loop_header >= 0)
23919 			verbose(env, "%3d ", env->insn_aux_data[i].loop_header);
23920 		else
23921 			verbose(env, "    ");
23922 		verbose(env, "%3d ", env->idoms[i]);
23923 		verbose(env, "%3d: ", i);
23924 		for (j = BPF_REG_0; j < BPF_REG_10; ++j)
23925 			if (insn_aux[i].live_regs_before & BIT(j))
23926 				verbose(env, "%d", j);
23927 			else
23928 				verbose(env, ".");
23929 		verbose(env, " ");
23930 		pos = env->log.end_pos;
23931 		bpf_verbose_insn(env, &insns[i]);
23932 		insn_pos = env->log.end_pos;
23933 		if (insn_aux[i].zext_dst)
23934 			verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' ');
23935 		verbose(env, "\n");
23936 		if (bpf_is_ldimm64(&insns[i]))
23937 			i++;
23938 	}
23939 }
23940 
23941 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
23942 	      struct bpf_log_attr *attr_log)
23943 {
23944 	u64 start_time = ktime_get_ns();
23945 	struct bpf_verifier_env *env;
23946 	int i, len, ret = -EINVAL, err;
23947 	bool is_priv;
23948 
23949 	BTF_TYPE_EMIT(enum bpf_features);
23950 
23951 	/* no program is valid */
23952 	if (ARRAY_SIZE(bpf_verifier_ops) == 0)
23953 		return -EINVAL;
23954 
23955 	/* 'struct bpf_verifier_env' can be global, but since it's not small,
23956 	 * allocate/free it every time bpf_check() is called
23957 	 */
23958 	env = kvzalloc_obj(struct bpf_verifier_env, GFP_KERNEL_ACCOUNT);
23959 	if (!env)
23960 		return -ENOMEM;
23961 
23962 	env->bt.env = env;
23963 	env->prog = *prog;
23964 	env->ops = bpf_verifier_ops[env->prog->type];
23965 	env->stack_limit = bpf_prog_stack_limit(env->prog);
23966 	if (bpf_prog_reenters_datapath(env->prog))
23967 		env->stack_limit = MAX_BPF_STACK;
23968 
23969 	env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token);
23970 	env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token);
23971 	env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token);
23972 	env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token);
23973 	env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF);
23974 	env->signature = attr->signature;
23975 
23976 	/* user could have requested verbose verifier output
23977 	 * and supplied buffer to store the verification trace
23978 	 */
23979 	ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size);
23980 	if (ret)
23981 		goto err_free_env;
23982 	ret = bpf_diag_init(env);
23983 	if (ret)
23984 		goto err_prep;
23985 	if (env->prog->insnsi[env->prog->len - 1].code == (BPF_LD | BPF_IMM | BPF_DW)) {
23986 		verbose(env, "invalid bpf_ld_imm64 insn\n");
23987 		ret = -EINVAL;
23988 		goto err_prep;
23989 	}
23990 
23991 	ret = process_fd_array(env, attr, uattr);
23992 	if (ret)
23993 		goto err_prep;
23994 
23995 	if (env->signature) {
23996 		ret = bpf_prog_verify_signature(env, attr, uattr.is_kernel);
23997 		if (ret)
23998 			goto err_prep;
23999 	}
24000 
24001 	ret = security_bpf_prog_load(env->prog, attr, env->prog->aux->token,
24002 				     uattr.is_kernel);
24003 	if (ret)
24004 		goto err_prep;
24005 
24006 	bpf_get_btf_vmlinux();
24007 
24008 	/* Serialize verification of unprivileged programs. */
24009 	if (!is_priv)
24010 		mutex_lock(&bpf_verifier_lock);
24011 
24012 	len = env->insn_aux_data_len = env->prog->len;
24013 	env->insn_aux_data =
24014 		__vmalloc(array_size(sizeof(struct bpf_insn_aux_data), len),
24015 			  GFP_KERNEL_ACCOUNT | __GFP_ZERO);
24016 	ret = -ENOMEM;
24017 	if (!env->insn_aux_data)
24018 		goto skip_full_check;
24019 	for (i = 0; i < len; i++)
24020 		env->insn_aux_data[i].orig_idx = i;
24021 	env->succ = bpf_iarray_realloc(NULL, 2);
24022 	if (!env->succ)
24023 		goto skip_full_check;
24024 
24025 	mark_verifier_state_clean(env);
24026 
24027 	if (IS_ERR(btf_vmlinux)) {
24028 		/* Either gcc or pahole or kernel are broken. */
24029 		verbose(env, "in-kernel BTF is malformed\n");
24030 		ret = PTR_ERR(btf_vmlinux);
24031 		goto skip_full_check;
24032 	}
24033 
24034 	env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
24035 	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
24036 		env->strict_alignment = true;
24037 	if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
24038 		env->strict_alignment = false;
24039 
24040 	if (is_priv)
24041 		env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
24042 	env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;
24043 
24044 	env->explored_states = kvzalloc_objs(struct list_head,
24045 					     state_htab_size(env),
24046 					     GFP_KERNEL_ACCOUNT);
24047 	ret = -ENOMEM;
24048 	if (!env->explored_states)
24049 		goto skip_full_check;
24050 
24051 	for (i = 0; i < state_htab_size(env); i++)
24052 		INIT_LIST_HEAD(&env->explored_states[i]);
24053 	INIT_LIST_HEAD(&env->free_list);
24054 
24055 	/* Prepare BTF and func_info needed to discover all subprograms. */
24056 	ret = bpf_prepare_btf_info(env, attr, uattr);
24057 	if (ret < 0)
24058 		goto skip_full_check;
24059 
24060 	/* Apply CO-RE before validating the program's instruction layout. */
24061 	ret = bpf_check_core_relo(env, attr, uattr);
24062 	if (ret < 0)
24063 		goto skip_full_check;
24064 
24065 	/* Discover all subprograms and collect the properties needed by BTF validation. */
24066 	ret = add_subprogs(env);
24067 	if (ret < 0)
24068 		goto skip_full_check;
24069 
24070 	find_subprog_properties(env);
24071 
24072 	/* Validate BTF before reporting subprogram layout errors. */
24073 	ret = bpf_check_btf_info(env, attr, uattr);
24074 	if (ret < 0)
24075 		goto skip_full_check;
24076 
24077 	ret = check_subprogs(env);
24078 	if (ret < 0)
24079 		goto skip_full_check;
24080 
24081 	/* Validate instructions and resolve the program's referenced resources. */
24082 	ret = check_and_resolve_insns(env);
24083 	if (ret < 0)
24084 		goto skip_full_check;
24085 
24086 	/* Find pointers to functions in the read-only maps of the program. */
24087 	ret = resolve_func_ptrs(env);
24088 	if (ret < 0)
24089 		goto skip_full_check;
24090 
24091 	/* Build kfunc prototypes after resolving program resources. */
24092 	ret = add_kfuncs(env);
24093 	if (ret < 0)
24094 		goto skip_full_check;
24095 
24096 	if (bpf_prog_is_offloaded(env->prog->aux)) {
24097 		ret = bpf_prog_offload_verifier_prep(env->prog);
24098 		if (ret)
24099 			goto skip_full_check;
24100 	}
24101 
24102 	ret = bpf_check_cfg(env);
24103 	if (ret < 0)
24104 		goto skip_full_check;
24105 
24106 	ret = check_unwind_insns(env);
24107 	if (ret < 0)
24108 		goto skip_full_check;
24109 
24110 	ret = bpf_compute_postorder(env);
24111 	if (ret < 0)
24112 		goto skip_full_check;
24113 
24114 	ret = bpf_stack_liveness_init(env);
24115 	if (ret)
24116 		goto skip_full_check;
24117 
24118 	ret = check_attach_btf_id(env);
24119 	if (ret)
24120 		goto skip_full_check;
24121 
24122 	ret = bpf_compute_const_regs(env);
24123 	if (ret < 0)
24124 		goto skip_full_check;
24125 
24126 	ret = bpf_prune_dead_branches(env);
24127 	if (ret < 0)
24128 		goto skip_full_check;
24129 
24130 	ret = sort_subprogs_topo(env);
24131 	if (ret < 0)
24132 		goto skip_full_check;
24133 
24134 	ret = bpf_compute_scc(env);
24135 	if (ret < 0)
24136 		goto skip_full_check;
24137 
24138 	/* must precede the first bpf_ret_reg_pair() user below */
24139 	ret = bpf_compute_subprog_ret_regs(env);
24140 	if (ret < 0)
24141 		goto skip_full_check;
24142 
24143 	prepare_subprog_args(env);
24144 
24145 	ret = bpf_compute_idoms(env);
24146 	if (ret < 0)
24147 		goto skip_full_check;
24148 
24149 	ret = bpf_compute_loops(env);
24150 	if (ret < 0)
24151 		goto skip_full_check;
24152 
24153 	ret = bpf_compute_live_registers(env);
24154 	if (ret < 0)
24155 		goto skip_full_check;
24156 
24157 	if (env->log.level & BPF_LOG_LEVEL2)
24158 		log_program(env);
24159 
24160 	ret = bpf_init_scev(env);
24161 	if (ret < 0)
24162 		goto skip_full_check;
24163 
24164 	ret = bpf_compute_scev(env);
24165 	if (ret < 0)
24166 		goto skip_full_check;
24167 
24168 	bpf_free_subprog_jts(env);
24169 
24170 	ret = mark_fastcall_patterns(env);
24171 	if (ret < 0)
24172 		goto skip_full_check;
24173 
24174 	ret = do_check_main(env);
24175 	ret = ret ?: do_check_subprogs(env);
24176 
24177 	/* reject recursion through the callx edges found by the main pass */
24178 	if (ret == 0 && env->callx_edges)
24179 		ret = sort_subprogs_topo(env);
24180 
24181 	if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux))
24182 		ret = bpf_prog_offload_finalize(env);
24183 
24184 skip_full_check:
24185 	kvfree(env->explored_states);
24186 
24187 	/* might decrease stack depth, keep it before passes that
24188 	 * allocate additional slots.
24189 	 */
24190 	if (ret == 0)
24191 		ret = bpf_remove_fastcall_spills_fills(env);
24192 
24193 	if (ret == 0)
24194 		ret = check_max_stack_depth(env);
24195 
24196 	/* instruction rewrites happen after this point */
24197 	if (ret == 0)
24198 		ret = bpf_optimize_bpf_loop(env);
24199 
24200 	if (is_priv) {
24201 		if (ret == 0)
24202 			bpf_opt_hard_wire_dead_code_branches(env);
24203 		if (ret == 0)
24204 			ret = bpf_opt_remove_dead_code(env);
24205 		if (ret == 0)
24206 			ret = bpf_opt_remove_nops(env);
24207 	} else {
24208 		if (ret == 0)
24209 			sanitize_dead_code(env);
24210 	}
24211 
24212 	if (ret == 0)
24213 		/* program is valid, convert *(u32*)(ctx + off) accesses */
24214 		ret = bpf_convert_ctx_accesses(env);
24215 
24216 	if (ret == 0)
24217 		ret = bpf_do_misc_fixups(env);
24218 
24219 	/* do 32-bit optimization after insn patching has done so those patched
24220 	 * insns could be handled correctly.
24221 	 */
24222 	if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) {
24223 		ret = bpf_opt_subreg_zext_lo32_rnd_hi32(env, attr);
24224 		env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret
24225 								     : false;
24226 	}
24227 
24228 	if (ret == 0)
24229 		ret = bpf_fixup_call_args(env);
24230 
24231 	env->verification_time = ktime_get_ns() - start_time;
24232 	print_verification_stats(env);
24233 	env->prog->aux->verified_insns = env->insn_processed;
24234 
24235 	/* preserve original error even if log finalization is successful */
24236 	err = bpf_log_attr_finalize(attr_log, &env->log);
24237 	if (err)
24238 		ret = err;
24239 
24240 	if (ret)
24241 		goto err_release_maps;
24242 
24243 	if (env->used_map_cnt) {
24244 		/* if program passed verifier, update used_maps in bpf_prog_info */
24245 		env->prog->aux->used_maps = kmalloc_objs(env->used_maps[0],
24246 							 env->used_map_cnt,
24247 							 GFP_KERNEL_ACCOUNT);
24248 
24249 		if (!env->prog->aux->used_maps) {
24250 			ret = -ENOMEM;
24251 			goto err_release_maps;
24252 		}
24253 
24254 		memcpy(env->prog->aux->used_maps, env->used_maps,
24255 		       sizeof(env->used_maps[0]) * env->used_map_cnt);
24256 		env->prog->aux->used_map_cnt = env->used_map_cnt;
24257 	}
24258 	if (env->used_btf_cnt) {
24259 		/* if program passed verifier, update used_btfs in bpf_prog_aux */
24260 		env->prog->aux->used_btfs = kmalloc_objs(env->used_btfs[0],
24261 							 env->used_btf_cnt,
24262 							 GFP_KERNEL_ACCOUNT);
24263 		if (!env->prog->aux->used_btfs) {
24264 			ret = -ENOMEM;
24265 			goto err_release_maps;
24266 		}
24267 
24268 		memcpy(env->prog->aux->used_btfs, env->used_btfs,
24269 		       sizeof(env->used_btfs[0]) * env->used_btf_cnt);
24270 		env->prog->aux->used_btf_cnt = env->used_btf_cnt;
24271 	}
24272 	if (env->used_map_cnt || env->used_btf_cnt) {
24273 		/* program is valid. Convert pseudo bpf_ld_imm64 into generic
24274 		 * bpf_ld_imm64 instructions
24275 		 */
24276 		convert_pseudo_ld_imm64(env);
24277 	}
24278 
24279 	adjust_btf_func(env);
24280 
24281 	/* extension progs temporarily inherit the attach_type of their targets
24282 	   for verification purposes, so set it back to zero before returning
24283 	 */
24284 	if (env->prog->type == BPF_PROG_TYPE_EXT)
24285 		env->prog->expected_attach_type = 0;
24286 
24287 	env->prog = __bpf_prog_select_runtime(env, env->prog, &ret);
24288 
24289 err_release_maps:
24290 	if (ret)
24291 		release_insn_arrays(env);
24292 	if (!env->prog->aux->used_maps)
24293 		/* if we didn't copy map pointers into bpf_prog_info, release
24294 		 * them now. Otherwise free_used_maps() will release them.
24295 		 */
24296 		release_maps(env);
24297 	if (!env->prog->aux->used_btfs)
24298 		release_btfs(env);
24299 
24300 	*prog = env->prog;
24301 
24302 	module_put(env->attach_btf_mod);
24303 	if (!is_priv)
24304 		mutex_unlock(&bpf_verifier_lock);
24305 	goto err_free_env;
24306 err_prep:
24307 	err = bpf_log_attr_finalize(attr_log, &env->log);
24308 	if (err)
24309 		ret = err;
24310 	release_insn_arrays(env);
24311 	release_maps(env);
24312 	release_btfs(env);
24313 err_free_env:
24314 	bpf_free_subprog_jts(env);
24315 	if (env->insn_aux_data)
24316 		bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len);
24317 	vfree(env->insn_aux_data);
24318 	kvfree(env->fd_array);
24319 	bpf_free_scev(env);
24320 	bpf_stack_liveness_free(env);
24321 	kvfree(env->cfg.postorder_nums);
24322 	kvfree(env->cfg.insn_postorder);
24323 	kvfree(env->scc_info);
24324 	kvfree(env->succ);
24325 	kvfree(env->gotox_tmp_buf);
24326 	kvfree(env->callx_edges);
24327 	kvfree(env->func_ptrs);
24328 	bpf_diag_free(env);
24329 	kvfree(env->idoms);
24330 	kvfree(env);
24331 	return ret;
24332 }
24333